TSK-476: Bitrix import fixes (#2548)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

common/config/rush/pnpm-lock.yaml

@@ -11768,7 +11768,7 @@ packages:
     dev: false
 
   file:projects/bitrix-resources.tgz_a1d864769aaf53d09b76fe134ab55e60:
-    resolution: {integrity: sha512-Wv/ATljkeBewaCxRNMoYjoX07HOl1PniT0kkDZUrqkxPW+fjqMSB2DrU0D/5CS39UAC4jOtTPV1yMCqz7r5smw==, tarball: file:projects/bitrix-resources.tgz}
+    resolution: {integrity: sha512-vXwCwfcCLy3cb6Dj8p8+XWglgI6aTUdr4sN5xzKqaL5soj1JwmEyWhHvafBJMWDpk8UsJUEGNB5Jpiw24n+q9A==, tarball: file:projects/bitrix-resources.tgz}
     id: file:projects/bitrix-resources.tgz
     name: '@rush-temp/bitrix-resources'
     version: 0.0.0
@@ -11807,13 +11807,14 @@ packages:
     dev: false
 
   file:projects/bitrix.tgz:
-    resolution: {integrity: sha512-LKlmOtIMjMgNCPKafSUsJm10okaehcJvYA4mrIrmNR9lHnbldRg3GlaglyOtXQR1fK3/KY/HTlOKzgP4xFnzpw==, tarball: file:projects/bitrix.tgz}
+    resolution: {integrity: sha512-DdNHQSCosOfpZzQR2nd3BOx33UVHTggxpLaGcJh0Jd4yW7FoKISS7xwBnQbHGtUNczHM9R0pNvKoVdpNNBHhnQ==, tarball: file:projects/bitrix.tgz}
     name: '@rush-temp/bitrix'
     version: 0.0.0
     dependencies:
       '@2bad/bitrix': 2.5.0
       '@rushstack/heft': 0.47.11
       '@types/heft-jest': 1.0.3
+      '@types/qs': 6.9.7
       '@typescript-eslint/eslint-plugin': 5.42.1_d506b9be61cb4ac2646ecbc6e0680464
       '@typescript-eslint/parser': 5.42.1_eslint@8.27.0+typescript@4.8.4
       eslint: 8.27.0
@@ -11821,7 +11822,9 @@ packages:
       eslint-plugin-import: 2.26.0_eslint@8.27.0
       eslint-plugin-n: 15.5.1_eslint@8.27.0
       eslint-plugin-promise: 6.1.1_eslint@8.27.0
+      fast-equals: 2.0.4
       prettier: 2.7.1
+      qs: 6.11.0
       typescript: 4.8.4
     transitivePeerDependencies:
       - supports-color
@@ -13090,7 +13093,7 @@ packages:
     dev: false
 
   file:projects/model-all.tgz_typescript@4.8.4:
-    resolution: {integrity: sha512-AejnJ5pKXrdrHEdXX2AoOtbSX2mx0ceRbISCPyuvr1O9w8VKnmp43TGLOZEKVY3D+dmnjKj1+2+A+bAxMCUsQQ==, tarball: file:projects/model-all.tgz}
+    resolution: {integrity: sha512-KExZMGU3rzrjnf1pAwSlS4n0IEYrCONE9uXtbBQMokbcfQ/3YbeDGcDrvRdl7okj89wE47sUS088LY/22UzUMw==, tarball: file:projects/model-all.tgz}
     id: file:projects/model-all.tgz
     name: '@rush-temp/model-all'
     version: 0.0.0
@@ -14284,7 +14287,7 @@ packages:
     dev: false
 
   file:projects/openai.tgz:
-    resolution: {integrity: sha512-nplFlMK8VBiE2EBm11HhSpX74sZapWZ30sEWOHZfAv8AZqCINY0FOHP0Jwc+AVFtMzClx8NTCo6pgaNTLs0ixA==, tarball: file:projects/openai.tgz}
+    resolution: {integrity: sha512-O+mwmpb/LfabOFEsid8hIWuJpSombaoFJf3xTlH+pmi47h54Pwn8CsSjY3aZ2PdtvVjY5TK5vIQxKHxGNe3YiQ==, tarball: file:projects/openai.tgz}
     name: '@rush-temp/openai'
     version: 0.0.0
     dependencies:
@@ -14539,7 +14542,7 @@ packages:
     dev: false
 
   file:projects/pod-server.tgz:
-    resolution: {integrity: sha512-4IjhfCueH7UYdgcWfpeiciOQADpXCKj1C41rP6ZgHaIWzUErhMl2X/T5C2SQfp0nnbvpPWqWeqSPYYK2GGGkdw==, tarball: file:projects/pod-server.tgz}
+    resolution: {integrity: sha512-1gSQAnD3H3VvnxG3bDk/x+MKlmiR+dk1730YE7SjA0AmWcIaa40QqKPgCLpdhCrtDT/fwr0TFVjr3Bo8Z/vCHQ==, tarball: file:projects/pod-server.tgz}
     name: '@rush-temp/pod-server'
     version: 0.0.0
     dependencies:
@@ -14646,7 +14649,7 @@ packages:
     dev: false
 
   file:projects/prod.tgz_a9366fc5abe1de81e350f3e9b2acb628:
-    resolution: {integrity: sha512-wfzTNxTVOW15y2ZLXjebNfstBDQSIdSevMj0AorZ1SUWUVqQ8ZkO8XOI2wFwQdiGJplBTG0zVIDpZFdYBWF6mg==, tarball: file:projects/prod.tgz}
+    resolution: {integrity: sha512-oHJixn8gWZB93sHdI2epzD4DeR9Ff8y8njVa9Kr+QFk2p7IGH68FCTTA90Q61ysV7k07e/OED1ClxZ5hAthtgA==, tarball: file:projects/prod.tgz}
     id: file:projects/prod.tgz
     name: '@rush-temp/prod'
     version: 0.0.0
@@ -16286,7 +16289,7 @@ packages:
     dev: false
 
   file:projects/tool.tgz:
-    resolution: {integrity: sha512-VVNc6+f2BssTik3I3y+lBrBVkMz6sHygnom2/PvO/ZwEVKeWDN2rwpzZxj6n+RlPeCCvvcII9sXpPOAoJAnxnA==, tarball: file:projects/tool.tgz}
+    resolution: {integrity: sha512-OqWoKCsumcmAHkfsi+zF+Lt6uoamsg5W+/K3SGlPwovAIeo/Tyc+F0Sm9EKUe1OKX0FT9+dcQstcHRigNFSNQA==, tarball: file:projects/tool.tgz}
     name: '@rush-temp/tool'
     version: 0.0.0
     dependencies:

dev/prod/tsconfig.json

@@ -3,7 +3,7 @@
     "outDir": "./dist/",
     "noImplicitAny": true,
     "module": "esnext",
-    "target": "es2016",
+    "target": "es2021",
     "allowJs": true,
     "sourceMap": true,
     "skipLibCheck": true,
@@ -11,7 +11,8 @@
     "allowSyntheticDefaultImports": true,
     "lib": [
       "es2016",
-      "dom"
+      "dom",
+      "ES2021.String"
     ]
   }
 }

packages/presentation/src/components/MessageBox.svelte

@@ -23,8 +23,10 @@
   export let message: IntlString
   export let params: Record<string, any> = {}
   export let canSubmit = true
+  export let action: (() => Promise<void>) | undefined = undefined
 
   const dispatch = createEventDispatcher()
+  let processing = false
 </script>
 
 <div class="msgbox-container">
@@ -36,10 +38,28 @@
       label={presentation.string.Ok}
       size={'small'}
       kind={'primary'}
-      on:click={() => dispatch('close', true)}
+      loading={processing}
+      on:click={() => {
+        processing = true
+        if (action !== undefined) {
+          action().then(() => {
+            processing = false
+            dispatch('close', true)
+          })
+        } else {
+          dispatch('close', true)
+          processing = false
+        }
+      }}
     />
     {#if canSubmit}
-      <Button label={presentation.string.Cancel} size={'small'} on:click={() => dispatch('close', false)} />
+      <Button
+        label={presentation.string.Cancel}
+        size={'small'}
+        on:click={() => {
+          dispatch('close', false)
+        }}
+      />
     {/if}
   </div>
 </div>

packages/presentation/src/components/message/Nodes.svelte

@@ -41,7 +41,7 @@
     {:else if node.nodeName === 'HR'}
       <hr />
     {:else if node.nodeName === 'IMG'}
-      <div class="max-h-60 max-w-60 img">{@html node.outerHTML}</div>
+      <div class="max-h-60 max-w-60">{@html node.outerHTML}</div>
     {:else if node.nodeName === 'H1'}
       <h1><svelte:self nodes={node.childNodes} /></h1>
     {:else if node.nodeName === 'H2'}

packages/presentation/src/utils.ts

@@ -206,7 +206,7 @@ export type AttributeCategory = 'object' | 'attribute' | 'inplace' | 'collection
 /**
  * @public
  */
-export const AttributeCategoryOrder = { attribute: 0, inplace: 1, collection: 2, array: 2 }
+export const AttributeCategoryOrder = { attribute: 0, inplace: 1, collection: 2, array: 2, object: 3 }
 
 /**
  * @public

plugins/attachment-resources/package.json

@@ -46,5 +46,9 @@
     "filesize": "^8.0.3",
     "@hcengineering/preference": "^0.6.2",
     "@hcengineering/activity": "^0.6.0"
+  },
+  "repository": "https://github.com/hcengineering/anticrm",
+  "publishConfig": {
+    "registry": "https://npm.pkg.github.com"
   }
 }

plugins/bitrix-resources/package.json

@@ -27,7 +27,8 @@
     "prettier-plugin-svelte": "^2.8.0",
     "prettier": "^2.7.1",
     "svelte-check": "^2.8.0",
-    "typescript": "^4.3.5"
+    "typescript": "^4.3.5",
+    "@types/qs": "~6.9.7"
   },
   "dependencies": {
     "@hcengineering/platform": "^0.6.8",
@@ -38,6 +39,7 @@
     "@hcengineering/text-editor": "^0.6.0",
     "@hcengineering/contact": "^0.6.9",
     "@hcengineering/lead": "^0.6.0",
+    "@hcengineering/login": "^0.6.1",
     "@hcengineering/setting": "^0.6.2",
     "@hcengineering/core": "^0.6.20",
     "@hcengineering/attachment": "^0.6.1",
@@ -48,9 +50,12 @@
     "@hcengineering/chunter": "^0.6.2",
     "p-queue": "~7.3.0",
     "qs": "~6.11.0",
-    "@types/qs": "~6.9.7",
     "@hcengineering/tags": "^0.6.3",
     "@hcengineering/tags-resources": "^0.6.0",
     "fast-equals": "^2.0.3"
+  },
+  "repository": "https://github.com/hcengineering/anticrm",
+  "publishConfig": {
+    "registry": "https://npm.pkg.github.com"
   }
 }

plugins/bitrix-resources/src/components/BitrixConfigure.svelte

@@ -20,18 +20,18 @@
   import { createEventDispatcher, onMount } from 'svelte'
   import bitrix from '../plugin'
 
-  import { BitrixEntityMapping } from '@hcengineering/bitrix'
+  import { BitrixClient, BitrixEntityMapping, BitrixProfile, StatusValue } from '@hcengineering/bitrix'
   import { Button, eventToHTMLElement, IconAdd, Label, showPopup } from '@hcengineering/ui'
-  import { BitrixClient } from '../client'
+
-  import { BitrixProfile, StatusValue } from '../types'
   import CreateMapping from './CreateMapping.svelte'
   import EntiryMapping from './EntityMapping.svelte'
+  import { bitrixQueue } from '../queue'
 
   export let integration: Integration
 
   const dispatch = createEventDispatcher()
 
-  const bitrixClient = new BitrixClient(integration.value)
+  const bitrixClient = new BitrixClient(integration.value, (op) => bitrixQueue.add(op))
 
   let profile: BitrixProfile | undefined
 

plugins/bitrix-resources/src/components/BitrixFieldLookup.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-  import { BitrixEntityMapping, Fields, FieldValue } from '@hcengineering/bitrix'
+  import { BitrixClient, BitrixEntityMapping, Fields, FieldValue } from '@hcengineering/bitrix'
   import core, { Enum, Ref } from '@hcengineering/core'
   import { getEmbeddedLabel } from '@hcengineering/platform'
   import { createQuery, getClient } from '@hcengineering/presentation'
@@ -16,7 +16,6 @@
     showPopup
   } from '@hcengineering/ui'
   import Grid from '@hcengineering/ui/src/components/Grid.svelte'
-  import { BitrixClient } from '../client'
 
   import EnumPopup from './EnumPopup.svelte'
 

plugins/bitrix-resources/src/components/BitrixImport.svelte

@@ -1,12 +1,12 @@
 <script lang="ts">
-  import { BitrixEntityMapping } from '@hcengineering/bitrix'
+  import { BitrixClient, BitrixEntityMapping } from '@hcengineering/bitrix'
   import { getEmbeddedLabel } from '@hcengineering/platform'
   import { Card, createQuery } from '@hcengineering/presentation'
   import setting, { Integration } from '@hcengineering/setting'
   import { Label } from '@hcengineering/ui'
   import { createEventDispatcher } from 'svelte'
-  import { BitrixClient } from '../client'
   import bitrix from '../plugin'
+  import { bitrixQueue } from '../queue'
   import FieldMappingSynchronizer from './FieldMappingSynchronizer.svelte'
 
   const mappingQuery = createQuery()
@@ -37,7 +37,8 @@
     integration = res.shift()
   })
 
-  $: bitrixClient = integration !== undefined ? new BitrixClient(integration.value) : undefined
+  $: bitrixClient =
+    integration !== undefined ? new BitrixClient(integration.value, (op) => bitrixQueue.add(op)) : undefined
   let loading = false
 </script>
 

plugins/bitrix-resources/src/components/CreateMapping.svelte

@@ -13,9 +13,14 @@
   const client = getClient()
 
   async function save (): Promise<void> {
-    client.createDoc(bitrix.class.EntityMapping, bitrix.space.Mappings, {
+    await client.createDoc<BitrixEntityMapping>(bitrix.class.EntityMapping, bitrix.space.Mappings, {
       ofClass,
-      type
+      type,
+      comments: true,
+      attachments: true,
+      bitrixFields: {},
+      fields: 0,
+      activity: false
     })
   }
 

plugins/bitrix-resources/src/components/EntityMapping.svelte

@@ -1,11 +1,18 @@
 <script lang="ts">
-  import { BitrixEntityMapping, BitrixFieldMapping, Fields, mappingTypes } from '@hcengineering/bitrix'
+  import {
+    BitrixClient,
+    BitrixEntityMapping,
+    BitrixFieldMapping,
+    Fields,
+    mappingTypes,
+    StatusValue,
+    toClassRef
+  } from '@hcengineering/bitrix'
   import { Class, Doc, Ref } from '@hcengineering/core'
   import { getEmbeddedLabel } from '@hcengineering/platform'
   import { createQuery, getClient } from '@hcengineering/presentation'
   import { ClassSetting } from '@hcengineering/setting-resources'
   import { Button, Expandable, Icon, IconDelete, IconEdit, Label, showPopup } from '@hcengineering/ui'
-  import { BitrixClient } from '../client'
   import bitrix from '../plugin'
 
   import AttributeMapper from './AttributeMapper.svelte'
@@ -13,8 +20,6 @@
 
   import CheckBox from '@hcengineering/ui/src/components/CheckBox.svelte'
   import { deepEqual } from 'fast-equals'
-  import { StatusValue } from '../types'
-  import { toClassRef } from '../utils'
   import BitrixFieldLookup from './BitrixFieldLookup.svelte'
   import CreateMappingAttribute from './CreateMappingAttribute.svelte'
 

plugins/bitrix-resources/src/components/FieldMappingSynchronizer.svelte

@@ -1,35 +1,19 @@
 <script lang="ts">
-  import bitrix, {
+  import {
+    BitrixClient,
     BitrixEntityMapping,
-    BitrixEntityType,
     BitrixFieldMapping,
-    BitrixSyncDoc
+    performSynchronization,
+    toClassRef
   } from '@hcengineering/bitrix'
-  import chunter, { Comment } from '@hcengineering/chunter'
+  import contact from '@hcengineering/contact'
-  import contact, { combineName, EmployeeAccount } from '@hcengineering/contact'
+  import core, { Class, Doc, Ref, Space, WithLookup } from '@hcengineering/core'
-  import core, {
+  import login from '@hcengineering/login'
-    AccountRole,
+  import { getEmbeddedLabel, getMetadata } from '@hcengineering/platform'
-    ApplyOperations,
-    AttachedDoc,
-    Class,
-    Data,
-    Doc,
-    DocumentUpdate,
-    generateId,
-    Mixin,
-    Ref,
-    Space,
-    WithLookup
-  } from '@hcengineering/core'
-  import { getEmbeddedLabel } from '@hcengineering/platform'
   import { getClient, SpaceSelect } from '@hcengineering/presentation'
-  import { TagElement } from '@hcengineering/tags'
   import { Button, Expandable, Icon, Label } from '@hcengineering/ui'
   import DropdownLabels from '@hcengineering/ui/src/components/DropdownLabels.svelte'
   import { NumberEditor } from '@hcengineering/view-resources'
-  import { deepEqual } from 'fast-equals'
-  import { BitrixClient } from '../client'
-  import { convert, ConvertResult, toClassRef } from '../utils'
   import FieldMappingPresenter from './FieldMappingPresenter.svelte'
 
   export let mapping: WithLookup<BitrixEntityMapping>
@@ -43,275 +27,38 @@
     return p
   }, {} as Record<Ref<Class<Doc>>, BitrixFieldMapping[]>)
 
-  let direction: 'ASC' | 'DSC' = 'DSC'
+  let direction: 'ASC' | 'DSC' = 'ASC'
-  let limit = 200
+  let limit = 1
   let space: Ref<Space> | undefined
 
   export let loading = false
   let state = ''
 
-  async function updateDoc (client: ApplyOperations, doc: Doc, raw: Doc | Data<Doc>): Promise<void> {
-    // We need to update fields if they are different.
-    const documentUpdate: DocumentUpdate<Doc> = {}
-    for (const [k, v] of Object.entries(raw)) {
-      if (['_class', '_id', 'modifiedBy', 'modifiedOn', 'space'].includes(k)) {
-        continue
-      }
-      if (!deepEqual((doc as any)[k], v)) {
-        ;(documentUpdate as any)[k] = v
-      }
-    }
-    if (Object.keys(documentUpdate).length > 0) {
-      await client.update(doc, documentUpdate)
-    }
-  }
-
   let docsProcessed = 0
-  let total = 0
-
-  async function syncPlatform (documents: ConvertResult[]): Promise<void> {
-    const existingDocuments = await client.findAll(mapping.ofClass, {
-      [bitrix.mixin.BitrixSyncDoc + '.bitrixId']: { $in: documents.map((it) => it.document.bitrixId) }
-    })
-    const hierarchy = client.getHierarchy()
-    for (const d of documents) {
-      const existing = existingDocuments.find(
-        (it) => hierarchy.as(it, bitrix.mixin.BitrixSyncDoc).bitrixId === d.document.bitrixId
-      )
-      const applyOp = client.apply('bitrix')
-      if (existing !== undefined) {
-        // We need to update fields if they are different.
-        await updateDoc(applyOp, existing, d.document)
-
-        // Check and update mixins
-        for (const [m, mv] of Object.entries(d.mixins)) {
-          const mRef = m as Ref<Mixin<Doc>>
-          if (hierarchy.hasMixin(existing, mRef)) {
-            await applyOp.createMixin(
-              d.document._id,
-              d.document._class,
-              d.document.space,
-              m as Ref<Mixin<Doc>>,
-              mv,
-              d.document.modifiedOn,
-              d.document.modifiedBy
-            )
-          } else {
-            const existingM = hierarchy.as(existing, mRef)
-            await updateDoc(applyOp, existingM, mv)
-          }
-        }
-      } else {
-        await applyOp.createDoc(
-          d.document._class,
-          d.document.space,
-          d.document,
-          d.document._id,
-          d.document.modifiedOn,
-          d.document.modifiedBy
-        )
-
-        await applyOp.createMixin<Doc, BitrixSyncDoc>(
-          d.document._id,
-          d.document._class,
-          d.document.space,
-          bitrix.mixin.BitrixSyncDoc,
-          {
-            type: d.document.type,
-            bitrixId: d.document.bitrixId
-          },
-          d.document.modifiedOn,
-          d.document.modifiedBy
-        )
-        for (const [m, mv] of Object.entries(d.mixins)) {
-          await applyOp.createMixin(
-            d.document._id,
-            d.document._class,
-            d.document.space,
-            m as Ref<Mixin<Doc>>,
-            mv,
-            d.document.modifiedOn,
-            d.document.modifiedBy
-          )
-        }
-        for (const ed of d.extraDocs) {
-          if (applyOp.getHierarchy().isDerived(ed._class, core.class.AttachedDoc)) {
-            const adoc = ed as AttachedDoc
-            await applyOp.addCollection(
-              adoc._class,
-              adoc.space,
-              adoc.attachedTo,
-              adoc.attachedToClass,
-              adoc.collection,
-              adoc,
-              adoc._id,
-              d.document.modifiedOn,
-              d.document.modifiedBy
-            )
-          } else {
-            await applyOp.createDoc(ed._class, ed.space, ed, ed._id, d.document.modifiedOn, d.document.modifiedBy)
-          }
-        }
-
-        if (d.comments !== undefined) {
-          const comments = await d.comments
-          if (comments !== undefined && comments.length > 0) {
-            const existingComments = await client.findAll(chunter.class.Comment, {
-              attachedTo: d.document._id,
-              [bitrix.mixin.BitrixSyncDoc + '.bitrixId']: { $in: comments.map((it) => it.bitrixId) }
-            })
-
-            for (const comment of comments) {
-              const existing = existingComments.find(
-                (it) => hierarchy.as<Doc, BitrixSyncDoc>(it, bitrix.mixin.BitrixSyncDoc).bitrixId === comment.bitrixId
-              )
-              if (existing !== undefined) {
-                // We need to update fields if they are different.
-                await updateDoc(applyOp, existing, comment)
-              } else {
-                await applyOp.addCollection(
-                  comment._class,
-                  comment.space,
-                  comment.attachedTo,
-                  comment.attachedToClass,
-                  comment.collection,
-                  comment,
-                  comment._id,
-                  comment.modifiedOn,
-                  comment.modifiedBy
-                )
-
-                await applyOp.createMixin<Doc, BitrixSyncDoc>(
-                  comment._id,
-                  comment._class,
-                  comment.space,
-                  bitrix.mixin.BitrixSyncDoc,
-                  {
-                    type: d.document.type,
-                    bitrixId: d.document.bitrixId
-                  },
-                  comment.modifiedOn,
-                  comment.modifiedBy
-                )
-              }
-            }
-          }
-        }
-      }
-      await applyOp.commit()
-      docsProcessed++
-      state = `processed: ${docsProcessed}/${total}`
-    }
-  }
 
   async function doSync (): Promise<void> {
     loading = true
-
+    const uploadUrl = (window.location.origin + getMetadata(login.metadata.UploadUrl)) as string
-    const commentFields = await bitrixClient.call(BitrixEntityType.Comment + '.fields', {})
+    const token = (getMetadata(login.metadata.LoginToken) as string) ?? ''
-
-    const commentFieldKeys = Object.keys(commentFields.result)
-
-    const allEmployee = await client.findAll(contact.class.EmployeeAccount, {})
-
-    const userList = new Map<string, Ref<EmployeeAccount>>()
-
-    // Fill all users and create new ones, if required.
-    let totalUsers = 1
-    let next = 0
-    while (userList.size < totalUsers) {
-      const users = await bitrixClient.call('user.search', { start: next })
-      next = users.next
-      totalUsers = users.total
-      for (const u of users.result) {
-        let accountId = allEmployee.find((it) => it.email === u.EMAIL)?._id
-        if (accountId === undefined) {
-          const employeeId = await client.createDoc(contact.class.Employee, contact.space.Contacts, {
-            name: combineName(u.NAME, u.LAST_NAME),
-            avatar: u.PERSONAL_PHOTO,
-            active: u.ACTIVE,
-            city: u.PERSONAL_CITY
-          })
-          accountId = await client.createDoc(contact.class.EmployeeAccount, core.space.Model, {
-            email: u.EMAIL,
-            name: combineName(u.NAME, u.LAST_NAME),
-            employee: employeeId,
-            role: AccountRole.User
-          })
-        }
-        userList.set(u.ID, accountId)
-      }
-    }
-
     try {
-      if (space === undefined || mapping.$lookup?.fields === undefined) {
+      await performSynchronization({
-        return
+        bitrixClient,
-      }
+        client,
-      let processed = 0
+        direction,
-      const tagElements: Map<Ref<Class<Doc>>, TagElement[]> = new Map()
+        limit,
-
+        space,
-      let added = 0
+        mapping,
-
+        loginInfo: {
-      while (added <= limit) {
+          token,
-        const sel = ['*', 'UF_*']
+          email: '',
-        if (mapping.type === BitrixEntityType.Lead) {
+          endpoint: ''
-          sel.push('EMAIL')
+        },
-          sel.push('IM')
+        frontUrl: uploadUrl,
+        monitor: (total: number) => {
+          docsProcessed++
+          state = `processed: ${docsProcessed}/${total ?? 1}`
         }
-        const result = await bitrixClient.call(mapping.type + '.list', {
+      })
-          select: sel,
-          order: { ID: direction },
-          start: processed
-        })
-
-        const extraDocs: Doc[] = []
-
-        const convertResults: ConvertResult[] = []
-        const fields = mapping.$lookup?.fields as BitrixFieldMapping[]
-
-        for (const r of result.result) {
-          // Convert documents.
-          const res = await convert(client, mapping, space, fields, r, extraDocs, tagElements, userList)
-          if (mapping.comments) {
-            res.comments = bitrixClient
-              .call(BitrixEntityType.Comment + '.list', {
-                filter: {
-                  ENTITY_ID: res.document.bitrixId,
-                  ENTITY_TYPE: mapping.type.replace('crm.', '')
-                },
-                select: commentFieldKeys,
-                order: { ID: direction }
-              })
-              .then((comments) => {
-                return comments.result.map(
-                  (it: any) =>
-                    ({
-                      _id: generateId(),
-                      _class: chunter.class.Comment,
-                      message: it.COMMENT,
-                      bitrixId: it.ID,
-                      type: it.ENTITY_TYPE,
-                      attachedTo: res.document._id,
-                      attachedToClass: res.document._class,
-                      collection: 'comments',
-                      space: res.document.space,
-                      modifiedBy: userList.get(it.AUTHOR_ID) ?? core.account.System,
-                      modifiedOn: new Date(userList.get(it.CREATED) ?? new Date().toString()).getTime()
-                    } as Comment)
-                )
-              })
-          }
-          convertResults.push(res)
-          extraDocs.push(...res.extraDocs)
-          added++
-          if (added > limit) {
-            break
-          }
-        }
-        total = result.total
-        await syncPlatform(convertResults)
-
-        processed = result.next
-      }
     } catch (err: any) {
       state = err.message
       console.error(err)

plugins/bitrix-resources/src/queue.ts

@@ -0,0 +1,6 @@
+import PQueue from 'p-queue'
+
+export const bitrixQueue = new PQueue({
+  intervalCap: 2,
+  interval: 1000
+})

plugins/bitrix-resources/src/types.ts

@@ -1,45 +0,0 @@
-/**
- * @public
- */
-export interface BitrixProfile {
-  ID: string
-  ADMIN: boolean
-  NAME: string
-  LAST_NAME: string
-  PERSONAL_GENDER: string
-  PERSONAL_PHOTO: string
-  TIME_ZONE: string
-  TIME_ZONE_OFFSET: number
-}
-
-export type NumberString = string
-export type ISODate = string
-export type BoolString = 'Y' | 'N'
-export type GenderString = 'M' | 'F' | ''
-
-export interface MultiField {
-  readonly ID: NumberString
-  readonly VALUE_TYPE: string
-  readonly VALUE: string
-  readonly TYPE_ID: string
-}
-export type MultiFieldArray = ReadonlyArray<Pick<MultiField, 'VALUE' | 'VALUE_TYPE'>>
-
-export interface StatusValue {
-  CATEGORY_ID: string | null
-  COLOR: string | null
-  ENTITY_ID: string | null
-  ID: number
-  NAME: string
-  NAME_INIT: string | null
-  SEMANTICS: string | null
-  SORT: string | null
-  STATUS_ID: string | null
-  SYSTEM: 'Y' | 'N'
-}
-
-export interface BitrixResult {
-  result: any
-  next: number
-  total: number
-}

plugins/bitrix/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hcengineering/bitrix",
-  "version": "0.6.1",
+  "version": "0.6.4",
   "main": "lib/index.js",
   "author": "Anticrm Platform Contributors",
   "license": "EPL-2.0",
@@ -23,15 +23,19 @@
     "eslint-config-standard-with-typescript": "^23.0.0",
     "prettier": "^2.7.1",
     "@rushstack/heft": "^0.47.9",
-    "typescript": "^4.3.5"
+    "typescript": "^4.3.5",    
+    "@types/qs": "~6.9.7"
   },
   "dependencies": {
     "@hcengineering/platform": "^0.6.8",
     "@hcengineering/core": "^0.6.20",
-    "@hcengineering/ui": "^0.6.3",
     "@hcengineering/preference": "^0.6.2",
     "@hcengineering/tags": "^0.6.3",
-    "@hcengineering/contact": "^0.6.9"
+    "@hcengineering/contact": "^0.6.9",
+    "@hcengineering/chunter": "^0.6.2",
+    "@hcengineering/attachment": "^0.6.1",
+    "fast-equals": "^2.0.3",
+    "qs": "~6.11.0"
   },
   "repository": "https://github.com/hcengineering/anticrm",
   "publishConfig": {

plugins/bitrix/src/client.ts

@@ -1,16 +1,16 @@
-import PQueue from 'p-queue'
 import { stringify as toQuery } from 'qs'
 import { BitrixResult } from './types'
 
-const queue = new PQueue({
+/**
-  intervalCap: 2,
+ * @public
-  interval: 1000
+ *
-})
+ * Require a proper rate limiter to function properly.
+ */
 export class BitrixClient {
-  constructor (readonly url: string) {}
+  constructor (readonly url: string, readonly rateLimiter: <T>(op: () => Promise<T>) => Promise<T>) {}
 
   async call (method: string, params: any): Promise<BitrixResult> {
-    return await queue.add(async () => {
+    return await this.rateLimiter(async () => {
       let query: string = toQuery(params)
       if (query.length > 0) {
         query = `?${query}`

plugins/bitrix/src/index.ts

@@ -13,161 +13,10 @@
 // limitations under the License.
 //
 
-import { ChannelProvider } from '@hcengineering/contact'
+import type { Class, Mixin, Ref, Space } from '@hcengineering/core'
-import type { AttachedDoc, Class, Doc, Mixin, Ref, Space } from '@hcengineering/core'
+import type { Plugin, Resource } from '@hcengineering/platform'
-import type { Plugin } from '@hcengineering/platform'
 import { Asset, plugin } from '@hcengineering/platform'
-import { ExpertKnowledge, InitialKnowledge, MeaningfullKnowledge } from '@hcengineering/tags'
+import { BitrixEntityMapping, BitrixFieldMapping, BitrixSyncDoc } from './types'
-import { AnyComponent } from '@hcengineering/ui'
-
-/**
- * @public
- */
-export interface BitrixSyncDoc extends Doc {
-  type: string
-  bitrixId: string
-}
-
-/**
- * @public
- */
-export enum BitrixEntityType {
-  Comment = 'crm.timeline.comment',
-  Binding = 'crm.timeline.bindings',
-  Lead = 'crm.lead',
-  Activity = 'crm.activity',
-  Company = 'crm.company'
-}
-
-/**
- * @public
- */
-export const mappingTypes = [
-  { label: 'Leads', id: BitrixEntityType.Lead },
-  // { label: 'Comments', id: BitrixEntityType.Comment },
-  { label: 'Company', id: BitrixEntityType.Company }
-  // { label: 'Activity', id: BitrixEntityType.Activity }
-]
-
-/**
- * @public
- */
-export interface FieldValue {
-  type: string
-  statusType?: string
-  isRequired: boolean
-  isReadOnly: boolean
-  isImmutable: boolean
-  isMultiple: boolean
-  isDynamic: boolean
-  title: string
-
-  formLabel?: string
-  filterLabel?: string
-  items?: Array<{
-    ID: string
-    VALUE: string
-  }>
-}
-
-/**
- * @public
- */
-export interface Fields {
-  [key: string]: FieldValue
-}
-
-/**
- * @public
- */
-export interface BitrixEntityMapping extends Doc {
-  ofClass: Ref<Class<Doc>>
-  type: string
-  bitrixFields: Fields
-
-  fields: number
-
-  comments: boolean
-  activity: boolean
-  attachments: boolean
-}
-/**
- * @public
- */
-export enum MappingOperation {
-  CopyValue,
-  CreateTag, // Create tag
-  CreateChannel, // Create channel
-  DownloadAttachment
-}
-/**
- * @public
- */
-export interface CopyPattern {
-  text: string
-  field?: string
-  alternatives?: string[]
-}
-/**
- * @public
- */
-export interface CopyValueOperation {
-  kind: MappingOperation.CopyValue
-  patterns: CopyPattern[]
-}
-
-/**
- * @public
- */
-export interface TagField {
-  weight: InitialKnowledge | MeaningfullKnowledge | ExpertKnowledge
-
-  field: string
-  split: string // If defined values from field will be split to check for multiple values.
-}
-/**
- * @public
- */
-export interface CreateTagOperation {
-  kind: MappingOperation.CreateTag
-
-  fields: TagField[]
-}
-
-/**
- * @public
- */
-export interface ChannelFieldMapping {
-  provider: Ref<ChannelProvider>
-  field: string
-}
-
-/**
- * @public
- */
-export interface CreateChannelOperation {
-  kind: MappingOperation.CreateChannel
-  fields: ChannelFieldMapping[]
-}
-
-/**
- * @public
- */
-export interface DownloadAttachmentOperation {
-  kind: MappingOperation.DownloadAttachment
-
-  fields: { field: string }[]
-}
-
-/**
- * @public
- */
-export interface BitrixFieldMapping extends AttachedDoc {
-  ofClass: Ref<Class<Doc>> // Specify mixin if applicable
-  attributeName: string
-
-  operation: CopyValueOperation | CreateTagOperation | CreateChannelOperation | DownloadAttachmentOperation
-}
 
 /**
  * @public
@@ -183,7 +32,7 @@ export default plugin(bitrixId, {
     FieldMapping: '' as Ref<Class<BitrixFieldMapping>>
   },
   component: {
-    BitrixIntegration: '' as AnyComponent
+    BitrixIntegration: '' as Resource<any>
   },
   icon: {
     Bitrix: '' as Asset
@@ -192,3 +41,8 @@ export default plugin(bitrixId, {
     Mappings: '' as Ref<Space>
   }
 })
+
+export * from './client'
+export * from './sync'
+export * from './types'
+export * from './utils'

plugins/bitrix/src/sync.ts

@@ -0,0 +1,448 @@
+import attachment, { Attachment } from '@hcengineering/attachment'
+import chunter, { Comment } from '@hcengineering/chunter'
+import contact, { combineName, EmployeeAccount } from '@hcengineering/contact'
+import core, {
+  AccountRole,
+  ApplyOperations,
+  AttachedDoc,
+  Class,
+  Data,
+  Doc,
+  DocumentUpdate,
+  generateId,
+  Mixin,
+  Ref,
+  Space,
+  TxOperations,
+  WithLookup
+} from '@hcengineering/core'
+import tags, { TagElement } from '@hcengineering/tags'
+import { deepEqual } from 'fast-equals'
+import { BitrixClient } from './client'
+import { BitrixEntityMapping, BitrixEntityType, BitrixFieldMapping, BitrixSyncDoc, LoginInfo } from './types'
+import { convert, ConvertResult } from './utils'
+import bitrix from './index'
+
+async function updateDoc (client: ApplyOperations, doc: Doc, raw: Doc | Data<Doc>): Promise<void> {
+  // We need to update fields if they are different.
+  const documentUpdate: DocumentUpdate<Doc> = {}
+  for (const [k, v] of Object.entries(raw)) {
+    if (['_class', '_id', 'modifiedBy', 'modifiedOn', 'space'].includes(k)) {
+      continue
+    }
+    if (!deepEqual((doc as any)[k], v)) {
+      ;(documentUpdate as any)[k] = v
+    }
+  }
+  if (Object.keys(documentUpdate).length > 0) {
+    await client.update(doc, documentUpdate)
+  }
+}
+
+/**
+ * @public
+ */
+export async function syncPlatform (
+  client: TxOperations,
+  mapping: BitrixEntityMapping,
+  documents: ConvertResult[],
+  info: LoginInfo,
+  frontUrl: string,
+  monitor?: (doc: ConvertResult) => void
+): Promise<void> {
+  const existingDocuments = await client.findAll(mapping.ofClass, {
+    [bitrix.mixin.BitrixSyncDoc + '.bitrixId']: { $in: documents.map((it) => it.document.bitrixId) }
+  })
+  const hierarchy = client.getHierarchy()
+  let syncronized = 0
+  for (const d of documents) {
+    try {
+      const existing = existingDocuments.find(
+        (it) => hierarchy.as<Doc, BitrixSyncDoc>(it, bitrix.mixin.BitrixSyncDoc).bitrixId === d.document.bitrixId
+      )
+      const applyOp = client.apply('bitrix')
+      if (existing !== undefined) {
+        // We need update doucment id.
+        d.document._id = existing._id as Ref<BitrixSyncDoc>
+        // We need to update fields if they are different.
+        await updateDoc(applyOp, existing, d.document)
+
+        // Check and update mixins
+        for (const [m, mv] of Object.entries(d.mixins)) {
+          const mRef = m as Ref<Mixin<Doc>>
+          if (hierarchy.hasMixin(existing, mRef)) {
+            await applyOp.createMixin(
+              d.document._id,
+              d.document._class,
+              d.document.space,
+              m as Ref<Mixin<Doc>>,
+              mv,
+              d.document.modifiedOn,
+              d.document.modifiedBy
+            )
+          } else {
+            const existingM = hierarchy.as(existing, mRef)
+            await updateDoc(applyOp, existingM, mv)
+          }
+        }
+      } else {
+        await applyOp.createDoc(
+          d.document._class,
+          d.document.space,
+          d.document,
+          d.document._id,
+          d.document.modifiedOn,
+          d.document.modifiedBy
+        )
+
+        await applyOp.createMixin<Doc, BitrixSyncDoc>(
+          d.document._id,
+          d.document._class,
+          d.document.space,
+          bitrix.mixin.BitrixSyncDoc,
+          {
+            type: d.document.type,
+            bitrixId: d.document.bitrixId
+          },
+          d.document.modifiedOn,
+          d.document.modifiedBy
+        )
+        for (const [m, mv] of Object.entries(d.mixins)) {
+          await applyOp.createMixin(
+            d.document._id,
+            d.document._class,
+            d.document.space,
+            m as Ref<Mixin<Doc>>,
+            mv,
+            d.document.modifiedOn,
+            d.document.modifiedBy
+          )
+        }
+        for (const ed of d.extraDocs) {
+          if (applyOp.getHierarchy().isDerived(ed._class, core.class.AttachedDoc)) {
+            const adoc = ed as AttachedDoc
+            await applyOp.addCollection(
+              adoc._class,
+              adoc.space,
+              adoc.attachedTo,
+              adoc.attachedToClass,
+              adoc.collection,
+              adoc,
+              adoc._id,
+              d.document.modifiedOn,
+              d.document.modifiedBy
+            )
+          } else {
+            await applyOp.createDoc(ed._class, ed.space, ed, ed._id, d.document.modifiedOn, d.document.modifiedBy)
+          }
+        }
+
+        for (const ed of d.blobs) {
+          const attachmentId: Ref<Attachment> = generateId()
+
+          const data = new FormData()
+          data.append('file', ed)
+          const resp = await fetch(frontUrl + '/files', {
+            method: 'POST',
+            headers: {
+              Authorization: 'Bearer ' + info.token
+            },
+            body: data
+          })
+          if (resp.status === 200) {
+            const uuid = await resp.text()
+
+            await applyOp.addCollection(
+              attachment.class.Attachment,
+              d.document.space,
+              d.document._id,
+              d.document._class,
+              'attachments',
+              {
+                file: uuid,
+                lastModified: ed.lastModified,
+                name: ed.name,
+                size: ed.size,
+                type: ed.type
+              },
+              attachmentId,
+              d.document.modifiedOn,
+              d.document.modifiedBy
+            )
+          }
+        }
+
+        if (d.comments !== undefined) {
+          const comments = await d.comments
+          if (comments !== undefined && comments.length > 0) {
+            const existingComments = await client.findAll(chunter.class.Comment, {
+              attachedTo: d.document._id,
+              [bitrix.mixin.BitrixSyncDoc + '.bitrixId']: { $in: comments.map((it) => it.bitrixId) }
+            })
+
+            for (const comment of comments) {
+              const existing = existingComments.find(
+                (it) => hierarchy.as<Doc, BitrixSyncDoc>(it, bitrix.mixin.BitrixSyncDoc).bitrixId === comment.bitrixId
+              )
+              if (existing !== undefined) {
+                // We need to update fields if they are different.
+                await updateDoc(applyOp, existing, comment)
+              } else {
+                await applyOp.addCollection(
+                  comment._class,
+                  comment.space,
+                  comment.attachedTo,
+                  comment.attachedToClass,
+                  comment.collection,
+                  comment,
+                  comment._id,
+                  comment.modifiedOn,
+                  comment.modifiedBy
+                )
+
+                await applyOp.createMixin<Doc, BitrixSyncDoc>(
+                  comment._id,
+                  comment._class,
+                  comment.space,
+                  bitrix.mixin.BitrixSyncDoc,
+                  {
+                    type: d.document.type,
+                    bitrixId: d.document.bitrixId
+                  },
+                  comment.modifiedOn,
+                  comment.modifiedBy
+                )
+              }
+            }
+          }
+        }
+      }
+      await applyOp.commit()
+    } catch (err: any) {
+      console.error(err)
+    }
+    console.log('SYNC:', syncronized, documents.length - syncronized)
+    syncronized++
+    monitor?.(d)
+  }
+}
+
+/**
+ * @public
+ */
+export function processComment (comment: string): string {
+  comment = comment.replaceAll('\n', '\n</br>')
+  comment = comment.replaceAll(/\[(\/?[^[\]]+)]/gi, (text: string, args: string) => {
+    if (args.startsWith('/URL')) {
+      return '</a>'
+    }
+
+    if (args.startsWith('URL=')) {
+      return `<a href="${args.substring(4)}">`
+    }
+    if (args.includes('/FONT')) {
+      return '</span>'
+    }
+    if (args.includes('FONT')) {
+      return `<span style="font: ${args.substring(4)};">`
+    }
+
+    if (args.includes('/SIZE')) {
+      return '</span>'
+    }
+    if (args.includes('SIZE')) {
+      return `<span style="font-size: ${args.substring(4)};">`
+    }
+
+    if (args.includes('/COLOR')) {
+      return '</span>'
+    }
+    if (args.includes('COLOR')) {
+      return `<span style="color: ${args.substring(5)};">`
+    }
+
+    if (args.includes('/IMG')) {
+      return '"/>'
+    }
+    if (args.includes('IMG')) {
+      return `<img ${args.substring(3)} src="`
+    }
+
+    if (args.includes('/TABLE')) {
+      return '</table>'
+    }
+    if (args.includes('TABLE')) {
+      return '<table>'
+    }
+
+    return `<${args}>`
+  })
+  return comment
+}
+
+/**
+ * @public
+ */
+export async function performSynchronization (ops: {
+  client: TxOperations
+  bitrixClient: BitrixClient
+  space: Ref<Space> | undefined
+  mapping: WithLookup<BitrixEntityMapping>
+  limit: number
+  direction: 'ASC' | 'DSC'
+  frontUrl: string
+  loginInfo: LoginInfo
+  monitor: (total: number) => void
+}): Promise<void> {
+  const commentFields = await ops.bitrixClient.call(BitrixEntityType.Comment + '.fields', {})
+
+  const commentFieldKeys = Object.keys(commentFields.result)
+
+  const allEmployee = await ops.client.findAll(contact.class.EmployeeAccount, {})
+
+  const userList = new Map<string, Ref<EmployeeAccount>>()
+
+  // Fill all users and create new ones, if required.
+  let totalUsers = 1
+  let next = 0
+  while (userList.size < totalUsers) {
+    const users = await ops.bitrixClient.call('user.search', { start: next })
+    next = users.next
+    totalUsers = users.total
+    for (const u of users.result) {
+      let accountId = allEmployee.find((it) => it.email === u.EMAIL)?._id
+      if (accountId === undefined) {
+        const employeeId = await ops.client.createDoc(contact.class.Employee, contact.space.Contacts, {
+          name: combineName(u.NAME, u.LAST_NAME),
+          avatar: u.PERSONAL_PHOTO,
+          active: u.ACTIVE,
+          city: u.PERSONAL_CITY
+        })
+        accountId = await ops.client.createDoc(contact.class.EmployeeAccount, core.space.Model, {
+          email: u.EMAIL,
+          name: combineName(u.NAME, u.LAST_NAME),
+          employee: employeeId,
+          role: AccountRole.User
+        })
+      }
+      userList.set(u.ID, accountId)
+    }
+  }
+
+  try {
+    if (ops.space === undefined || ops.mapping.$lookup?.fields === undefined) {
+      return
+    }
+    let processed = 0
+    const tagElements: Map<Ref<Class<Doc>>, TagElement[]> = new Map()
+
+    let added = 0
+
+    while (added < ops.limit) {
+      const sel = ['*', 'UF_*']
+      if (ops.mapping.type === BitrixEntityType.Lead) {
+        sel.push('EMAIL')
+        sel.push('IM')
+      }
+      const result = await ops.bitrixClient.call(ops.mapping.type + '.list', {
+        select: sel,
+        order: { ID: ops.direction },
+        start: processed
+      })
+
+      const extraDocs: Doc[] = []
+
+      const convertResults: ConvertResult[] = []
+      const fields = ops.mapping.$lookup?.fields as BitrixFieldMapping[]
+
+      const toProcess = result.result as any[]
+
+      const existingDocuments = await ops.client.findAll(
+        ops.mapping.ofClass,
+        {
+          [bitrix.mixin.BitrixSyncDoc + '.bitrixId']: { $in: toProcess.map((it) => `${it.ID as string}`) }
+        },
+        {
+          projection: {
+            _id: 1,
+            [bitrix.mixin.BitrixSyncDoc + '.bitrixId']: 1
+          }
+        }
+      )
+      const defaultCategories = await ops.client.findAll(tags.class.TagCategory, {
+        default: true
+      })
+      let synchronized = 0
+      while (toProcess.length > 0) {
+        console.log('LOAD:', synchronized, toProcess.length)
+        synchronized++
+        const [r] = toProcess.slice(0, 1)
+        // Convert documents.
+        try {
+          const res = await convert(
+            ops.client,
+            ops.mapping,
+            ops.space,
+            fields,
+            r,
+            extraDocs,
+            tagElements,
+            userList,
+            existingDocuments,
+            defaultCategories
+          )
+          if (ops.mapping.comments) {
+            res.comments = await ops.bitrixClient
+              .call(BitrixEntityType.Comment + '.list', {
+                filter: {
+                  ENTITY_ID: res.document.bitrixId,
+                  ENTITY_TYPE: ops.mapping.type.replace('crm.', '')
+                },
+                select: commentFieldKeys,
+                order: { ID: ops.direction }
+              })
+              .then((comments) => {
+                return comments.result.map((it: any) => {
+                  const c: Comment & { bitrixId: string, type: string } = {
+                    _id: generateId(),
+                    _class: chunter.class.Comment,
+                    message: processComment(it.COMMENT as string),
+                    bitrixId: it.ID,
+                    type: it.ENTITY_TYPE,
+                    attachedTo: res.document._id,
+                    attachedToClass: res.document._class,
+                    collection: 'comments',
+                    space: res.document.space,
+                    modifiedBy: userList.get(it.AUTHOR_ID) ?? core.account.System,
+                    modifiedOn: new Date(it.CREATED ?? new Date().toString()).getTime()
+                  }
+                  return c
+                })
+              })
+          }
+          convertResults.push(res)
+          extraDocs.push(...res.extraDocs)
+          added++
+          if (added >= ops.limit) {
+            break
+          }
+        } catch (err: any) {
+          console.log('failed to obtain data for', r)
+          await new Promise((resolve) => {
+            // Sleep for a while
+            setTimeout(resolve, 1000)
+          })
+        }
+        toProcess.splice(0, 1)
+      }
+      const total = result.total
+      await syncPlatform(ops.client, ops.mapping, convertResults, ops.loginInfo, ops.frontUrl, () => {
+        ops.monitor?.(total)
+      })
+
+      processed = result.next
+    }
+  } catch (err: any) {
+    console.error(err)
+  }
+}

plugins/bitrix/src/types.ts

@@ -0,0 +1,232 @@
+import { ChannelProvider } from '@hcengineering/contact'
+import { AttachedDoc, Class, Doc, Ref } from '@hcengineering/core'
+import { ExpertKnowledge, InitialKnowledge, MeaningfullKnowledge } from '@hcengineering/tags'
+
+/**
+ * @public
+ */
+export interface BitrixProfile {
+  ID: string
+  ADMIN: boolean
+  NAME: string
+  LAST_NAME: string
+  PERSONAL_GENDER: string
+  PERSONAL_PHOTO: string
+  TIME_ZONE: string
+  TIME_ZONE_OFFSET: number
+}
+
+/**
+ * @public
+ */
+export type NumberString = string
+/**
+ * @public
+ */
+export type ISODate = string
+/**
+ * @public
+ */
+export type BoolString = 'Y' | 'N'
+/**
+ * @public
+ */
+export type GenderString = 'M' | 'F' | ''
+
+/**
+ * @public
+ */
+export interface MultiField {
+  readonly ID: NumberString
+  readonly VALUE_TYPE: string
+  readonly VALUE: string
+  readonly TYPE_ID: string
+}
+/**
+ * @public
+ */
+export type MultiFieldArray = ReadonlyArray<Pick<MultiField, 'VALUE' | 'VALUE_TYPE'>>
+
+/**
+ * @public
+ */
+export interface StatusValue {
+  CATEGORY_ID: string | null
+  COLOR: string | null
+  ENTITY_ID: string | null
+  ID: number
+  NAME: string
+  NAME_INIT: string | null
+  SEMANTICS: string | null
+  SORT: string | null
+  STATUS_ID: string | null
+  SYSTEM: 'Y' | 'N'
+}
+
+/**
+ * @public
+ */
+
+export interface BitrixResult {
+  result: any
+  next: number
+  total: number
+}
+
+/**
+ * @public
+ */
+export interface LoginInfo {
+  endpoint: string
+  email: string
+  token: string
+}
+
+/**
+ * @public
+ */
+export interface BitrixSyncDoc extends Doc {
+  type: string
+  bitrixId: string
+}
+
+/**
+ * @public
+ */
+export enum BitrixEntityType {
+  Comment = 'crm.timeline.comment',
+  Binding = 'crm.timeline.bindings',
+  Lead = 'crm.lead',
+  Activity = 'crm.activity',
+  Company = 'crm.company'
+}
+
+/**
+ * @public
+ */
+export const mappingTypes = [
+  { label: 'Leads', id: BitrixEntityType.Lead },
+  // { label: 'Comments', id: BitrixEntityType.Comment },
+  { label: 'Company', id: BitrixEntityType.Company }
+  // { label: 'Activity', id: BitrixEntityType.Activity }
+]
+
+/**
+ * @public
+ */
+export interface FieldValue {
+  type: string
+  statusType?: string
+  isRequired: boolean
+  isReadOnly: boolean
+  isImmutable: boolean
+  isMultiple: boolean
+  isDynamic: boolean
+  title: string
+
+  formLabel?: string
+  filterLabel?: string
+  items?: Array<{
+    ID: string
+    VALUE: string
+  }>
+}
+
+/**
+ * @public
+ */
+export interface Fields {
+  [key: string]: FieldValue
+}
+
+/**
+ * @public
+ */
+export interface BitrixEntityMapping extends Doc {
+  ofClass: Ref<Class<Doc>>
+  type: string
+  bitrixFields: Fields
+
+  fields: number
+
+  comments: boolean
+  activity: boolean
+  attachments: boolean
+}
+/**
+ * @public
+ */
+export enum MappingOperation {
+  CopyValue,
+  CreateTag, // Create tag
+  CreateChannel, // Create channel
+  DownloadAttachment
+}
+/**
+ * @public
+ */
+export interface CopyPattern {
+  text: string
+  field?: string
+  alternatives?: string[]
+}
+/**
+ * @public
+ */
+export interface CopyValueOperation {
+  kind: MappingOperation.CopyValue
+  patterns: CopyPattern[]
+}
+
+/**
+ * @public
+ */
+export interface TagField {
+  weight: InitialKnowledge | MeaningfullKnowledge | ExpertKnowledge
+
+  field: string
+  split: string // If defined values from field will be split to check for multiple values.
+}
+/**
+ * @public
+ */
+export interface CreateTagOperation {
+  kind: MappingOperation.CreateTag
+
+  fields: TagField[]
+}
+
+/**
+ * @public
+ */
+export interface ChannelFieldMapping {
+  provider: Ref<ChannelProvider>
+  field: string
+}
+
+/**
+ * @public
+ */
+export interface CreateChannelOperation {
+  kind: MappingOperation.CreateChannel
+  fields: ChannelFieldMapping[]
+}
+
+/**
+ * @public
+ */
+export interface DownloadAttachmentOperation {
+  kind: MappingOperation.DownloadAttachment
+
+  fields: { field: string }[]
+}
+
+/**
+ * @public
+ */
+export interface BitrixFieldMapping extends AttachedDoc {
+  ofClass: Ref<Class<Doc>> // Specify mixin if applicable
+  attributeName: string
+
+  operation: CopyValueOperation | CreateTagOperation | CreateChannelOperation | DownloadAttachmentOperation
+}

plugins/bitrix/src/utils.ts

@@ -1,3 +1,7 @@
+import { Comment } from '@hcengineering/chunter'
+import contact, { Channel, EmployeeAccount } from '@hcengineering/contact'
+import core, { AnyAttribute, Class, Client, Data, Doc, generateId, Mixin, Ref, Space } from '@hcengineering/core'
+import tags, { TagCategory, TagElement, TagReference } from '@hcengineering/tags'
 import {
   BitrixEntityMapping,
   BitrixFieldMapping,
@@ -5,14 +9,14 @@ import {
   CopyValueOperation,
   CreateChannelOperation,
   CreateTagOperation,
+  DownloadAttachmentOperation,
   MappingOperation
-} from '@hcengineering/bitrix'
+} from '.'
-import { Comment } from '@hcengineering/chunter'
+import bitrix from './index'
-import contact, { Channel, EmployeeAccount } from '@hcengineering/contact'
-import core, { AnyAttribute, Class, Client, Data, Doc, generateId, Mixin, Ref, Space } from '@hcengineering/core'
-import tags, { TagElement, TagReference } from '@hcengineering/tags'
-import { getColorNumberByText } from '@hcengineering/ui'
 
+/**
+ * @public
+ */
 export function collectFields (fieldMapping: BitrixFieldMapping[]): string[] {
   const fields: string[] = ['ID']
   for (const f of fieldMapping) {
@@ -33,14 +37,20 @@ export function collectFields (fieldMapping: BitrixFieldMapping[]): string[] {
   return fields
 }
 
+/**
+ * @public
+ */
 export interface ConvertResult {
   document: BitrixSyncDoc // Document we should achive
   mixins: Record<Ref<Mixin<Doc>>, Data<Doc>> // Mixins of document we will achive
   extraDocs: Doc[] // Extra documents we will achive, comments etc.
   blobs: File[] //
-  comments?: Promise<Array<BitrixSyncDoc & Comment>>
+  comments?: Array<BitrixSyncDoc & Comment>
 }
 
+/**
+ * @public
+ */
 export async function convert (
   client: Client,
   entity: BitrixEntityMapping,
@@ -49,19 +59,26 @@ export async function convert (
   rawDocument: any,
   prevExtra: Doc[], // <<-- a list of previous extra documents, so for example TagElement will be reused, if present for more what one item and required to be created
   tagElements: Map<Ref<Class<Doc>>, TagElement[]>, // TagElement cache.
-  userList: Map<string, Ref<EmployeeAccount>>
+  userList: Map<string, Ref<EmployeeAccount>>,
+  existingDocuments: Doc[],
+  defaultCategories: TagCategory[],
+  blobProvider?: (blobRef: any) => Promise<Blob | undefined>
 ): Promise<ConvertResult> {
   const hierarchy = client.getHierarchy()
+  const bitrixId = `${rawDocument.ID as string}`
   const document: BitrixSyncDoc = {
     _id: generateId(),
     type: entity.type,
-    bitrixId: `${rawDocument.ID as string}`,
+    bitrixId,
     _class: entity.ofClass,
     space,
     modifiedOn: new Date(rawDocument.DATE_CREATE).getTime(),
     modifiedBy: userList.get(rawDocument.CREATED_BY_ID) ?? core.account.System
   }
 
+  const existingId = existingDocuments.find((it) => (it as any)[bitrix.mixin.BitrixSyncDoc].bitrixId === bitrixId)
+    ?._id as Ref<BitrixSyncDoc>
+
   // Obtain a proper modified by for document
 
   const newExtraDocs: Doc[] = []
@@ -91,6 +108,10 @@ export async function convert (
         if (Array.isArray(lval)) {
           return lval.map((it) => it.VALUE)
         }
+      } else if (bfield.type === 'file') {
+        if (Array.isArray(lval)) {
+          return lval.map((it) => ({ id: it.id, file: it.downloadUrl }))
+        }
       } else if (bfield.type === 'string' || bfield.type === 'url') {
         if (bfield.isMultiple && Array.isArray(lval)) {
           return lval.join(', ')
@@ -135,6 +156,26 @@ export async function convert (
     }
     return r.join('').trim()
   }
+  const getDownloadValue = async (attr: AnyAttribute, operation: DownloadAttachmentOperation): Promise<any> => {
+    const r: Array<string | number | boolean | Date> = []
+    for (const o of operation.fields) {
+      const lval = extractValue(o.field)
+      if (lval != null) {
+        if (Array.isArray(lval)) {
+          r.push(...lval)
+        } else {
+          r.push(lval)
+        }
+      }
+    }
+    if (r.length === 1) {
+      return r[0]
+    }
+    if (r.length === 0) {
+      return
+    }
+    return r.join('').trim()
+  }
 
   const getChannelValue = async (attr: AnyAttribute, operation: CreateChannelOperation): Promise<any> => {
     for (const f of operation.fields) {
@@ -164,21 +205,22 @@ export async function convert (
   const getTagValue = async (attr: AnyAttribute, operation: CreateTagOperation): Promise<any> => {
     const elements =
       tagElements.get(attr.attributeOf) ??
-      (await client.findAll(tags.class.TagElement, {
+      (await client.findAll<TagElement>(tags.class.TagElement, {
         targetClass: attr.attributeOf
       }))
 
-    const references = await client.findAll(tags.class.TagReference, {
+    const references =
-      attachedTo: document._id
+      existingId !== undefined
-    })
+        ? await client.findAll<TagReference>(tags.class.TagReference, {
+          attachedTo: existingId
+        })
+        : []
     // Add tags creation requests from previous conversions.
     elements.push(...prevExtra.filter((it) => it._class === tags.class.TagElement).map((it) => it as TagElement))
 
     tagElements.set(attr.attributeOf, elements)
-    const defaultCategory = await client.findOne(tags.class.TagCategory, {
+    const defaultCategory = defaultCategories.find((it) => it.targetClass === attr.attributeOf)
-      targetClass: attr.attributeOf,
+
-      default: true
-    })
     if (defaultCategory === undefined) {
       console.error('could not proceed tags without default category')
       return
@@ -206,7 +248,7 @@ export async function convert (
             _id: generateId(),
             _class: tags.class.TagElement,
             category: defaultCategory._id,
-            color: getColorNumberByText(vv),
+            color: 1,
             description: '',
             title: vv,
             targetClass: attr.attributeOf,
@@ -218,12 +260,12 @@ export async function convert (
         }
         const ref: TagReference = {
           _id: generateId(),
-          attachedTo: document._id,
+          attachedTo: existingId ?? document._id,
           attachedToClass: attr.attributeOf,
           collection: attr.name,
           _class: tags.class.TagReference,
           tag: tag._id,
-          color: getColorNumberByText(vv),
+          color: 1,
           title: vv,
           weight: o.weight,
           modifiedBy: document.modifiedBy,
@@ -256,10 +298,32 @@ export async function convert (
       case MappingOperation.CreateTag:
         value = await getTagValue(attr, f.operation)
         break
+      case MappingOperation.DownloadAttachment: {
+        const blobRef: { file: string, id: string } = await getDownloadValue(attr, f.operation)
+        if (blobRef !== undefined) {
+          const response = await blobProvider?.(blobRef)
+          if (response !== undefined) {
+            let fname = blobRef.id
+            switch (response.type) {
+              case 'application/pdf':
+                fname += '.pdf'
+                break
+              case 'application/msword':
+                fname += '.doc'
+                break
+            }
+            blobs.push(new File([response], fname, { type: response.type }))
+          }
+        }
+        break
+      }
     }
     if (value !== undefined) {
       if (hierarchy.isMixin(attr.attributeOf)) {
-        mixins[attr.attributeOf] = { ...mixins[attr.attributeOf], [attr.name]: value }
+        mixins[attr.attributeOf] = {
+          ...mixins[attr.attributeOf],
+          [attr.name]: value
+        }
       } else {
         ;(document as any)[attr.name] = value
       }

plugins/tracker-resources/src/components/projects/ProjectPresenter.svelte

@@ -28,7 +28,11 @@
 
 {#if value}
   <!-- svelte-ignore a11y-click-events-have-key-events -->
-  <span title={value.label} class="fs-bold caption-color overflow-label clear-mins" on:click={navigateToProject}>
+  <span
+    title={value.label}
+    class="cursor-pointer fs-bold caption-color overflow-label clear-mins"
+    on:click={navigateToProject}
+  >
     {value.label}
   </span>
 {/if}

plugins/view-resources/src/actionImpl.ts

@@ -16,7 +16,7 @@ import MoveView from './components/Move.svelte'
 import { contextStore } from './context'
 import view from './plugin'
 import { FocusSelection, focusStore, previewDocument, SelectDirection, selectionStore } from './selection'
-import { deleteObject } from './utils'
+import { deleteObjects } from './utils'
 
 /**
  * Action to be used for copying text to clipboard.
@@ -53,23 +53,19 @@ async function CopyTextToClipboard (
   }
 }
 
-function Delete (object: Doc): void {
+function Delete (object: Doc | Doc[]): void {
   showPopup(
     MessageBox,
     {
       label: view.string.DeleteObject,
       message: view.string.DeleteObjectConfirm,
-      params: { count: Array.isArray(object) ? object.length : 1 }
+      params: { count: Array.isArray(object) ? object.length : 1 },
-    },
+      action: async () => {
-    undefined,
-    (result?: boolean) => {
-      if (result === true) {
         const objs = Array.isArray(object) ? object : [object]
-        for (const o of objs) {
+        await deleteObjects(getClient(), objs).catch((err) => console.error(err))
-          deleteObject(getClient(), o).catch((err) => console.error(err))
-        }
       }
-    }
+    },
+    undefined
   )
 }
 

plugins/view-resources/src/components/EditDoc.svelte

@@ -76,7 +76,10 @@
     query.unsubscribe()
   }
 
-  $: if (_class) {
+  let oldClass: Ref<Class<Doc>>
+
+  $: if (_class !== oldClass) {
+    oldClass = _class
     mainEditor = undefined
   }
 
@@ -152,9 +155,9 @@
   }
 
   let mainEditor: MixinEditor | undefined
-  $: getEditorOrDefault(realObjectClass, showAllMixins)
+  $: getEditorOrDefault(realObjectClass, showAllMixins, _id)
 
-  async function getEditorOrDefault (_class: Ref<Class<Doc>>, showAllMixins: boolean): Promise<void> {
+  async function getEditorOrDefault (_class: Ref<Class<Doc>>, showAllMixins: boolean, _id: Ref<Doc>): Promise<void> {
     parentClass = getParentClass(_class)
     mainEditor = await getEditor(_class)
     updateKeys(showAllMixins)
@@ -167,11 +170,16 @@
       array: view.mixin.ArrayEditor,
       collection: view.mixin.CollectionEditor,
       inplace: view.mixin.InlineAttributEditor,
-      attribute: view.mixin.AttributeEditor
+      attribute: view.mixin.AttributeEditor,
+      object: undefined
     }
     const mixinRef = mix[attrClass.category]
-    const editorMixin = hierarchy.as(clazz, mixinRef)
+    if (mixinRef) {
-    return editorMixin.editor
+      const editorMixin = hierarchy.as(clazz, mixinRef)
+      return editorMixin.editor
+    } else {
+      return undefined
+    }
   }
 
   function getIcon (_class: Ref<Class<Obj>> | undefined): Asset | undefined {

plugins/view-resources/src/utils.ts

@@ -301,6 +301,21 @@ export async function deleteObject (client: TxOperations, object: Doc): Promise<
   }
 }
 
+export async function deleteObjects (client: TxOperations, objects: Doc[]): Promise<void> {
+  const ops = client.apply('delete')
+  for (const object of objects) {
+    if (client.getHierarchy().isDerived(object._class, core.class.AttachedDoc)) {
+      const adoc = object as AttachedDoc
+      await ops
+        .removeCollection(object._class, object.space, adoc._id, adoc.attachedTo, adoc.attachedToClass, adoc.collection)
+        .catch((err) => console.error(err))
+    } else {
+      await ops.removeDoc(object._class, object.space, object._id).catch((err) => console.error(err))
+    }
+  }
+  await ops.commit()
+}
+
 export function getMixinStyle (id: Ref<Class<Doc>>, selected: boolean): string {
   const color = getPlatformColorForText(id as string)
   return `

pods/account/src/index.ts

@@ -101,6 +101,15 @@ export function serveAccount (methods: Record<string, AccountMethod>, productId
   const close = (): void => {
     server.close()
   }
+
+  process.on('uncaughtException', (e) => {
+    console.error(e)
+  })
+
+  process.on('unhandledRejection', (reason, promise) => {
+    console.error('Unhandled Rejection at:', promise, 'reason:', reason)
+  })
+
   process.on('SIGINT', close)
   process.on('SIGTERM', close)
   process.on('exit', close)

server/account/src/index.ts

@@ -210,6 +210,7 @@ async function getAccountInfo (db: Db, email: string, password: string): Promise
  * @returns
  */
 export async function login (db: Db, productId: string, email: string, password: string): Promise<LoginInfo> {
+  console.log(`login attempt:${email}`)
   await getAccountInfo(db, email, password)
   const result = {
     endpoint: getEndpoint(),