TSK-476: Bitrix import fixes (#2548)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev 2023-01-27 19:14:40 +07:00 committed by GitHub
parent dd68380e7e
commit 474cb9c887
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 948 additions and 562 deletions

View File

@ -11768,7 +11768,7 @@ packages:
dev: false
file:projects/bitrix-resources.tgz_a1d864769aaf53d09b76fe134ab55e60:
resolution: {integrity: sha512-Wv/ATljkeBewaCxRNMoYjoX07HOl1PniT0kkDZUrqkxPW+fjqMSB2DrU0D/5CS39UAC4jOtTPV1yMCqz7r5smw==, tarball: file:projects/bitrix-resources.tgz}
resolution: {integrity: sha512-vXwCwfcCLy3cb6Dj8p8+XWglgI6aTUdr4sN5xzKqaL5soj1JwmEyWhHvafBJMWDpk8UsJUEGNB5Jpiw24n+q9A==, tarball: file:projects/bitrix-resources.tgz}
id: file:projects/bitrix-resources.tgz
name: '@rush-temp/bitrix-resources'
version: 0.0.0
@ -11807,13 +11807,14 @@ packages:
dev: false
file:projects/bitrix.tgz:
resolution: {integrity: sha512-LKlmOtIMjMgNCPKafSUsJm10okaehcJvYA4mrIrmNR9lHnbldRg3GlaglyOtXQR1fK3/KY/HTlOKzgP4xFnzpw==, tarball: file:projects/bitrix.tgz}
resolution: {integrity: sha512-DdNHQSCosOfpZzQR2nd3BOx33UVHTggxpLaGcJh0Jd4yW7FoKISS7xwBnQbHGtUNczHM9R0pNvKoVdpNNBHhnQ==, tarball: file:projects/bitrix.tgz}
name: '@rush-temp/bitrix'
version: 0.0.0
dependencies:
'@2bad/bitrix': 2.5.0
'@rushstack/heft': 0.47.11
'@types/heft-jest': 1.0.3
'@types/qs': 6.9.7
'@typescript-eslint/eslint-plugin': 5.42.1_d506b9be61cb4ac2646ecbc6e0680464
'@typescript-eslint/parser': 5.42.1_eslint@8.27.0+typescript@4.8.4
eslint: 8.27.0
@ -11821,7 +11822,9 @@ packages:
eslint-plugin-import: 2.26.0_eslint@8.27.0
eslint-plugin-n: 15.5.1_eslint@8.27.0
eslint-plugin-promise: 6.1.1_eslint@8.27.0
fast-equals: 2.0.4
prettier: 2.7.1
qs: 6.11.0
typescript: 4.8.4
transitivePeerDependencies:
- supports-color
@ -13090,7 +13093,7 @@ packages:
dev: false
file:projects/model-all.tgz_typescript@4.8.4:
resolution: {integrity: sha512-AejnJ5pKXrdrHEdXX2AoOtbSX2mx0ceRbISCPyuvr1O9w8VKnmp43TGLOZEKVY3D+dmnjKj1+2+A+bAxMCUsQQ==, tarball: file:projects/model-all.tgz}
resolution: {integrity: sha512-KExZMGU3rzrjnf1pAwSlS4n0IEYrCONE9uXtbBQMokbcfQ/3YbeDGcDrvRdl7okj89wE47sUS088LY/22UzUMw==, tarball: file:projects/model-all.tgz}
id: file:projects/model-all.tgz
name: '@rush-temp/model-all'
version: 0.0.0
@ -14284,7 +14287,7 @@ packages:
dev: false
file:projects/openai.tgz:
resolution: {integrity: sha512-nplFlMK8VBiE2EBm11HhSpX74sZapWZ30sEWOHZfAv8AZqCINY0FOHP0Jwc+AVFtMzClx8NTCo6pgaNTLs0ixA==, tarball: file:projects/openai.tgz}
resolution: {integrity: sha512-O+mwmpb/LfabOFEsid8hIWuJpSombaoFJf3xTlH+pmi47h54Pwn8CsSjY3aZ2PdtvVjY5TK5vIQxKHxGNe3YiQ==, tarball: file:projects/openai.tgz}
name: '@rush-temp/openai'
version: 0.0.0
dependencies:
@ -14539,7 +14542,7 @@ packages:
dev: false
file:projects/pod-server.tgz:
resolution: {integrity: sha512-4IjhfCueH7UYdgcWfpeiciOQADpXCKj1C41rP6ZgHaIWzUErhMl2X/T5C2SQfp0nnbvpPWqWeqSPYYK2GGGkdw==, tarball: file:projects/pod-server.tgz}
resolution: {integrity: sha512-1gSQAnD3H3VvnxG3bDk/x+MKlmiR+dk1730YE7SjA0AmWcIaa40QqKPgCLpdhCrtDT/fwr0TFVjr3Bo8Z/vCHQ==, tarball: file:projects/pod-server.tgz}
name: '@rush-temp/pod-server'
version: 0.0.0
dependencies:
@ -14646,7 +14649,7 @@ packages:
dev: false
file:projects/prod.tgz_a9366fc5abe1de81e350f3e9b2acb628:
resolution: {integrity: sha512-wfzTNxTVOW15y2ZLXjebNfstBDQSIdSevMj0AorZ1SUWUVqQ8ZkO8XOI2wFwQdiGJplBTG0zVIDpZFdYBWF6mg==, tarball: file:projects/prod.tgz}
resolution: {integrity: sha512-oHJixn8gWZB93sHdI2epzD4DeR9Ff8y8njVa9Kr+QFk2p7IGH68FCTTA90Q61ysV7k07e/OED1ClxZ5hAthtgA==, tarball: file:projects/prod.tgz}
id: file:projects/prod.tgz
name: '@rush-temp/prod'
version: 0.0.0
@ -16286,7 +16289,7 @@ packages:
dev: false
file:projects/tool.tgz:
resolution: {integrity: sha512-VVNc6+f2BssTik3I3y+lBrBVkMz6sHygnom2/PvO/ZwEVKeWDN2rwpzZxj6n+RlPeCCvvcII9sXpPOAoJAnxnA==, tarball: file:projects/tool.tgz}
resolution: {integrity: sha512-OqWoKCsumcmAHkfsi+zF+Lt6uoamsg5W+/K3SGlPwovAIeo/Tyc+F0Sm9EKUe1OKX0FT9+dcQstcHRigNFSNQA==, tarball: file:projects/tool.tgz}
name: '@rush-temp/tool'
version: 0.0.0
dependencies:

View File

@ -3,7 +3,7 @@
"outDir": "./dist/",
"noImplicitAny": true,
"module": "esnext",
"target": "es2016",
"target": "es2021",
"allowJs": true,
"sourceMap": true,
"skipLibCheck": true,
@ -11,7 +11,8 @@
"allowSyntheticDefaultImports": true,
"lib": [
"es2016",
"dom"
"dom",
"ES2021.String"
]
}
}

View File

@ -23,8 +23,10 @@
export let message: IntlString
export let params: Record<string, any> = {}
export let canSubmit = true
export let action: (() => Promise<void>) | undefined = undefined
const dispatch = createEventDispatcher()
let processing = false
</script>
<div class="msgbox-container">
@ -36,10 +38,28 @@
label={presentation.string.Ok}
size={'small'}
kind={'primary'}
on:click={() => dispatch('close', true)}
loading={processing}
on:click={() => {
processing = true
if (action !== undefined) {
action().then(() => {
processing = false
dispatch('close', true)
})
} else {
dispatch('close', true)
processing = false
}
}}
/>
{#if canSubmit}
<Button label={presentation.string.Cancel} size={'small'} on:click={() => dispatch('close', false)} />
<Button
label={presentation.string.Cancel}
size={'small'}
on:click={() => {
dispatch('close', false)
}}
/>
{/if}
</div>
</div>

View File

@ -41,7 +41,7 @@
{:else if node.nodeName === 'HR'}
<hr />
{:else if node.nodeName === 'IMG'}
<div class="max-h-60 max-w-60 img">{@html node.outerHTML}</div>
<div class="max-h-60 max-w-60">{@html node.outerHTML}</div>
{:else if node.nodeName === 'H1'}
<h1><svelte:self nodes={node.childNodes} /></h1>
{:else if node.nodeName === 'H2'}

View File

@ -206,7 +206,7 @@ export type AttributeCategory = 'object' | 'attribute' | 'inplace' | 'collection
/**
* @public
*/
export const AttributeCategoryOrder = { attribute: 0, inplace: 1, collection: 2, array: 2 }
export const AttributeCategoryOrder = { attribute: 0, inplace: 1, collection: 2, array: 2, object: 3 }
/**
* @public

View File

@ -46,5 +46,9 @@
"filesize": "^8.0.3",
"@hcengineering/preference": "^0.6.2",
"@hcengineering/activity": "^0.6.0"
},
"repository": "https://github.com/hcengineering/anticrm",
"publishConfig": {
"registry": "https://npm.pkg.github.com"
}
}

View File

@ -27,7 +27,8 @@
"prettier-plugin-svelte": "^2.8.0",
"prettier": "^2.7.1",
"svelte-check": "^2.8.0",
"typescript": "^4.3.5"
"typescript": "^4.3.5",
"@types/qs": "~6.9.7"
},
"dependencies": {
"@hcengineering/platform": "^0.6.8",
@ -38,6 +39,7 @@
"@hcengineering/text-editor": "^0.6.0",
"@hcengineering/contact": "^0.6.9",
"@hcengineering/lead": "^0.6.0",
"@hcengineering/login": "^0.6.1",
"@hcengineering/setting": "^0.6.2",
"@hcengineering/core": "^0.6.20",
"@hcengineering/attachment": "^0.6.1",
@ -48,9 +50,12 @@
"@hcengineering/chunter": "^0.6.2",
"p-queue": "~7.3.0",
"qs": "~6.11.0",
"@types/qs": "~6.9.7",
"@hcengineering/tags": "^0.6.3",
"@hcengineering/tags-resources": "^0.6.0",
"fast-equals": "^2.0.3"
},
"repository": "https://github.com/hcengineering/anticrm",
"publishConfig": {
"registry": "https://npm.pkg.github.com"
}
}

View File

@ -20,18 +20,18 @@
import { createEventDispatcher, onMount } from 'svelte'
import bitrix from '../plugin'
import { BitrixEntityMapping } from '@hcengineering/bitrix'
import { BitrixClient, BitrixEntityMapping, BitrixProfile, StatusValue } from '@hcengineering/bitrix'
import { Button, eventToHTMLElement, IconAdd, Label, showPopup } from '@hcengineering/ui'
import { BitrixClient } from '../client'
import { BitrixProfile, StatusValue } from '../types'
import CreateMapping from './CreateMapping.svelte'
import EntiryMapping from './EntityMapping.svelte'
import { bitrixQueue } from '../queue'
export let integration: Integration
const dispatch = createEventDispatcher()
const bitrixClient = new BitrixClient(integration.value)
const bitrixClient = new BitrixClient(integration.value, (op) => bitrixQueue.add(op))
let profile: BitrixProfile | undefined

View File

@ -1,5 +1,5 @@
<script lang="ts">
import { BitrixEntityMapping, Fields, FieldValue } from '@hcengineering/bitrix'
import { BitrixClient, BitrixEntityMapping, Fields, FieldValue } from '@hcengineering/bitrix'
import core, { Enum, Ref } from '@hcengineering/core'
import { getEmbeddedLabel } from '@hcengineering/platform'
import { createQuery, getClient } from '@hcengineering/presentation'
@ -16,7 +16,6 @@
showPopup
} from '@hcengineering/ui'
import Grid from '@hcengineering/ui/src/components/Grid.svelte'
import { BitrixClient } from '../client'
import EnumPopup from './EnumPopup.svelte'

View File

@ -1,12 +1,12 @@
<script lang="ts">
import { BitrixEntityMapping } from '@hcengineering/bitrix'
import { BitrixClient, BitrixEntityMapping } from '@hcengineering/bitrix'
import { getEmbeddedLabel } from '@hcengineering/platform'
import { Card, createQuery } from '@hcengineering/presentation'
import setting, { Integration } from '@hcengineering/setting'
import { Label } from '@hcengineering/ui'
import { createEventDispatcher } from 'svelte'
import { BitrixClient } from '../client'
import bitrix from '../plugin'
import { bitrixQueue } from '../queue'
import FieldMappingSynchronizer from './FieldMappingSynchronizer.svelte'
const mappingQuery = createQuery()
@ -37,7 +37,8 @@
integration = res.shift()
})
$: bitrixClient = integration !== undefined ? new BitrixClient(integration.value) : undefined
$: bitrixClient =
integration !== undefined ? new BitrixClient(integration.value, (op) => bitrixQueue.add(op)) : undefined
let loading = false
</script>

View File

@ -13,9 +13,14 @@
const client = getClient()
async function save (): Promise<void> {
client.createDoc(bitrix.class.EntityMapping, bitrix.space.Mappings, {
await client.createDoc<BitrixEntityMapping>(bitrix.class.EntityMapping, bitrix.space.Mappings, {
ofClass,
type
type,
comments: true,
attachments: true,
bitrixFields: {},
fields: 0,
activity: false
})
}

View File

@ -1,11 +1,18 @@
<script lang="ts">
import { BitrixEntityMapping, BitrixFieldMapping, Fields, mappingTypes } from '@hcengineering/bitrix'
import {
BitrixClient,
BitrixEntityMapping,
BitrixFieldMapping,
Fields,
mappingTypes,
StatusValue,
toClassRef
} from '@hcengineering/bitrix'
import { Class, Doc, Ref } from '@hcengineering/core'
import { getEmbeddedLabel } from '@hcengineering/platform'
import { createQuery, getClient } from '@hcengineering/presentation'
import { ClassSetting } from '@hcengineering/setting-resources'
import { Button, Expandable, Icon, IconDelete, IconEdit, Label, showPopup } from '@hcengineering/ui'
import { BitrixClient } from '../client'
import bitrix from '../plugin'
import AttributeMapper from './AttributeMapper.svelte'
@ -13,8 +20,6 @@
import CheckBox from '@hcengineering/ui/src/components/CheckBox.svelte'
import { deepEqual } from 'fast-equals'
import { StatusValue } from '../types'
import { toClassRef } from '../utils'
import BitrixFieldLookup from './BitrixFieldLookup.svelte'
import CreateMappingAttribute from './CreateMappingAttribute.svelte'

View File

@ -1,35 +1,19 @@
<script lang="ts">
import bitrix, {
import {
BitrixClient,
BitrixEntityMapping,
BitrixEntityType,
BitrixFieldMapping,
BitrixSyncDoc
performSynchronization,
toClassRef
} from '@hcengineering/bitrix'
import chunter, { Comment } from '@hcengineering/chunter'
import contact, { combineName, EmployeeAccount } from '@hcengineering/contact'
import core, {
AccountRole,
ApplyOperations,
AttachedDoc,
Class,
Data,
Doc,
DocumentUpdate,
generateId,
Mixin,
Ref,
Space,
WithLookup
} from '@hcengineering/core'
import { getEmbeddedLabel } from '@hcengineering/platform'
import contact from '@hcengineering/contact'
import core, { Class, Doc, Ref, Space, WithLookup } from '@hcengineering/core'
import login from '@hcengineering/login'
import { getEmbeddedLabel, getMetadata } from '@hcengineering/platform'
import { getClient, SpaceSelect } from '@hcengineering/presentation'
import { TagElement } from '@hcengineering/tags'
import { Button, Expandable, Icon, Label } from '@hcengineering/ui'
import DropdownLabels from '@hcengineering/ui/src/components/DropdownLabels.svelte'
import { NumberEditor } from '@hcengineering/view-resources'
import { deepEqual } from 'fast-equals'
import { BitrixClient } from '../client'
import { convert, ConvertResult, toClassRef } from '../utils'
import FieldMappingPresenter from './FieldMappingPresenter.svelte'
export let mapping: WithLookup<BitrixEntityMapping>
@ -43,275 +27,38 @@
return p
}, {} as Record<Ref<Class<Doc>>, BitrixFieldMapping[]>)
let direction: 'ASC' | 'DSC' = 'DSC'
let limit = 200
let direction: 'ASC' | 'DSC' = 'ASC'
let limit = 1
let space: Ref<Space> | undefined
export let loading = false
let state = ''
async function updateDoc (client: ApplyOperations, doc: Doc, raw: Doc | Data<Doc>): Promise<void> {
// We need to update fields if they are different.
const documentUpdate: DocumentUpdate<Doc> = {}
for (const [k, v] of Object.entries(raw)) {
if (['_class', '_id', 'modifiedBy', 'modifiedOn', 'space'].includes(k)) {
continue
}
if (!deepEqual((doc as any)[k], v)) {
;(documentUpdate as any)[k] = v
}
}
if (Object.keys(documentUpdate).length > 0) {
await client.update(doc, documentUpdate)
}
}
let docsProcessed = 0
let total = 0
async function syncPlatform (documents: ConvertResult[]): Promise<void> {
const existingDocuments = await client.findAll(mapping.ofClass, {
[bitrix.mixin.BitrixSyncDoc + '.bitrixId']: { $in: documents.map((it) => it.document.bitrixId) }
})
const hierarchy = client.getHierarchy()
for (const d of documents) {
const existing = existingDocuments.find(
(it) => hierarchy.as(it, bitrix.mixin.BitrixSyncDoc).bitrixId === d.document.bitrixId
)
const applyOp = client.apply('bitrix')
if (existing !== undefined) {
// We need to update fields if they are different.
await updateDoc(applyOp, existing, d.document)
// Check and update mixins
for (const [m, mv] of Object.entries(d.mixins)) {
const mRef = m as Ref<Mixin<Doc>>
if (hierarchy.hasMixin(existing, mRef)) {
await applyOp.createMixin(
d.document._id,
d.document._class,
d.document.space,
m as Ref<Mixin<Doc>>,
mv,
d.document.modifiedOn,
d.document.modifiedBy
)
} else {
const existingM = hierarchy.as(existing, mRef)
await updateDoc(applyOp, existingM, mv)
}
}
} else {
await applyOp.createDoc(
d.document._class,
d.document.space,
d.document,
d.document._id,
d.document.modifiedOn,
d.document.modifiedBy
)
await applyOp.createMixin<Doc, BitrixSyncDoc>(
d.document._id,
d.document._class,
d.document.space,
bitrix.mixin.BitrixSyncDoc,
{
type: d.document.type,
bitrixId: d.document.bitrixId
},
d.document.modifiedOn,
d.document.modifiedBy
)
for (const [m, mv] of Object.entries(d.mixins)) {
await applyOp.createMixin(
d.document._id,
d.document._class,
d.document.space,
m as Ref<Mixin<Doc>>,
mv,
d.document.modifiedOn,
d.document.modifiedBy
)
}
for (const ed of d.extraDocs) {
if (applyOp.getHierarchy().isDerived(ed._class, core.class.AttachedDoc)) {
const adoc = ed as AttachedDoc
await applyOp.addCollection(
adoc._class,
adoc.space,
adoc.attachedTo,
adoc.attachedToClass,
adoc.collection,
adoc,
adoc._id,
d.document.modifiedOn,
d.document.modifiedBy
)
} else {
await applyOp.createDoc(ed._class, ed.space, ed, ed._id, d.document.modifiedOn, d.document.modifiedBy)
}
}
if (d.comments !== undefined) {
const comments = await d.comments
if (comments !== undefined && comments.length > 0) {
const existingComments = await client.findAll(chunter.class.Comment, {
attachedTo: d.document._id,
[bitrix.mixin.BitrixSyncDoc + '.bitrixId']: { $in: comments.map((it) => it.bitrixId) }
})
for (const comment of comments) {
const existing = existingComments.find(
(it) => hierarchy.as<Doc, BitrixSyncDoc>(it, bitrix.mixin.BitrixSyncDoc).bitrixId === comment.bitrixId
)
if (existing !== undefined) {
// We need to update fields if they are different.
await updateDoc(applyOp, existing, comment)
} else {
await applyOp.addCollection(
comment._class,
comment.space,
comment.attachedTo,
comment.attachedToClass,
comment.collection,
comment,
comment._id,
comment.modifiedOn,
comment.modifiedBy
)
await applyOp.createMixin<Doc, BitrixSyncDoc>(
comment._id,
comment._class,
comment.space,
bitrix.mixin.BitrixSyncDoc,
{
type: d.document.type,
bitrixId: d.document.bitrixId
},
comment.modifiedOn,
comment.modifiedBy
)
}
}
}
}
}
await applyOp.commit()
docsProcessed++
state = `processed: ${docsProcessed}/${total}`
}
}
async function doSync (): Promise<void> {
loading = true
const commentFields = await bitrixClient.call(BitrixEntityType.Comment + '.fields', {})
const commentFieldKeys = Object.keys(commentFields.result)
const allEmployee = await client.findAll(contact.class.EmployeeAccount, {})
const userList = new Map<string, Ref<EmployeeAccount>>()
// Fill all users and create new ones, if required.
let totalUsers = 1
let next = 0
while (userList.size < totalUsers) {
const users = await bitrixClient.call('user.search', { start: next })
next = users.next
totalUsers = users.total
for (const u of users.result) {
let accountId = allEmployee.find((it) => it.email === u.EMAIL)?._id
if (accountId === undefined) {
const employeeId = await client.createDoc(contact.class.Employee, contact.space.Contacts, {
name: combineName(u.NAME, u.LAST_NAME),
avatar: u.PERSONAL_PHOTO,
active: u.ACTIVE,
city: u.PERSONAL_CITY
})
accountId = await client.createDoc(contact.class.EmployeeAccount, core.space.Model, {
email: u.EMAIL,
name: combineName(u.NAME, u.LAST_NAME),
employee: employeeId,
role: AccountRole.User
})
}
userList.set(u.ID, accountId)
}
}
const uploadUrl = (window.location.origin + getMetadata(login.metadata.UploadUrl)) as string
const token = (getMetadata(login.metadata.LoginToken) as string) ?? ''
try {
if (space === undefined || mapping.$lookup?.fields === undefined) {
return
}
let processed = 0
const tagElements: Map<Ref<Class<Doc>>, TagElement[]> = new Map()
let added = 0
while (added <= limit) {
const sel = ['*', 'UF_*']
if (mapping.type === BitrixEntityType.Lead) {
sel.push('EMAIL')
sel.push('IM')
await performSynchronization({
bitrixClient,
client,
direction,
limit,
space,
mapping,
loginInfo: {
token,
email: '',
endpoint: ''
},
frontUrl: uploadUrl,
monitor: (total: number) => {
docsProcessed++
state = `processed: ${docsProcessed}/${total ?? 1}`
}
const result = await bitrixClient.call(mapping.type + '.list', {
select: sel,
order: { ID: direction },
start: processed
})
const extraDocs: Doc[] = []
const convertResults: ConvertResult[] = []
const fields = mapping.$lookup?.fields as BitrixFieldMapping[]
for (const r of result.result) {
// Convert documents.
const res = await convert(client, mapping, space, fields, r, extraDocs, tagElements, userList)
if (mapping.comments) {
res.comments = bitrixClient
.call(BitrixEntityType.Comment + '.list', {
filter: {
ENTITY_ID: res.document.bitrixId,
ENTITY_TYPE: mapping.type.replace('crm.', '')
},
select: commentFieldKeys,
order: { ID: direction }
})
.then((comments) => {
return comments.result.map(
(it: any) =>
({
_id: generateId(),
_class: chunter.class.Comment,
message: it.COMMENT,
bitrixId: it.ID,
type: it.ENTITY_TYPE,
attachedTo: res.document._id,
attachedToClass: res.document._class,
collection: 'comments',
space: res.document.space,
modifiedBy: userList.get(it.AUTHOR_ID) ?? core.account.System,
modifiedOn: new Date(userList.get(it.CREATED) ?? new Date().toString()).getTime()
} as Comment)
)
})
}
convertResults.push(res)
extraDocs.push(...res.extraDocs)
added++
if (added > limit) {
break
}
}
total = result.total
await syncPlatform(convertResults)
processed = result.next
}
})
} catch (err: any) {
state = err.message
console.error(err)

View File

@ -0,0 +1,6 @@
import PQueue from 'p-queue'
export const bitrixQueue = new PQueue({
intervalCap: 2,
interval: 1000
})

View File

@ -1,45 +0,0 @@
/**
* @public
*/
export interface BitrixProfile {
ID: string
ADMIN: boolean
NAME: string
LAST_NAME: string
PERSONAL_GENDER: string
PERSONAL_PHOTO: string
TIME_ZONE: string
TIME_ZONE_OFFSET: number
}
export type NumberString = string
export type ISODate = string
export type BoolString = 'Y' | 'N'
export type GenderString = 'M' | 'F' | ''
export interface MultiField {
readonly ID: NumberString
readonly VALUE_TYPE: string
readonly VALUE: string
readonly TYPE_ID: string
}
export type MultiFieldArray = ReadonlyArray<Pick<MultiField, 'VALUE' | 'VALUE_TYPE'>>
export interface StatusValue {
CATEGORY_ID: string | null
COLOR: string | null
ENTITY_ID: string | null
ID: number
NAME: string
NAME_INIT: string | null
SEMANTICS: string | null
SORT: string | null
STATUS_ID: string | null
SYSTEM: 'Y' | 'N'
}
export interface BitrixResult {
result: any
next: number
total: number
}

View File

@ -1,6 +1,6 @@
{
"name": "@hcengineering/bitrix",
"version": "0.6.1",
"version": "0.6.4",
"main": "lib/index.js",
"author": "Anticrm Platform Contributors",
"license": "EPL-2.0",
@ -23,15 +23,19 @@
"eslint-config-standard-with-typescript": "^23.0.0",
"prettier": "^2.7.1",
"@rushstack/heft": "^0.47.9",
"typescript": "^4.3.5"
"typescript": "^4.3.5",
"@types/qs": "~6.9.7"
},
"dependencies": {
"@hcengineering/platform": "^0.6.8",
"@hcengineering/core": "^0.6.20",
"@hcengineering/ui": "^0.6.3",
"@hcengineering/preference": "^0.6.2",
"@hcengineering/tags": "^0.6.3",
"@hcengineering/contact": "^0.6.9"
"@hcengineering/contact": "^0.6.9",
"@hcengineering/chunter": "^0.6.2",
"@hcengineering/attachment": "^0.6.1",
"fast-equals": "^2.0.3",
"qs": "~6.11.0"
},
"repository": "https://github.com/hcengineering/anticrm",
"publishConfig": {

View File

@ -1,16 +1,16 @@
import PQueue from 'p-queue'
import { stringify as toQuery } from 'qs'
import { BitrixResult } from './types'
const queue = new PQueue({
intervalCap: 2,
interval: 1000
})
/**
* @public
*
* Require a proper rate limiter to function properly.
*/
export class BitrixClient {
constructor (readonly url: string) {}
constructor (readonly url: string, readonly rateLimiter: <T>(op: () => Promise<T>) => Promise<T>) {}
async call (method: string, params: any): Promise<BitrixResult> {
return await queue.add(async () => {
return await this.rateLimiter(async () => {
let query: string = toQuery(params)
if (query.length > 0) {
query = `?${query}`

View File

@ -13,161 +13,10 @@
// limitations under the License.
//
import { ChannelProvider } from '@hcengineering/contact'
import type { AttachedDoc, Class, Doc, Mixin, Ref, Space } from '@hcengineering/core'
import type { Plugin } from '@hcengineering/platform'
import type { Class, Mixin, Ref, Space } from '@hcengineering/core'
import type { Plugin, Resource } from '@hcengineering/platform'
import { Asset, plugin } from '@hcengineering/platform'
import { ExpertKnowledge, InitialKnowledge, MeaningfullKnowledge } from '@hcengineering/tags'
import { AnyComponent } from '@hcengineering/ui'
/**
* @public
*/
export interface BitrixSyncDoc extends Doc {
type: string
bitrixId: string
}
/**
* @public
*/
export enum BitrixEntityType {
Comment = 'crm.timeline.comment',
Binding = 'crm.timeline.bindings',
Lead = 'crm.lead',
Activity = 'crm.activity',
Company = 'crm.company'
}
/**
* @public
*/
export const mappingTypes = [
{ label: 'Leads', id: BitrixEntityType.Lead },
// { label: 'Comments', id: BitrixEntityType.Comment },
{ label: 'Company', id: BitrixEntityType.Company }
// { label: 'Activity', id: BitrixEntityType.Activity }
]
/**
* @public
*/
export interface FieldValue {
type: string
statusType?: string
isRequired: boolean
isReadOnly: boolean
isImmutable: boolean
isMultiple: boolean
isDynamic: boolean
title: string
formLabel?: string
filterLabel?: string
items?: Array<{
ID: string
VALUE: string
}>
}
/**
* @public
*/
export interface Fields {
[key: string]: FieldValue
}
/**
* @public
*/
export interface BitrixEntityMapping extends Doc {
ofClass: Ref<Class<Doc>>
type: string
bitrixFields: Fields
fields: number
comments: boolean
activity: boolean
attachments: boolean
}
/**
* @public
*/
export enum MappingOperation {
CopyValue,
CreateTag, // Create tag
CreateChannel, // Create channel
DownloadAttachment
}
/**
* @public
*/
export interface CopyPattern {
text: string
field?: string
alternatives?: string[]
}
/**
* @public
*/
export interface CopyValueOperation {
kind: MappingOperation.CopyValue
patterns: CopyPattern[]
}
/**
* @public
*/
export interface TagField {
weight: InitialKnowledge | MeaningfullKnowledge | ExpertKnowledge
field: string
split: string // If defined values from field will be split to check for multiple values.
}
/**
* @public
*/
export interface CreateTagOperation {
kind: MappingOperation.CreateTag
fields: TagField[]
}
/**
* @public
*/
export interface ChannelFieldMapping {
provider: Ref<ChannelProvider>
field: string
}
/**
* @public
*/
export interface CreateChannelOperation {
kind: MappingOperation.CreateChannel
fields: ChannelFieldMapping[]
}
/**
* @public
*/
export interface DownloadAttachmentOperation {
kind: MappingOperation.DownloadAttachment
fields: { field: string }[]
}
/**
* @public
*/
export interface BitrixFieldMapping extends AttachedDoc {
ofClass: Ref<Class<Doc>> // Specify mixin if applicable
attributeName: string
operation: CopyValueOperation | CreateTagOperation | CreateChannelOperation | DownloadAttachmentOperation
}
import { BitrixEntityMapping, BitrixFieldMapping, BitrixSyncDoc } from './types'
/**
* @public
@ -183,7 +32,7 @@ export default plugin(bitrixId, {
FieldMapping: '' as Ref<Class<BitrixFieldMapping>>
},
component: {
BitrixIntegration: '' as AnyComponent
BitrixIntegration: '' as Resource<any>
},
icon: {
Bitrix: '' as Asset
@ -192,3 +41,8 @@ export default plugin(bitrixId, {
Mappings: '' as Ref<Space>
}
})
export * from './client'
export * from './sync'
export * from './types'
export * from './utils'

448
plugins/bitrix/src/sync.ts Normal file
View File

@ -0,0 +1,448 @@
import attachment, { Attachment } from '@hcengineering/attachment'
import chunter, { Comment } from '@hcengineering/chunter'
import contact, { combineName, EmployeeAccount } from '@hcengineering/contact'
import core, {
AccountRole,
ApplyOperations,
AttachedDoc,
Class,
Data,
Doc,
DocumentUpdate,
generateId,
Mixin,
Ref,
Space,
TxOperations,
WithLookup
} from '@hcengineering/core'
import tags, { TagElement } from '@hcengineering/tags'
import { deepEqual } from 'fast-equals'
import { BitrixClient } from './client'
import { BitrixEntityMapping, BitrixEntityType, BitrixFieldMapping, BitrixSyncDoc, LoginInfo } from './types'
import { convert, ConvertResult } from './utils'
import bitrix from './index'
async function updateDoc (client: ApplyOperations, doc: Doc, raw: Doc | Data<Doc>): Promise<void> {
// We need to update fields if they are different.
const documentUpdate: DocumentUpdate<Doc> = {}
for (const [k, v] of Object.entries(raw)) {
if (['_class', '_id', 'modifiedBy', 'modifiedOn', 'space'].includes(k)) {
continue
}
if (!deepEqual((doc as any)[k], v)) {
;(documentUpdate as any)[k] = v
}
}
if (Object.keys(documentUpdate).length > 0) {
await client.update(doc, documentUpdate)
}
}
/**
* @public
*/
export async function syncPlatform (
client: TxOperations,
mapping: BitrixEntityMapping,
documents: ConvertResult[],
info: LoginInfo,
frontUrl: string,
monitor?: (doc: ConvertResult) => void
): Promise<void> {
const existingDocuments = await client.findAll(mapping.ofClass, {
[bitrix.mixin.BitrixSyncDoc + '.bitrixId']: { $in: documents.map((it) => it.document.bitrixId) }
})
const hierarchy = client.getHierarchy()
let syncronized = 0
for (const d of documents) {
try {
const existing = existingDocuments.find(
(it) => hierarchy.as<Doc, BitrixSyncDoc>(it, bitrix.mixin.BitrixSyncDoc).bitrixId === d.document.bitrixId
)
const applyOp = client.apply('bitrix')
if (existing !== undefined) {
// We need update doucment id.
d.document._id = existing._id as Ref<BitrixSyncDoc>
// We need to update fields if they are different.
await updateDoc(applyOp, existing, d.document)
// Check and update mixins
for (const [m, mv] of Object.entries(d.mixins)) {
const mRef = m as Ref<Mixin<Doc>>
if (hierarchy.hasMixin(existing, mRef)) {
await applyOp.createMixin(
d.document._id,
d.document._class,
d.document.space,
m as Ref<Mixin<Doc>>,
mv,
d.document.modifiedOn,
d.document.modifiedBy
)
} else {
const existingM = hierarchy.as(existing, mRef)
await updateDoc(applyOp, existingM, mv)
}
}
} else {
await applyOp.createDoc(
d.document._class,
d.document.space,
d.document,
d.document._id,
d.document.modifiedOn,
d.document.modifiedBy
)
await applyOp.createMixin<Doc, BitrixSyncDoc>(
d.document._id,
d.document._class,
d.document.space,
bitrix.mixin.BitrixSyncDoc,
{
type: d.document.type,
bitrixId: d.document.bitrixId
},
d.document.modifiedOn,
d.document.modifiedBy
)
for (const [m, mv] of Object.entries(d.mixins)) {
await applyOp.createMixin(
d.document._id,
d.document._class,
d.document.space,
m as Ref<Mixin<Doc>>,
mv,
d.document.modifiedOn,
d.document.modifiedBy
)
}
for (const ed of d.extraDocs) {
if (applyOp.getHierarchy().isDerived(ed._class, core.class.AttachedDoc)) {
const adoc = ed as AttachedDoc
await applyOp.addCollection(
adoc._class,
adoc.space,
adoc.attachedTo,
adoc.attachedToClass,
adoc.collection,
adoc,
adoc._id,
d.document.modifiedOn,
d.document.modifiedBy
)
} else {
await applyOp.createDoc(ed._class, ed.space, ed, ed._id, d.document.modifiedOn, d.document.modifiedBy)
}
}
for (const ed of d.blobs) {
const attachmentId: Ref<Attachment> = generateId()
const data = new FormData()
data.append('file', ed)
const resp = await fetch(frontUrl + '/files', {
method: 'POST',
headers: {
Authorization: 'Bearer ' + info.token
},
body: data
})
if (resp.status === 200) {
const uuid = await resp.text()
await applyOp.addCollection(
attachment.class.Attachment,
d.document.space,
d.document._id,
d.document._class,
'attachments',
{
file: uuid,
lastModified: ed.lastModified,
name: ed.name,
size: ed.size,
type: ed.type
},
attachmentId,
d.document.modifiedOn,
d.document.modifiedBy
)
}
}
if (d.comments !== undefined) {
const comments = await d.comments
if (comments !== undefined && comments.length > 0) {
const existingComments = await client.findAll(chunter.class.Comment, {
attachedTo: d.document._id,
[bitrix.mixin.BitrixSyncDoc + '.bitrixId']: { $in: comments.map((it) => it.bitrixId) }
})
for (const comment of comments) {
const existing = existingComments.find(
(it) => hierarchy.as<Doc, BitrixSyncDoc>(it, bitrix.mixin.BitrixSyncDoc).bitrixId === comment.bitrixId
)
if (existing !== undefined) {
// We need to update fields if they are different.
await updateDoc(applyOp, existing, comment)
} else {
await applyOp.addCollection(
comment._class,
comment.space,
comment.attachedTo,
comment.attachedToClass,
comment.collection,
comment,
comment._id,
comment.modifiedOn,
comment.modifiedBy
)
await applyOp.createMixin<Doc, BitrixSyncDoc>(
comment._id,
comment._class,
comment.space,
bitrix.mixin.BitrixSyncDoc,
{
type: d.document.type,
bitrixId: d.document.bitrixId
},
comment.modifiedOn,
comment.modifiedBy
)
}
}
}
}
}
await applyOp.commit()
} catch (err: any) {
console.error(err)
}
console.log('SYNC:', syncronized, documents.length - syncronized)
syncronized++
monitor?.(d)
}
}
/**
* @public
*/
export function processComment (comment: string): string {
comment = comment.replaceAll('\n', '\n</br>')
comment = comment.replaceAll(/\[(\/?[^[\]]+)]/gi, (text: string, args: string) => {
if (args.startsWith('/URL')) {
return '</a>'
}
if (args.startsWith('URL=')) {
return `<a href="${args.substring(4)}">`
}
if (args.includes('/FONT')) {
return '</span>'
}
if (args.includes('FONT')) {
return `<span style="font: ${args.substring(4)};">`
}
if (args.includes('/SIZE')) {
return '</span>'
}
if (args.includes('SIZE')) {
return `<span style="font-size: ${args.substring(4)};">`
}
if (args.includes('/COLOR')) {
return '</span>'
}
if (args.includes('COLOR')) {
return `<span style="color: ${args.substring(5)};">`
}
if (args.includes('/IMG')) {
return '"/>'
}
if (args.includes('IMG')) {
return `<img ${args.substring(3)} src="`
}
if (args.includes('/TABLE')) {
return '</table>'
}
if (args.includes('TABLE')) {
return '<table>'
}
return `<${args}>`
})
return comment
}
/**
* @public
*/
export async function performSynchronization (ops: {
client: TxOperations
bitrixClient: BitrixClient
space: Ref<Space> | undefined
mapping: WithLookup<BitrixEntityMapping>
limit: number
direction: 'ASC' | 'DSC'
frontUrl: string
loginInfo: LoginInfo
monitor: (total: number) => void
}): Promise<void> {
const commentFields = await ops.bitrixClient.call(BitrixEntityType.Comment + '.fields', {})
const commentFieldKeys = Object.keys(commentFields.result)
const allEmployee = await ops.client.findAll(contact.class.EmployeeAccount, {})
const userList = new Map<string, Ref<EmployeeAccount>>()
// Fill all users and create new ones, if required.
let totalUsers = 1
let next = 0
while (userList.size < totalUsers) {
const users = await ops.bitrixClient.call('user.search', { start: next })
next = users.next
totalUsers = users.total
for (const u of users.result) {
let accountId = allEmployee.find((it) => it.email === u.EMAIL)?._id
if (accountId === undefined) {
const employeeId = await ops.client.createDoc(contact.class.Employee, contact.space.Contacts, {
name: combineName(u.NAME, u.LAST_NAME),
avatar: u.PERSONAL_PHOTO,
active: u.ACTIVE,
city: u.PERSONAL_CITY
})
accountId = await ops.client.createDoc(contact.class.EmployeeAccount, core.space.Model, {
email: u.EMAIL,
name: combineName(u.NAME, u.LAST_NAME),
employee: employeeId,
role: AccountRole.User
})
}
userList.set(u.ID, accountId)
}
}
try {
if (ops.space === undefined || ops.mapping.$lookup?.fields === undefined) {
return
}
let processed = 0
const tagElements: Map<Ref<Class<Doc>>, TagElement[]> = new Map()
let added = 0
while (added < ops.limit) {
const sel = ['*', 'UF_*']
if (ops.mapping.type === BitrixEntityType.Lead) {
sel.push('EMAIL')
sel.push('IM')
}
const result = await ops.bitrixClient.call(ops.mapping.type + '.list', {
select: sel,
order: { ID: ops.direction },
start: processed
})
const extraDocs: Doc[] = []
const convertResults: ConvertResult[] = []
const fields = ops.mapping.$lookup?.fields as BitrixFieldMapping[]
const toProcess = result.result as any[]
const existingDocuments = await ops.client.findAll(
ops.mapping.ofClass,
{
[bitrix.mixin.BitrixSyncDoc + '.bitrixId']: { $in: toProcess.map((it) => `${it.ID as string}`) }
},
{
projection: {
_id: 1,
[bitrix.mixin.BitrixSyncDoc + '.bitrixId']: 1
}
}
)
const defaultCategories = await ops.client.findAll(tags.class.TagCategory, {
default: true
})
let synchronized = 0
while (toProcess.length > 0) {
console.log('LOAD:', synchronized, toProcess.length)
synchronized++
const [r] = toProcess.slice(0, 1)
// Convert documents.
try {
const res = await convert(
ops.client,
ops.mapping,
ops.space,
fields,
r,
extraDocs,
tagElements,
userList,
existingDocuments,
defaultCategories
)
if (ops.mapping.comments) {
res.comments = await ops.bitrixClient
.call(BitrixEntityType.Comment + '.list', {
filter: {
ENTITY_ID: res.document.bitrixId,
ENTITY_TYPE: ops.mapping.type.replace('crm.', '')
},
select: commentFieldKeys,
order: { ID: ops.direction }
})
.then((comments) => {
return comments.result.map((it: any) => {
const c: Comment & { bitrixId: string, type: string } = {
_id: generateId(),
_class: chunter.class.Comment,
message: processComment(it.COMMENT as string),
bitrixId: it.ID,
type: it.ENTITY_TYPE,
attachedTo: res.document._id,
attachedToClass: res.document._class,
collection: 'comments',
space: res.document.space,
modifiedBy: userList.get(it.AUTHOR_ID) ?? core.account.System,
modifiedOn: new Date(it.CREATED ?? new Date().toString()).getTime()
}
return c
})
})
}
convertResults.push(res)
extraDocs.push(...res.extraDocs)
added++
if (added >= ops.limit) {
break
}
} catch (err: any) {
console.log('failed to obtain data for', r)
await new Promise((resolve) => {
// Sleep for a while
setTimeout(resolve, 1000)
})
}
toProcess.splice(0, 1)
}
const total = result.total
await syncPlatform(ops.client, ops.mapping, convertResults, ops.loginInfo, ops.frontUrl, () => {
ops.monitor?.(total)
})
processed = result.next
}
} catch (err: any) {
console.error(err)
}
}

232
plugins/bitrix/src/types.ts Normal file
View File

@ -0,0 +1,232 @@
import { ChannelProvider } from '@hcengineering/contact'
import { AttachedDoc, Class, Doc, Ref } from '@hcengineering/core'
import { ExpertKnowledge, InitialKnowledge, MeaningfullKnowledge } from '@hcengineering/tags'
/**
* @public
*/
export interface BitrixProfile {
ID: string
ADMIN: boolean
NAME: string
LAST_NAME: string
PERSONAL_GENDER: string
PERSONAL_PHOTO: string
TIME_ZONE: string
TIME_ZONE_OFFSET: number
}
/**
* @public
*/
export type NumberString = string
/**
* @public
*/
export type ISODate = string
/**
* @public
*/
export type BoolString = 'Y' | 'N'
/**
* @public
*/
export type GenderString = 'M' | 'F' | ''
/**
* @public
*/
export interface MultiField {
readonly ID: NumberString
readonly VALUE_TYPE: string
readonly VALUE: string
readonly TYPE_ID: string
}
/**
* @public
*/
export type MultiFieldArray = ReadonlyArray<Pick<MultiField, 'VALUE' | 'VALUE_TYPE'>>
/**
* @public
*/
export interface StatusValue {
CATEGORY_ID: string | null
COLOR: string | null
ENTITY_ID: string | null
ID: number
NAME: string
NAME_INIT: string | null
SEMANTICS: string | null
SORT: string | null
STATUS_ID: string | null
SYSTEM: 'Y' | 'N'
}
/**
* @public
*/
export interface BitrixResult {
result: any
next: number
total: number
}
/**
* @public
*/
export interface LoginInfo {
endpoint: string
email: string
token: string
}
/**
* @public
*/
export interface BitrixSyncDoc extends Doc {
type: string
bitrixId: string
}
/**
* @public
*/
export enum BitrixEntityType {
Comment = 'crm.timeline.comment',
Binding = 'crm.timeline.bindings',
Lead = 'crm.lead',
Activity = 'crm.activity',
Company = 'crm.company'
}
/**
* @public
*/
export const mappingTypes = [
{ label: 'Leads', id: BitrixEntityType.Lead },
// { label: 'Comments', id: BitrixEntityType.Comment },
{ label: 'Company', id: BitrixEntityType.Company }
// { label: 'Activity', id: BitrixEntityType.Activity }
]
/**
* @public
*/
export interface FieldValue {
type: string
statusType?: string
isRequired: boolean
isReadOnly: boolean
isImmutable: boolean
isMultiple: boolean
isDynamic: boolean
title: string
formLabel?: string
filterLabel?: string
items?: Array<{
ID: string
VALUE: string
}>
}
/**
* @public
*/
export interface Fields {
[key: string]: FieldValue
}
/**
* @public
*/
export interface BitrixEntityMapping extends Doc {
ofClass: Ref<Class<Doc>>
type: string
bitrixFields: Fields
fields: number
comments: boolean
activity: boolean
attachments: boolean
}
/**
* @public
*/
export enum MappingOperation {
CopyValue,
CreateTag, // Create tag
CreateChannel, // Create channel
DownloadAttachment
}
/**
* @public
*/
export interface CopyPattern {
text: string
field?: string
alternatives?: string[]
}
/**
* @public
*/
export interface CopyValueOperation {
kind: MappingOperation.CopyValue
patterns: CopyPattern[]
}
/**
* @public
*/
export interface TagField {
weight: InitialKnowledge | MeaningfullKnowledge | ExpertKnowledge
field: string
split: string // If defined values from field will be split to check for multiple values.
}
/**
* @public
*/
export interface CreateTagOperation {
kind: MappingOperation.CreateTag
fields: TagField[]
}
/**
* @public
*/
export interface ChannelFieldMapping {
provider: Ref<ChannelProvider>
field: string
}
/**
* @public
*/
export interface CreateChannelOperation {
kind: MappingOperation.CreateChannel
fields: ChannelFieldMapping[]
}
/**
* @public
*/
export interface DownloadAttachmentOperation {
kind: MappingOperation.DownloadAttachment
fields: { field: string }[]
}
/**
* @public
*/
export interface BitrixFieldMapping extends AttachedDoc {
ofClass: Ref<Class<Doc>> // Specify mixin if applicable
attributeName: string
operation: CopyValueOperation | CreateTagOperation | CreateChannelOperation | DownloadAttachmentOperation
}

View File

@ -1,3 +1,7 @@
import { Comment } from '@hcengineering/chunter'
import contact, { Channel, EmployeeAccount } from '@hcengineering/contact'
import core, { AnyAttribute, Class, Client, Data, Doc, generateId, Mixin, Ref, Space } from '@hcengineering/core'
import tags, { TagCategory, TagElement, TagReference } from '@hcengineering/tags'
import {
BitrixEntityMapping,
BitrixFieldMapping,
@ -5,14 +9,14 @@ import {
CopyValueOperation,
CreateChannelOperation,
CreateTagOperation,
DownloadAttachmentOperation,
MappingOperation
} from '@hcengineering/bitrix'
import { Comment } from '@hcengineering/chunter'
import contact, { Channel, EmployeeAccount } from '@hcengineering/contact'
import core, { AnyAttribute, Class, Client, Data, Doc, generateId, Mixin, Ref, Space } from '@hcengineering/core'
import tags, { TagElement, TagReference } from '@hcengineering/tags'
import { getColorNumberByText } from '@hcengineering/ui'
} from '.'
import bitrix from './index'
/**
* @public
*/
export function collectFields (fieldMapping: BitrixFieldMapping[]): string[] {
const fields: string[] = ['ID']
for (const f of fieldMapping) {
@ -33,14 +37,20 @@ export function collectFields (fieldMapping: BitrixFieldMapping[]): string[] {
return fields
}
/**
* @public
*/
export interface ConvertResult {
document: BitrixSyncDoc // Document we should achive
mixins: Record<Ref<Mixin<Doc>>, Data<Doc>> // Mixins of document we will achive
extraDocs: Doc[] // Extra documents we will achive, comments etc.
blobs: File[] //
comments?: Promise<Array<BitrixSyncDoc & Comment>>
comments?: Array<BitrixSyncDoc & Comment>
}
/**
* @public
*/
export async function convert (
client: Client,
entity: BitrixEntityMapping,
@ -49,19 +59,26 @@ export async function convert (
rawDocument: any,
prevExtra: Doc[], // <<-- a list of previous extra documents, so for example TagElement will be reused, if present for more what one item and required to be created
tagElements: Map<Ref<Class<Doc>>, TagElement[]>, // TagElement cache.
userList: Map<string, Ref<EmployeeAccount>>
userList: Map<string, Ref<EmployeeAccount>>,
existingDocuments: Doc[],
defaultCategories: TagCategory[],
blobProvider?: (blobRef: any) => Promise<Blob | undefined>
): Promise<ConvertResult> {
const hierarchy = client.getHierarchy()
const bitrixId = `${rawDocument.ID as string}`
const document: BitrixSyncDoc = {
_id: generateId(),
type: entity.type,
bitrixId: `${rawDocument.ID as string}`,
bitrixId,
_class: entity.ofClass,
space,
modifiedOn: new Date(rawDocument.DATE_CREATE).getTime(),
modifiedBy: userList.get(rawDocument.CREATED_BY_ID) ?? core.account.System
}
const existingId = existingDocuments.find((it) => (it as any)[bitrix.mixin.BitrixSyncDoc].bitrixId === bitrixId)
?._id as Ref<BitrixSyncDoc>
// Obtain a proper modified by for document
const newExtraDocs: Doc[] = []
@ -91,6 +108,10 @@ export async function convert (
if (Array.isArray(lval)) {
return lval.map((it) => it.VALUE)
}
} else if (bfield.type === 'file') {
if (Array.isArray(lval)) {
return lval.map((it) => ({ id: it.id, file: it.downloadUrl }))
}
} else if (bfield.type === 'string' || bfield.type === 'url') {
if (bfield.isMultiple && Array.isArray(lval)) {
return lval.join(', ')
@ -135,6 +156,26 @@ export async function convert (
}
return r.join('').trim()
}
const getDownloadValue = async (attr: AnyAttribute, operation: DownloadAttachmentOperation): Promise<any> => {
const r: Array<string | number | boolean | Date> = []
for (const o of operation.fields) {
const lval = extractValue(o.field)
if (lval != null) {
if (Array.isArray(lval)) {
r.push(...lval)
} else {
r.push(lval)
}
}
}
if (r.length === 1) {
return r[0]
}
if (r.length === 0) {
return
}
return r.join('').trim()
}
const getChannelValue = async (attr: AnyAttribute, operation: CreateChannelOperation): Promise<any> => {
for (const f of operation.fields) {
@ -164,21 +205,22 @@ export async function convert (
const getTagValue = async (attr: AnyAttribute, operation: CreateTagOperation): Promise<any> => {
const elements =
tagElements.get(attr.attributeOf) ??
(await client.findAll(tags.class.TagElement, {
(await client.findAll<TagElement>(tags.class.TagElement, {
targetClass: attr.attributeOf
}))
const references = await client.findAll(tags.class.TagReference, {
attachedTo: document._id
})
const references =
existingId !== undefined
? await client.findAll<TagReference>(tags.class.TagReference, {
attachedTo: existingId
})
: []
// Add tags creation requests from previous conversions.
elements.push(...prevExtra.filter((it) => it._class === tags.class.TagElement).map((it) => it as TagElement))
tagElements.set(attr.attributeOf, elements)
const defaultCategory = await client.findOne(tags.class.TagCategory, {
targetClass: attr.attributeOf,
default: true
})
const defaultCategory = defaultCategories.find((it) => it.targetClass === attr.attributeOf)
if (defaultCategory === undefined) {
console.error('could not proceed tags without default category')
return
@ -206,7 +248,7 @@ export async function convert (
_id: generateId(),
_class: tags.class.TagElement,
category: defaultCategory._id,
color: getColorNumberByText(vv),
color: 1,
description: '',
title: vv,
targetClass: attr.attributeOf,
@ -218,12 +260,12 @@ export async function convert (
}
const ref: TagReference = {
_id: generateId(),
attachedTo: document._id,
attachedTo: existingId ?? document._id,
attachedToClass: attr.attributeOf,
collection: attr.name,
_class: tags.class.TagReference,
tag: tag._id,
color: getColorNumberByText(vv),
color: 1,
title: vv,
weight: o.weight,
modifiedBy: document.modifiedBy,
@ -256,10 +298,32 @@ export async function convert (
case MappingOperation.CreateTag:
value = await getTagValue(attr, f.operation)
break
case MappingOperation.DownloadAttachment: {
const blobRef: { file: string, id: string } = await getDownloadValue(attr, f.operation)
if (blobRef !== undefined) {
const response = await blobProvider?.(blobRef)
if (response !== undefined) {
let fname = blobRef.id
switch (response.type) {
case 'application/pdf':
fname += '.pdf'
break
case 'application/msword':
fname += '.doc'
break
}
blobs.push(new File([response], fname, { type: response.type }))
}
}
break
}
}
if (value !== undefined) {
if (hierarchy.isMixin(attr.attributeOf)) {
mixins[attr.attributeOf] = { ...mixins[attr.attributeOf], [attr.name]: value }
mixins[attr.attributeOf] = {
...mixins[attr.attributeOf],
[attr.name]: value
}
} else {
;(document as any)[attr.name] = value
}

View File

@ -28,7 +28,11 @@
{#if value}
<!-- svelte-ignore a11y-click-events-have-key-events -->
<span title={value.label} class="fs-bold caption-color overflow-label clear-mins" on:click={navigateToProject}>
<span
title={value.label}
class="cursor-pointer fs-bold caption-color overflow-label clear-mins"
on:click={navigateToProject}
>
{value.label}
</span>
{/if}

View File

@ -16,7 +16,7 @@ import MoveView from './components/Move.svelte'
import { contextStore } from './context'
import view from './plugin'
import { FocusSelection, focusStore, previewDocument, SelectDirection, selectionStore } from './selection'
import { deleteObject } from './utils'
import { deleteObjects } from './utils'
/**
* Action to be used for copying text to clipboard.
@ -53,23 +53,19 @@ async function CopyTextToClipboard (
}
}
function Delete (object: Doc): void {
function Delete (object: Doc | Doc[]): void {
showPopup(
MessageBox,
{
label: view.string.DeleteObject,
message: view.string.DeleteObjectConfirm,
params: { count: Array.isArray(object) ? object.length : 1 }
},
undefined,
(result?: boolean) => {
if (result === true) {
params: { count: Array.isArray(object) ? object.length : 1 },
action: async () => {
const objs = Array.isArray(object) ? object : [object]
for (const o of objs) {
deleteObject(getClient(), o).catch((err) => console.error(err))
}
await deleteObjects(getClient(), objs).catch((err) => console.error(err))
}
}
},
undefined
)
}

View File

@ -76,7 +76,10 @@
query.unsubscribe()
}
$: if (_class) {
let oldClass: Ref<Class<Doc>>
$: if (_class !== oldClass) {
oldClass = _class
mainEditor = undefined
}
@ -152,9 +155,9 @@
}
let mainEditor: MixinEditor | undefined
$: getEditorOrDefault(realObjectClass, showAllMixins)
$: getEditorOrDefault(realObjectClass, showAllMixins, _id)
async function getEditorOrDefault (_class: Ref<Class<Doc>>, showAllMixins: boolean): Promise<void> {
async function getEditorOrDefault (_class: Ref<Class<Doc>>, showAllMixins: boolean, _id: Ref<Doc>): Promise<void> {
parentClass = getParentClass(_class)
mainEditor = await getEditor(_class)
updateKeys(showAllMixins)
@ -167,11 +170,16 @@
array: view.mixin.ArrayEditor,
collection: view.mixin.CollectionEditor,
inplace: view.mixin.InlineAttributEditor,
attribute: view.mixin.AttributeEditor
attribute: view.mixin.AttributeEditor,
object: undefined
}
const mixinRef = mix[attrClass.category]
const editorMixin = hierarchy.as(clazz, mixinRef)
return editorMixin.editor
if (mixinRef) {
const editorMixin = hierarchy.as(clazz, mixinRef)
return editorMixin.editor
} else {
return undefined
}
}
function getIcon (_class: Ref<Class<Obj>> | undefined): Asset | undefined {

View File

@ -301,6 +301,21 @@ export async function deleteObject (client: TxOperations, object: Doc): Promise<
}
}
export async function deleteObjects (client: TxOperations, objects: Doc[]): Promise<void> {
const ops = client.apply('delete')
for (const object of objects) {
if (client.getHierarchy().isDerived(object._class, core.class.AttachedDoc)) {
const adoc = object as AttachedDoc
await ops
.removeCollection(object._class, object.space, adoc._id, adoc.attachedTo, adoc.attachedToClass, adoc.collection)
.catch((err) => console.error(err))
} else {
await ops.removeDoc(object._class, object.space, object._id).catch((err) => console.error(err))
}
}
await ops.commit()
}
export function getMixinStyle (id: Ref<Class<Doc>>, selected: boolean): string {
const color = getPlatformColorForText(id as string)
return `

View File

@ -101,6 +101,15 @@ export function serveAccount (methods: Record<string, AccountMethod>, productId
const close = (): void => {
server.close()
}
process.on('uncaughtException', (e) => {
console.error(e)
})
process.on('unhandledRejection', (reason, promise) => {
console.error('Unhandled Rejection at:', promise, 'reason:', reason)
})
process.on('SIGINT', close)
process.on('SIGTERM', close)
process.on('exit', close)

View File

@ -210,6 +210,7 @@ async function getAccountInfo (db: Db, email: string, password: string): Promise
* @returns
*/
export async function login (db: Db, productId: string, email: string, password: string): Promise<LoginInfo> {
console.log(`login attempt:${email}`)
await getAccountInfo(db, email, password)
const result = {
endpoint: getEndpoint(),