hcengineering/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2024-11-22 21:50:34 +03:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

TSK-734: Fix Bitrix email import (#2700)

Browse Source 
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev 2023-03-02 01:38:39 +07:00 committed by GitHub
parent 8e69063994
commit 18e9b76c61
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 183 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
plugins/activity-resources/src/utils.ts
View File

@ -47,7 +47,7 @@ async function createPseudoViewlet (
}
const docClass: Class<Doc> = client.getModel().getObject(doc._class)
let trLabel = await translate(docClass.label, {})
let trLabel = docClass.label !== undefined ? await translate(docClass.label, {}) : undefined
if (dtx.collectionAttribute !== undefined) {
const itemLabel = (dtx.collectionAttribute.type as Collection<AttachedDoc>).itemLabel
if (itemLabel !== undefined) {

184
plugins/bitrix/src/sync.ts
View File

@ -1,7 +1,8 @@
import attachment, { Attachment } from '@hcengineering/attachment'
import chunter, { Comment } from '@hcengineering/chunter'
import contact, { combineName, Contact, EmployeeAccount } from '@hcengineering/contact'
import contact, { Channel, combineName, Contact, EmployeeAccount } from '@hcengineering/contact'
import core, {
Account,
AccountRole,
ApplyOperations,
AttachedDoc,
@ -18,10 +19,12 @@ import core, {
MixinUpdate,
Ref,
Space,
Timestamp,
TxOperations,
TxProcessor,
WithLookup
} from '@hcengineering/core'
import gmail, { Message } from '@hcengineering/gmail'
import tags, { TagElement } from '@hcengineering/tags'
import { deepEqual } from 'fast-equals'
import { BitrixClient } from './client'
@ -38,7 +41,7 @@ import {
} from './types'
import { convert, ConvertResult } from './utils'
async function updateDoc (client: ApplyOperations, doc: Doc, raw: Doc | Data<Doc>): Promise<Doc> {
async function updateDoc (client: ApplyOperations, doc: Doc, raw: Doc | Data<Doc>, date: Timestamp): Promise<Doc> {
// We need to update fields if they are different.
const documentUpdate: DocumentUpdate<Doc> = {}
for (const [k, v] of Object.entries(raw)) {
@ -51,7 +54,7 @@ async function updateDoc (client: ApplyOperations, doc: Doc, raw: Doc | Data<Doc
}
}
if (Object.keys(documentUpdate).length > 0) {
await client.update(doc, documentUpdate)
await client.update(doc, documentUpdate, false, date, doc.modifiedBy)
TxProcessor.applyUpdate(doc, documentUpdate)
}
return doc
@ -61,12 +64,14 @@ async function updateMixin (
client: ApplyOperations,
doc: Doc,
raw: Doc | Data<Doc>,
mixin: Ref<Class<Mixin<Doc>>>
mixin: Ref<Class<Mixin<Doc>>>,
modifiedBy: Ref<Account>,
modifiedOn: Timestamp
): Promise<Doc> {
// We need to update fields if they are different.
if (!client.getHierarchy().hasMixin(doc, mixin)) {
await client.createMixin(doc._id, doc._class, doc.space, mixin, raw as MixinData<Doc, Doc>)
await client.createMixin(doc._id, doc._class, doc.space, mixin, raw as MixinData<Doc, Doc>, modifiedOn, modifiedBy)
return doc
}
@ -81,7 +86,7 @@ async function updateMixin (
}
}
if (Object.keys(documentUpdate).length > 0) {
await client.updateMixin(doc._id, doc._class, doc.space, mixin, documentUpdate)
await client.updateMixin(doc._id, doc._class, doc.space, mixin, documentUpdate, modifiedOn, modifiedBy)
}
return doc
}
@ -129,37 +134,26 @@ export async function syncDocument (
)
}
// Find all attachemnt documents to existing.
// Find all attachment documents to existing.
const byClass = new Map<Ref<Class<Doc>>, (AttachedDoc & BitrixSyncDoc)[]>()
const idMapping = new Map<Ref<Doc>, Ref<Doc>>()
for (const d of resultDoc.extraSync) {
byClass.set(d._class, [...(byClass.get(d._class) ?? []), d])
}
for (const [cl, vals] of byClass.entries()) {
if (applyOp.getHierarchy().isDerived(cl, core.class.AttachedDoc)) {
const existingByClass = await client.findAll(cl, {
attachedTo: resultDoc.document._id
})
await syncClass(applyOp, cl, vals, idMapping, resultDoc.document._id)
}
for (const valValue of vals) {
const existingIdx = existingByClass.findIndex(
(it) => hierarchy.as<Doc, BitrixSyncDoc>(it, bitrix.mixin.BitrixSyncDoc).bitrixId === valValue.bitrixId
)
// Update document id, for existing document.
valValue.attachedTo = resultDoc.document._id
let existing: Doc | undefined
if (existingIdx >= 0) {
existing = existingByClass.splice(existingIdx, 1).shift()
}
await updateAttachedDoc(existing, applyOp, valValue)
}
// Remove previous merged documents, probable they are deleted in bitrix or wrongly migrated.
for (const doc of existingByClass) {
await client.remove(doc)
}
}
// Sync gmail documents
const emailAccount = resultDoc.extraSync.find(
(it) =>
it._class === contact.class.Channel && (it as unknown as Channel).provider === contact.channelProvider.Email
)
if (resultDoc.gmailDocuments.length > 0 && emailAccount !== undefined) {
const emailReadId = idMapping.get(emailAccount._id) ?? emailAccount._id
await syncClass(applyOp, gmail.class.Message, resultDoc.gmailDocuments, idMapping, emailReadId)
}
const existingBlobs = await client.findAll(attachment.class.Attachment, {
@ -229,6 +223,46 @@ export async function syncDocument (
}
monitor?.(resultDoc)
async function syncClass (
applyOp: ApplyOperations,
cl: Ref<Class<Doc>>,
vals: (AttachedDoc & BitrixSyncDoc)[],
idMapping: Map<Ref<Doc>, Ref<Doc>>,
attachedTo: Ref<Doc>
): Promise<void> {
if (applyOp.getHierarchy().isDerived(cl, core.class.AttachedDoc)) {
const existingByClass = await client.findAll(cl, {
attachedTo
})
for (const valValue of vals) {
const id = idMapping.get(valValue.attachedTo)
if (id !== undefined) {
valValue.attachedTo = id
} else {
// Update document id, for existing document.
valValue.attachedTo = resultDoc.document._id
}
const existingIdx = existingByClass.findIndex(
(it) => hierarchy.as<Doc, BitrixSyncDoc>(it, bitrix.mixin.BitrixSyncDoc).bitrixId === valValue.bitrixId
)
let existing: Doc | undefined
if (existingIdx >= 0) {
existing = existingByClass.splice(existingIdx, 1).shift()
if (existing !== undefined) {
idMapping.set(valValue._id, existing._id)
}
}
await updateAttachedDoc(existing, applyOp, valValue)
}
// Remove previous merged documents, probable they are deleted in bitrix or wrongly migrated.
for (const doc of existingByClass) {
await applyOp.remove(doc)
}
}
}
async function updateAttachedDoc (
existing: WithLookup<Doc> | undefined,
applyOp: ApplyOperations,
@ -236,7 +270,7 @@ export async function syncDocument (
): Promise<void> {
if (existing !== undefined) {
// We need to update fields if they are different.
existing = await updateDoc(applyOp, existing, valValue)
existing = await updateDoc(applyOp, existing, valValue, Date.now())
const existingM = hierarchy.as(existing, bitrix.mixin.BitrixSyncDoc)
await updateMixin(
applyOp,
@ -246,7 +280,9 @@ export async function syncDocument (
bitrixId: valValue.bitrixId,
rawData: valValue.rawData
},
bitrix.mixin.BitrixSyncDoc
bitrix.mixin.BitrixSyncDoc,
valValue.modifiedBy,
valValue.modifiedOn
)
} else {
const { bitrixId, rawData, ...data } = valValue
@ -283,7 +319,7 @@ export async function syncDocument (
// We need update doucment id.
resultDoc.document._id = existing._id as Ref<BitrixSyncDoc>
// We need to update fields if they are different.
return (await updateDoc(applyOp, existing, resultDoc.document)) as BitrixSyncDoc
return (await updateDoc(applyOp, existing, resultDoc.document, resultDoc.document.modifiedOn)) as BitrixSyncDoc
// Go over extra documents.
} else {
const { bitrixId, rawData, ...data } = resultDoc.document
@ -323,7 +359,7 @@ async function updateMixins (
)
} else {
const existingM = hierarchy.as(existing, mRef)
await updateMixin(applyOp, existingM, mv, mRef)
await updateMixin(applyOp, existingM, mv, mRef, resultDoc.modifiedBy, resultDoc.modifiedOn)
}
}
}
@ -676,12 +712,13 @@ async function downloadComments (
order: { ID: ops.direction }
})
for (const it of commentsData.result) {
const c: Comment & { bitrixId: string, type: string } = {
const c: Comment & BitrixSyncDoc = {
_id: generateId(),
_class: chunter.class.Comment,
message: processComment(it.COMMENT as string),
bitrixId: `${it.ID as string}`,
type: it.ENTITY_TYPE,
rawData: it,
attachedTo: res.document._id,
attachedToClass: res.document._class,
collection: 'comments',
@ -729,46 +766,51 @@ async function downloadComments (
}
res.extraSync.push(c)
}
const communications = await ops.bitrixClient.call('crm.activity.list', {
order: { ID: 'DESC' },
filter: {
OWNER_ID: res.document.bitrixId,
OWNER_TYPE: ownerType.ID
},
select: ['*', 'COMMUNICATIONS']
})
const cr = Array.isArray(communications.result)
? (communications.result as BitrixActivity[])
: [communications.result as BitrixActivity]
for (const comm of cr) {
const cummunications = comm.COMMUNICATIONS?.map((it) => it.ENTITY_SETTINGS?.LEAD_TITLE ?? '')
let message = `<p>
<span style="color: var(--primary-color-skyblue);">e-mail: ${cummunications?.join(',') ?? ''}</span><br/>\n
<span style="color: var(--primary-color-skyblue);">Subject: ${comm.SUBJECT}</span><br/>\n`
for (const [k, v] of Object.entries(comm.SETTINGS?.EMAIL_META ?? {}).concat(
Object.entries(comm.SETTINGS?.MESSAGE_HEADERS ?? {})
)) {
if (v.trim().length > 0) {
message += `<span style="color: var(--primary-color-skyblue);">${k}: ${v}</span><br/>\n`
const emailAccount = res.extraSync.find(
(it) => it._class === contact.class.Channel && (it as unknown as Channel).provider === contact.channelProvider.Email
)
if (emailAccount !== undefined) {
const communications = await ops.bitrixClient.call('crm.activity.list', {
order: { ID: 'DESC' },
filter: {
OWNER_ID: res.document.bitrixId,
OWNER_TYPE_ID: ownerType.ID
},
select: ['*', 'COMMUNICATIONS']
})
const cr = Array.isArray(communications.result)
? (communications.result as BitrixActivity[])
: [communications.result as BitrixActivity]
for (const comm of cr) {
if (comm.PROVIDER_TYPE_ID === 'EMAIL') {
const parser = new DOMParser()
const c: Message & BitrixSyncDoc = {
_id: generateId(),
_class: gmail.class.Message,
content: comm.DESCRIPTION,
textContent:
parser.parseFromString(comm.DESCRIPTION, 'text/html').textContent?.split('\n').slice(0, 3).join('\n') ?? '',
incoming: comm.DIRECTION === '1',
sendOn: new Date(comm.CREATED ?? new Date().toString()).getTime(),
subject: comm.SUBJECT,
bitrixId: `${comm.ID}`,
rawData: comm,
from: comm.SETTINGS?.EMAIL_META?.from ?? '',
to: comm.SETTINGS?.EMAIL_META?.to ?? '',
replyTo: comm.SETTINGS?.EMAIL_META?.replyTo ?? comm.SETTINGS?.MESSAGE_HEADERS?.['Reply-To'] ?? '',
messageId: comm.SETTINGS?.MESSAGE_HEADERS?.['Message-Id'] ?? '',
attachedTo: emailAccount._id as unknown as Ref<Channel>,
attachedToClass: emailAccount._class,
collection: 'items',
space: res.document.space,
modifiedBy: userList.get(comm.AUTHOR_ID) ?? core.account.System,
modifiedOn: new Date(comm.CREATED ?? new Date().toString()).getTime()
}
res.gmailDocuments.push(c)
}
}
message += '</p>' + comm.DESCRIPTION
const c: Comment & { bitrixId: string, type: string } = {
_id: generateId(),
_class: chunter.class.Comment,
message,
bitrixId: `${comm.ID}`,
type: 'email',
attachedTo: res.document._id,
attachedToClass: res.document._class,
collection: 'comments',
space: res.document.space,
modifiedBy: userList.get(comm.AUTHOR_ID) ?? core.account.System,
modifiedOn: new Date(comm.CREATED ?? new Date().toString()).getTime()
}
res.extraSync.push(c)
}
}

42
plugins/bitrix/src/types.ts
View File

@ -266,25 +266,49 @@ export interface BitrixFieldMapping extends AttachedDoc {
| FindReferenceOperation
}
/**
* @public
*/
export interface BitrixCommunication {
ID: string
TYPE: 'EMAIL' | 'TASK'
VALUE: string // "a@gmail.com",
ENTITY_ID: string // "89013",
ENTITY_TYPE_ID: string // "1",
ENTITY_SETTINGS: {
HONORIFIC: string
NAME: string
SECOND_NAME: string
LAST_NAME: string
LEAD_TITLE: string
}
}
/**
* @public
*/
export interface BitrixActivity {
ID: string
SUBJECT: string
COMMUNICATIONS?: {
ENTITY_SETTINGS?: {
LAST_NAME: string
NAME: string
LEAD_TITLE: string
}
}[]
PROVIDER_TYPE_ID: 'EMAIL' | 'TASK'
COMMUNICATIONS?: BitrixCommunication[]
DESCRIPTION: string
DIRECTION: '1' | '2'
AUTHOR_ID: string
CREATED: number
SETTINGS?: {
MESSAGE_HEADERS?: Record<string, string>
EMAIL_META?: Record<string, string>
MESSAGE_HEADERS?: Record<string, string> & {
'Message-Id': string // "<crm.activity.226613-8PWA4M@a.com>",
'Reply-To': string // "manager@a.com"
}
EMAIL_META?: Record<string, string> & {
__email: string // some email
from: string // From email address
replyTo: string // '
to: string // To email address
cc: string
bcc: string
}
}
}
/**

43
plugins/bitrix/src/utils.ts
View File

@ -13,6 +13,7 @@ import core, {
Space,
WithLookup
} from '@hcengineering/core'
import { Message } from '@hcengineering/gmail'
import tags, { TagCategory, TagElement, TagReference } from '@hcengineering/tags'
import bitrix, {
BitrixEntityMapping,
@ -68,6 +69,7 @@ export interface ConvertResult {
mixins: Record<Ref<Mixin<Doc>>, Data<Doc>> // Mixins of document we will sync
extraDocs: Doc[] // Extra documents we will sync, etc.
extraSync: (AttachedDoc & BitrixSyncDoc)[] // Extra documents we will sync, etc.
gmailDocuments: (Message & BitrixSyncDoc)[]
blobs: [Attachment & BitrixSyncDoc, () => Promise<File | undefined>, (file: File, attach: Attachment) => void][]
syncRequests: BitrixSyncRequest[]
}
@ -134,8 +136,10 @@ export async function convert (
}
return lval
} else if (bfield.type === 'crm_multifield') {
if (Array.isArray(lval)) {
return lval.map((it) => it.VALUE)
if (lval != null && Array.isArray(lval)) {
return lval.map((it) => ({ value: it.VALUE, type: it.VALUE_TYPE.toLowerCase() }))
} else if (lval != null) {
return [{ value: lval.VALUE, type: lval.VALUE_TYPE.toLowerCase() }]
}
} else if (bfield.type === 'file') {
if (Array.isArray(lval) && bfield.isMultiple) {
@ -236,14 +240,38 @@ export async function convert (
if (lval != null && lval !== '') {
const vals = Array.isArray(lval) ? lval : [lval]
for (const llVal of vals) {
const svalue = typeof llVal === 'string' ? llVal : `${JSON.stringify(llVal)}`
if (f.include != null || f.exclude != null) {
if (f.include !== undefined && svalue.match(f.include) == null) {
let svalue: string = typeof llVal === 'string' ? llVal : llVal.value
if (typeof llVal === 'string') {
if (f.include != null || f.exclude != null) {
if (f.include !== undefined && svalue.match(f.include) == null) {
continue
}
if (f.exclude !== undefined && svalue.match(f.exclude) != null) {
continue
}
}
} else {
// TYPE matching to category.
if (f.provider === contact.channelProvider.Telegram && llVal.type !== 'telegram') {
continue
}
if (f.exclude !== undefined && svalue.match(f.exclude) != null) {
if (f.provider === contact.channelProvider.Whatsapp && llVal.type !== 'whatsapp') {
continue
}
if (f.provider === contact.channelProvider.Twitter && llVal.type !== 'twitter') {
continue
}
if (f.provider === contact.channelProvider.LinkedIn && llVal.type !== 'linkedin') {
continue
}
// Fixes
if (f.provider === contact.channelProvider.Telegram) {
if (!svalue.startsWith('@') && !/^\d+/.test(svalue)) {
svalue = '@' + svalue
}
}
}
const c: Channel & BitrixSyncDoc = {
_id: generateId(),
@ -452,7 +480,8 @@ export async function convert (
extraDocs: newExtraDocs,
blobs,
rawData: rawDocument,
syncRequests
syncRequests,
gmailDocuments: []
}
}