hcengineering/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2024-12-22 11:01:54 +03:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Fix release notes and new csv import (#2116)

Browse Source 
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev 2022-06-21 08:32:33 +07:00 committed by GitHub
parent 1091ced274
commit af0b0f3ec2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 578 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
changelog.md
View File

@ -1,6 +1,8 @@
# Changelog
## 0.6.28 (upcoming)
## 0.6.29 (upcoming)
## 0.6.28
Core:
@ -17,6 +19,7 @@ Lead:
- Lead presentation changed to number.
- Title column for leads.
- Fix New Lead action for Organization.
- Duplicate Organization detection
## 0.6.27

30
common/config/rush/pnpm-lock.yaml
View File

@ -205,6 +205,7 @@ specifiers:
'@types/compression': ~1.7.2
'@types/cors': ^2.8.12
'@types/deep-equal': ^1.0.1
'@types/email-addresses': ^3.0.0
'@types/express': ^4.17.13
'@types/express-fileupload': ^1.1.7
'@types/faker': ~5.5.9
@ -241,6 +242,7 @@ specifiers:
dotenv: ~16.0.0
dotenv-webpack: ^7.0.2
elastic-apm-node: ~3.26.0
email-addresses: ^5.0.0
esbuild: ^0.12.26
eslint: ^7.32.0
eslint-config-standard-with-typescript: ^21.0.1
@ -263,6 +265,7 @@ specifiers:
koa-bodyparser: ^4.3.0
koa-router: ^10.1.1
lexorank: ~1.0.4
libphonenumber-js: ^1.9.46
mime-types: ~2.1.34
mini-css-extract-plugin: ^2.2.0
minio: ^7.0.26
@ -502,6 +505,7 @@ dependencies:
'@types/compression': 1.7.2
'@types/cors': 2.8.12
'@types/deep-equal': 1.0.1
'@types/email-addresses': 3.0.0
'@types/express': 4.17.13
'@types/express-fileupload': 1.2.2
'@types/faker': 5.5.9
@ -538,6 +542,7 @@ dependencies:
dotenv: 16.0.1
dotenv-webpack: 7.1.0_webpack@5.73.0
elastic-apm-node: 3.26.0
email-addresses: 5.0.0
esbuild: 0.12.29
eslint: 7.32.0
eslint-config-standard-with-typescript: 21.0.1_99a5fe2f2ae1dc64d6b59974c931eb2a
@ -560,6 +565,7 @@ dependencies:
koa-bodyparser: 4.3.0
koa-router: 10.1.1
lexorank: 1.0.4
libphonenumber-js: 1.10.6
mime-types: 2.1.35
mini-css-extract-plugin: 2.6.0_webpack@5.73.0
minio: 7.0.28
@ -2223,6 +2229,13 @@ packages:
resolution: {integrity: sha512-mMUu4nWHLBlHtxXY17Fg6+ucS/MnndyOWyOe7MmwkoMYxvfQU2ajtRaEvqSUv+aVkMqH/C0NCI8UoVfRNQ10yg==}
dev: false
/@types/email-addresses/3.0.0:
resolution: {integrity: sha512-jGUOSgpOEWhTH4tMCj56NZenkzER259nJ5NGRvxXld3X7Lai/lxC3QNfDM0rVGMkj+WhANMpvIf195tgwnE7wQ==}
deprecated: This is a stub types definition for email-addresses (https://github.com/jackbowman/email-addresses). email-addresses provides its own type definitions, so you don't need @types/email-addresses installed!
dependencies:
email-addresses: 5.0.0
dev: false
/@types/eslint-scope/3.7.3:
resolution: {integrity: sha512-PB3ldyrcnAicT35TWPs5IcwKD8S333HMaa2VVv4+wdvebJkjWuW/xESoB8IwRcog8HYVYamb1g/R31Qv5Bx03g==}
dependencies:
@ -4463,6 +4476,10 @@ packages:
minimalistic-crypto-utils: 1.0.1
dev: false
/email-addresses/5.0.0:
resolution: {integrity: sha512-4OIPYlA6JXqtVn8zpHpGiI7vE6EQOAg16aGnDMIAlZVinnoZ8208tW1hAbjWydgN/4PLTT9q+O1K6AH/vALJGw==}
dev: false
/emittery/0.8.1:
resolution: {integrity: sha512-uDfvUjVrfGJJhymx/kz6prltenw1u7WrCg1oa94zYY8xxVpLLUu045LAT0dhDZdXG58/EpPL/5kA180fQ/qudg==}
engines: {node: '>=10'}
@ -6879,6 +6896,10 @@ packages:
resolution: {integrity: sha512-CMgA8AMJIX/QfoYHKyjg0hv9W1SGL2xRkt0uLyhT9xKKRj73fHi+IhsrB3W36wwk4I0iz8YlKHfdW14QDwerMA==}
dev: false
/libphonenumber-js/1.10.6:
resolution: {integrity: sha512-CIjT100/SmntsUjsLVs2t3ufeN4KdNXUxhD07tH153pdbaCWuAjv0jK/gPuywR3IImB/U/MQM+x9RfhMs5XZiA==}
dev: false
/lilconfig/2.0.5:
resolution: {integrity: sha512-xaYmXZtTHPAw5m+xLN8ab9C+3a8YmV3asNSPOATITbtwrfbwaLJj8h66H1WMIpALCkqsIzK3h7oQ+PdX+LQ9Eg==}
engines: {node: '>=10'}
@ -10964,7 +10985,7 @@ packages:
dev: false
file:projects/hr-resources.tgz_1e3963ebf0ceeb25b2fa6a1cc87e253c:
resolution: {integrity: sha512-sYnkYAs2h/gH6VD8c6+QBHMemRZfIMVthpYzi1+TXliv/EmebzxdFyPCPjLbxgQxpq4mUwuMLlAusMIrmpVNrg==, tarball: file:projects/hr-resources.tgz}
resolution: {integrity: sha512-XEARExHSGYAtE9S/t40BikzGTpOvffiQ07PdMv7XooZao5H2Rq9INzFCfe78vORfpmR0qaR8EhxBHyyJQbJ5ng==, tarball: file:projects/hr-resources.tgz}
id: file:projects/hr-resources.tgz
name: '@rush-temp/hr-resources'
version: 0.0.0
@ -11626,7 +11647,7 @@ packages:
dev: false
file:projects/model-hr.tgz_typescript@4.7.2:
resolution: {integrity: sha512-HtAgUigvoyvjSpGUHwcRfTqM+RpMKyvDrieHpRBIvFans/2Kg3FPlacTHa9vhBilcze0pNBDrlSXyGuKMBLl8g==, tarball: file:projects/model-hr.tgz}
resolution: {integrity: sha512-N+3GtMbOYJZCiiIrw5lwGQ9tkHgePYpxLRtnZdCN15CkFY1hbFx55NeTDhp5rMq09FeWjxuBSSuXJyWu96rIMg==, tarball: file:projects/model-hr.tgz}
id: file:projects/model-hr.tgz
name: '@rush-temp/model-hr'
version: 0.0.0
@ -14440,12 +14461,13 @@ packages:
dev: false
file:projects/tool.tgz:
resolution: {integrity: sha512-gvdRnHRhzst5gkHEFg9GVc3D04b+BAks8qPdrVxHQSWvWiiUmh3kG+uNWQKklSa9rtRzlSHUrf6NvUvgEg2c5w==, tarball: file:projects/tool.tgz}
resolution: {integrity: sha512-JO6pn5WreSM6pO8MAbVM0lDk6nYKwfX7fLjfpcBv8AQ2C0ZHuGxbFGOA75Y1ccsq3fB4n35KMgBCUKnecCBZKQ==, tarball: file:projects/tool.tgz}
name: '@rush-temp/tool'
version: 0.0.0
dependencies:
'@elastic/elasticsearch': 7.17.0
'@rushstack/heft': 0.45.5
'@types/email-addresses': 3.0.0
'@types/heft-jest': 1.0.2
'@types/mime-types': 2.1.1
'@types/minio': 7.0.13
@ -14458,6 +14480,7 @@ packages:
commander: 8.3.0
cross-env: 7.0.3
csv-parse: 5.1.0
email-addresses: 5.0.0
esbuild: 0.12.29
eslint: 7.32.0
eslint-config-standard-with-typescript: 21.0.1_99a5fe2f2ae1dc64d6b59974c931eb2a
@ -14466,6 +14489,7 @@ packages:
eslint-plugin-promise: 5.2.0_eslint@7.32.0
fast-equals: 2.0.4
got: 11.8.5
libphonenumber-js: 1.10.6
mime-types: 2.1.35
minio: 7.0.28
mongodb: 4.6.0

7
dev/tool/package.json
View File

@ -39,7 +39,8 @@
"@types/ws": "^8.2.1",
"@types/xml2js": "~0.4.9",
"@types/mime-types": "~2.1.1",
"@types/request": "~2.48.8"
"@types/request": "~2.48.8",
"@types/email-addresses": "^3.0.0"
},
"dependencies": {
"mongodb": "^4.1.1",
@ -110,6 +111,8 @@
"@anticrm/tags": "~0.6.2",
"@anticrm/server-backup": "~0.6.0",
"csv-parse": "~5.1.0",
"@anticrm/lead": "~0.6.0"
"@anticrm/lead": "~0.6.0",
"email-addresses": "^5.0.0",
"libphonenumber-js": "^1.9.46"
}
}

18
dev/tool/src/index.ts
View File

@ -36,7 +36,9 @@ import { Db, MongoClient } from 'mongodb'
import { exit } from 'process'
import { rebuildElastic } from './elastic'
import { importXml } from './importer'
import { importLead } from './lead-importer'
import { removeDuplicates } from './leads/duplicates'
import { importLead } from './leads/lead-importer'
import { importLead2 } from './leads/lead-importer2'
import { updateCandidates } from './recruit'
import { clearTelegramHistory } from './telegram'
import { diffWorkspace, dumpWorkspace, restoreWorkspace } from './workspace'
@ -315,6 +317,20 @@ program
return await importLead(transactorUrl, workspace, fileName)
})
program
.command('import-lead-csv2 <workspace> <fileName>')
.description('Import LEAD csv customer organizations')
.action(async (workspace, fileName, cmd) => {
return await importLead2(transactorUrl, workspace, fileName)
})
program
.command('lead-duplicates <workspace>')
.description('Find and remove duplicate organizations.')
.action(async (workspace, cmd) => {
return await removeDuplicates(transactorUrl, workspace)
})
program
.command('generate-token <name> <workspace>')
.description('generate token')

92
dev/tool/src/leads/classes.ts Normal file
View File

@ -0,0 +1,92 @@
//
// Copyright © 2022 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import core, { AnyAttribute, Data, Enum, EnumOf, Ref, TxOperations } from '@anticrm/core'
import lead from '@anticrm/lead'
import { getEmbeddedLabel } from '@anticrm/platform'
import { FieldType } from './types'
export async function updateClasses (
client: TxOperations,
records: any[],
fieldMapping: Record<string, FieldType>
): Promise<void> {
const allAttrs = client.getHierarchy().getAllAttributes(lead.mixin.Customer)
for (const [k, v] of Object.entries(fieldMapping)) {
if (v.type === undefined) {
continue
}
let attr = allAttrs.get(v.name)
if (attr === undefined) {
try {
if (!client.getHierarchy().isDerived(v.type, core.class.Type)) {
// Skip channels mapping
continue
}
} catch (any) {
continue
}
// Create attr
const data: Data<AnyAttribute> = {
attributeOf: lead.mixin.Customer,
name: v.name,
label: getEmbeddedLabel(k),
isCustom: true,
type: {
_class: v.type,
label: v.label ?? core.string.String
}
}
if (client.getHierarchy().isDerived(v.type, core.class.EnumOf)) {
;(data.type as EnumOf).of = `lead:class:${(v as any).enumName as string}` as Ref<Enum>
}
const attrId = (lead.mixin.Customer + '.' + v.name) as Ref<AnyAttribute>
await client.createDoc(core.class.Attribute, core.space.Model, data, attrId)
attr = await client.findOne(core.class.Attribute, { _id: attrId })
}
// Check update Enum/Values
if (client.getHierarchy().isDerived(v.type, core.class.EnumOf)) {
const enumName = (v as any).enumName as string
const enumId = `lead:class:${enumName}` as Ref<Enum>
let enumClass = await client.findOne(core.class.Enum, { _id: enumId })
if (enumClass === undefined) {
await client.createDoc(
core.class.Enum,
core.space.Model,
{
name: enumName,
enumValues: []
},
enumId
)
enumClass = client.getModel().getObject(enumId)
}
// Check values
const mapv = (v?: string): string =>
(v?.toString() ?? '').trim().length === 0 ? 'не задано' : (v?.toString() ?? '').trim()
const values = records
.map((it) => it[v.fName ?? k])
.map(mapv)
.filter((it, idx, arr) => arr.indexOf(it) === idx)
for (const v of values) {
if (!enumClass.enumValues.includes(v)) {
await client.update(enumClass, {
$push: { enumValues: v }
})
}
}
}
}
}

60
dev/tool/src/leads/duplicates.ts Normal file
View File

@ -0,0 +1,60 @@
//
// Copyright © 2022 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import contact, { EmployeeAccount } from '@anticrm/contact'
import { Ref, TxOperations } from '@anticrm/core'
import { connect } from '@anticrm/server-tool'
import { deepEqual } from 'fast-equals'
export async function removeDuplicates (transactorUrl: string, dbName: string): Promise<void> {
const connection = await connect(transactorUrl, dbName)
try {
console.log('loading cvs document...')
const client = new TxOperations(connection, 'core:account:lead-importer' as Ref<EmployeeAccount>)
const organizationNames = await client.findAll(contact.class.Organization, {})
const unicOrg = organizationNames.filter((it, idx, arr) => idx === arr.findIndex((nit) => it.name === nit.name))
let total = 0
for (const org of unicOrg) {
const sameName = organizationNames.filter((it) => it.name === org.name)
if (sameName.length > 1) {
console.log('duplicate orgs', org.name)
total += sameName.length - 1
} else {
continue
}
const target = sameName[0]
for (const oi of sameName.slice(1)) {
const { _id: tid, modifiedOn: _1, ...tdata } = target
const { _id: oid, modifiedOn: _2, ...oiddata } = oi
if (deepEqual(tdata, oiddata)) {
// If same we could remove oid
await client.remove(oi)
console.log('removed', oid)
}
}
}
console.log('Total duplicates', total)
} catch (err: any) {
console.error(err)
} finally {
await connection.close()
}
}

170
dev/tool/src/lead-importer.ts → dev/tool/src/leads/lead-importer.ts
View File

@ -1,6 +1,5 @@
//
// Copyright © 2020, 2021 Anticrm Platform Contributors.
// Copyright © 2021 Hardcore Engineering Inc.
// Copyright © 2022 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
@ -15,41 +14,16 @@
//
import contact, { Contact, EmployeeAccount, Organization } from '@anticrm/contact'
import core, {
AnyAttribute,
Class,
Data,
Doc,
DocumentUpdate,
Enum,
EnumOf,
MixinUpdate,
PropertyType,
Ref,
SortingOrder,
TxOperations,
Type
} from '@anticrm/core'
import core, { Class, DocumentUpdate, MixinUpdate, Ref, SortingOrder, TxOperations } from '@anticrm/core'
import lead, { Customer, Funnel, Lead } from '@anticrm/lead'
import { getEmbeddedLabel } from '@anticrm/platform'
import { connect } from '@anticrm/server-tool'
import task, { calcRank, DoneState, genRanks, State } from '@anticrm/task'
import { parse } from 'csv-parse'
import { readFile } from 'fs/promises'
import { updateClasses } from './classes'
import { CustomCustomer, FieldType } from './types'
import { filled } from './utils'
function filled (obj: any, uniqKeys: string[]): any {
const result: Record<string, any> = {}
for (const [k, v] of Object.entries(obj)) {
if (typeof v === 'string' && v.trim().length === 0) {
continue
}
if (!uniqKeys.includes(k)) {
uniqKeys.push(k)
}
result[k] = v
}
return result
}
import { parse } from 'csv-parse'
const names = {
orgName: 'Название компании',
@ -74,19 +48,7 @@ const names = {
responsible: 'Ответственный'
}
interface CustomCustomer extends Customer {
annualTurnover?: string
currency?: string
comment?: string
source_ka: string
employees_count?: number
activity_area: string
company_type: string
processing_status: string
responsible: string
}
const fieldMapping = {
const fieldMapping: Record<string, FieldType> = {
'Название компании': { name: 'name', type: core.class.TypeString, label: core.string.String },
'Годовой оборот': { name: 'annual.turnover', type: core.class.TypeNumber, label: core.string.Number },
Валюта: {
@ -99,7 +61,7 @@ const fieldMapping = {
'Кем создана': { name: 'created_by', type: undefined, label: core.string.String },
'Кем изменена': { name: 'modified_by', type: undefined, label: core.string.String },
'Источник (где связь) для КА': { name: 'source_ka', type: core.class.TypeString, label: core.string.String },
'Корпоративный сайт': { name: 'corporate_site', type: contact.channelProvider.Homepage, label: undefined },
'Корпоративный сайт': { name: 'corporate_site', type: undefined, label: undefined },
'Кол-во сотрудников': {
name: 'employees_count',
type: core.class.EnumOf,
@ -126,7 +88,6 @@ const fieldMapping = {
},
'Корпоративный e-mail': {
name: 'corporate_email',
type: contact.channelProvider.Email,
label: core.string.String
},
Ответственный: {
@ -136,67 +97,7 @@ const fieldMapping = {
label: core.string.Enum
}
}
async function uodateClasses (client: TxOperations, records: any[]): Promise<void> {
const allAttrs = client.getHierarchy().getAllAttributes(lead.mixin.Customer)
for (const [k, v] of Object.entries(fieldMapping)) {
let attr = allAttrs.get(v.name)
if (attr === undefined) {
try {
if (!client.getHierarchy().isDerived(v.type as Ref<Class<Doc>>, core.class.Type)) {
// Skip channels mapping
continue
}
} catch (any) {
continue
}
// Create attr
const data: Data<AnyAttribute> = {
attributeOf: lead.mixin.Customer,
name: v.name,
label: getEmbeddedLabel(k),
isCustom: true,
type: {
_class: v.type as Ref<Class<Type<PropertyType>>>,
label: v.label ?? core.string.String
}
}
if (client.getHierarchy().isDerived(v.type as Ref<Class<Doc>>, core.class.EnumOf)) {
;(data.type as EnumOf).of = `lead:class:${(v as any).enumName as string}` as Ref<Enum>
}
const attrId = (lead.mixin.Customer + '.' + v.name) as Ref<AnyAttribute>
await client.createDoc(core.class.Attribute, core.space.Model, data, attrId)
attr = await client.findOne(core.class.Attribute, { _id: attrId })
}
// Check update Enum/Values
if (client.getHierarchy().isDerived(v.type as Ref<Class<Doc>>, core.class.EnumOf)) {
const enumName = (v as any).enumName as string
const enumId = `lead:class:${enumName}` as Ref<Enum>
let enumClass = await client.findOne(core.class.Enum, { _id: enumId })
if (enumClass === undefined) {
await client.createDoc(
core.class.Enum,
core.space.Model,
{
name: enumName,
enumValues: []
},
enumId
)
enumClass = client.getModel().getObject(enumId)
}
// Check values
const values = records.map((it) => it[k]).filter((it, idx, arr) => arr.indexOf(it) === idx)
for (const v of values) {
const vv = (v ?? '').trim().length === 0 ? 'не задано' : v
if (!enumClass.enumValues.includes(vv)) {
await client.update(enumClass, {
$push: { enumValues: vv }
})
}
}
}
}
}
async function updateStates<T extends State | DoneState> (
client: TxOperations,
states: string[],
@ -224,6 +125,31 @@ async function updateStates<T extends State | DoneState> (
}
}
export async function parseCSV (csvData: string): Promise<any[]> {
return await new Promise((resolve, reject) => {
parse(
csvData,
{
delimiter: ';',
columns: true,
quote: '"',
bom: true,
cast: true,
autoParse: true,
castDate: false,
skipEmptyLines: true,
skipRecordsWithEmptyValues: true
},
(err, records) => {
if (err !== undefined) {
console.error(err)
reject(err)
}
resolve(records)
}
)
})
}
export async function importLead (transactorUrl: string, dbName: string, csvFile: string): Promise<void> {
const connection = await connect(transactorUrl, dbName)
@ -238,7 +164,7 @@ export async function importLead (transactorUrl: string, dbName: string, csvFile
const client = new TxOperations(connection, 'core:account:lead-importer' as Ref<EmployeeAccount>)
await uodateClasses(client, records)
await updateClasses(client, records, fieldMapping)
await createCustomers(client, filledFields)
const importedFunnelId = await createFunnel(records, client)
@ -464,29 +390,3 @@ async function createCustomers (client: TxOperations, filledFields: any[]): Prom
}
}
}
async function parseCSV (csvData: string): Promise<any[]> {
return await new Promise((resolve, reject) => {
parse(
csvData,
{
delimiter: ';',
columns: true,
quote: '"',
bom: true,
cast: true,
autoParse: true,
castDate: false,
skipEmptyLines: true,
skipRecordsWithEmptyValues: true
},
(err, records) => {
if (err !== undefined) {
console.error(err)
reject(err)
}
resolve(records)
}
)
})
}

267
dev/tool/src/leads/lead-importer2.ts Normal file
View File

@ -0,0 +1,267 @@
//
// Copyright © 2022 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import contact, { Contact, EmployeeAccount, Organization } from '@anticrm/contact'
import core, { generateId, MixinUpdate, Ref, TxOperations } from '@anticrm/core'
import lead from '@anticrm/lead'
import { connect } from '@anticrm/server-tool'
import { readFile } from 'fs/promises'
import { updateClasses } from './classes'
import { CustomCustomer, FieldType } from './types'
import { filled } from './utils'
import { parse } from 'csv-parse'
import parsePhoneNumber from 'libphonenumber-js'
import addrs from 'email-addresses'
const names = {
orgName: 'название компании',
corporate_site: 'сайт компании',
contact_person: 'контактное лицо',
contacts: 'контакты',
dialog_area: 'место диалога: ССеть, почта, ТГ',
sales_manager: 'seles-менеджер',
bot_account: 'бот аккаунт',
description: 'договорённости, доп. инфа о компаниии, вакансии',
hh_link: 'ссылка hh'
}
const fieldMapping: Record<string, FieldType> = {
'Место диалога': {
name: 'dialog_area',
type: core.class.EnumOf,
enumName: 'CompanyDialogArea',
label: core.string.Enum,
fName: names.dialog_area
},
'Sales Manager': {
name: 'sales_manager',
type: core.class.EnumOf,
enumName: 'CompanySalesManager',
label: core.string.Enum,
fName: names.sales_manager
},
'Бот Аккаунт': {
name: 'bot_account',
type: core.class.EnumOf,
enumName: 'CompanyBotAccount',
label: core.string.Enum,
fName: names.bot_account
},
'Контактное Лицо': {
name: 'contact_person',
type: core.class.TypeString,
label: core.string.String
},
Контакты: {
name: 'contacts',
type: core.class.TypeString,
label: core.string.String
}
}
export async function parseCSV (csvData: string): Promise<any[]> {
return await new Promise((resolve, reject) => {
parse(
csvData,
{
delimiter: ',',
columns: true,
quote: '"',
bom: true,
cast: true,
autoParse: true,
castDate: false,
skipEmptyLines: true,
skipRecordsWithEmptyValues: true
},
(err, records) => {
if (err !== undefined) {
console.error(err)
reject(err)
}
resolve(records)
}
)
})
}
export async function importLead2 (transactorUrl: string, dbName: string, csvFile: string): Promise<void> {
const connection = await connect(transactorUrl, dbName)
try {
console.log('loading cvs document...')
const csvData = await readFile(csvFile, 'utf-8')
const records: any[] = await parseCSV(csvData)
const uniqKeys: string[] = []
const filledFields = records.map((it) => filled(it, uniqKeys))
// console.log(filledFields)
const client = new TxOperations(connection, 'core:account:lead-importer' as Ref<EmployeeAccount>)
await updateClasses(client, records, fieldMapping)
await createCustomers(client, filledFields)
} catch (err: any) {
console.error(err)
} finally {
await connection.close()
}
}
async function createCustomers (client: TxOperations, filledFields: any[]): Promise<void> {
for (const record of filledFields) {
let orgId: Ref<Organization> = generateId()
const orgName = record[names.orgName]
if ((orgName?.toString() ?? '').trim().length === 0) {
continue
}
const org = await client.findOne(lead.mixin.Customer, { name: orgName })
console.log('processing', orgName)
if (org === undefined) {
await client.createDoc(
contact.class.Organization,
contact.space.Contacts,
{
name: orgName,
city: '',
members: 0
},
orgId as unknown as Ref<Organization>
)
await client.createMixin<Contact, CustomCustomer>(
orgId,
contact.class.Organization,
contact.space.Contacts,
lead.mixin.Customer,
{
description: record[names.description],
dialog_area: record[names.dialog_area],
sales_manager: record[names.sales_manager],
bot_account: record[names.bot_account],
contacts: record[names.contacts],
contact_person: record[names.contact_person]
}
)
} else {
orgId = org._id as unknown as Ref<Organization>
const upd: MixinUpdate<Contact, CustomCustomer> = {}
const newValues = {
description: record[names.description],
dialog_area: record[names.dialog_area],
sales_manager: record[names.sales_manager],
bot_account: record[names.bot_account],
contact_person: record[names.contact_person]
}
for (const [k, v] of Object.entries(newValues)) {
if ((org as any)[k] !== v && v !== undefined) {
;(upd as any)[k] = v
}
}
if (Object.keys(upd).length > 0) {
await client.updateMixin<Contact, CustomCustomer>(
orgId,
contact.class.Organization,
contact.space.Contacts,
lead.mixin.Customer,
upd
)
}
}
const contacts: string = record[names.contacts]
let dta: string = contacts?.toString() ?? ''
if (dta.length > 0) {
dta = dta.replace('.ru ', '.ru\n')
dta = dta.replace('.com ', '.com\n')
dta = dta.replace('+', '\n+')
dta = dta.replace(',', '\n')
dta = dta.replace('|', '\n')
dta = dta.replace(' ', '\n')
dta = dta.replace('Email:', '\n')
dta = dta.replace('Email:', '\n')
const res = dta
.split('\n')
.map((it) => it.trim())
.filter((it) => it.length > 0)
// console.log('\t', res)
for (const r of res) {
const phone = parsePhoneNumber(r)
if (phone !== undefined) {
const iphone = phone.formatInternational()
const channels = await client.findAll(contact.class.Channel, { attachedTo: orgId })
const emailPr = channels.find((it) => it.value === iphone)
if (emailPr === undefined) {
await client.addCollection(
contact.class.Channel,
contact.space.Contacts,
orgId,
contact.class.Organization,
'channels',
{
value: iphone,
provider: contact.channelProvider.Phone
}
)
}
}
const parsedddr = addrs.parseOneAddress(r)
if (parsedddr != null) {
const email = r.trim()
if (email !== undefined) {
const channels = await client.findAll(contact.class.Channel, { attachedTo: orgId })
const emailPr = channels.find((it) => it.value === email)
if (emailPr === undefined) {
await client.addCollection(
contact.class.Channel,
contact.space.Contacts,
orgId,
contact.class.Organization,
'channels',
{
value: email,
provider: contact.channelProvider.Email
}
)
}
}
}
}
}
const site = record[names.corporate_site]
if (site !== undefined) {
const channels = await client.findAll(contact.class.Channel, { attachedTo: orgId })
const sitePr = channels.find((it) => it.value === site)
if (sitePr === undefined) {
await client.addCollection(
contact.class.Channel,
contact.space.Contacts,
orgId,
contact.class.Organization,
'channels',
{
value: site,
provider: contact.channelProvider.Homepage
}
)
}
}
}
}

43
dev/tool/src/leads/types.ts Normal file
View File

@ -0,0 +1,43 @@
//
// Copyright © 2022 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { Class, Doc, Ref } from '@anticrm/core'
import { Customer } from '@anticrm/lead'
import { IntlString } from '@anticrm/platform'
export interface CustomCustomer extends Customer {
annualTurnover?: string
currency?: string
comment?: string
source_ka?: string
employees_count?: number
activity_area?: string
company_type?: string
processing_status?: string
responsible?: string
dialog_area?: string
sales_manager?: string
bot_account?: string
contacts?: string
contact_person?: string
}
export interface FieldType {
name: string
type?: Ref<Class<Doc>>
label?: IntlString
enumName?: string
fName?: string
}

28
dev/tool/src/leads/utils.ts Normal file
View File

@ -0,0 +1,28 @@
//
// Copyright © 2022 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
export function filled (obj: any, uniqKeys: string[]): any {
const result: Record<string, any> = {}
for (const [k, v] of Object.entries(obj)) {
if (typeof v === 'string' && v.trim().length === 0) {
continue
}
if (!uniqKeys.includes(k)) {
uniqKeys.push(k)
}
result[k] = v
}
return result
}