From af0b0f3ec225a3fd98f23a9b86949bce3eaec506 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Tue, 21 Jun 2022 08:32:33 +0700 Subject: [PATCH] Fix release notes and new csv import (#2116) Signed-off-by: Andrey Sobolev --- changelog.md | 5 +- common/config/rush/pnpm-lock.yaml | 30 ++- dev/tool/package.json | 7 +- dev/tool/src/index.ts | 18 +- dev/tool/src/leads/classes.ts | 92 ++++++++ dev/tool/src/leads/duplicates.ts | 60 +++++ dev/tool/src/{ => leads}/lead-importer.ts | 170 +++----------- dev/tool/src/leads/lead-importer2.ts | 267 ++++++++++++++++++++++ dev/tool/src/leads/types.ts | 43 ++++ dev/tool/src/leads/utils.ts | 28 +++ 10 files changed, 578 insertions(+), 142 deletions(-) create mode 100644 dev/tool/src/leads/classes.ts create mode 100644 dev/tool/src/leads/duplicates.ts rename dev/tool/src/{ => leads}/lead-importer.ts (79%) create mode 100644 dev/tool/src/leads/lead-importer2.ts create mode 100644 dev/tool/src/leads/types.ts create mode 100644 dev/tool/src/leads/utils.ts diff --git a/changelog.md b/changelog.md index 89db372a9a..2906cab869 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,8 @@ # Changelog -## 0.6.28 (upcoming) +## 0.6.29 (upcoming) + +## 0.6.28 Core: @@ -17,6 +19,7 @@ Lead: - Lead presentation changed to number. - Title column for leads. - Fix New Lead action for Organization. +- Duplicate Organization detection ## 0.6.27 diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index ad8f9537ae..4aa222cfe3 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -205,6 +205,7 @@ specifiers: '@types/compression': ~1.7.2 '@types/cors': ^2.8.12 '@types/deep-equal': ^1.0.1 + '@types/email-addresses': ^3.0.0 '@types/express': ^4.17.13 '@types/express-fileupload': ^1.1.7 '@types/faker': ~5.5.9 @@ -241,6 +242,7 @@ specifiers: dotenv: ~16.0.0 dotenv-webpack: ^7.0.2 elastic-apm-node: ~3.26.0 + email-addresses: ^5.0.0 esbuild: ^0.12.26 eslint: ^7.32.0 eslint-config-standard-with-typescript: ^21.0.1 @@ -263,6 +265,7 @@ specifiers: koa-bodyparser: ^4.3.0 koa-router: ^10.1.1 lexorank: ~1.0.4 + libphonenumber-js: ^1.9.46 mime-types: ~2.1.34 mini-css-extract-plugin: ^2.2.0 minio: ^7.0.26 @@ -502,6 +505,7 @@ dependencies: '@types/compression': 1.7.2 '@types/cors': 2.8.12 '@types/deep-equal': 1.0.1 + '@types/email-addresses': 3.0.0 '@types/express': 4.17.13 '@types/express-fileupload': 1.2.2 '@types/faker': 5.5.9 @@ -538,6 +542,7 @@ dependencies: dotenv: 16.0.1 dotenv-webpack: 7.1.0_webpack@5.73.0 elastic-apm-node: 3.26.0 + email-addresses: 5.0.0 esbuild: 0.12.29 eslint: 7.32.0 eslint-config-standard-with-typescript: 21.0.1_99a5fe2f2ae1dc64d6b59974c931eb2a @@ -560,6 +565,7 @@ dependencies: koa-bodyparser: 4.3.0 koa-router: 10.1.1 lexorank: 1.0.4 + libphonenumber-js: 1.10.6 mime-types: 2.1.35 mini-css-extract-plugin: 2.6.0_webpack@5.73.0 minio: 7.0.28 @@ -2223,6 +2229,13 @@ packages: resolution: {integrity: sha512-mMUu4nWHLBlHtxXY17Fg6+ucS/MnndyOWyOe7MmwkoMYxvfQU2ajtRaEvqSUv+aVkMqH/C0NCI8UoVfRNQ10yg==} dev: false + /@types/email-addresses/3.0.0: + resolution: {integrity: sha512-jGUOSgpOEWhTH4tMCj56NZenkzER259nJ5NGRvxXld3X7Lai/lxC3QNfDM0rVGMkj+WhANMpvIf195tgwnE7wQ==} + deprecated: This is a stub types definition for email-addresses (https://github.com/jackbowman/email-addresses). email-addresses provides its own type definitions, so you don't need @types/email-addresses installed! + dependencies: + email-addresses: 5.0.0 + dev: false + /@types/eslint-scope/3.7.3: resolution: {integrity: sha512-PB3ldyrcnAicT35TWPs5IcwKD8S333HMaa2VVv4+wdvebJkjWuW/xESoB8IwRcog8HYVYamb1g/R31Qv5Bx03g==} dependencies: @@ -4463,6 +4476,10 @@ packages: minimalistic-crypto-utils: 1.0.1 dev: false + /email-addresses/5.0.0: + resolution: {integrity: sha512-4OIPYlA6JXqtVn8zpHpGiI7vE6EQOAg16aGnDMIAlZVinnoZ8208tW1hAbjWydgN/4PLTT9q+O1K6AH/vALJGw==} + dev: false + /emittery/0.8.1: resolution: {integrity: sha512-uDfvUjVrfGJJhymx/kz6prltenw1u7WrCg1oa94zYY8xxVpLLUu045LAT0dhDZdXG58/EpPL/5kA180fQ/qudg==} engines: {node: '>=10'} @@ -6879,6 +6896,10 @@ packages: resolution: {integrity: sha512-CMgA8AMJIX/QfoYHKyjg0hv9W1SGL2xRkt0uLyhT9xKKRj73fHi+IhsrB3W36wwk4I0iz8YlKHfdW14QDwerMA==} dev: false + /libphonenumber-js/1.10.6: + resolution: {integrity: sha512-CIjT100/SmntsUjsLVs2t3ufeN4KdNXUxhD07tH153pdbaCWuAjv0jK/gPuywR3IImB/U/MQM+x9RfhMs5XZiA==} + dev: false + /lilconfig/2.0.5: resolution: {integrity: sha512-xaYmXZtTHPAw5m+xLN8ab9C+3a8YmV3asNSPOATITbtwrfbwaLJj8h66H1WMIpALCkqsIzK3h7oQ+PdX+LQ9Eg==} engines: {node: '>=10'} @@ -10964,7 +10985,7 @@ packages: dev: false file:projects/hr-resources.tgz_1e3963ebf0ceeb25b2fa6a1cc87e253c: - resolution: {integrity: sha512-sYnkYAs2h/gH6VD8c6+QBHMemRZfIMVthpYzi1+TXliv/EmebzxdFyPCPjLbxgQxpq4mUwuMLlAusMIrmpVNrg==, tarball: file:projects/hr-resources.tgz} + resolution: {integrity: sha512-XEARExHSGYAtE9S/t40BikzGTpOvffiQ07PdMv7XooZao5H2Rq9INzFCfe78vORfpmR0qaR8EhxBHyyJQbJ5ng==, tarball: file:projects/hr-resources.tgz} id: file:projects/hr-resources.tgz name: '@rush-temp/hr-resources' version: 0.0.0 @@ -11626,7 +11647,7 @@ packages: dev: false file:projects/model-hr.tgz_typescript@4.7.2: - resolution: {integrity: sha512-HtAgUigvoyvjSpGUHwcRfTqM+RpMKyvDrieHpRBIvFans/2Kg3FPlacTHa9vhBilcze0pNBDrlSXyGuKMBLl8g==, tarball: file:projects/model-hr.tgz} + resolution: {integrity: sha512-N+3GtMbOYJZCiiIrw5lwGQ9tkHgePYpxLRtnZdCN15CkFY1hbFx55NeTDhp5rMq09FeWjxuBSSuXJyWu96rIMg==, tarball: file:projects/model-hr.tgz} id: file:projects/model-hr.tgz name: '@rush-temp/model-hr' version: 0.0.0 @@ -14440,12 +14461,13 @@ packages: dev: false file:projects/tool.tgz: - resolution: {integrity: sha512-gvdRnHRhzst5gkHEFg9GVc3D04b+BAks8qPdrVxHQSWvWiiUmh3kG+uNWQKklSa9rtRzlSHUrf6NvUvgEg2c5w==, tarball: file:projects/tool.tgz} + resolution: {integrity: sha512-JO6pn5WreSM6pO8MAbVM0lDk6nYKwfX7fLjfpcBv8AQ2C0ZHuGxbFGOA75Y1ccsq3fB4n35KMgBCUKnecCBZKQ==, tarball: file:projects/tool.tgz} name: '@rush-temp/tool' version: 0.0.0 dependencies: '@elastic/elasticsearch': 7.17.0 '@rushstack/heft': 0.45.5 + '@types/email-addresses': 3.0.0 '@types/heft-jest': 1.0.2 '@types/mime-types': 2.1.1 '@types/minio': 7.0.13 @@ -14458,6 +14480,7 @@ packages: commander: 8.3.0 cross-env: 7.0.3 csv-parse: 5.1.0 + email-addresses: 5.0.0 esbuild: 0.12.29 eslint: 7.32.0 eslint-config-standard-with-typescript: 21.0.1_99a5fe2f2ae1dc64d6b59974c931eb2a @@ -14466,6 +14489,7 @@ packages: eslint-plugin-promise: 5.2.0_eslint@7.32.0 fast-equals: 2.0.4 got: 11.8.5 + libphonenumber-js: 1.10.6 mime-types: 2.1.35 minio: 7.0.28 mongodb: 4.6.0 diff --git a/dev/tool/package.json b/dev/tool/package.json index 5a3acbe242..39a619c6dd 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -39,7 +39,8 @@ "@types/ws": "^8.2.1", "@types/xml2js": "~0.4.9", "@types/mime-types": "~2.1.1", - "@types/request": "~2.48.8" + "@types/request": "~2.48.8", + "@types/email-addresses": "^3.0.0" }, "dependencies": { "mongodb": "^4.1.1", @@ -110,6 +111,8 @@ "@anticrm/tags": "~0.6.2", "@anticrm/server-backup": "~0.6.0", "csv-parse": "~5.1.0", - "@anticrm/lead": "~0.6.0" + "@anticrm/lead": "~0.6.0", + "email-addresses": "^5.0.0", + "libphonenumber-js": "^1.9.46" } } diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 1a484b81d7..f6ad9bdb53 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -36,7 +36,9 @@ import { Db, MongoClient } from 'mongodb' import { exit } from 'process' import { rebuildElastic } from './elastic' import { importXml } from './importer' -import { importLead } from './lead-importer' +import { removeDuplicates } from './leads/duplicates' +import { importLead } from './leads/lead-importer' +import { importLead2 } from './leads/lead-importer2' import { updateCandidates } from './recruit' import { clearTelegramHistory } from './telegram' import { diffWorkspace, dumpWorkspace, restoreWorkspace } from './workspace' @@ -315,6 +317,20 @@ program return await importLead(transactorUrl, workspace, fileName) }) +program + .command('import-lead-csv2 ') + .description('Import LEAD csv customer organizations') + .action(async (workspace, fileName, cmd) => { + return await importLead2(transactorUrl, workspace, fileName) + }) + +program + .command('lead-duplicates ') + .description('Find and remove duplicate organizations.') + .action(async (workspace, cmd) => { + return await removeDuplicates(transactorUrl, workspace) + }) + program .command('generate-token ') .description('generate token') diff --git a/dev/tool/src/leads/classes.ts b/dev/tool/src/leads/classes.ts new file mode 100644 index 0000000000..d9e01ef3e6 --- /dev/null +++ b/dev/tool/src/leads/classes.ts @@ -0,0 +1,92 @@ +// +// Copyright © 2022 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import core, { AnyAttribute, Data, Enum, EnumOf, Ref, TxOperations } from '@anticrm/core' +import lead from '@anticrm/lead' +import { getEmbeddedLabel } from '@anticrm/platform' +import { FieldType } from './types' + +export async function updateClasses ( + client: TxOperations, + records: any[], + fieldMapping: Record +): Promise { + const allAttrs = client.getHierarchy().getAllAttributes(lead.mixin.Customer) + for (const [k, v] of Object.entries(fieldMapping)) { + if (v.type === undefined) { + continue + } + let attr = allAttrs.get(v.name) + if (attr === undefined) { + try { + if (!client.getHierarchy().isDerived(v.type, core.class.Type)) { + // Skip channels mapping + continue + } + } catch (any) { + continue + } + // Create attr + const data: Data = { + attributeOf: lead.mixin.Customer, + name: v.name, + label: getEmbeddedLabel(k), + isCustom: true, + type: { + _class: v.type, + label: v.label ?? core.string.String + } + } + if (client.getHierarchy().isDerived(v.type, core.class.EnumOf)) { + ;(data.type as EnumOf).of = `lead:class:${(v as any).enumName as string}` as Ref + } + const attrId = (lead.mixin.Customer + '.' + v.name) as Ref + await client.createDoc(core.class.Attribute, core.space.Model, data, attrId) + attr = await client.findOne(core.class.Attribute, { _id: attrId }) + } + // Check update Enum/Values + if (client.getHierarchy().isDerived(v.type, core.class.EnumOf)) { + const enumName = (v as any).enumName as string + const enumId = `lead:class:${enumName}` as Ref + let enumClass = await client.findOne(core.class.Enum, { _id: enumId }) + if (enumClass === undefined) { + await client.createDoc( + core.class.Enum, + core.space.Model, + { + name: enumName, + enumValues: [] + }, + enumId + ) + enumClass = client.getModel().getObject(enumId) + } + // Check values + const mapv = (v?: string): string => + (v?.toString() ?? '').trim().length === 0 ? 'не задано' : (v?.toString() ?? '').trim() + const values = records + .map((it) => it[v.fName ?? k]) + .map(mapv) + .filter((it, idx, arr) => arr.indexOf(it) === idx) + for (const v of values) { + if (!enumClass.enumValues.includes(v)) { + await client.update(enumClass, { + $push: { enumValues: v } + }) + } + } + } + } +} diff --git a/dev/tool/src/leads/duplicates.ts b/dev/tool/src/leads/duplicates.ts new file mode 100644 index 0000000000..46a471798b --- /dev/null +++ b/dev/tool/src/leads/duplicates.ts @@ -0,0 +1,60 @@ +// +// Copyright © 2022 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import contact, { EmployeeAccount } from '@anticrm/contact' +import { Ref, TxOperations } from '@anticrm/core' +import { connect } from '@anticrm/server-tool' +import { deepEqual } from 'fast-equals' + +export async function removeDuplicates (transactorUrl: string, dbName: string): Promise { + const connection = await connect(transactorUrl, dbName) + + try { + console.log('loading cvs document...') + + const client = new TxOperations(connection, 'core:account:lead-importer' as Ref) + + const organizationNames = await client.findAll(contact.class.Organization, {}) + + const unicOrg = organizationNames.filter((it, idx, arr) => idx === arr.findIndex((nit) => it.name === nit.name)) + let total = 0 + for (const org of unicOrg) { + const sameName = organizationNames.filter((it) => it.name === org.name) + + if (sameName.length > 1) { + console.log('duplicate orgs', org.name) + total += sameName.length - 1 + } else { + continue + } + + const target = sameName[0] + for (const oi of sameName.slice(1)) { + const { _id: tid, modifiedOn: _1, ...tdata } = target + const { _id: oid, modifiedOn: _2, ...oiddata } = oi + if (deepEqual(tdata, oiddata)) { + // If same we could remove oid + await client.remove(oi) + console.log('removed', oid) + } + } + } + console.log('Total duplicates', total) + } catch (err: any) { + console.error(err) + } finally { + await connection.close() + } +} diff --git a/dev/tool/src/lead-importer.ts b/dev/tool/src/leads/lead-importer.ts similarity index 79% rename from dev/tool/src/lead-importer.ts rename to dev/tool/src/leads/lead-importer.ts index 575bf92f0e..d8e9476fa7 100644 --- a/dev/tool/src/lead-importer.ts +++ b/dev/tool/src/leads/lead-importer.ts @@ -1,6 +1,5 @@ // -// Copyright © 2020, 2021 Anticrm Platform Contributors. -// Copyright © 2021 Hardcore Engineering Inc. +// Copyright © 2022 Hardcore Engineering Inc. // // Licensed under the Eclipse Public License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. You may @@ -15,41 +14,16 @@ // import contact, { Contact, EmployeeAccount, Organization } from '@anticrm/contact' -import core, { - AnyAttribute, - Class, - Data, - Doc, - DocumentUpdate, - Enum, - EnumOf, - MixinUpdate, - PropertyType, - Ref, - SortingOrder, - TxOperations, - Type -} from '@anticrm/core' +import core, { Class, DocumentUpdate, MixinUpdate, Ref, SortingOrder, TxOperations } from '@anticrm/core' import lead, { Customer, Funnel, Lead } from '@anticrm/lead' -import { getEmbeddedLabel } from '@anticrm/platform' import { connect } from '@anticrm/server-tool' import task, { calcRank, DoneState, genRanks, State } from '@anticrm/task' -import { parse } from 'csv-parse' import { readFile } from 'fs/promises' +import { updateClasses } from './classes' +import { CustomCustomer, FieldType } from './types' +import { filled } from './utils' -function filled (obj: any, uniqKeys: string[]): any { - const result: Record = {} - for (const [k, v] of Object.entries(obj)) { - if (typeof v === 'string' && v.trim().length === 0) { - continue - } - if (!uniqKeys.includes(k)) { - uniqKeys.push(k) - } - result[k] = v - } - return result -} +import { parse } from 'csv-parse' const names = { orgName: 'Название компании', @@ -74,19 +48,7 @@ const names = { responsible: 'Ответственный' } -interface CustomCustomer extends Customer { - annualTurnover?: string - currency?: string - comment?: string - source_ka: string - employees_count?: number - activity_area: string - company_type: string - processing_status: string - responsible: string -} - -const fieldMapping = { +const fieldMapping: Record = { 'Название компании': { name: 'name', type: core.class.TypeString, label: core.string.String }, 'Годовой оборот': { name: 'annual.turnover', type: core.class.TypeNumber, label: core.string.Number }, Валюта: { @@ -99,7 +61,7 @@ const fieldMapping = { 'Кем создана': { name: 'created_by', type: undefined, label: core.string.String }, 'Кем изменена': { name: 'modified_by', type: undefined, label: core.string.String }, 'Источник (где связь) для КА': { name: 'source_ka', type: core.class.TypeString, label: core.string.String }, - 'Корпоративный сайт': { name: 'corporate_site', type: contact.channelProvider.Homepage, label: undefined }, + 'Корпоративный сайт': { name: 'corporate_site', type: undefined, label: undefined }, 'Кол-во сотрудников': { name: 'employees_count', type: core.class.EnumOf, @@ -126,7 +88,6 @@ const fieldMapping = { }, 'Корпоративный e-mail': { name: 'corporate_email', - type: contact.channelProvider.Email, label: core.string.String }, Ответственный: { @@ -136,67 +97,7 @@ const fieldMapping = { label: core.string.Enum } } -async function uodateClasses (client: TxOperations, records: any[]): Promise { - const allAttrs = client.getHierarchy().getAllAttributes(lead.mixin.Customer) - for (const [k, v] of Object.entries(fieldMapping)) { - let attr = allAttrs.get(v.name) - if (attr === undefined) { - try { - if (!client.getHierarchy().isDerived(v.type as Ref>, core.class.Type)) { - // Skip channels mapping - continue - } - } catch (any) { - continue - } - // Create attr - const data: Data = { - attributeOf: lead.mixin.Customer, - name: v.name, - label: getEmbeddedLabel(k), - isCustom: true, - type: { - _class: v.type as Ref>>, - label: v.label ?? core.string.String - } - } - if (client.getHierarchy().isDerived(v.type as Ref>, core.class.EnumOf)) { - ;(data.type as EnumOf).of = `lead:class:${(v as any).enumName as string}` as Ref - } - const attrId = (lead.mixin.Customer + '.' + v.name) as Ref - await client.createDoc(core.class.Attribute, core.space.Model, data, attrId) - attr = await client.findOne(core.class.Attribute, { _id: attrId }) - } - // Check update Enum/Values - if (client.getHierarchy().isDerived(v.type as Ref>, core.class.EnumOf)) { - const enumName = (v as any).enumName as string - const enumId = `lead:class:${enumName}` as Ref - let enumClass = await client.findOne(core.class.Enum, { _id: enumId }) - if (enumClass === undefined) { - await client.createDoc( - core.class.Enum, - core.space.Model, - { - name: enumName, - enumValues: [] - }, - enumId - ) - enumClass = client.getModel().getObject(enumId) - } - // Check values - const values = records.map((it) => it[k]).filter((it, idx, arr) => arr.indexOf(it) === idx) - for (const v of values) { - const vv = (v ?? '').trim().length === 0 ? 'не задано' : v - if (!enumClass.enumValues.includes(vv)) { - await client.update(enumClass, { - $push: { enumValues: vv } - }) - } - } - } - } -} + async function updateStates ( client: TxOperations, states: string[], @@ -224,6 +125,31 @@ async function updateStates ( } } +export async function parseCSV (csvData: string): Promise { + return await new Promise((resolve, reject) => { + parse( + csvData, + { + delimiter: ';', + columns: true, + quote: '"', + bom: true, + cast: true, + autoParse: true, + castDate: false, + skipEmptyLines: true, + skipRecordsWithEmptyValues: true + }, + (err, records) => { + if (err !== undefined) { + console.error(err) + reject(err) + } + resolve(records) + } + ) + }) +} export async function importLead (transactorUrl: string, dbName: string, csvFile: string): Promise { const connection = await connect(transactorUrl, dbName) @@ -238,7 +164,7 @@ export async function importLead (transactorUrl: string, dbName: string, csvFile const client = new TxOperations(connection, 'core:account:lead-importer' as Ref) - await uodateClasses(client, records) + await updateClasses(client, records, fieldMapping) await createCustomers(client, filledFields) const importedFunnelId = await createFunnel(records, client) @@ -464,29 +390,3 @@ async function createCustomers (client: TxOperations, filledFields: any[]): Prom } } } - -async function parseCSV (csvData: string): Promise { - return await new Promise((resolve, reject) => { - parse( - csvData, - { - delimiter: ';', - columns: true, - quote: '"', - bom: true, - cast: true, - autoParse: true, - castDate: false, - skipEmptyLines: true, - skipRecordsWithEmptyValues: true - }, - (err, records) => { - if (err !== undefined) { - console.error(err) - reject(err) - } - resolve(records) - } - ) - }) -} diff --git a/dev/tool/src/leads/lead-importer2.ts b/dev/tool/src/leads/lead-importer2.ts new file mode 100644 index 0000000000..950cd5d2a7 --- /dev/null +++ b/dev/tool/src/leads/lead-importer2.ts @@ -0,0 +1,267 @@ +// +// Copyright © 2022 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import contact, { Contact, EmployeeAccount, Organization } from '@anticrm/contact' +import core, { generateId, MixinUpdate, Ref, TxOperations } from '@anticrm/core' +import lead from '@anticrm/lead' +import { connect } from '@anticrm/server-tool' +import { readFile } from 'fs/promises' +import { updateClasses } from './classes' +import { CustomCustomer, FieldType } from './types' +import { filled } from './utils' + +import { parse } from 'csv-parse' +import parsePhoneNumber from 'libphonenumber-js' +import addrs from 'email-addresses' + +const names = { + orgName: 'название компании', + corporate_site: 'сайт компании', + contact_person: 'контактное лицо', + contacts: 'контакты', + dialog_area: 'место диалога: ССеть, почта, ТГ', + sales_manager: 'seles-менеджер', + bot_account: 'бот аккаунт', + description: 'договорённости, доп. инфа о компаниии, вакансии', + hh_link: 'ссылка hh' +} + +const fieldMapping: Record = { + 'Место диалога': { + name: 'dialog_area', + type: core.class.EnumOf, + enumName: 'CompanyDialogArea', + label: core.string.Enum, + fName: names.dialog_area + }, + 'Sales Manager': { + name: 'sales_manager', + type: core.class.EnumOf, + enumName: 'CompanySalesManager', + label: core.string.Enum, + fName: names.sales_manager + }, + 'Бот Аккаунт': { + name: 'bot_account', + type: core.class.EnumOf, + enumName: 'CompanyBotAccount', + label: core.string.Enum, + fName: names.bot_account + }, + 'Контактное Лицо': { + name: 'contact_person', + type: core.class.TypeString, + label: core.string.String + }, + Контакты: { + name: 'contacts', + type: core.class.TypeString, + label: core.string.String + } +} + +export async function parseCSV (csvData: string): Promise { + return await new Promise((resolve, reject) => { + parse( + csvData, + { + delimiter: ',', + columns: true, + quote: '"', + bom: true, + cast: true, + autoParse: true, + castDate: false, + skipEmptyLines: true, + skipRecordsWithEmptyValues: true + }, + (err, records) => { + if (err !== undefined) { + console.error(err) + reject(err) + } + resolve(records) + } + ) + }) +} + +export async function importLead2 (transactorUrl: string, dbName: string, csvFile: string): Promise { + const connection = await connect(transactorUrl, dbName) + + try { + console.log('loading cvs document...') + + const csvData = await readFile(csvFile, 'utf-8') + const records: any[] = await parseCSV(csvData) + const uniqKeys: string[] = [] + const filledFields = records.map((it) => filled(it, uniqKeys)) + // console.log(filledFields) + + const client = new TxOperations(connection, 'core:account:lead-importer' as Ref) + + await updateClasses(client, records, fieldMapping) + + await createCustomers(client, filledFields) + } catch (err: any) { + console.error(err) + } finally { + await connection.close() + } +} + +async function createCustomers (client: TxOperations, filledFields: any[]): Promise { + for (const record of filledFields) { + let orgId: Ref = generateId() + const orgName = record[names.orgName] + if ((orgName?.toString() ?? '').trim().length === 0) { + continue + } + const org = await client.findOne(lead.mixin.Customer, { name: orgName }) + console.log('processing', orgName) + if (org === undefined) { + await client.createDoc( + contact.class.Organization, + contact.space.Contacts, + { + name: orgName, + city: '', + members: 0 + }, + orgId as unknown as Ref + ) + await client.createMixin( + orgId, + contact.class.Organization, + contact.space.Contacts, + lead.mixin.Customer, + { + description: record[names.description], + dialog_area: record[names.dialog_area], + sales_manager: record[names.sales_manager], + bot_account: record[names.bot_account], + contacts: record[names.contacts], + contact_person: record[names.contact_person] + } + ) + } else { + orgId = org._id as unknown as Ref + const upd: MixinUpdate = {} + const newValues = { + description: record[names.description], + dialog_area: record[names.dialog_area], + sales_manager: record[names.sales_manager], + bot_account: record[names.bot_account], + contact_person: record[names.contact_person] + } + for (const [k, v] of Object.entries(newValues)) { + if ((org as any)[k] !== v && v !== undefined) { + ;(upd as any)[k] = v + } + } + if (Object.keys(upd).length > 0) { + await client.updateMixin( + orgId, + contact.class.Organization, + contact.space.Contacts, + lead.mixin.Customer, + upd + ) + } + } + + const contacts: string = record[names.contacts] + + let dta: string = contacts?.toString() ?? '' + if (dta.length > 0) { + dta = dta.replace('.ru ', '.ru\n') + dta = dta.replace('.com ', '.com\n') + dta = dta.replace('+', '\n+') + dta = dta.replace(',', '\n') + dta = dta.replace('|', '\n') + dta = dta.replace(' ', '\n') + dta = dta.replace('Email:', '\n') + dta = dta.replace('Email:', '\n') + const res = dta + .split('\n') + .map((it) => it.trim()) + .filter((it) => it.length > 0) + // console.log('\t', res) + for (const r of res) { + const phone = parsePhoneNumber(r) + if (phone !== undefined) { + const iphone = phone.formatInternational() + const channels = await client.findAll(contact.class.Channel, { attachedTo: orgId }) + const emailPr = channels.find((it) => it.value === iphone) + + if (emailPr === undefined) { + await client.addCollection( + contact.class.Channel, + contact.space.Contacts, + orgId, + contact.class.Organization, + 'channels', + { + value: iphone, + provider: contact.channelProvider.Phone + } + ) + } + } + + const parsedddr = addrs.parseOneAddress(r) + if (parsedddr != null) { + const email = r.trim() + if (email !== undefined) { + const channels = await client.findAll(contact.class.Channel, { attachedTo: orgId }) + const emailPr = channels.find((it) => it.value === email) + if (emailPr === undefined) { + await client.addCollection( + contact.class.Channel, + contact.space.Contacts, + orgId, + contact.class.Organization, + 'channels', + { + value: email, + provider: contact.channelProvider.Email + } + ) + } + } + } + } + } + + const site = record[names.corporate_site] + if (site !== undefined) { + const channels = await client.findAll(contact.class.Channel, { attachedTo: orgId }) + const sitePr = channels.find((it) => it.value === site) + if (sitePr === undefined) { + await client.addCollection( + contact.class.Channel, + contact.space.Contacts, + orgId, + contact.class.Organization, + 'channels', + { + value: site, + provider: contact.channelProvider.Homepage + } + ) + } + } + } +} diff --git a/dev/tool/src/leads/types.ts b/dev/tool/src/leads/types.ts new file mode 100644 index 0000000000..8840df74ec --- /dev/null +++ b/dev/tool/src/leads/types.ts @@ -0,0 +1,43 @@ +// +// Copyright © 2022 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import { Class, Doc, Ref } from '@anticrm/core' +import { Customer } from '@anticrm/lead' +import { IntlString } from '@anticrm/platform' + +export interface CustomCustomer extends Customer { + annualTurnover?: string + currency?: string + comment?: string + source_ka?: string + employees_count?: number + activity_area?: string + company_type?: string + processing_status?: string + responsible?: string + + dialog_area?: string + sales_manager?: string + bot_account?: string + contacts?: string + contact_person?: string +} + +export interface FieldType { + name: string + type?: Ref> + label?: IntlString + enumName?: string + fName?: string +} diff --git a/dev/tool/src/leads/utils.ts b/dev/tool/src/leads/utils.ts new file mode 100644 index 0000000000..1a519befb0 --- /dev/null +++ b/dev/tool/src/leads/utils.ts @@ -0,0 +1,28 @@ +// +// Copyright © 2022 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +export function filled (obj: any, uniqKeys: string[]): any { + const result: Record = {} + for (const [k, v] of Object.entries(obj)) { + if (typeof v === 'string' && v.trim().length === 0) { + continue + } + if (!uniqKeys.includes(k)) { + uniqKeys.push(k) + } + result[k] = v + } + return result +}