Fix release notes and new csv import (#2116)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

changelog.md

@@ -1,6 +1,8 @@
 # Changelog
 
-## 0.6.28 (upcoming)
+## 0.6.29 (upcoming)
+
+## 0.6.28
 
 Core:
 
@@ -17,6 +19,7 @@ Lead:
 - Lead presentation changed to number.
 - Title column for leads.
 - Fix New Lead action for Organization.
+- Duplicate Organization detection
 
 ## 0.6.27
 

common/config/rush/pnpm-lock.yaml

@@ -205,6 +205,7 @@ specifiers:
   '@types/compression': ~1.7.2
   '@types/cors': ^2.8.12
   '@types/deep-equal': ^1.0.1
+  '@types/email-addresses': ^3.0.0
   '@types/express': ^4.17.13
   '@types/express-fileupload': ^1.1.7
   '@types/faker': ~5.5.9
@@ -241,6 +242,7 @@ specifiers:
   dotenv: ~16.0.0
   dotenv-webpack: ^7.0.2
   elastic-apm-node: ~3.26.0
+  email-addresses: ^5.0.0
   esbuild: ^0.12.26
   eslint: ^7.32.0
   eslint-config-standard-with-typescript: ^21.0.1
@@ -263,6 +265,7 @@ specifiers:
   koa-bodyparser: ^4.3.0
   koa-router: ^10.1.1
   lexorank: ~1.0.4
+  libphonenumber-js: ^1.9.46
   mime-types: ~2.1.34
   mini-css-extract-plugin: ^2.2.0
   minio: ^7.0.26
@@ -502,6 +505,7 @@ dependencies:
   '@types/compression': 1.7.2
   '@types/cors': 2.8.12
   '@types/deep-equal': 1.0.1
+  '@types/email-addresses': 3.0.0
   '@types/express': 4.17.13
   '@types/express-fileupload': 1.2.2
   '@types/faker': 5.5.9
@@ -538,6 +542,7 @@ dependencies:
   dotenv: 16.0.1
   dotenv-webpack: 7.1.0_webpack@5.73.0
   elastic-apm-node: 3.26.0
+  email-addresses: 5.0.0
   esbuild: 0.12.29
   eslint: 7.32.0
   eslint-config-standard-with-typescript: 21.0.1_99a5fe2f2ae1dc64d6b59974c931eb2a
@@ -560,6 +565,7 @@ dependencies:
   koa-bodyparser: 4.3.0
   koa-router: 10.1.1
   lexorank: 1.0.4
+  libphonenumber-js: 1.10.6
   mime-types: 2.1.35
   mini-css-extract-plugin: 2.6.0_webpack@5.73.0
   minio: 7.0.28
@@ -2223,6 +2229,13 @@ packages:
     resolution: {integrity: sha512-mMUu4nWHLBlHtxXY17Fg6+ucS/MnndyOWyOe7MmwkoMYxvfQU2ajtRaEvqSUv+aVkMqH/C0NCI8UoVfRNQ10yg==}
     dev: false
 
+  /@types/email-addresses/3.0.0:
+    resolution: {integrity: sha512-jGUOSgpOEWhTH4tMCj56NZenkzER259nJ5NGRvxXld3X7Lai/lxC3QNfDM0rVGMkj+WhANMpvIf195tgwnE7wQ==}
+    deprecated: This is a stub types definition for email-addresses (https://github.com/jackbowman/email-addresses). email-addresses provides its own type definitions, so you don't need @types/email-addresses installed!
+    dependencies:
+      email-addresses: 5.0.0
+    dev: false
+
   /@types/eslint-scope/3.7.3:
     resolution: {integrity: sha512-PB3ldyrcnAicT35TWPs5IcwKD8S333HMaa2VVv4+wdvebJkjWuW/xESoB8IwRcog8HYVYamb1g/R31Qv5Bx03g==}
     dependencies:
@@ -4463,6 +4476,10 @@ packages:
       minimalistic-crypto-utils: 1.0.1
     dev: false
 
+  /email-addresses/5.0.0:
+    resolution: {integrity: sha512-4OIPYlA6JXqtVn8zpHpGiI7vE6EQOAg16aGnDMIAlZVinnoZ8208tW1hAbjWydgN/4PLTT9q+O1K6AH/vALJGw==}
+    dev: false
+
   /emittery/0.8.1:
     resolution: {integrity: sha512-uDfvUjVrfGJJhymx/kz6prltenw1u7WrCg1oa94zYY8xxVpLLUu045LAT0dhDZdXG58/EpPL/5kA180fQ/qudg==}
     engines: {node: '>=10'}
@@ -6879,6 +6896,10 @@ packages:
     resolution: {integrity: sha512-CMgA8AMJIX/QfoYHKyjg0hv9W1SGL2xRkt0uLyhT9xKKRj73fHi+IhsrB3W36wwk4I0iz8YlKHfdW14QDwerMA==}
     dev: false
 
+  /libphonenumber-js/1.10.6:
+    resolution: {integrity: sha512-CIjT100/SmntsUjsLVs2t3ufeN4KdNXUxhD07tH153pdbaCWuAjv0jK/gPuywR3IImB/U/MQM+x9RfhMs5XZiA==}
+    dev: false
+
   /lilconfig/2.0.5:
     resolution: {integrity: sha512-xaYmXZtTHPAw5m+xLN8ab9C+3a8YmV3asNSPOATITbtwrfbwaLJj8h66H1WMIpALCkqsIzK3h7oQ+PdX+LQ9Eg==}
     engines: {node: '>=10'}
@@ -10964,7 +10985,7 @@ packages:
     dev: false
 
   file:projects/hr-resources.tgz_1e3963ebf0ceeb25b2fa6a1cc87e253c:
-    resolution: {integrity: sha512-sYnkYAs2h/gH6VD8c6+QBHMemRZfIMVthpYzi1+TXliv/EmebzxdFyPCPjLbxgQxpq4mUwuMLlAusMIrmpVNrg==, tarball: file:projects/hr-resources.tgz}
+    resolution: {integrity: sha512-XEARExHSGYAtE9S/t40BikzGTpOvffiQ07PdMv7XooZao5H2Rq9INzFCfe78vORfpmR0qaR8EhxBHyyJQbJ5ng==, tarball: file:projects/hr-resources.tgz}
     id: file:projects/hr-resources.tgz
     name: '@rush-temp/hr-resources'
     version: 0.0.0
@@ -11626,7 +11647,7 @@ packages:
     dev: false
 
   file:projects/model-hr.tgz_typescript@4.7.2:
-    resolution: {integrity: sha512-HtAgUigvoyvjSpGUHwcRfTqM+RpMKyvDrieHpRBIvFans/2Kg3FPlacTHa9vhBilcze0pNBDrlSXyGuKMBLl8g==, tarball: file:projects/model-hr.tgz}
+    resolution: {integrity: sha512-N+3GtMbOYJZCiiIrw5lwGQ9tkHgePYpxLRtnZdCN15CkFY1hbFx55NeTDhp5rMq09FeWjxuBSSuXJyWu96rIMg==, tarball: file:projects/model-hr.tgz}
     id: file:projects/model-hr.tgz
     name: '@rush-temp/model-hr'
     version: 0.0.0
@@ -14440,12 +14461,13 @@ packages:
     dev: false
 
   file:projects/tool.tgz:
-    resolution: {integrity: sha512-gvdRnHRhzst5gkHEFg9GVc3D04b+BAks8qPdrVxHQSWvWiiUmh3kG+uNWQKklSa9rtRzlSHUrf6NvUvgEg2c5w==, tarball: file:projects/tool.tgz}
+    resolution: {integrity: sha512-JO6pn5WreSM6pO8MAbVM0lDk6nYKwfX7fLjfpcBv8AQ2C0ZHuGxbFGOA75Y1ccsq3fB4n35KMgBCUKnecCBZKQ==, tarball: file:projects/tool.tgz}
     name: '@rush-temp/tool'
     version: 0.0.0
     dependencies:
       '@elastic/elasticsearch': 7.17.0
       '@rushstack/heft': 0.45.5
+      '@types/email-addresses': 3.0.0
       '@types/heft-jest': 1.0.2
       '@types/mime-types': 2.1.1
       '@types/minio': 7.0.13
@@ -14458,6 +14480,7 @@ packages:
       commander: 8.3.0
       cross-env: 7.0.3
       csv-parse: 5.1.0
+      email-addresses: 5.0.0
       esbuild: 0.12.29
       eslint: 7.32.0
       eslint-config-standard-with-typescript: 21.0.1_99a5fe2f2ae1dc64d6b59974c931eb2a
@@ -14466,6 +14489,7 @@ packages:
       eslint-plugin-promise: 5.2.0_eslint@7.32.0
       fast-equals: 2.0.4
       got: 11.8.5
+      libphonenumber-js: 1.10.6
       mime-types: 2.1.35
       minio: 7.0.28
       mongodb: 4.6.0

dev/tool/package.json

@@ -39,7 +39,8 @@
     "@types/ws": "^8.2.1",
     "@types/xml2js": "~0.4.9",
     "@types/mime-types": "~2.1.1",
-    "@types/request": "~2.48.8"
+    "@types/request": "~2.48.8",
+    "@types/email-addresses": "^3.0.0"
   },
   "dependencies": {
     "mongodb": "^4.1.1",
@@ -110,6 +111,8 @@
     "@anticrm/tags": "~0.6.2",
     "@anticrm/server-backup": "~0.6.0",
     "csv-parse": "~5.1.0",
-    "@anticrm/lead": "~0.6.0"
+    "@anticrm/lead": "~0.6.0",    
+    "email-addresses": "^5.0.0",
+    "libphonenumber-js": "^1.9.46"
   }
 }

dev/tool/src/index.ts

@@ -36,7 +36,9 @@ import { Db, MongoClient } from 'mongodb'
 import { exit } from 'process'
 import { rebuildElastic } from './elastic'
 import { importXml } from './importer'
-import { importLead } from './lead-importer'
+import { removeDuplicates } from './leads/duplicates'
+import { importLead } from './leads/lead-importer'
+import { importLead2 } from './leads/lead-importer2'
 import { updateCandidates } from './recruit'
 import { clearTelegramHistory } from './telegram'
 import { diffWorkspace, dumpWorkspace, restoreWorkspace } from './workspace'
@@ -315,6 +317,20 @@ program
     return await importLead(transactorUrl, workspace, fileName)
   })
 
+program
+  .command('import-lead-csv2 <workspace> <fileName>')
+  .description('Import LEAD csv customer organizations')
+  .action(async (workspace, fileName, cmd) => {
+    return await importLead2(transactorUrl, workspace, fileName)
+  })
+
+program
+  .command('lead-duplicates <workspace>')
+  .description('Find and remove duplicate organizations.')
+  .action(async (workspace, cmd) => {
+    return await removeDuplicates(transactorUrl, workspace)
+  })
+
 program
   .command('generate-token <name> <workspace>')
   .description('generate token')

dev/tool/src/leads/classes.ts

@@ -0,0 +1,92 @@
+//
+// Copyright © 2022 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import core, { AnyAttribute, Data, Enum, EnumOf, Ref, TxOperations } from '@anticrm/core'
+import lead from '@anticrm/lead'
+import { getEmbeddedLabel } from '@anticrm/platform'
+import { FieldType } from './types'
+
+export async function updateClasses (
+  client: TxOperations,
+  records: any[],
+  fieldMapping: Record<string, FieldType>
+): Promise<void> {
+  const allAttrs = client.getHierarchy().getAllAttributes(lead.mixin.Customer)
+  for (const [k, v] of Object.entries(fieldMapping)) {
+    if (v.type === undefined) {
+      continue
+    }
+    let attr = allAttrs.get(v.name)
+    if (attr === undefined) {
+      try {
+        if (!client.getHierarchy().isDerived(v.type, core.class.Type)) {
+          // Skip channels mapping
+          continue
+        }
+      } catch (any) {
+        continue
+      }
+      // Create attr
+      const data: Data<AnyAttribute> = {
+        attributeOf: lead.mixin.Customer,
+        name: v.name,
+        label: getEmbeddedLabel(k),
+        isCustom: true,
+        type: {
+          _class: v.type,
+          label: v.label ?? core.string.String
+        }
+      }
+      if (client.getHierarchy().isDerived(v.type, core.class.EnumOf)) {
+        ;(data.type as EnumOf).of = `lead:class:${(v as any).enumName as string}` as Ref<Enum>
+      }
+      const attrId = (lead.mixin.Customer + '.' + v.name) as Ref<AnyAttribute>
+      await client.createDoc(core.class.Attribute, core.space.Model, data, attrId)
+      attr = await client.findOne(core.class.Attribute, { _id: attrId })
+    }
+    // Check update Enum/Values
+    if (client.getHierarchy().isDerived(v.type, core.class.EnumOf)) {
+      const enumName = (v as any).enumName as string
+      const enumId = `lead:class:${enumName}` as Ref<Enum>
+      let enumClass = await client.findOne(core.class.Enum, { _id: enumId })
+      if (enumClass === undefined) {
+        await client.createDoc(
+          core.class.Enum,
+          core.space.Model,
+          {
+            name: enumName,
+            enumValues: []
+          },
+          enumId
+        )
+        enumClass = client.getModel().getObject(enumId)
+      }
+      // Check values
+      const mapv = (v?: string): string =>
+        (v?.toString() ?? '').trim().length === 0 ? 'не задано' : (v?.toString() ?? '').trim()
+      const values = records
+        .map((it) => it[v.fName ?? k])
+        .map(mapv)
+        .filter((it, idx, arr) => arr.indexOf(it) === idx)
+      for (const v of values) {
+        if (!enumClass.enumValues.includes(v)) {
+          await client.update(enumClass, {
+            $push: { enumValues: v }
+          })
+        }
+      }
+    }
+  }
+}

dev/tool/src/leads/duplicates.ts

@@ -0,0 +1,60 @@
+//
+// Copyright © 2022 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import contact, { EmployeeAccount } from '@anticrm/contact'
+import { Ref, TxOperations } from '@anticrm/core'
+import { connect } from '@anticrm/server-tool'
+import { deepEqual } from 'fast-equals'
+
+export async function removeDuplicates (transactorUrl: string, dbName: string): Promise<void> {
+  const connection = await connect(transactorUrl, dbName)
+
+  try {
+    console.log('loading cvs document...')
+
+    const client = new TxOperations(connection, 'core:account:lead-importer' as Ref<EmployeeAccount>)
+
+    const organizationNames = await client.findAll(contact.class.Organization, {})
+
+    const unicOrg = organizationNames.filter((it, idx, arr) => idx === arr.findIndex((nit) => it.name === nit.name))
+    let total = 0
+    for (const org of unicOrg) {
+      const sameName = organizationNames.filter((it) => it.name === org.name)
+
+      if (sameName.length > 1) {
+        console.log('duplicate orgs', org.name)
+        total += sameName.length - 1
+      } else {
+        continue
+      }
+
+      const target = sameName[0]
+      for (const oi of sameName.slice(1)) {
+        const { _id: tid, modifiedOn: _1, ...tdata } = target
+        const { _id: oid, modifiedOn: _2, ...oiddata } = oi
+        if (deepEqual(tdata, oiddata)) {
+          // If same we could remove oid
+          await client.remove(oi)
+          console.log('removed', oid)
+        }
+      }
+    }
+    console.log('Total duplicates', total)
+  } catch (err: any) {
+    console.error(err)
+  } finally {
+    await connection.close()
+  }
+}

dev/tool/src/leads/lead-importer.ts

@@ -1,6 +1,5 @@
 //
-// Copyright © 2020, 2021 Anticrm Platform Contributors.
+// Copyright © 2022 Hardcore Engineering Inc.
-// Copyright © 2021 Hardcore Engineering Inc.
 //
 // Licensed under the Eclipse Public License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License. You may
@@ -15,41 +14,16 @@
 //
 
 import contact, { Contact, EmployeeAccount, Organization } from '@anticrm/contact'
-import core, {
+import core, { Class, DocumentUpdate, MixinUpdate, Ref, SortingOrder, TxOperations } from '@anticrm/core'
-  AnyAttribute,
-  Class,
-  Data,
-  Doc,
-  DocumentUpdate,
-  Enum,
-  EnumOf,
-  MixinUpdate,
-  PropertyType,
-  Ref,
-  SortingOrder,
-  TxOperations,
-  Type
-} from '@anticrm/core'
 import lead, { Customer, Funnel, Lead } from '@anticrm/lead'
-import { getEmbeddedLabel } from '@anticrm/platform'
 import { connect } from '@anticrm/server-tool'
 import task, { calcRank, DoneState, genRanks, State } from '@anticrm/task'
-import { parse } from 'csv-parse'
 import { readFile } from 'fs/promises'
+import { updateClasses } from './classes'
+import { CustomCustomer, FieldType } from './types'
+import { filled } from './utils'
 
-function filled (obj: any, uniqKeys: string[]): any {
+import { parse } from 'csv-parse'
-  const result: Record<string, any> = {}
-  for (const [k, v] of Object.entries(obj)) {
-    if (typeof v === 'string' && v.trim().length === 0) {
-      continue
-    }
-    if (!uniqKeys.includes(k)) {
-      uniqKeys.push(k)
-    }
-    result[k] = v
-  }
-  return result
-}
 
 const names = {
   orgName: 'Название компании',
@@ -74,19 +48,7 @@ const names = {
   responsible: 'Ответственный'
 }
 
-interface CustomCustomer extends Customer {
+const fieldMapping: Record<string, FieldType> = {
-  annualTurnover?: string
-  currency?: string
-  comment?: string
-  source_ka: string
-  employees_count?: number
-  activity_area: string
-  company_type: string
-  processing_status: string
-  responsible: string
-}
-
-const fieldMapping = {
   'Название компании': { name: 'name', type: core.class.TypeString, label: core.string.String },
   'Годовой оборот': { name: 'annual.turnover', type: core.class.TypeNumber, label: core.string.Number },
   Валюта: {
@@ -99,7 +61,7 @@ const fieldMapping = {
   'Кем создана': { name: 'created_by', type: undefined, label: core.string.String },
   'Кем изменена': { name: 'modified_by', type: undefined, label: core.string.String },
   'Источник (где связь) для КА': { name: 'source_ka', type: core.class.TypeString, label: core.string.String },
-  'Корпоративный сайт': { name: 'corporate_site', type: contact.channelProvider.Homepage, label: undefined },
+  'Корпоративный сайт': { name: 'corporate_site', type: undefined, label: undefined },
   'Кол-во сотрудников': {
     name: 'employees_count',
     type: core.class.EnumOf,
@@ -126,7 +88,6 @@ const fieldMapping = {
   },
   'Корпоративный e-mail': {
     name: 'corporate_email',
-    type: contact.channelProvider.Email,
     label: core.string.String
   },
   Ответственный: {
@@ -136,67 +97,7 @@ const fieldMapping = {
     label: core.string.Enum
   }
 }
-async function uodateClasses (client: TxOperations, records: any[]): Promise<void> {
+
-  const allAttrs = client.getHierarchy().getAllAttributes(lead.mixin.Customer)
-  for (const [k, v] of Object.entries(fieldMapping)) {
-    let attr = allAttrs.get(v.name)
-    if (attr === undefined) {
-      try {
-        if (!client.getHierarchy().isDerived(v.type as Ref<Class<Doc>>, core.class.Type)) {
-          // Skip channels mapping
-          continue
-        }
-      } catch (any) {
-        continue
-      }
-      // Create attr
-      const data: Data<AnyAttribute> = {
-        attributeOf: lead.mixin.Customer,
-        name: v.name,
-        label: getEmbeddedLabel(k),
-        isCustom: true,
-        type: {
-          _class: v.type as Ref<Class<Type<PropertyType>>>,
-          label: v.label ?? core.string.String
-        }
-      }
-      if (client.getHierarchy().isDerived(v.type as Ref<Class<Doc>>, core.class.EnumOf)) {
-        ;(data.type as EnumOf).of = `lead:class:${(v as any).enumName as string}` as Ref<Enum>
-      }
-      const attrId = (lead.mixin.Customer + '.' + v.name) as Ref<AnyAttribute>
-      await client.createDoc(core.class.Attribute, core.space.Model, data, attrId)
-      attr = await client.findOne(core.class.Attribute, { _id: attrId })
-    }
-    // Check update Enum/Values
-    if (client.getHierarchy().isDerived(v.type as Ref<Class<Doc>>, core.class.EnumOf)) {
-      const enumName = (v as any).enumName as string
-      const enumId = `lead:class:${enumName}` as Ref<Enum>
-      let enumClass = await client.findOne(core.class.Enum, { _id: enumId })
-      if (enumClass === undefined) {
-        await client.createDoc(
-          core.class.Enum,
-          core.space.Model,
-          {
-            name: enumName,
-            enumValues: []
-          },
-          enumId
-        )
-        enumClass = client.getModel().getObject(enumId)
-      }
-      // Check values
-      const values = records.map((it) => it[k]).filter((it, idx, arr) => arr.indexOf(it) === idx)
-      for (const v of values) {
-        const vv = (v ?? '').trim().length === 0 ? 'не задано' : v
-        if (!enumClass.enumValues.includes(vv)) {
-          await client.update(enumClass, {
-            $push: { enumValues: vv }
-          })
-        }
-      }
-    }
-  }
-}
 async function updateStates<T extends State | DoneState> (
   client: TxOperations,
   states: string[],
@@ -224,6 +125,31 @@ async function updateStates<T extends State | DoneState> (
   }
 }
 
+export async function parseCSV (csvData: string): Promise<any[]> {
+  return await new Promise((resolve, reject) => {
+    parse(
+      csvData,
+      {
+        delimiter: ';',
+        columns: true,
+        quote: '"',
+        bom: true,
+        cast: true,
+        autoParse: true,
+        castDate: false,
+        skipEmptyLines: true,
+        skipRecordsWithEmptyValues: true
+      },
+      (err, records) => {
+        if (err !== undefined) {
+          console.error(err)
+          reject(err)
+        }
+        resolve(records)
+      }
+    )
+  })
+}
 export async function importLead (transactorUrl: string, dbName: string, csvFile: string): Promise<void> {
   const connection = await connect(transactorUrl, dbName)
 
@@ -238,7 +164,7 @@ export async function importLead (transactorUrl: string, dbName: string, csvFile
 
     const client = new TxOperations(connection, 'core:account:lead-importer' as Ref<EmployeeAccount>)
 
-    await uodateClasses(client, records)
+    await updateClasses(client, records, fieldMapping)
     await createCustomers(client, filledFields)
 
     const importedFunnelId = await createFunnel(records, client)
@@ -464,29 +390,3 @@ async function createCustomers (client: TxOperations, filledFields: any[]): Prom
     }
   }
 }
-
-async function parseCSV (csvData: string): Promise<any[]> {
-  return await new Promise((resolve, reject) => {
-    parse(
-      csvData,
-      {
-        delimiter: ';',
-        columns: true,
-        quote: '"',
-        bom: true,
-        cast: true,
-        autoParse: true,
-        castDate: false,
-        skipEmptyLines: true,
-        skipRecordsWithEmptyValues: true
-      },
-      (err, records) => {
-        if (err !== undefined) {
-          console.error(err)
-          reject(err)
-        }
-        resolve(records)
-      }
-    )
-  })
-}

dev/tool/src/leads/lead-importer2.ts

@@ -0,0 +1,267 @@
+//
+// Copyright © 2022 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import contact, { Contact, EmployeeAccount, Organization } from '@anticrm/contact'
+import core, { generateId, MixinUpdate, Ref, TxOperations } from '@anticrm/core'
+import lead from '@anticrm/lead'
+import { connect } from '@anticrm/server-tool'
+import { readFile } from 'fs/promises'
+import { updateClasses } from './classes'
+import { CustomCustomer, FieldType } from './types'
+import { filled } from './utils'
+
+import { parse } from 'csv-parse'
+import parsePhoneNumber from 'libphonenumber-js'
+import addrs from 'email-addresses'
+
+const names = {
+  orgName: 'название компании',
+  corporate_site: 'сайт компании',
+  contact_person: 'контактное лицо',
+  contacts: 'контакты',
+  dialog_area: 'место диалога: ССеть, почта, ТГ',
+  sales_manager: 'seles-менеджер',
+  bot_account: 'бот аккаунт',
+  description: 'договорённости, доп. инфа о компаниии, вакансии',
+  hh_link: 'ссылка hh'
+}
+
+const fieldMapping: Record<string, FieldType> = {
+  'Место диалога': {
+    name: 'dialog_area',
+    type: core.class.EnumOf,
+    enumName: 'CompanyDialogArea',
+    label: core.string.Enum,
+    fName: names.dialog_area
+  },
+  'Sales Manager': {
+    name: 'sales_manager',
+    type: core.class.EnumOf,
+    enumName: 'CompanySalesManager',
+    label: core.string.Enum,
+    fName: names.sales_manager
+  },
+  'Бот Аккаунт': {
+    name: 'bot_account',
+    type: core.class.EnumOf,
+    enumName: 'CompanyBotAccount',
+    label: core.string.Enum,
+    fName: names.bot_account
+  },
+  'Контактное Лицо': {
+    name: 'contact_person',
+    type: core.class.TypeString,
+    label: core.string.String
+  },
+  Контакты: {
+    name: 'contacts',
+    type: core.class.TypeString,
+    label: core.string.String
+  }
+}
+
+export async function parseCSV (csvData: string): Promise<any[]> {
+  return await new Promise((resolve, reject) => {
+    parse(
+      csvData,
+      {
+        delimiter: ',',
+        columns: true,
+        quote: '"',
+        bom: true,
+        cast: true,
+        autoParse: true,
+        castDate: false,
+        skipEmptyLines: true,
+        skipRecordsWithEmptyValues: true
+      },
+      (err, records) => {
+        if (err !== undefined) {
+          console.error(err)
+          reject(err)
+        }
+        resolve(records)
+      }
+    )
+  })
+}
+
+export async function importLead2 (transactorUrl: string, dbName: string, csvFile: string): Promise<void> {
+  const connection = await connect(transactorUrl, dbName)
+
+  try {
+    console.log('loading cvs document...')
+
+    const csvData = await readFile(csvFile, 'utf-8')
+    const records: any[] = await parseCSV(csvData)
+    const uniqKeys: string[] = []
+    const filledFields = records.map((it) => filled(it, uniqKeys))
+    // console.log(filledFields)
+
+    const client = new TxOperations(connection, 'core:account:lead-importer' as Ref<EmployeeAccount>)
+
+    await updateClasses(client, records, fieldMapping)
+
+    await createCustomers(client, filledFields)
+  } catch (err: any) {
+    console.error(err)
+  } finally {
+    await connection.close()
+  }
+}
+
+async function createCustomers (client: TxOperations, filledFields: any[]): Promise<void> {
+  for (const record of filledFields) {
+    let orgId: Ref<Organization> = generateId()
+    const orgName = record[names.orgName]
+    if ((orgName?.toString() ?? '').trim().length === 0) {
+      continue
+    }
+    const org = await client.findOne(lead.mixin.Customer, { name: orgName })
+    console.log('processing', orgName)
+    if (org === undefined) {
+      await client.createDoc(
+        contact.class.Organization,
+        contact.space.Contacts,
+        {
+          name: orgName,
+          city: '',
+          members: 0
+        },
+        orgId as unknown as Ref<Organization>
+      )
+      await client.createMixin<Contact, CustomCustomer>(
+        orgId,
+        contact.class.Organization,
+        contact.space.Contacts,
+        lead.mixin.Customer,
+        {
+          description: record[names.description],
+          dialog_area: record[names.dialog_area],
+          sales_manager: record[names.sales_manager],
+          bot_account: record[names.bot_account],
+          contacts: record[names.contacts],
+          contact_person: record[names.contact_person]
+        }
+      )
+    } else {
+      orgId = org._id as unknown as Ref<Organization>
+      const upd: MixinUpdate<Contact, CustomCustomer> = {}
+      const newValues = {
+        description: record[names.description],
+        dialog_area: record[names.dialog_area],
+        sales_manager: record[names.sales_manager],
+        bot_account: record[names.bot_account],
+        contact_person: record[names.contact_person]
+      }
+      for (const [k, v] of Object.entries(newValues)) {
+        if ((org as any)[k] !== v && v !== undefined) {
+          ;(upd as any)[k] = v
+        }
+      }
+      if (Object.keys(upd).length > 0) {
+        await client.updateMixin<Contact, CustomCustomer>(
+          orgId,
+          contact.class.Organization,
+          contact.space.Contacts,
+          lead.mixin.Customer,
+          upd
+        )
+      }
+    }
+
+    const contacts: string = record[names.contacts]
+
+    let dta: string = contacts?.toString() ?? ''
+    if (dta.length > 0) {
+      dta = dta.replace('.ru ', '.ru\n')
+      dta = dta.replace('.com ', '.com\n')
+      dta = dta.replace('+', '\n+')
+      dta = dta.replace(',', '\n')
+      dta = dta.replace('|', '\n')
+      dta = dta.replace('  ', '\n')
+      dta = dta.replace('Email:', '\n')
+      dta = dta.replace('Email:', '\n')
+      const res = dta
+        .split('\n')
+        .map((it) => it.trim())
+        .filter((it) => it.length > 0)
+      // console.log('\t', res)
+      for (const r of res) {
+        const phone = parsePhoneNumber(r)
+        if (phone !== undefined) {
+          const iphone = phone.formatInternational()
+          const channels = await client.findAll(contact.class.Channel, { attachedTo: orgId })
+          const emailPr = channels.find((it) => it.value === iphone)
+
+          if (emailPr === undefined) {
+            await client.addCollection(
+              contact.class.Channel,
+              contact.space.Contacts,
+              orgId,
+              contact.class.Organization,
+              'channels',
+              {
+                value: iphone,
+                provider: contact.channelProvider.Phone
+              }
+            )
+          }
+        }
+
+        const parsedddr = addrs.parseOneAddress(r)
+        if (parsedddr != null) {
+          const email = r.trim()
+          if (email !== undefined) {
+            const channels = await client.findAll(contact.class.Channel, { attachedTo: orgId })
+            const emailPr = channels.find((it) => it.value === email)
+            if (emailPr === undefined) {
+              await client.addCollection(
+                contact.class.Channel,
+                contact.space.Contacts,
+                orgId,
+                contact.class.Organization,
+                'channels',
+                {
+                  value: email,
+                  provider: contact.channelProvider.Email
+                }
+              )
+            }
+          }
+        }
+      }
+    }
+
+    const site = record[names.corporate_site]
+    if (site !== undefined) {
+      const channels = await client.findAll(contact.class.Channel, { attachedTo: orgId })
+      const sitePr = channels.find((it) => it.value === site)
+      if (sitePr === undefined) {
+        await client.addCollection(
+          contact.class.Channel,
+          contact.space.Contacts,
+          orgId,
+          contact.class.Organization,
+          'channels',
+          {
+            value: site,
+            provider: contact.channelProvider.Homepage
+          }
+        )
+      }
+    }
+  }
+}

dev/tool/src/leads/types.ts

@@ -0,0 +1,43 @@
+//
+// Copyright © 2022 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+import { Class, Doc, Ref } from '@anticrm/core'
+import { Customer } from '@anticrm/lead'
+import { IntlString } from '@anticrm/platform'
+
+export interface CustomCustomer extends Customer {
+  annualTurnover?: string
+  currency?: string
+  comment?: string
+  source_ka?: string
+  employees_count?: number
+  activity_area?: string
+  company_type?: string
+  processing_status?: string
+  responsible?: string
+
+  dialog_area?: string
+  sales_manager?: string
+  bot_account?: string
+  contacts?: string
+  contact_person?: string
+}
+
+export interface FieldType {
+  name: string
+  type?: Ref<Class<Doc>>
+  label?: IntlString
+  enumName?: string
+  fName?: string
+}

dev/tool/src/leads/utils.ts

@@ -0,0 +1,28 @@
+//
+// Copyright © 2022 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+export function filled (obj: any, uniqKeys: string[]): any {
+  const result: Record<string, any> = {}
+  for (const [k, v] of Object.entries(obj)) {
+    if (typeof v === 'string' && v.trim().length === 0) {
+      continue
+    }
+    if (!uniqKeys.includes(k)) {
+      uniqKeys.push(k)
+    }
+    result[k] = v
+  }
+  return result
+}