Allow to create organizations from CSV (#2260)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

dev/tool/src/csv/lead-importer.ts

@@ -98,7 +98,7 @@ const fieldMapping: Record<string, FieldType> = {
   }
 }
 
-async function updateStates<T extends State | DoneState> (
+export async function updateStates<T extends State | DoneState> (
   client: TxOperations,
   states: string[],
   _class: Ref<Class<T>>,

dev/tool/src/csv/org-importer.ts

@@ -0,0 +1,551 @@
+//
+// Copyright © 2022 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import contact, { Contact, EmployeeAccount, Organization } from '@anticrm/contact'
+import core, {
+  BackupClient,
+  BlobData,
+  Client,
+  DOMAIN_BLOB,
+  generateId,
+  MixinUpdate,
+  Ref,
+  SortingOrder,
+  TxOperations,
+  WithLookup
+} from '@anticrm/core'
+import lead, { Customer, Funnel, Lead } from '@anticrm/lead'
+import { connect } from '@anticrm/server-tool'
+import { readFile } from 'fs/promises'
+import { updateClasses } from './classes'
+import { CustomCustomer, FieldType } from './types'
+import { filled, getValid, updateChannel } from './utils'
+
+import task, { calcRank, DoneState, Sequence, State } from '@anticrm/task'
+import { parse } from 'csv-parse'
+import { updateStates } from './lead-importer'
+import got from 'got'
+import mimetypes from 'mime-types'
+import attachment from '@anticrm/model-attachment'
+
+const names = {
+  companyName: 'Company Name',
+
+  companyType: 'Company Type',
+  processingStatus: 'Статус обработки',
+
+  employees: 'Employees',
+  corporate_website: 'Corporate Website',
+  corporate_email: 'Корпоративный e-mail',
+  workEmail: 'Work E-mail',
+
+  responsible: 'Responsible',
+  industry: 'Industry',
+  currency: 'Currency',
+  comment: 'Comment',
+  createdBy: 'Created by',
+  created: 'Created',
+  modifiedBy: 'Modified by',
+  modified: 'Modified',
+  trelloLink: 'Ссылка на Trello',
+  research: 'Ресеч',
+  startCollaboration: 'Начало сотрудничества',
+  geography: 'География',
+  interestRate: 'Ставка вознаграждения',
+  agreement: 'Договор',
+  technologyStack: 'Стек технологий заказчика',
+  fieldOfActivity: 'Сфера деятельности (Другое)',
+  hrLanguage: 'Язык общения с заказчиком',
+  selectionFeatures: 'Особенности подбора',
+  source: 'Источник',
+  clientActivityStatus: 'Статус активности клиента',
+  valCompany: 'ВАЛ от компании (евро)'
+}
+
+const fieldMapping: Record<string, FieldType> = {
+  Status: {
+    name: 'status',
+    type: core.class.EnumOf,
+    enumName: 'CompanyStatus',
+    label: core.string.Enum,
+    fName: names.companyType
+  },
+  [names.employees]: {
+    name: 'employees',
+    type: core.class.EnumOf,
+    enumName: 'CompanyEmployees',
+    label: core.string.Enum,
+    fName: names.employees
+  },
+  [names.responsible]: {
+    name: 'responsible',
+    type: core.class.EnumOf,
+    enumName: 'CompanyResponsible',
+    label: core.string.Enum,
+    fName: names.responsible
+  },
+  [names.industry]: {
+    name: 'industry',
+    type: core.class.EnumOf,
+    enumName: 'CompanyIndustry',
+    label: core.string.Enum,
+    fName: names.industry
+  },
+  [names.currency]: {
+    name: 'currency',
+    type: core.class.EnumOf,
+    enumName: 'CompanyCurrency',
+    label: core.string.Enum,
+    fName: names.currency
+  },
+  [names.research]: {
+    name: 'research',
+    type: core.class.EnumOf,
+    enumName: 'CompanyResearch',
+    label: core.string.Enum,
+    fName: names.research
+  },
+  [names.geography]: {
+    name: 'geography',
+    type: core.class.EnumOf,
+    enumName: 'CompanyGeography',
+    label: core.string.Enum,
+    fName: names.geography
+  },
+  [names.interestRate]: {
+    name: 'interestRate',
+    type: core.class.EnumOf,
+    enumName: 'CompanyInterestRate',
+    label: core.string.Enum,
+    fName: names.interestRate
+  },
+  [names.agreement]: {
+    name: 'agreement',
+    type: core.class.TypeString,
+    label: core.string.String,
+    fName: names.agreement
+  },
+  [names.comment]: {
+    name: 'mainComment',
+    type: core.class.TypeString,
+    label: core.string.String,
+    fName: names.comment
+  },
+  [names.trelloLink]: {
+    name: 'trelloLink',
+    type: core.class.TypeString,
+    label: core.string.String,
+    fName: names.trelloLink
+  },
+  [names.technologyStack]: {
+    name: 'technologyStack',
+    type: core.class.TypeString,
+    label: core.string.String,
+    fName: names.technologyStack
+  },
+  [names.fieldOfActivity]: {
+    name: 'fieldOfActivity',
+    type: core.class.TypeString,
+    label: core.string.String,
+    fName: names.fieldOfActivity
+  },
+  [names.hrLanguage]: {
+    name: 'hrLanguage',
+    type: core.class.EnumOf,
+    enumName: 'CompanyHRLanguage',
+    label: core.string.Enum,
+    fName: names.interestRate
+  },
+  [names.selectionFeatures]: {
+    name: 'selectionFeatures',
+    type: core.class.EnumOf,
+    enumName: 'CompanyInterestRate',
+    label: core.string.Enum,
+    fName: names.interestRate
+  },
+  [names.source]: {
+    name: 'source',
+    type: core.class.EnumOf,
+    enumName: 'CompanySource',
+    label: core.string.Enum,
+    fName: names.source
+  },
+  [names.valCompany]: {
+    name: 'interestRate',
+    type: core.class.TypeString,
+    label: core.string.String,
+    fName: names.valCompany
+  },
+  [names.startCollaboration]: {
+    name: 'startCollaboration',
+    type: core.class.TypeString,
+    label: core.string.Enum,
+    fName: names.startCollaboration
+  },
+  [names.clientActivityStatus]: {
+    name: 'clientActivityStatus',
+    type: core.class.EnumOf,
+    enumName: 'CompanyClientActivity',
+    label: core.string.Enum,
+    fName: names.clientActivityStatus
+  }
+}
+
+export async function parseCSV (csvData: string): Promise<any[]> {
+  return await new Promise((resolve, reject) => {
+    parse(
+      csvData,
+      {
+        delimiter: ';',
+        columns: true,
+        quote: '"',
+        bom: true,
+        cast: true,
+        autoParse: true,
+        castDate: false,
+        skipEmptyLines: true,
+        skipRecordsWithEmptyValues: true
+      },
+      (err, records) => {
+        if (err !== undefined) {
+          console.error(err)
+          reject(err)
+        }
+        resolve(records)
+      }
+    )
+  })
+}
+
+export async function importOrgs (transactorUrl: string, dbName: string, csvFile: string): Promise<void> {
+  const connection = (await connect(transactorUrl, dbName, undefined, {
+    mode: 'backup'
+  })) as unknown as Client & BackupClient
+  try {
+    console.log('loading cvs document...')
+
+    const csvData = await readFile(csvFile, 'utf-8')
+    const records: any[] = await parseCSV(csvData)
+    const uniqKeys: string[] = []
+    const filledFields = records.map((it) => filled(it, uniqKeys))
+    // console.log(filledFields)
+
+    const client = new TxOperations(connection, 'core:account:lead-importer' as Ref<EmployeeAccount>)
+
+    await updateClasses(client, records, fieldMapping)
+
+    const funnelId = await createFunnel(records, client)
+
+    const statusKaValues = records
+      .map((it) => it[names.processingStatus])
+      .filter((it, idx, arr) => arr.indexOf(it) === idx)
+    console.log(statusKaValues)
+
+    const wonStateFrom = 'Договор на рассмотрении'
+
+    const states = [
+      'Переговоры / назначен звонок',
+      'Обсуждение условий в переписке',
+      wonStateFrom,
+      'Направлено первичное письмо по e-mail',
+      'Направлен запрос в LinkedIn / другом канале'
+    ]
+    const wonStates = ['Договор заключен']
+    const lostStates = ['Получен отказ', 'Перестал отвечать']
+
+    const statesMap = new Map<string, State | DoneState>()
+
+    // Create update/states
+    await updateStates(client, states, task.class.State, funnelId, statesMap)
+    await updateStates(client, wonStates, task.class.WonState, funnelId, statesMap)
+    await updateStates(client, lostStates, task.class.LostState, funnelId, statesMap)
+
+    if ((await client.findOne(task.class.Kanban, { attachedTo: funnelId })) === undefined) {
+      await client.createDoc(task.class.Kanban, funnelId, {
+        attachedTo: funnelId
+      })
+    }
+
+    // Ok we have all states, let's create Leads.
+    const sequence = await client.findOne(task.class.Sequence, { attachedTo: lead.class.Lead })
+    if (sequence === undefined) {
+      throw new Error('sequence object not found')
+    }
+
+    await createOrganizations(client, filledFields, connection, funnelId, statesMap, wonStateFrom, sequence)
+  } catch (err: any) {
+    console.error(err)
+  } finally {
+    await connection.close()
+  }
+}
+
+async function createFunnel (records: any[], client: TxOperations): Promise<Ref<Funnel>> {
+  const importedFunnelId = 'imported-funnel' as Ref<Funnel>
+  let funnel = await client.findOne(lead.class.Funnel, { _id: importedFunnelId })
+  if (funnel === undefined) {
+    // No funnel, let's create one.
+    await client.createDoc(
+      lead.class.Funnel,
+      core.space.Space,
+      {
+        name: 'Organizations',
+        archived: false,
+        members: [],
+        private: false,
+        description: ''
+      },
+      importedFunnelId
+    )
+    funnel = await client.findOne(lead.class.Funnel, { _id: importedFunnelId })
+  }
+  return importedFunnelId
+}
+
+export interface CustomOrg extends Customer {
+  status: any
+  clientActivityStatus: any
+
+  employees: any
+
+  responsible: any
+  industry: any
+  currency: any
+  trelloLink: string
+  research: any
+  startCollaboration: string
+  geography: any
+  interestRate: any
+  agreement: string
+  technologyStack: string
+  fieldOfActivity: string
+  hrLanguage: any
+  selectionFeatures: any
+  source: any
+  valCompany: any
+}
+
+async function createOrganizations (
+  client: TxOperations,
+  filledFields: any[],
+  connection: Client & BackupClient,
+  funnelId: Ref<Funnel>,
+  statesMap: Map<string, State | DoneState>,
+  wonStateFrom: string,
+  sequence: WithLookup<Sequence>
+): Promise<void> {
+  for (const record of filledFields) {
+    let orgId: Ref<Organization> = generateId()
+    const orgName = record[names.companyName]
+    if ((orgName?.toString() ?? '').trim().length === 0) {
+      continue
+    }
+    const org = await client.findOne(lead.mixin.Customer, { name: orgName })
+    console.log('processing', orgName)
+    if (org === undefined) {
+      await client.createDoc(
+        contact.class.Organization,
+        contact.space.Contacts,
+        {
+          name: orgName,
+          city: '',
+          members: 0
+        },
+        orgId as unknown as Ref<Organization>
+      )
+      await client.createMixin<Contact, CustomOrg>(
+        orgId,
+        contact.class.Organization,
+        contact.space.Contacts,
+        lead.mixin.Customer,
+        {
+          description: record[names.comment],
+          status: record[names.companyType],
+          clientActivityStatus: record[names.clientActivityStatus],
+
+          employees: record[names.employees],
+
+          responsible: record[names.responsible],
+          industry: record[names.industry],
+          currency: record[names.currency],
+          trelloLink: record[names.trelloLink],
+          research: record[names.research],
+          startCollaboration: record[names.startCollaboration],
+          geography: record[names.geography],
+          interestRate: record[names.interestRate],
+          agreement: record[names.agreement],
+          technologyStack: record[names.technologyStack],
+          fieldOfActivity: record[names.fieldOfActivity],
+          hrLanguage: record[names.hrLanguage],
+          selectionFeatures: record[names.selectionFeatures],
+          source: record[names.source],
+          valCompany: record[names.valCompany]
+        }
+      )
+    } else {
+      orgId = org._id as unknown as Ref<Organization>
+      const upd: MixinUpdate<Contact, CustomCustomer> = {}
+      const newValues = {
+        description: record[names.comment],
+        status: record[names.companyType],
+        clientActivityStatus: record[names.clientActivityStatus],
+
+        employees: record[names.employees],
+
+        responsible: record[names.responsible],
+        industry: record[names.industry],
+        currency: record[names.currency],
+        trelloLink: record[names.trelloLink],
+        research: record[names.research],
+        startCollaboration: record[names.startCollaboration],
+        geography: record[names.geography],
+        interestRate: record[names.interestRate],
+        agreement: record[names.agreement],
+        technologyStack: record[names.technologyStack],
+        fieldOfActivity: record[names.fieldOfActivity],
+        hrLanguage: record[names.hrLanguage],
+        selectionFeatures: record[names.selectionFeatures],
+        source: record[names.source],
+        valCompany: record[names.valCompany]
+      }
+      for (const [k, v] of Object.entries(newValues)) {
+        if ((org as any)[k] !== v && v !== undefined) {
+          ;(upd as any)[k] = v
+        }
+      }
+      if (Object.keys(upd).length > 0) {
+        await client.updateMixin<Contact, CustomCustomer>(
+          orgId,
+          contact.class.Organization,
+          contact.space.Contacts,
+          lead.mixin.Customer,
+          upd
+        )
+      }
+    }
+
+    await updateChannel(
+      client,
+      orgId,
+      getValid(record, names.corporate_website),
+      contact.channelProvider.Homepage,
+      contact.class.Organization
+    )
+    await updateChannel(
+      client,
+      orgId,
+      getValid(record, names.corporate_email),
+      contact.channelProvider.Email,
+      contact.class.Organization
+    )
+    await updateChannel(
+      client,
+      orgId,
+      getValid(record, names.workEmail),
+      contact.channelProvider.Email,
+      contact.class.Organization
+    )
+
+    const state = statesMap.get(record[names.processingStatus])
+    if (state !== undefined) {
+      const leadState =
+        state._class === task.class.State
+          ? (state._id as Ref<State>)
+          : (statesMap.get(wonStateFrom) as unknown as Ref<State>)
+      const doneState =
+        state._class === task.class.WonState || state._class === task.class.LostState
+          ? (state._id as Ref<DoneState>)
+          : null
+
+      const orgLeadId = `imported-lead-${record.ID as string}` as Ref<Lead>
+      const orgLead = await client.findOne(lead.class.Lead, { _id: orgLeadId })
+
+      if (orgLead === undefined) {
+        const lastOne = await client.findOne(
+          lead.class.Lead,
+          { state: leadState },
+          { sort: { rank: SortingOrder.Descending } }
+        )
+        const incResult = await client.update(sequence, { $inc: { sequence: 1 } }, true)
+        await client.addCollection(
+          lead.class.Lead,
+          funnelId,
+          orgId,
+          lead.mixin.Customer,
+          'leads',
+          {
+            title: orgName,
+            number: (incResult as any).object.sequence,
+            rank: calcRank(lastOne, undefined),
+            assignee: null,
+            startDate: null,
+            dueDate: null,
+            state: leadState,
+            doneState
+          },
+          orgLeadId
+        )
+      }
+    }
+
+    const agreement = record[names.agreement] as string
+
+    if (agreement !== undefined) {
+      const agreements = agreement.split(',')
+      for (const r of agreements) {
+        try {
+          const url = (r ?? '').trim()
+          if (url.startsWith('http')) {
+            const lastpos = url.lastIndexOf('/')
+            const fname = url.substring(lastpos + 1)
+
+            const buffer = await got(url).buffer()
+            const blobId = (orgId + '_' + generateId()) as Ref<BlobData>
+            const type = mimetypes.contentType(fname)
+            const data: BlobData = {
+              _id: blobId,
+              space: contact.space.Contacts,
+              modifiedBy: client.txFactory.account,
+              modifiedOn: Date.now(),
+              _class: core.class.BlobData,
+              name: fname,
+              size: buffer.length,
+              type: type !== false ? type : 'unknown',
+              base64Data: buffer.toString('base64')
+            }
+            await connection.upload(DOMAIN_BLOB, [data])
+
+            await client.addCollection(
+              attachment.class.Attachment,
+              contact.space.Contacts,
+              orgId,
+              contact.class.Organization,
+              'attachments',
+              {
+                file: blobId,
+                name: fname,
+                size: buffer.length,
+                type: type !== false ? type : 'unknown',
+                lastModified: Date.now()
+              }
+            )
+          }
+        } catch (err) {
+          console.log(err)
+        }
+      }
+    }
+  }
+}

dev/tool/src/csv/talant-importer.ts

@@ -13,7 +13,8 @@
 // limitations under the License.
 //
 
-import contact, { ChannelProvider, combineName, Contact, EmployeeAccount, Person } from '@anticrm/contact'
+import attachment, { Attachment } from '@anticrm/attachment'
+import contact, { combineName, Contact, EmployeeAccount, Person } from '@anticrm/contact'
 import core, {
   AnyAttribute,
   BackupClient,
@@ -34,19 +35,18 @@ import core, {
 } from '@anticrm/core'
 import { Asset, getEmbeddedLabel, IntlString } from '@anticrm/platform'
 import recruit, { Candidate } from '@anticrm/recruit'
+import { ReconiDocument } from '@anticrm/rekoni'
+import { generateToken } from '@anticrm/server-token'
 import { connect } from '@anticrm/server-tool'
 import setting from '@anticrm/setting'
 import { readFile } from 'fs/promises'
-import { parseCSV } from './parseCSV'
-import { FieldType } from './types'
-import { filled } from './utils'
 import got from 'got'
 import mimetypes from 'mime-types'
-import attachment, { Attachment } from '@anticrm/attachment'
-import { generateToken } from '@anticrm/server-token'
 import { recognize, updateContacts, updateSkills } from '../recruit'
-import { ReconiDocument } from '@anticrm/rekoni'
 import { findOrUpdateAttached } from '../utils'
+import { parseCSV } from './parseCSV'
+import { FieldType } from './types'
+import { filled, getValid, updateChannel } from './utils'
 
 const names = {
   status: 'Status',
@@ -585,52 +585,29 @@ async function createTalants (
       dataLocationDetails
     )
 
-    function getValid (...names: string[]): string | undefined {
-      for (const o of names) {
-        const v = record[o]
-        if (v !== undefined && typeof v === 'string' && v.trim().length > 0) {
-          return v
-        }
-      }
-    }
-
-    async function updateChannel (value: string | undefined, provider: Ref<ChannelProvider>): Promise<void> {
-      if (value === undefined) {
-        return
-      }
-      const channels = await client.findAll(contact.class.Channel, { attachedTo: candidateId })
-      const emailPr = channels.find((it) => it.value === value)
-      if (emailPr === undefined) {
-        await client.addCollection(
-          contact.class.Channel,
-          contact.space.Contacts,
-          candidateId,
-          contact.class.Person,
-          'channels',
-          {
-            value,
-            provider
-          }
-        )
-      }
-    }
-
     await updateChannel(
+      client,
+      candidateId,
       getValid(record, names.workEmail, names.homeEmail, names.newsletterEmail, names.otherEmail),
       contact.channelProvider.Email
     )
-    await updateChannel(getValid(record, names.webSite), contact.channelProvider.Homepage)
+    await updateChannel(client, candidateId, getValid(record, names.webSite), contact.channelProvider.Homepage)
-    await updateChannel(getValid(record, names.phone, names.phoneNumber), contact.channelProvider.Phone)
+    await updateChannel(
-    await updateChannel(getValid(record, names.telegram), contact.channelProvider.Telegram)
+      client,
+      candidateId,
+      getValid(record, names.phone, names.phoneNumber),
+      contact.channelProvider.Phone
+    )
+    await updateChannel(client, candidateId, getValid(record, names.telegram), contact.channelProvider.Telegram)
 
     const ghval = getValid(record, names.githubPortfolio)
     if (ghval?.includes('https://github.com') ?? false) {
-      await updateChannel(ghval, contact.channelProvider.GitHub)
+      await updateChannel(client, candidateId, ghval, contact.channelProvider.GitHub)
     }
 
     const profile = getValid(record, names.profile)
     if (profile?.includes('linkedin.com') ?? false) {
-      await updateChannel(profile, contact.channelProvider.LinkedIn)
+      await updateChannel(client, candidateId, profile, contact.channelProvider.LinkedIn)
     }
 
     const resume = record[names.resume] as string

dev/tool/src/csv/utils.ts

@@ -13,6 +13,9 @@
 // limitations under the License.
 //
 
+import contact, { ChannelProvider, Contact } from '@anticrm/contact'
+import { Class, Doc, Ref, TxOperations } from '@anticrm/core'
+
 export function filled (obj: any, uniqKeys: string[]): any {
   const result: Record<string, any> = {}
   for (const [k, v] of Object.entries(obj)) {
@@ -26,3 +29,31 @@ export function filled (obj: any, uniqKeys: string[]): any {
   }
   return result
 }
+
+export async function updateChannel (
+  client: TxOperations,
+  attachedTo: Ref<Contact>,
+  value: string | undefined,
+  provider: Ref<ChannelProvider>,
+  attachToClass: Ref<Class<Doc>> = contact.class.Person
+): Promise<void> {
+  if (value === undefined) {
+    return
+  }
+  const channels = await client.findAll(contact.class.Channel, { attachedTo })
+  const valueCh = channels.find((it) => it.value === value)
+  if (valueCh === undefined) {
+    await client.addCollection(contact.class.Channel, contact.space.Contacts, attachedTo, attachToClass, 'channels', {
+      value,
+      provider
+    })
+  }
+}
+export function getValid (record: any, ...names: string[]): string | undefined {
+  for (const o of names) {
+    const v = record[o]
+    if (v !== undefined && typeof v === 'string' && v.trim().length > 0) {
+      return v
+    }
+  }
+}

dev/tool/src/index.ts

@@ -39,6 +39,7 @@ import { exit } from 'process'
 import { removeDuplicates } from './csv/duplicates'
 import { importLead } from './csv/lead-importer'
 import { importLead2 } from './csv/lead-importer2'
+import { importOrgs } from './csv/org-importer'
 import { importTalants } from './csv/talant-importer'
 import { rebuildElastic } from './elastic'
 import { importXml } from './importer'
@@ -387,6 +388,13 @@ program
     return await importTalants(transactorUrl, workspace, fileName, rekoniUrl)
   })
 
+program
+  .command('import-org-csv <workspace> <fileName>')
+  .description('Import Organizations csv')
+  .action(async (workspace, fileName, cmd) => {
+    return await importOrgs(transactorUrl, workspace, fileName)
+  })
+
 program
   .command('lead-duplicates <workspace>')
   .description('Find and remove duplicate organizations.')