From 849f94ef904d3910b095bf6f7de5a3453f9dfd79 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Fri, 26 Aug 2022 15:20:20 +0700 Subject: [PATCH] Allow to create organizations from CSV (#2260) Signed-off-by: Andrey Sobolev --- dev/tool/src/csv/lead-importer.ts | 2 +- dev/tool/src/csv/org-importer.ts | 551 ++++++++++++++++++++++++++++ dev/tool/src/csv/talant-importer.ts | 61 +-- dev/tool/src/csv/utils.ts | 31 ++ dev/tool/src/index.ts | 8 + 5 files changed, 610 insertions(+), 43 deletions(-) create mode 100644 dev/tool/src/csv/org-importer.ts diff --git a/dev/tool/src/csv/lead-importer.ts b/dev/tool/src/csv/lead-importer.ts index 87c94a790a..dafa844d36 100644 --- a/dev/tool/src/csv/lead-importer.ts +++ b/dev/tool/src/csv/lead-importer.ts @@ -98,7 +98,7 @@ const fieldMapping: Record = { } } -async function updateStates ( +export async function updateStates ( client: TxOperations, states: string[], _class: Ref>, diff --git a/dev/tool/src/csv/org-importer.ts b/dev/tool/src/csv/org-importer.ts new file mode 100644 index 0000000000..b05fbb11a7 --- /dev/null +++ b/dev/tool/src/csv/org-importer.ts @@ -0,0 +1,551 @@ +// +// Copyright © 2022 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import contact, { Contact, EmployeeAccount, Organization } from '@anticrm/contact' +import core, { + BackupClient, + BlobData, + Client, + DOMAIN_BLOB, + generateId, + MixinUpdate, + Ref, + SortingOrder, + TxOperations, + WithLookup +} from '@anticrm/core' +import lead, { Customer, Funnel, Lead } from '@anticrm/lead' +import { connect } from '@anticrm/server-tool' +import { readFile } from 'fs/promises' +import { updateClasses } from './classes' +import { CustomCustomer, FieldType } from './types' +import { filled, getValid, updateChannel } from './utils' + +import task, { calcRank, DoneState, Sequence, State } from '@anticrm/task' +import { parse } from 'csv-parse' +import { updateStates } from './lead-importer' +import got from 'got' +import mimetypes from 'mime-types' +import attachment from '@anticrm/model-attachment' + +const names = { + companyName: 'Company Name', + + companyType: 'Company Type', + processingStatus: 'Статус обработки', + + employees: 'Employees', + corporate_website: 'Corporate Website', + corporate_email: 'Корпоративный e-mail', + workEmail: 'Work E-mail', + + responsible: 'Responsible', + industry: 'Industry', + currency: 'Currency', + comment: 'Comment', + createdBy: 'Created by', + created: 'Created', + modifiedBy: 'Modified by', + modified: 'Modified', + trelloLink: 'Ссылка на Trello', + research: 'Ресеч', + startCollaboration: 'Начало сотрудничества', + geography: 'География', + interestRate: 'Ставка вознаграждения', + agreement: 'Договор', + technologyStack: 'Стек технологий заказчика', + fieldOfActivity: 'Сфера деятельности (Другое)', + hrLanguage: 'Язык общения с заказчиком', + selectionFeatures: 'Особенности подбора', + source: 'Источник', + clientActivityStatus: 'Статус активности клиента', + valCompany: 'ВАЛ от компании (евро)' +} + +const fieldMapping: Record = { + Status: { + name: 'status', + type: core.class.EnumOf, + enumName: 'CompanyStatus', + label: core.string.Enum, + fName: names.companyType + }, + [names.employees]: { + name: 'employees', + type: core.class.EnumOf, + enumName: 'CompanyEmployees', + label: core.string.Enum, + fName: names.employees + }, + [names.responsible]: { + name: 'responsible', + type: core.class.EnumOf, + enumName: 'CompanyResponsible', + label: core.string.Enum, + fName: names.responsible + }, + [names.industry]: { + name: 'industry', + type: core.class.EnumOf, + enumName: 'CompanyIndustry', + label: core.string.Enum, + fName: names.industry + }, + [names.currency]: { + name: 'currency', + type: core.class.EnumOf, + enumName: 'CompanyCurrency', + label: core.string.Enum, + fName: names.currency + }, + [names.research]: { + name: 'research', + type: core.class.EnumOf, + enumName: 'CompanyResearch', + label: core.string.Enum, + fName: names.research + }, + [names.geography]: { + name: 'geography', + type: core.class.EnumOf, + enumName: 'CompanyGeography', + label: core.string.Enum, + fName: names.geography + }, + [names.interestRate]: { + name: 'interestRate', + type: core.class.EnumOf, + enumName: 'CompanyInterestRate', + label: core.string.Enum, + fName: names.interestRate + }, + [names.agreement]: { + name: 'agreement', + type: core.class.TypeString, + label: core.string.String, + fName: names.agreement + }, + [names.comment]: { + name: 'mainComment', + type: core.class.TypeString, + label: core.string.String, + fName: names.comment + }, + [names.trelloLink]: { + name: 'trelloLink', + type: core.class.TypeString, + label: core.string.String, + fName: names.trelloLink + }, + [names.technologyStack]: { + name: 'technologyStack', + type: core.class.TypeString, + label: core.string.String, + fName: names.technologyStack + }, + [names.fieldOfActivity]: { + name: 'fieldOfActivity', + type: core.class.TypeString, + label: core.string.String, + fName: names.fieldOfActivity + }, + [names.hrLanguage]: { + name: 'hrLanguage', + type: core.class.EnumOf, + enumName: 'CompanyHRLanguage', + label: core.string.Enum, + fName: names.interestRate + }, + [names.selectionFeatures]: { + name: 'selectionFeatures', + type: core.class.EnumOf, + enumName: 'CompanyInterestRate', + label: core.string.Enum, + fName: names.interestRate + }, + [names.source]: { + name: 'source', + type: core.class.EnumOf, + enumName: 'CompanySource', + label: core.string.Enum, + fName: names.source + }, + [names.valCompany]: { + name: 'interestRate', + type: core.class.TypeString, + label: core.string.String, + fName: names.valCompany + }, + [names.startCollaboration]: { + name: 'startCollaboration', + type: core.class.TypeString, + label: core.string.Enum, + fName: names.startCollaboration + }, + [names.clientActivityStatus]: { + name: 'clientActivityStatus', + type: core.class.EnumOf, + enumName: 'CompanyClientActivity', + label: core.string.Enum, + fName: names.clientActivityStatus + } +} + +export async function parseCSV (csvData: string): Promise { + return await new Promise((resolve, reject) => { + parse( + csvData, + { + delimiter: ';', + columns: true, + quote: '"', + bom: true, + cast: true, + autoParse: true, + castDate: false, + skipEmptyLines: true, + skipRecordsWithEmptyValues: true + }, + (err, records) => { + if (err !== undefined) { + console.error(err) + reject(err) + } + resolve(records) + } + ) + }) +} + +export async function importOrgs (transactorUrl: string, dbName: string, csvFile: string): Promise { + const connection = (await connect(transactorUrl, dbName, undefined, { + mode: 'backup' + })) as unknown as Client & BackupClient + try { + console.log('loading cvs document...') + + const csvData = await readFile(csvFile, 'utf-8') + const records: any[] = await parseCSV(csvData) + const uniqKeys: string[] = [] + const filledFields = records.map((it) => filled(it, uniqKeys)) + // console.log(filledFields) + + const client = new TxOperations(connection, 'core:account:lead-importer' as Ref) + + await updateClasses(client, records, fieldMapping) + + const funnelId = await createFunnel(records, client) + + const statusKaValues = records + .map((it) => it[names.processingStatus]) + .filter((it, idx, arr) => arr.indexOf(it) === idx) + console.log(statusKaValues) + + const wonStateFrom = 'Договор на рассмотрении' + + const states = [ + 'Переговоры / назначен звонок', + 'Обсуждение условий в переписке', + wonStateFrom, + 'Направлено первичное письмо по e-mail', + 'Направлен запрос в LinkedIn / другом канале' + ] + const wonStates = ['Договор заключен'] + const lostStates = ['Получен отказ', 'Перестал отвечать'] + + const statesMap = new Map() + + // Create update/states + await updateStates(client, states, task.class.State, funnelId, statesMap) + await updateStates(client, wonStates, task.class.WonState, funnelId, statesMap) + await updateStates(client, lostStates, task.class.LostState, funnelId, statesMap) + + if ((await client.findOne(task.class.Kanban, { attachedTo: funnelId })) === undefined) { + await client.createDoc(task.class.Kanban, funnelId, { + attachedTo: funnelId + }) + } + + // Ok we have all states, let's create Leads. + const sequence = await client.findOne(task.class.Sequence, { attachedTo: lead.class.Lead }) + if (sequence === undefined) { + throw new Error('sequence object not found') + } + + await createOrganizations(client, filledFields, connection, funnelId, statesMap, wonStateFrom, sequence) + } catch (err: any) { + console.error(err) + } finally { + await connection.close() + } +} + +async function createFunnel (records: any[], client: TxOperations): Promise> { + const importedFunnelId = 'imported-funnel' as Ref + let funnel = await client.findOne(lead.class.Funnel, { _id: importedFunnelId }) + if (funnel === undefined) { + // No funnel, let's create one. + await client.createDoc( + lead.class.Funnel, + core.space.Space, + { + name: 'Organizations', + archived: false, + members: [], + private: false, + description: '' + }, + importedFunnelId + ) + funnel = await client.findOne(lead.class.Funnel, { _id: importedFunnelId }) + } + return importedFunnelId +} + +export interface CustomOrg extends Customer { + status: any + clientActivityStatus: any + + employees: any + + responsible: any + industry: any + currency: any + trelloLink: string + research: any + startCollaboration: string + geography: any + interestRate: any + agreement: string + technologyStack: string + fieldOfActivity: string + hrLanguage: any + selectionFeatures: any + source: any + valCompany: any +} + +async function createOrganizations ( + client: TxOperations, + filledFields: any[], + connection: Client & BackupClient, + funnelId: Ref, + statesMap: Map, + wonStateFrom: string, + sequence: WithLookup +): Promise { + for (const record of filledFields) { + let orgId: Ref = generateId() + const orgName = record[names.companyName] + if ((orgName?.toString() ?? '').trim().length === 0) { + continue + } + const org = await client.findOne(lead.mixin.Customer, { name: orgName }) + console.log('processing', orgName) + if (org === undefined) { + await client.createDoc( + contact.class.Organization, + contact.space.Contacts, + { + name: orgName, + city: '', + members: 0 + }, + orgId as unknown as Ref + ) + await client.createMixin( + orgId, + contact.class.Organization, + contact.space.Contacts, + lead.mixin.Customer, + { + description: record[names.comment], + status: record[names.companyType], + clientActivityStatus: record[names.clientActivityStatus], + + employees: record[names.employees], + + responsible: record[names.responsible], + industry: record[names.industry], + currency: record[names.currency], + trelloLink: record[names.trelloLink], + research: record[names.research], + startCollaboration: record[names.startCollaboration], + geography: record[names.geography], + interestRate: record[names.interestRate], + agreement: record[names.agreement], + technologyStack: record[names.technologyStack], + fieldOfActivity: record[names.fieldOfActivity], + hrLanguage: record[names.hrLanguage], + selectionFeatures: record[names.selectionFeatures], + source: record[names.source], + valCompany: record[names.valCompany] + } + ) + } else { + orgId = org._id as unknown as Ref + const upd: MixinUpdate = {} + const newValues = { + description: record[names.comment], + status: record[names.companyType], + clientActivityStatus: record[names.clientActivityStatus], + + employees: record[names.employees], + + responsible: record[names.responsible], + industry: record[names.industry], + currency: record[names.currency], + trelloLink: record[names.trelloLink], + research: record[names.research], + startCollaboration: record[names.startCollaboration], + geography: record[names.geography], + interestRate: record[names.interestRate], + agreement: record[names.agreement], + technologyStack: record[names.technologyStack], + fieldOfActivity: record[names.fieldOfActivity], + hrLanguage: record[names.hrLanguage], + selectionFeatures: record[names.selectionFeatures], + source: record[names.source], + valCompany: record[names.valCompany] + } + for (const [k, v] of Object.entries(newValues)) { + if ((org as any)[k] !== v && v !== undefined) { + ;(upd as any)[k] = v + } + } + if (Object.keys(upd).length > 0) { + await client.updateMixin( + orgId, + contact.class.Organization, + contact.space.Contacts, + lead.mixin.Customer, + upd + ) + } + } + + await updateChannel( + client, + orgId, + getValid(record, names.corporate_website), + contact.channelProvider.Homepage, + contact.class.Organization + ) + await updateChannel( + client, + orgId, + getValid(record, names.corporate_email), + contact.channelProvider.Email, + contact.class.Organization + ) + await updateChannel( + client, + orgId, + getValid(record, names.workEmail), + contact.channelProvider.Email, + contact.class.Organization + ) + + const state = statesMap.get(record[names.processingStatus]) + if (state !== undefined) { + const leadState = + state._class === task.class.State + ? (state._id as Ref) + : (statesMap.get(wonStateFrom) as unknown as Ref) + const doneState = + state._class === task.class.WonState || state._class === task.class.LostState + ? (state._id as Ref) + : null + + const orgLeadId = `imported-lead-${record.ID as string}` as Ref + const orgLead = await client.findOne(lead.class.Lead, { _id: orgLeadId }) + + if (orgLead === undefined) { + const lastOne = await client.findOne( + lead.class.Lead, + { state: leadState }, + { sort: { rank: SortingOrder.Descending } } + ) + const incResult = await client.update(sequence, { $inc: { sequence: 1 } }, true) + await client.addCollection( + lead.class.Lead, + funnelId, + orgId, + lead.mixin.Customer, + 'leads', + { + title: orgName, + number: (incResult as any).object.sequence, + rank: calcRank(lastOne, undefined), + assignee: null, + startDate: null, + dueDate: null, + state: leadState, + doneState + }, + orgLeadId + ) + } + } + + const agreement = record[names.agreement] as string + + if (agreement !== undefined) { + const agreements = agreement.split(',') + for (const r of agreements) { + try { + const url = (r ?? '').trim() + if (url.startsWith('http')) { + const lastpos = url.lastIndexOf('/') + const fname = url.substring(lastpos + 1) + + const buffer = await got(url).buffer() + const blobId = (orgId + '_' + generateId()) as Ref + const type = mimetypes.contentType(fname) + const data: BlobData = { + _id: blobId, + space: contact.space.Contacts, + modifiedBy: client.txFactory.account, + modifiedOn: Date.now(), + _class: core.class.BlobData, + name: fname, + size: buffer.length, + type: type !== false ? type : 'unknown', + base64Data: buffer.toString('base64') + } + await connection.upload(DOMAIN_BLOB, [data]) + + await client.addCollection( + attachment.class.Attachment, + contact.space.Contacts, + orgId, + contact.class.Organization, + 'attachments', + { + file: blobId, + name: fname, + size: buffer.length, + type: type !== false ? type : 'unknown', + lastModified: Date.now() + } + ) + } + } catch (err) { + console.log(err) + } + } + } + } +} diff --git a/dev/tool/src/csv/talant-importer.ts b/dev/tool/src/csv/talant-importer.ts index 8667ad649c..65a8fec371 100644 --- a/dev/tool/src/csv/talant-importer.ts +++ b/dev/tool/src/csv/talant-importer.ts @@ -13,7 +13,8 @@ // limitations under the License. // -import contact, { ChannelProvider, combineName, Contact, EmployeeAccount, Person } from '@anticrm/contact' +import attachment, { Attachment } from '@anticrm/attachment' +import contact, { combineName, Contact, EmployeeAccount, Person } from '@anticrm/contact' import core, { AnyAttribute, BackupClient, @@ -34,19 +35,18 @@ import core, { } from '@anticrm/core' import { Asset, getEmbeddedLabel, IntlString } from '@anticrm/platform' import recruit, { Candidate } from '@anticrm/recruit' +import { ReconiDocument } from '@anticrm/rekoni' +import { generateToken } from '@anticrm/server-token' import { connect } from '@anticrm/server-tool' import setting from '@anticrm/setting' import { readFile } from 'fs/promises' -import { parseCSV } from './parseCSV' -import { FieldType } from './types' -import { filled } from './utils' import got from 'got' import mimetypes from 'mime-types' -import attachment, { Attachment } from '@anticrm/attachment' -import { generateToken } from '@anticrm/server-token' import { recognize, updateContacts, updateSkills } from '../recruit' -import { ReconiDocument } from '@anticrm/rekoni' import { findOrUpdateAttached } from '../utils' +import { parseCSV } from './parseCSV' +import { FieldType } from './types' +import { filled, getValid, updateChannel } from './utils' const names = { status: 'Status', @@ -585,52 +585,29 @@ async function createTalants ( dataLocationDetails ) - function getValid (...names: string[]): string | undefined { - for (const o of names) { - const v = record[o] - if (v !== undefined && typeof v === 'string' && v.trim().length > 0) { - return v - } - } - } - - async function updateChannel (value: string | undefined, provider: Ref): Promise { - if (value === undefined) { - return - } - const channels = await client.findAll(contact.class.Channel, { attachedTo: candidateId }) - const emailPr = channels.find((it) => it.value === value) - if (emailPr === undefined) { - await client.addCollection( - contact.class.Channel, - contact.space.Contacts, - candidateId, - contact.class.Person, - 'channels', - { - value, - provider - } - ) - } - } - await updateChannel( + client, + candidateId, getValid(record, names.workEmail, names.homeEmail, names.newsletterEmail, names.otherEmail), contact.channelProvider.Email ) - await updateChannel(getValid(record, names.webSite), contact.channelProvider.Homepage) - await updateChannel(getValid(record, names.phone, names.phoneNumber), contact.channelProvider.Phone) - await updateChannel(getValid(record, names.telegram), contact.channelProvider.Telegram) + await updateChannel(client, candidateId, getValid(record, names.webSite), contact.channelProvider.Homepage) + await updateChannel( + client, + candidateId, + getValid(record, names.phone, names.phoneNumber), + contact.channelProvider.Phone + ) + await updateChannel(client, candidateId, getValid(record, names.telegram), contact.channelProvider.Telegram) const ghval = getValid(record, names.githubPortfolio) if (ghval?.includes('https://github.com') ?? false) { - await updateChannel(ghval, contact.channelProvider.GitHub) + await updateChannel(client, candidateId, ghval, contact.channelProvider.GitHub) } const profile = getValid(record, names.profile) if (profile?.includes('linkedin.com') ?? false) { - await updateChannel(profile, contact.channelProvider.LinkedIn) + await updateChannel(client, candidateId, profile, contact.channelProvider.LinkedIn) } const resume = record[names.resume] as string diff --git a/dev/tool/src/csv/utils.ts b/dev/tool/src/csv/utils.ts index 1a519befb0..7eb87bbe51 100644 --- a/dev/tool/src/csv/utils.ts +++ b/dev/tool/src/csv/utils.ts @@ -13,6 +13,9 @@ // limitations under the License. // +import contact, { ChannelProvider, Contact } from '@anticrm/contact' +import { Class, Doc, Ref, TxOperations } from '@anticrm/core' + export function filled (obj: any, uniqKeys: string[]): any { const result: Record = {} for (const [k, v] of Object.entries(obj)) { @@ -26,3 +29,31 @@ export function filled (obj: any, uniqKeys: string[]): any { } return result } + +export async function updateChannel ( + client: TxOperations, + attachedTo: Ref, + value: string | undefined, + provider: Ref, + attachToClass: Ref> = contact.class.Person +): Promise { + if (value === undefined) { + return + } + const channels = await client.findAll(contact.class.Channel, { attachedTo }) + const valueCh = channels.find((it) => it.value === value) + if (valueCh === undefined) { + await client.addCollection(contact.class.Channel, contact.space.Contacts, attachedTo, attachToClass, 'channels', { + value, + provider + }) + } +} +export function getValid (record: any, ...names: string[]): string | undefined { + for (const o of names) { + const v = record[o] + if (v !== undefined && typeof v === 'string' && v.trim().length > 0) { + return v + } + } +} diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 4283726254..468359ecb4 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -39,6 +39,7 @@ import { exit } from 'process' import { removeDuplicates } from './csv/duplicates' import { importLead } from './csv/lead-importer' import { importLead2 } from './csv/lead-importer2' +import { importOrgs } from './csv/org-importer' import { importTalants } from './csv/talant-importer' import { rebuildElastic } from './elastic' import { importXml } from './importer' @@ -387,6 +388,13 @@ program return await importTalants(transactorUrl, workspace, fileName, rekoniUrl) }) +program + .command('import-org-csv ') + .description('Import Organizations csv') + .action(async (workspace, fileName, cmd) => { + return await importOrgs(transactorUrl, workspace, fileName) + }) + program .command('lead-duplicates ') .description('Find and remove duplicate organizations.')