Xml Importer (#701)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

.vscode/launch.json

@@ -51,7 +51,6 @@
       "name": "Debug tool",
       "type": "node",
       "request": "launch",
-      // "args": ["src/index.ts", "import-xml", "ws1", "/Users/haiodo/Develop/private/hardware/suho/Кандидаты/Кандидаты.xml"],
       "args": ["src/index.ts", "restore-workspace", "ws1", "../../temp/ws1/"],
       "env": {
         "MINIO_ACCESS_KEY":"minioadmin",

common/config/rush/pnpm-lock.yaml

@@ -116,6 +116,7 @@ specifiers:
   '@types/pdfkit': ~0.12.3
   '@types/toposort': ^2.0.3
   '@types/uuid': ^8.3.1
+  '@types/xml2js': ~0.4.9
   '@typescript-eslint/eslint-plugin': ^5.4.0
   autoprefixer: ^10.2.6
   commander: ^8.1.0
@@ -144,6 +145,7 @@ specifiers:
   koa-bodyparser: ^4.3.0
   koa-router: ^10.1.1
   lexorank: ~1.0.4
+  mime-types: ~2.1.34
   mini-css-extract-plugin: ^2.2.0
   minio: ^7.0.19
   node-html-parser: ^4.1.3
@@ -168,6 +170,7 @@ specifiers:
   webpack-bundle-analyzer: ^4.4.1
   webpack-cli: ^4.6.0
   webpack-dev-server: ^3.11.2
+  xml2js: ~0.4.23
 
 dependencies:
   '@elastic/elasticsearch': 7.16.0
@@ -285,6 +288,7 @@ dependencies:
   '@types/pdfkit': 0.12.3
   '@types/toposort': 2.0.3
   '@types/uuid': 8.3.3
+  '@types/xml2js': 0.4.9
   '@typescript-eslint/eslint-plugin': 5.7.0_eslint@7.32.0+typescript@4.5.4
   autoprefixer: 10.4.0_postcss@8.4.5
   commander: 8.3.0
@@ -313,6 +317,7 @@ dependencies:
   koa-bodyparser: 4.3.0
   koa-router: 10.1.1
   lexorank: 1.0.4
+  mime-types: 2.1.34
   mini-css-extract-plugin: 2.4.5_webpack@5.65.0
   minio: 7.0.25
   node-html-parser: 4.1.5
@@ -337,6 +342,7 @@ dependencies:
   webpack-bundle-analyzer: 4.5.0
   webpack-cli: 4.9.1_1cfd1380a4e9b8401fb780accef05e9c
   webpack-dev-server: 3.11.3_webpack-cli@4.9.1+webpack@5.65.0
+  xml2js: 0.4.23
 
 packages:
 
@@ -1713,6 +1719,10 @@ packages:
       '@types/koa': 2.13.4
     dev: false
 
+  /@types/mime-types/2.1.1:
+    resolution: {integrity: sha512-vXOTGVSLR2jMw440moWTC7H19iUyLtP3Z1YTj7cSsubOICinjMxFeb/V57v9QdyyPGbbWolUFSSmSiRSn94tFw==}
+    dev: false
+
   /@types/mime/1.3.2:
     resolution: {integrity: sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==}
     dev: false
@@ -1902,6 +1912,12 @@ packages:
       '@types/node': 17.0.0
     dev: false
 
+  /@types/xml2js/0.4.9:
+    resolution: {integrity: sha512-CHiCKIihl1pychwR2RNX5mAYmJDACgFVCMT5OArMaO3erzwXVcBqPcusr+Vl8yeeXukxZqtF8mZioqX+mpjjdw==}
+    dependencies:
+      '@types/node': 17.0.0
+    dev: false
+
   /@types/yargs-parser/20.2.1:
     resolution: {integrity: sha512-7tFImggNeNBVMsn0vLrpn1H1uPrUBdnARPTpZoitY37ZrdJREzf7I16tMrlK3hen349gr1NYh8CmZQa7CTG6Aw==}
     dev: false
@@ -11019,7 +11035,7 @@ packages:
     dev: false
 
   file:projects/dev-client-resources.tgz:
-    resolution: {integrity: sha512-d1gACtHmA2R8CbItLx5oJYfVQV7FvM4tQthnwsDeoIM9OUIlXaRVFhd/KlJf1MkqLIsOA4O+Gjvhih/+EtediA==, tarball: file:projects/dev-client-resources.tgz}
+    resolution: {integrity: sha512-hcwOdV5TgwA6yTNeZDtMwAaGEOYFO5VzqWTXk+sK+SPSiEN2yy6rb2Do1cfgL0wvSXLH/F2VT+nr1WAlI909lQ==, tarball: file:projects/dev-client-resources.tgz}
     name: '@rush-temp/dev-client-resources'
     version: 0.0.0
     dependencies:
@@ -12511,16 +12527,18 @@ packages:
     dev: false
 
   file:projects/tool.tgz:
-    resolution: {integrity: sha512-FQTXToKTNRIcen67zjbh/ZeibEPSskFtKhFZgTtUgB0TXhzJ0V+ZRcpnsXK+nND2DBBRYhtylQJmqWsoASRtFg==, tarball: file:projects/tool.tgz}
+    resolution: {integrity: sha512-kxAczrVzjVh47GTnScTkjjXIpSAkovYmCIxJSznbPqT4XOdtQ0F8eadl3h2mhqjKzgE7VoFOdTB5JJ3oy4+TUg==, tarball: file:projects/tool.tgz}
     name: '@rush-temp/tool'
     version: 0.0.0
     dependencies:
       '@elastic/elasticsearch': 7.16.0
       '@rushstack/heft': 0.41.8
       '@types/heft-jest': 1.0.2
+      '@types/mime-types': 2.1.1
       '@types/minio': 7.0.11
       '@types/node': 16.11.14
       '@types/ws': 8.2.2
+      '@types/xml2js': 0.4.9
       '@typescript-eslint/eslint-plugin': 5.7.0_c25e8c1f4f4f7aaed27aa6f9ce042237
       '@typescript-eslint/parser': 5.7.0_eslint@7.32.0+typescript@4.5.4
       commander: 8.3.0
@@ -12533,12 +12551,14 @@ packages:
       eslint-plugin-promise: 5.2.0_eslint@7.32.0
       fast-equals: 2.0.4
       jwt-simple: 0.5.6
+      mime-types: 2.1.34
       minio: 7.0.25
       mongodb: 4.2.2
       prettier: 2.5.1
       ts-node: 10.4.0_5d12c2add188ff0e728b4ade3dacd39b
       typescript: 4.5.4
       ws: 8.3.0
+      xml2js: 0.4.23
     transitivePeerDependencies:
       - '@swc/core'
       - '@swc/wasm'

dev/tool/package.json

@@ -34,7 +34,9 @@
     "prettier": "^2.4.1",
     "@rushstack/heft": "^0.41.1",
     "typescript": "^4.3.5",
-    "@types/ws": "^8.2.1"
+    "@types/ws": "^8.2.1",
+    "@types/xml2js": "~0.4.9",
+    "@types/mime-types": "~2.1.1"
   },
   "dependencies": {
     "mongodb": "^4.1.1",
@@ -62,6 +64,13 @@
     "@anticrm/server-chunter": "~0.6.1",
     "@anticrm/server-chunter-resources": "~0.6.0",
     "@anticrm/server-recruit": "~0.6.0",
-    "@anticrm/server-recruit-resources": "~0.6.0"
+    "@anticrm/server-recruit-resources": "~0.6.0",
+    "xml2js": "~0.4.23",
+    "@anticrm/model-recruit": "~0.6.0",
+    "@anticrm/recruit": "~0.6.2",
+    "@anticrm/task": "~0.6.0",
+    "@anticrm/chunter": "~0.6.1",
+    "mime-types": "~2.1.34",
+    "@anticrm/attachment": "~0.6.1"
   }
 }

dev/tool/src/elastic.ts

@@ -34,6 +34,7 @@ import core, {
 } from '@anticrm/core'
 import { createElasticAdapter } from '@anticrm/elastic'
 import { DOMAIN_ATTACHMENT } from '@anticrm/model-attachment'
+import { Attachment } from '@anticrm/attachment'
 import { createMongoAdapter, createMongoTxAdapter } from '@anticrm/mongo'
 import { addLocation } from '@anticrm/platform'
 import { serverChunterId } from '@anticrm/server-chunter'
@@ -91,29 +92,94 @@ async function dropElastic (elasticUrl: string, dbName: string): Promise<void> {
   await client.close()
 }
 
+export class ElasticTool {
+  mongoClient: MongoClient
+  elastic!: FullTextAdapter & {close: () => Promise<void>}
+  storage!: ServerStorage
+  db!: Db
+  constructor (readonly mongoUrl: string, readonly dbName: string, readonly minio: Client, readonly elasticUrl: string) {
+    addLocation(serverChunterId, () => import('@anticrm/server-chunter-resources'))
+    addLocation(serverRecruitId, () => import('@anticrm/server-recruit-resources'))
+    this.mongoClient = new MongoClient(mongoUrl)
+  }
+
+  async connect (): Promise<() => Promise<void>> {
+    await this.mongoClient.connect()
+
+    this.db = this.mongoClient.db(this.dbName)
+    this.elastic = await createElasticAdapter(this.elasticUrl, this.dbName)
+    this.storage = await createStorage(this.mongoUrl, this.elasticUrl, this.dbName)
+
+    return async () => {
+      await this.mongoClient.close()
+      await this.elastic.close()
+    }
+  }
+
+  async indexAttachment (
+    name: string
+  ): Promise<void> {
+    const doc: Attachment | null = await this.db.collection<Attachment>(DOMAIN_ATTACHMENT).findOne({ file: name })
+    if (doc == null) return
+
+    const buffer = await this.readMinioObject(name)
+    await this.indexAttachmentDoc(doc, buffer)
+  }
+
+  async indexAttachmentDoc (doc: Attachment, buffer: Buffer): Promise<void> {
+    const id: Ref<Doc> = (generateId() + '/attachments/') as Ref<Doc>
+
+    const indexedDoc: IndexedDoc = {
+      id: id,
+      _class: doc._class,
+      space: doc.space,
+      modifiedOn: doc.modifiedOn,
+      modifiedBy: 'core:account:System' as Ref<Account>,
+      attachedTo: doc.attachedTo,
+      data: buffer.toString('base64')
+    }
+
+    await this.elastic.index(indexedDoc)
+  }
+
+  private async readMinioObject (name: string): Promise<Buffer> {
+    const data = await this.minio.getObject(this.dbName, name)
+    const chunks: Buffer[] = []
+    await new Promise<void>((resolve) => {
+      data.on('readable', () => {
+        let chunk
+        while ((chunk = data.read()) !== null) {
+          const b = chunk as Buffer
+          chunks.push(b)
+        }
+      })
+
+      data.on('end', () => {
+        resolve()
+      })
+    })
+    return Buffer.concat(chunks)
+  }
+}
+
 async function restoreElastic (mongoUrl: string, dbName: string, minio: Client, elasticUrl: string): Promise<void> {
-  addLocation(serverChunterId, () => import('@anticrm/server-chunter-resources'))
+  const tool = new ElasticTool(mongoUrl, dbName, minio, elasticUrl)
-  addLocation(serverRecruitId, () => import('@anticrm/server-recruit-resources'))
+  const done = await tool.connect()
-  const mongoClient = new MongoClient(mongoUrl)
   try {
-    await mongoClient.connect()
+    const txes = (await tool.db.collection<Tx>(DOMAIN_TX).find().sort({ _id: 1 }).toArray())
-    const db = mongoClient.db(dbName)
-    const elastic = await createElasticAdapter(elasticUrl, dbName)
-    const storage = await createStorage(mongoUrl, elasticUrl, dbName)
-    const txes = (await db.collection<Tx>(DOMAIN_TX).find().sort({ _id: 1 }).toArray())
     const data = txes.filter((tx) => tx.objectSpace !== core.space.Model)
     const metricsCtx = new MeasureMetricsContext('elastic', {})
     for (const tx of data) {
-      await storage.tx(metricsCtx, tx)
+      await tool.storage.tx(metricsCtx, tx)
     }
     if (await minio.bucketExists(dbName)) {
       const minioObjects = await listMinioObjects(minio, dbName)
       for (const d of minioObjects) {
-        await indexAttachment(elastic, minio, db, dbName, d.name)
+        await tool.indexAttachment(d.name)
       }
     }
   } finally {
-    await mongoClient.close()
+    await done()
   }
 }
 
@@ -142,49 +208,6 @@ async function createStorage (mongoUrl: string, elasticUrl: string, workspace: s
   return await createServerStorage(conf)
 }
 
-async function indexAttachment (
-  elastic: FullTextAdapter,
-  minio: Client,
-  db: Db,
-  dbName: string,
-  name: string
-): Promise<void> {
-  const doc = await db.collection(DOMAIN_ATTACHMENT).findOne({
-    file: name
-  })
-  if (doc == null) return
-
-  const data = await minio.getObject(dbName, name)
-  const chunks: Buffer[] = []
-  await new Promise<void>((resolve) => {
-    data.on('readable', () => {
-      let chunk
-      while ((chunk = data.read()) !== null) {
-        const b = chunk as Buffer
-        chunks.push(b)
-      }
-    })
-
-    data.on('end', () => {
-      resolve()
-    })
-  })
-
-  const id: Ref<Doc> = (generateId() + '/attachments/') as Ref<Doc>
-
-  const indexedDoc: IndexedDoc = {
-    id: id,
-    _class: doc._class,
-    space: doc.space,
-    modifiedOn: doc.modifiedOn,
-    modifiedBy: 'core:account:System' as Ref<Account>,
-    attachedTo: doc.attachedTo,
-    data: Buffer.concat(chunks).toString('base64')
-  }
-
-  await elastic.index(indexedDoc)
-}
-
 async function createMongoReadOnlyAdapter (
   hierarchy: Hierarchy,
   url: string,

dev/tool/src/importer.ts

@@ -0,0 +1,440 @@
+//
+// Copyright © 2020, 2021 Anticrm Platform Contributors.
+// Copyright © 2021 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import attachment, { Attachment } from '@anticrm/attachment'
+import chunter, { Comment } from '@anticrm/chunter'
+import contact, { ChannelProvider } from '@anticrm/contact'
+import core, { AttachedData, AttachedDoc, Class, Data, Doc, DocumentUpdate, Ref, SortingOrder, Space, TxOperations, TxResult } from '@anticrm/core'
+import recruit from '@anticrm/model-recruit'
+import { Applicant, Candidate, Vacancy } from '@anticrm/recruit'
+import task, { calcRank, DoneState, genRanks, Kanban, State } from '@anticrm/task'
+import { deepEqual } from 'fast-equals'
+import { existsSync } from 'fs'
+import { readdir, readFile, stat } from 'fs/promises'
+import mime from 'mime-types'
+import { Client } from 'minio'
+import { dirname, join } from 'path'
+import { parseStringPromise } from 'xml2js'
+import { connect } from './connect'
+import { ElasticTool } from './elastic'
+
+const _ = {
+  candidates: 'Кандидаты',
+  candidate: 'Кандидат',
+
+  delete: 'ПометкаУдаления', // -
+
+  id: 'Код', // +
+  fullName: 'Наименование', // +
+  recruiter: 'Рекрутер',
+  birthdate: 'ДатаРождения',
+  phone: 'НомерТелефона', // +
+  email: 'ЭлектроннаяПочта', // +
+  date: 'Дата',
+
+  notifyStatus: 'УведомлятьОСменеСтатуса',
+
+  requiredPosition: 'ЖелаемаяДолжность', // +
+
+  vacancyKind: 'ВидВакансии', // +
+
+  city: 'Проживание', // +
+  comment: 'Комментарий', // +
+
+  socialLink: 'СсылкиНаСоцСети', // +
+  socialLink2: 'СсылкиНаСоцСети1', // +
+  socialLink3: 'СсылкиНаСоцСети2', // +
+  otherContact: 'Другие_КонтактыMyMail',
+  socialChannel: 'КаналСвязи', // +
+
+  area: 'Отрасль', // +
+  area2: 'Отрасль1', // +
+  area3: 'Отрасль2', // +
+  status: 'Статус',
+  socialContacted: 'Откликнулся', // +
+  framework: 'ДополнительныйФреймворкТехнологияПлатформаЛибоВторойЯП' // +
+}
+
+function get (data: any, key: string): string | undefined {
+  const v = data[key]
+  if (Array.isArray(v) && v.length === 1) {
+    return v[0]
+  }
+  return v
+}
+
+export async function importXml (
+  transactorUrl: string,
+  dbName: string,
+  minio: Client,
+  xmlFile: string,
+  mongoUrl: string,
+  elasticUrl: string
+): Promise<void> {
+  const connection = await connect(transactorUrl, dbName)
+
+  const tool = new ElasticTool(mongoUrl, dbName, minio, elasticUrl)
+  const done = await tool.connect()
+  try {
+    console.log('loading xml document...')
+
+    const xmlData = await readFile(xmlFile, 'utf-8')
+    const data = await parseStringPromise(xmlData)
+
+    const root = dirname(xmlFile)
+
+    const candidates = data[_.candidates][_.candidate]
+    console.log('Found candidates:', candidates.length)
+
+    // const attributes = new Set<string>()
+    const client = new TxOperations(connection, core.account.System)
+
+    const statuses = candidates.map((c: any) => get(c, _.status)).filter((c: any) => c !== undefined)
+      .filter(onlyUniq)
+
+    console.log(statuses)
+
+    const withStatus: {candidateId: Ref<Candidate>, status: string}[] = []
+    let pos = 0
+    const len = candidates.length as number
+    for (const c of candidates) {
+      pos++
+      const _id = get(c, _.id)
+      const _name = get(c, _.fullName)
+      if (_name === undefined || _id === undefined) {
+        console.log('error procesing', JSON.stringify(c, undefined, 2))
+        return
+      }
+      const candId = `tool-import-${_id}` as Ref<Candidate>
+      const _status = get(c, _.status)
+      if (_status !== undefined) {
+        withStatus.push({ candidateId: candId, status: _status })
+      }
+
+      await createCandidate(_name, pos, len, c, client, candId)
+
+      // Check and add attachments.
+      const candidateRoot = join(root, _id)
+
+      if (existsSync(candidateRoot)) {
+        const files = await readdir(candidateRoot)
+        for (const f of files) {
+          const attachId = (candId + f.toLowerCase()) as Ref<Attachment>
+          const type = mime.contentType(f)
+          if (typeof type === 'string') {
+            const fileName = join(candidateRoot, f)
+            const data = await readFile(fileName)
+            try {
+              await minio.statObject(dbName, attachId)
+            } catch (err: any) {
+            // No object, put new one.
+              await minio.putObject(dbName, attachId, data, data.length, {
+                'Content-Type': type
+              })
+            }
+
+            const stats = await stat(fileName)
+
+            const attachedDoc = await findOrUpdateAttached<Attachment>(client, recruit.space.CandidatesPublic, attachment.class.Attachment, attachId, {
+              name: f,
+              file: attachId,
+              type: type,
+              size: stats.size,
+              lastModified: stats.mtime.getTime()
+            }, {
+              attachedTo: candId,
+              attachedClass: recruit.class.Candidate,
+              collection: 'attachments'
+            })
+
+            await tool.indexAttachmentDoc(attachedDoc, data)
+          }
+        }
+      }
+    }
+
+    // Create/Update Vacancy and Applicants.
+
+    if (withStatus.length > 0) {
+      const { states, vacancyId } = await createUpdateVacancy(client, statuses)
+
+      // Applicant num sequence.
+      const sequence = await client.findOne(task.class.Sequence, { attachedTo: recruit.class.Applicant })
+      if (sequence === undefined) {
+        throw new Error('sequence object not found')
+      }
+
+      const rankGen = genRanks(withStatus.length)
+      const firstState = Array.from(states.values())[0]
+      for (const { candidateId, status } of withStatus) {
+        const incResult = await client.updateDoc(
+          task.class.Sequence,
+          task.space.Sequence,
+          sequence._id,
+          {
+            $inc: { sequence: 1 }
+          },
+          true
+        )
+
+        const rank = rankGen.next().value
+        const state = states.get(status) ?? firstState
+
+        if (rank === undefined) {
+          throw Error('Failed to generate rank')
+        }
+
+        const lastOne = await client.findOne(
+          recruit.class.Applicant,
+          { state },
+          { sort: { rank: SortingOrder.Descending } }
+        )
+        await createApplicant(vacancyId, candidateId, incResult, state, lastOne, client)
+      }
+    }
+  } catch (err: any) {
+    console.log(err)
+  } finally {
+    await done()
+    console.log('manual closing connection')
+    await connection.close()
+  }
+}
+const onlyUniq = (value: any, index: number, self: any[]): boolean => self.indexOf(value) === index
+
+async function createApplicant (vacancyId: Ref<Vacancy>, candidateId: Ref<Candidate>, incResult: TxResult, state: Ref<State>, lastOne: Applicant | undefined, client: TxOperations): Promise<void> {
+  const applicantId = `vacancy-${vacancyId}-${candidateId}` as Ref<Applicant>
+
+  const applicant: AttachedData<Applicant> = {
+    number: (incResult as any).object.sequence,
+    assignee: null,
+    state,
+    doneState: null,
+    rank: calcRank(lastOne, undefined)
+  }
+
+  // Update or create candidate
+  await findOrUpdateAttached(client, vacancyId, recruit.class.Applicant, applicantId, applicant, { attachedTo: candidateId, attachedClass: recruit.class.Candidate, collection: 'applications' })
+}
+
+async function createUpdateVacancy (client: TxOperations, statuses: any): Promise<{states: Map<string, Ref<State>>, vacancyId: Ref<Vacancy>}> {
+  const vacancy: Data<Vacancy> = {
+    name: 'Imported Vacancy',
+    description: '',
+    fullDescription: '',
+    location: '',
+    company: '',
+    members: [],
+    archived: false,
+    private: false
+  }
+  const vacancyId = 'imported-vacancy' as Ref<Vacancy>
+
+  console.log('Creating vacancy', vacancy.name)
+  // Update or create candidate
+  await findOrUpdate(client, core.space.Model, recruit.class.Vacancy, vacancyId, vacancy)
+
+  const states = await createUpdateSpaceKanban(vacancyId, client, statuses)
+
+  console.log('States generated', vacancy.name)
+  return { states, vacancyId }
+}
+
+async function createCandidate (_name: string, pos: number, len: number, c: any, client: TxOperations, candId: Ref<Candidate>): Promise<void> {
+  const names = _name.trim().split(' ')
+  console.log(`(${pos} pf ${len})`, names[0] + ',', names.slice(1).join(' '))
+
+  const { sourceFields, telegram, linkedin, github } = parseSocials(c)
+
+  const data: Data<Candidate> = {
+    name: names.slice(1).join(' ') + ', ' + names[0],
+    city: get(c, _.city) ?? '',
+    title: [
+      get(c, _.vacancyKind),
+      get(c, _.area)
+    ].filter(p => p !== undefined && p.trim().length > 0).filter(onlyUniq).join('/'),
+    source: [
+      get(c, _.socialContacted),
+      get(c, _.socialChannel),
+      sourceFields.filter(onlyUniq).join(', ')
+    ].filter(p => p !== undefined && p.trim().length > 0).filter(onlyUniq).join('/'),
+    channels: []
+  }
+
+  pushChannel(c, data, _.email, contact.channelProvider.Email)
+  pushChannel(c, data, _.phone, contact.channelProvider.Phone)
+
+  const commentData: string[] = []
+
+  addComment(commentData, c, _.socialContacted)
+  addComment(commentData, c, _.recruiter)
+  addComment(commentData, c, _.requiredPosition)
+  addComment(commentData, c, _.area2)
+  addComment(commentData, c, _.area3)
+  addComment(commentData, c, _.framework)
+  addComment(commentData, c, _.comment)
+
+  if (telegram !== undefined) {
+    data.channels.push({ provider: contact.channelProvider.Telegram, value: telegram })
+  }
+  if (linkedin !== undefined) {
+    data.channels.push({ provider: contact.channelProvider.LinkedIn, value: linkedin })
+  }
+  if (github !== undefined) {
+    data.channels.push({ provider: contact.channelProvider.GitHub, value: github })
+  }
+  await findOrUpdate(client, recruit.space.CandidatesPublic, recruit.class.Candidate, candId, data)
+
+  const commentId = (candId + '.description.comment') as Ref<Comment>
+
+  if (commentData.length > 0) {
+    await findOrUpdateAttached(client, recruit.space.CandidatesPublic, chunter.class.Comment, commentId, {
+      message: commentData.join('\n<br/>')
+    }, { attachedTo: candId, attachedClass: recruit.class.Candidate, collection: 'comments' })
+  }
+}
+
+function addComment (data: string[], c: any, key: string): void {
+  const val = get(c, key)
+  if (val !== undefined) {
+    data.push(`${key}: ${val}`.replace('\n', '\n<br/>'))
+  }
+}
+
+function parseSocials (c: any): { sourceFields: string[], telegram: string | undefined, linkedin: string | undefined, github: string | undefined } {
+  let telegram: string | undefined
+  let linkedin: string | undefined
+  let github: string | undefined
+
+  const sourceFields = ([
+    get(c, _.socialLink),
+    get(c, _.socialLink2),
+    get(c, _.socialLink3),
+    get(c, _.otherContact),
+    get(c, _.area2),
+    get(c, _.area3)
+  ].filter(p => p !== undefined && p.trim().length > 0) as string[]).filter(t => {
+    const lc = t.toLocaleLowerCase()
+    if (lc.startsWith('telegram')) {
+      telegram = t.substring(8).replace(':', '').trim()
+      return false
+    }
+    if (lc.includes('linkedin.')) {
+      linkedin = t.trim()
+      return false
+    }
+    if (lc.includes('github.com')) {
+      github = t.trim()
+      return false
+    }
+    return true
+  })
+  return { sourceFields, telegram, linkedin, github }
+}
+
+function pushChannel (c: any, data: Data<Candidate>, key: string, provider: Ref<ChannelProvider>): void {
+  const value = get(c, key)
+  if (value !== undefined) {
+    data.channels.push({ provider, value })
+  }
+}
+export async function findOrUpdate<T extends Doc> (client: TxOperations, space: Ref<Space>, _class: Ref<Class<T>>, objectId: Ref<T>, data: Data<T>): Promise<void> {
+  const existingObj = await client.findOne<Doc>(_class, { _id: objectId, space })
+  if (existingObj !== undefined) {
+    // Check some field changes
+    const { _id, _class, modifiedOn, modifiedBy, space, ...dta } = existingObj
+    if (!deepEqual(dta, data)) {
+      await client.updateDoc(_class, space, objectId, data)
+    }
+  } else {
+    await client.createDoc(_class, space, data, objectId)
+  }
+}
+
+function randColor (): string {
+  const letters = '0123456789ABCDEF'
+  let color = '#'
+  for (let i = 0; i < 6; i++) {
+    color += letters[Math.floor(Math.random() * 16)]
+  }
+  return color
+}
+
+async function createUpdateSpaceKanban (spaceId: Ref<Vacancy>, client: TxOperations, stateNames: string[]): Promise<Map<string, Ref<State>>> {
+  const states: Map<string, Ref<State>> = new Map()
+  const stateRanks = genRanks(stateNames.length)
+  for (const st of stateNames) {
+    const rank = stateRanks.next().value
+
+    if (rank === undefined) {
+      console.error('Failed to generate rank')
+      break
+    }
+
+    const sid = ('generated-' + spaceId + '.state.' + st.toLowerCase().replace(' ', '_')) as Ref<State>
+    await findOrUpdate(client, spaceId, task.class.State,
+      sid,
+      {
+        title: st,
+        color: randColor(),
+        rank
+      }
+    )
+    states.set(st, sid)
+  }
+
+  const doneStates = [
+    { class: task.class.WonState, title: 'Won' },
+    { class: task.class.LostState, title: 'Lost' }
+  ]
+  const doneStateRanks = genRanks(doneStates.length)
+  for (const st of doneStates) {
+    const rank = doneStateRanks.next().value
+
+    if (rank === undefined) {
+      console.error('Failed to generate rank')
+      break
+    }
+
+    const sid = ('generated-' + spaceId + '.done-state.' + st.title.toLowerCase().replace(' ', '_')) as Ref<DoneState>
+    await findOrUpdate(client, spaceId, st.class,
+      sid,
+      {
+        title: st.title,
+        rank
+      }
+    )
+  }
+
+  await findOrUpdate(client, spaceId,
+    task.class.Kanban,
+    ('generated-' + spaceId + '.kanban') as Ref<Kanban>,
+    {
+      attachedTo: spaceId
+    }
+  )
+  return states
+}
+async function findOrUpdateAttached<T extends AttachedDoc> (client: TxOperations, space: Ref<Space>, _class: Ref<Class<T>>, objectId: Ref<T>, data: AttachedData<T>, attached: {attachedTo: Ref<Doc>, attachedClass: Ref<Class<Doc>>, collection: string}): Promise<T> {
+  let existingObj = await client.findOne<Doc>(_class, { _id: objectId, space }) as T
+  if (existingObj !== undefined) {
+    await client.updateCollection(_class, space, objectId, attached.attachedTo, attached.attachedClass, attached.collection, data as unknown as DocumentUpdate<T>)
+  } else {
+    await client.addCollection(_class, space, attached.attachedTo, attached.attachedClass, attached.collection, data, objectId)
+    existingObj = { _id: objectId, _class, space, ...data, ...attached } as unknown as T
+  }
+  return existingObj
+}

dev/tool/src/index.ts

@@ -31,6 +31,7 @@ import { Client } from 'minio'
 import { Db, MongoClient } from 'mongodb'
 import { connect } from './connect'
 import { rebuildElastic } from './elastic'
+import { importXml } from './importer'
 import { clearTelegramHistory } from './telegram'
 import { diffWorkspace, dumpWorkspace, initWorkspace, restoreWorkspace, upgradeWorkspace } from './workspace'
 
@@ -247,4 +248,11 @@ program
     console.log('rebuild end')
   })
 
+program
+  .command('import-xml <workspace> <fileName>')
+  .description('dump workspace transactions and minio resources')
+  .action(async (workspace, fileName, cmd) => {
+    return await importXml(transactorUrl, workspace, minio, fileName, mongodbUri, elasticUrl)
+  })
+
 program.parse(process.argv)

plugins/view-resources/src/components/Table.svelte

@@ -43,7 +43,7 @@
     (result) => {
       objects = result
     },
-    { sort: { [sortKey]: sortOrder }, ...options }
+    { sort: { [sortKey]: sortOrder }, ...options, limit: 500 }
   )
 
   function getValue (doc: Doc, key: string): any {

server/elastic/src/adapter.ts

@@ -26,6 +26,10 @@ class ElasticAdapter implements FullTextAdapter {
   ) {
   }
 
+  async close (): Promise<void> {
+    await this.client.close()
+  }
+
   async search (
     search: string
   ): Promise<IndexedDoc[]> {
@@ -110,7 +114,7 @@ class ElasticAdapter implements FullTextAdapter {
 /**
  * @public
  */
-export async function createElasticAdapter (url: string, dbName: string): Promise<FullTextAdapter> {
+export async function createElasticAdapter (url: string, dbName: string): Promise<FullTextAdapter & {close: () => Promise<void>}> {
   const client = new Client({
     node: url
   })