mirror of
https://github.com/hcengineering/platform.git
synced 2024-12-22 11:01:54 +03:00
parent
07223ce5ae
commit
edbf42c435
1
.vscode/launch.json
vendored
1
.vscode/launch.json
vendored
@ -51,7 +51,6 @@
|
||||
"name": "Debug tool",
|
||||
"type": "node",
|
||||
"request": "launch",
|
||||
// "args": ["src/index.ts", "import-xml", "ws1", "/Users/haiodo/Develop/private/hardware/suho/Кандидаты/Кандидаты.xml"],
|
||||
"args": ["src/index.ts", "restore-workspace", "ws1", "../../temp/ws1/"],
|
||||
"env": {
|
||||
"MINIO_ACCESS_KEY":"minioadmin",
|
||||
|
@ -116,6 +116,7 @@ specifiers:
|
||||
'@types/pdfkit': ~0.12.3
|
||||
'@types/toposort': ^2.0.3
|
||||
'@types/uuid': ^8.3.1
|
||||
'@types/xml2js': ~0.4.9
|
||||
'@typescript-eslint/eslint-plugin': ^5.4.0
|
||||
autoprefixer: ^10.2.6
|
||||
commander: ^8.1.0
|
||||
@ -144,6 +145,7 @@ specifiers:
|
||||
koa-bodyparser: ^4.3.0
|
||||
koa-router: ^10.1.1
|
||||
lexorank: ~1.0.4
|
||||
mime-types: ~2.1.34
|
||||
mini-css-extract-plugin: ^2.2.0
|
||||
minio: ^7.0.19
|
||||
node-html-parser: ^4.1.3
|
||||
@ -168,6 +170,7 @@ specifiers:
|
||||
webpack-bundle-analyzer: ^4.4.1
|
||||
webpack-cli: ^4.6.0
|
||||
webpack-dev-server: ^3.11.2
|
||||
xml2js: ~0.4.23
|
||||
|
||||
dependencies:
|
||||
'@elastic/elasticsearch': 7.16.0
|
||||
@ -285,6 +288,7 @@ dependencies:
|
||||
'@types/pdfkit': 0.12.3
|
||||
'@types/toposort': 2.0.3
|
||||
'@types/uuid': 8.3.3
|
||||
'@types/xml2js': 0.4.9
|
||||
'@typescript-eslint/eslint-plugin': 5.7.0_eslint@7.32.0+typescript@4.5.4
|
||||
autoprefixer: 10.4.0_postcss@8.4.5
|
||||
commander: 8.3.0
|
||||
@ -313,6 +317,7 @@ dependencies:
|
||||
koa-bodyparser: 4.3.0
|
||||
koa-router: 10.1.1
|
||||
lexorank: 1.0.4
|
||||
mime-types: 2.1.34
|
||||
mini-css-extract-plugin: 2.4.5_webpack@5.65.0
|
||||
minio: 7.0.25
|
||||
node-html-parser: 4.1.5
|
||||
@ -337,6 +342,7 @@ dependencies:
|
||||
webpack-bundle-analyzer: 4.5.0
|
||||
webpack-cli: 4.9.1_1cfd1380a4e9b8401fb780accef05e9c
|
||||
webpack-dev-server: 3.11.3_webpack-cli@4.9.1+webpack@5.65.0
|
||||
xml2js: 0.4.23
|
||||
|
||||
packages:
|
||||
|
||||
@ -1713,6 +1719,10 @@ packages:
|
||||
'@types/koa': 2.13.4
|
||||
dev: false
|
||||
|
||||
/@types/mime-types/2.1.1:
|
||||
resolution: {integrity: sha512-vXOTGVSLR2jMw440moWTC7H19iUyLtP3Z1YTj7cSsubOICinjMxFeb/V57v9QdyyPGbbWolUFSSmSiRSn94tFw==}
|
||||
dev: false
|
||||
|
||||
/@types/mime/1.3.2:
|
||||
resolution: {integrity: sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==}
|
||||
dev: false
|
||||
@ -1902,6 +1912,12 @@ packages:
|
||||
'@types/node': 17.0.0
|
||||
dev: false
|
||||
|
||||
/@types/xml2js/0.4.9:
|
||||
resolution: {integrity: sha512-CHiCKIihl1pychwR2RNX5mAYmJDACgFVCMT5OArMaO3erzwXVcBqPcusr+Vl8yeeXukxZqtF8mZioqX+mpjjdw==}
|
||||
dependencies:
|
||||
'@types/node': 17.0.0
|
||||
dev: false
|
||||
|
||||
/@types/yargs-parser/20.2.1:
|
||||
resolution: {integrity: sha512-7tFImggNeNBVMsn0vLrpn1H1uPrUBdnARPTpZoitY37ZrdJREzf7I16tMrlK3hen349gr1NYh8CmZQa7CTG6Aw==}
|
||||
dev: false
|
||||
@ -11019,7 +11035,7 @@ packages:
|
||||
dev: false
|
||||
|
||||
file:projects/dev-client-resources.tgz:
|
||||
resolution: {integrity: sha512-d1gACtHmA2R8CbItLx5oJYfVQV7FvM4tQthnwsDeoIM9OUIlXaRVFhd/KlJf1MkqLIsOA4O+Gjvhih/+EtediA==, tarball: file:projects/dev-client-resources.tgz}
|
||||
resolution: {integrity: sha512-hcwOdV5TgwA6yTNeZDtMwAaGEOYFO5VzqWTXk+sK+SPSiEN2yy6rb2Do1cfgL0wvSXLH/F2VT+nr1WAlI909lQ==, tarball: file:projects/dev-client-resources.tgz}
|
||||
name: '@rush-temp/dev-client-resources'
|
||||
version: 0.0.0
|
||||
dependencies:
|
||||
@ -12511,16 +12527,18 @@ packages:
|
||||
dev: false
|
||||
|
||||
file:projects/tool.tgz:
|
||||
resolution: {integrity: sha512-FQTXToKTNRIcen67zjbh/ZeibEPSskFtKhFZgTtUgB0TXhzJ0V+ZRcpnsXK+nND2DBBRYhtylQJmqWsoASRtFg==, tarball: file:projects/tool.tgz}
|
||||
resolution: {integrity: sha512-kxAczrVzjVh47GTnScTkjjXIpSAkovYmCIxJSznbPqT4XOdtQ0F8eadl3h2mhqjKzgE7VoFOdTB5JJ3oy4+TUg==, tarball: file:projects/tool.tgz}
|
||||
name: '@rush-temp/tool'
|
||||
version: 0.0.0
|
||||
dependencies:
|
||||
'@elastic/elasticsearch': 7.16.0
|
||||
'@rushstack/heft': 0.41.8
|
||||
'@types/heft-jest': 1.0.2
|
||||
'@types/mime-types': 2.1.1
|
||||
'@types/minio': 7.0.11
|
||||
'@types/node': 16.11.14
|
||||
'@types/ws': 8.2.2
|
||||
'@types/xml2js': 0.4.9
|
||||
'@typescript-eslint/eslint-plugin': 5.7.0_c25e8c1f4f4f7aaed27aa6f9ce042237
|
||||
'@typescript-eslint/parser': 5.7.0_eslint@7.32.0+typescript@4.5.4
|
||||
commander: 8.3.0
|
||||
@ -12533,12 +12551,14 @@ packages:
|
||||
eslint-plugin-promise: 5.2.0_eslint@7.32.0
|
||||
fast-equals: 2.0.4
|
||||
jwt-simple: 0.5.6
|
||||
mime-types: 2.1.34
|
||||
minio: 7.0.25
|
||||
mongodb: 4.2.2
|
||||
prettier: 2.5.1
|
||||
ts-node: 10.4.0_5d12c2add188ff0e728b4ade3dacd39b
|
||||
typescript: 4.5.4
|
||||
ws: 8.3.0
|
||||
xml2js: 0.4.23
|
||||
transitivePeerDependencies:
|
||||
- '@swc/core'
|
||||
- '@swc/wasm'
|
||||
|
@ -34,7 +34,9 @@
|
||||
"prettier": "^2.4.1",
|
||||
"@rushstack/heft": "^0.41.1",
|
||||
"typescript": "^4.3.5",
|
||||
"@types/ws": "^8.2.1"
|
||||
"@types/ws": "^8.2.1",
|
||||
"@types/xml2js": "~0.4.9",
|
||||
"@types/mime-types": "~2.1.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"mongodb": "^4.1.1",
|
||||
@ -62,6 +64,13 @@
|
||||
"@anticrm/server-chunter": "~0.6.1",
|
||||
"@anticrm/server-chunter-resources": "~0.6.0",
|
||||
"@anticrm/server-recruit": "~0.6.0",
|
||||
"@anticrm/server-recruit-resources": "~0.6.0"
|
||||
"@anticrm/server-recruit-resources": "~0.6.0",
|
||||
"xml2js": "~0.4.23",
|
||||
"@anticrm/model-recruit": "~0.6.0",
|
||||
"@anticrm/recruit": "~0.6.2",
|
||||
"@anticrm/task": "~0.6.0",
|
||||
"@anticrm/chunter": "~0.6.1",
|
||||
"mime-types": "~2.1.34",
|
||||
"@anticrm/attachment": "~0.6.1"
|
||||
}
|
||||
}
|
||||
|
@ -34,6 +34,7 @@ import core, {
|
||||
} from '@anticrm/core'
|
||||
import { createElasticAdapter } from '@anticrm/elastic'
|
||||
import { DOMAIN_ATTACHMENT } from '@anticrm/model-attachment'
|
||||
import { Attachment } from '@anticrm/attachment'
|
||||
import { createMongoAdapter, createMongoTxAdapter } from '@anticrm/mongo'
|
||||
import { addLocation } from '@anticrm/platform'
|
||||
import { serverChunterId } from '@anticrm/server-chunter'
|
||||
@ -91,29 +92,94 @@ async function dropElastic (elasticUrl: string, dbName: string): Promise<void> {
|
||||
await client.close()
|
||||
}
|
||||
|
||||
export class ElasticTool {
|
||||
mongoClient: MongoClient
|
||||
elastic!: FullTextAdapter & {close: () => Promise<void>}
|
||||
storage!: ServerStorage
|
||||
db!: Db
|
||||
constructor (readonly mongoUrl: string, readonly dbName: string, readonly minio: Client, readonly elasticUrl: string) {
|
||||
addLocation(serverChunterId, () => import('@anticrm/server-chunter-resources'))
|
||||
addLocation(serverRecruitId, () => import('@anticrm/server-recruit-resources'))
|
||||
this.mongoClient = new MongoClient(mongoUrl)
|
||||
}
|
||||
|
||||
async connect (): Promise<() => Promise<void>> {
|
||||
await this.mongoClient.connect()
|
||||
|
||||
this.db = this.mongoClient.db(this.dbName)
|
||||
this.elastic = await createElasticAdapter(this.elasticUrl, this.dbName)
|
||||
this.storage = await createStorage(this.mongoUrl, this.elasticUrl, this.dbName)
|
||||
|
||||
return async () => {
|
||||
await this.mongoClient.close()
|
||||
await this.elastic.close()
|
||||
}
|
||||
}
|
||||
|
||||
async indexAttachment (
|
||||
name: string
|
||||
): Promise<void> {
|
||||
const doc: Attachment | null = await this.db.collection<Attachment>(DOMAIN_ATTACHMENT).findOne({ file: name })
|
||||
if (doc == null) return
|
||||
|
||||
const buffer = await this.readMinioObject(name)
|
||||
await this.indexAttachmentDoc(doc, buffer)
|
||||
}
|
||||
|
||||
async indexAttachmentDoc (doc: Attachment, buffer: Buffer): Promise<void> {
|
||||
const id: Ref<Doc> = (generateId() + '/attachments/') as Ref<Doc>
|
||||
|
||||
const indexedDoc: IndexedDoc = {
|
||||
id: id,
|
||||
_class: doc._class,
|
||||
space: doc.space,
|
||||
modifiedOn: doc.modifiedOn,
|
||||
modifiedBy: 'core:account:System' as Ref<Account>,
|
||||
attachedTo: doc.attachedTo,
|
||||
data: buffer.toString('base64')
|
||||
}
|
||||
|
||||
await this.elastic.index(indexedDoc)
|
||||
}
|
||||
|
||||
private async readMinioObject (name: string): Promise<Buffer> {
|
||||
const data = await this.minio.getObject(this.dbName, name)
|
||||
const chunks: Buffer[] = []
|
||||
await new Promise<void>((resolve) => {
|
||||
data.on('readable', () => {
|
||||
let chunk
|
||||
while ((chunk = data.read()) !== null) {
|
||||
const b = chunk as Buffer
|
||||
chunks.push(b)
|
||||
}
|
||||
})
|
||||
|
||||
data.on('end', () => {
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
return Buffer.concat(chunks)
|
||||
}
|
||||
}
|
||||
|
||||
async function restoreElastic (mongoUrl: string, dbName: string, minio: Client, elasticUrl: string): Promise<void> {
|
||||
addLocation(serverChunterId, () => import('@anticrm/server-chunter-resources'))
|
||||
addLocation(serverRecruitId, () => import('@anticrm/server-recruit-resources'))
|
||||
const mongoClient = new MongoClient(mongoUrl)
|
||||
const tool = new ElasticTool(mongoUrl, dbName, minio, elasticUrl)
|
||||
const done = await tool.connect()
|
||||
try {
|
||||
await mongoClient.connect()
|
||||
const db = mongoClient.db(dbName)
|
||||
const elastic = await createElasticAdapter(elasticUrl, dbName)
|
||||
const storage = await createStorage(mongoUrl, elasticUrl, dbName)
|
||||
const txes = (await db.collection<Tx>(DOMAIN_TX).find().sort({ _id: 1 }).toArray())
|
||||
const txes = (await tool.db.collection<Tx>(DOMAIN_TX).find().sort({ _id: 1 }).toArray())
|
||||
const data = txes.filter((tx) => tx.objectSpace !== core.space.Model)
|
||||
const metricsCtx = new MeasureMetricsContext('elastic', {})
|
||||
for (const tx of data) {
|
||||
await storage.tx(metricsCtx, tx)
|
||||
await tool.storage.tx(metricsCtx, tx)
|
||||
}
|
||||
if (await minio.bucketExists(dbName)) {
|
||||
const minioObjects = await listMinioObjects(minio, dbName)
|
||||
for (const d of minioObjects) {
|
||||
await indexAttachment(elastic, minio, db, dbName, d.name)
|
||||
await tool.indexAttachment(d.name)
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await mongoClient.close()
|
||||
await done()
|
||||
}
|
||||
}
|
||||
|
||||
@ -142,49 +208,6 @@ async function createStorage (mongoUrl: string, elasticUrl: string, workspace: s
|
||||
return await createServerStorage(conf)
|
||||
}
|
||||
|
||||
async function indexAttachment (
|
||||
elastic: FullTextAdapter,
|
||||
minio: Client,
|
||||
db: Db,
|
||||
dbName: string,
|
||||
name: string
|
||||
): Promise<void> {
|
||||
const doc = await db.collection(DOMAIN_ATTACHMENT).findOne({
|
||||
file: name
|
||||
})
|
||||
if (doc == null) return
|
||||
|
||||
const data = await minio.getObject(dbName, name)
|
||||
const chunks: Buffer[] = []
|
||||
await new Promise<void>((resolve) => {
|
||||
data.on('readable', () => {
|
||||
let chunk
|
||||
while ((chunk = data.read()) !== null) {
|
||||
const b = chunk as Buffer
|
||||
chunks.push(b)
|
||||
}
|
||||
})
|
||||
|
||||
data.on('end', () => {
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
|
||||
const id: Ref<Doc> = (generateId() + '/attachments/') as Ref<Doc>
|
||||
|
||||
const indexedDoc: IndexedDoc = {
|
||||
id: id,
|
||||
_class: doc._class,
|
||||
space: doc.space,
|
||||
modifiedOn: doc.modifiedOn,
|
||||
modifiedBy: 'core:account:System' as Ref<Account>,
|
||||
attachedTo: doc.attachedTo,
|
||||
data: Buffer.concat(chunks).toString('base64')
|
||||
}
|
||||
|
||||
await elastic.index(indexedDoc)
|
||||
}
|
||||
|
||||
async function createMongoReadOnlyAdapter (
|
||||
hierarchy: Hierarchy,
|
||||
url: string,
|
||||
|
440
dev/tool/src/importer.ts
Normal file
440
dev/tool/src/importer.ts
Normal file
@ -0,0 +1,440 @@
|
||||
//
|
||||
// Copyright © 2020, 2021 Anticrm Platform Contributors.
|
||||
// Copyright © 2021 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import attachment, { Attachment } from '@anticrm/attachment'
|
||||
import chunter, { Comment } from '@anticrm/chunter'
|
||||
import contact, { ChannelProvider } from '@anticrm/contact'
|
||||
import core, { AttachedData, AttachedDoc, Class, Data, Doc, DocumentUpdate, Ref, SortingOrder, Space, TxOperations, TxResult } from '@anticrm/core'
|
||||
import recruit from '@anticrm/model-recruit'
|
||||
import { Applicant, Candidate, Vacancy } from '@anticrm/recruit'
|
||||
import task, { calcRank, DoneState, genRanks, Kanban, State } from '@anticrm/task'
|
||||
import { deepEqual } from 'fast-equals'
|
||||
import { existsSync } from 'fs'
|
||||
import { readdir, readFile, stat } from 'fs/promises'
|
||||
import mime from 'mime-types'
|
||||
import { Client } from 'minio'
|
||||
import { dirname, join } from 'path'
|
||||
import { parseStringPromise } from 'xml2js'
|
||||
import { connect } from './connect'
|
||||
import { ElasticTool } from './elastic'
|
||||
|
||||
const _ = {
|
||||
candidates: 'Кандидаты',
|
||||
candidate: 'Кандидат',
|
||||
|
||||
delete: 'ПометкаУдаления', // -
|
||||
|
||||
id: 'Код', // +
|
||||
fullName: 'Наименование', // +
|
||||
recruiter: 'Рекрутер',
|
||||
birthdate: 'ДатаРождения',
|
||||
phone: 'НомерТелефона', // +
|
||||
email: 'ЭлектроннаяПочта', // +
|
||||
date: 'Дата',
|
||||
|
||||
notifyStatus: 'УведомлятьОСменеСтатуса',
|
||||
|
||||
requiredPosition: 'ЖелаемаяДолжность', // +
|
||||
|
||||
vacancyKind: 'ВидВакансии', // +
|
||||
|
||||
city: 'Проживание', // +
|
||||
comment: 'Комментарий', // +
|
||||
|
||||
socialLink: 'СсылкиНаСоцСети', // +
|
||||
socialLink2: 'СсылкиНаСоцСети1', // +
|
||||
socialLink3: 'СсылкиНаСоцСети2', // +
|
||||
otherContact: 'Другие_КонтактыMyMail',
|
||||
socialChannel: 'КаналСвязи', // +
|
||||
|
||||
area: 'Отрасль', // +
|
||||
area2: 'Отрасль1', // +
|
||||
area3: 'Отрасль2', // +
|
||||
status: 'Статус',
|
||||
socialContacted: 'Откликнулся', // +
|
||||
framework: 'ДополнительныйФреймворкТехнологияПлатформаЛибоВторойЯП' // +
|
||||
}
|
||||
|
||||
function get (data: any, key: string): string | undefined {
|
||||
const v = data[key]
|
||||
if (Array.isArray(v) && v.length === 1) {
|
||||
return v[0]
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
export async function importXml (
|
||||
transactorUrl: string,
|
||||
dbName: string,
|
||||
minio: Client,
|
||||
xmlFile: string,
|
||||
mongoUrl: string,
|
||||
elasticUrl: string
|
||||
): Promise<void> {
|
||||
const connection = await connect(transactorUrl, dbName)
|
||||
|
||||
const tool = new ElasticTool(mongoUrl, dbName, minio, elasticUrl)
|
||||
const done = await tool.connect()
|
||||
try {
|
||||
console.log('loading xml document...')
|
||||
|
||||
const xmlData = await readFile(xmlFile, 'utf-8')
|
||||
const data = await parseStringPromise(xmlData)
|
||||
|
||||
const root = dirname(xmlFile)
|
||||
|
||||
const candidates = data[_.candidates][_.candidate]
|
||||
console.log('Found candidates:', candidates.length)
|
||||
|
||||
// const attributes = new Set<string>()
|
||||
const client = new TxOperations(connection, core.account.System)
|
||||
|
||||
const statuses = candidates.map((c: any) => get(c, _.status)).filter((c: any) => c !== undefined)
|
||||
.filter(onlyUniq)
|
||||
|
||||
console.log(statuses)
|
||||
|
||||
const withStatus: {candidateId: Ref<Candidate>, status: string}[] = []
|
||||
let pos = 0
|
||||
const len = candidates.length as number
|
||||
for (const c of candidates) {
|
||||
pos++
|
||||
const _id = get(c, _.id)
|
||||
const _name = get(c, _.fullName)
|
||||
if (_name === undefined || _id === undefined) {
|
||||
console.log('error procesing', JSON.stringify(c, undefined, 2))
|
||||
return
|
||||
}
|
||||
const candId = `tool-import-${_id}` as Ref<Candidate>
|
||||
const _status = get(c, _.status)
|
||||
if (_status !== undefined) {
|
||||
withStatus.push({ candidateId: candId, status: _status })
|
||||
}
|
||||
|
||||
await createCandidate(_name, pos, len, c, client, candId)
|
||||
|
||||
// Check and add attachments.
|
||||
const candidateRoot = join(root, _id)
|
||||
|
||||
if (existsSync(candidateRoot)) {
|
||||
const files = await readdir(candidateRoot)
|
||||
for (const f of files) {
|
||||
const attachId = (candId + f.toLowerCase()) as Ref<Attachment>
|
||||
const type = mime.contentType(f)
|
||||
if (typeof type === 'string') {
|
||||
const fileName = join(candidateRoot, f)
|
||||
const data = await readFile(fileName)
|
||||
try {
|
||||
await minio.statObject(dbName, attachId)
|
||||
} catch (err: any) {
|
||||
// No object, put new one.
|
||||
await minio.putObject(dbName, attachId, data, data.length, {
|
||||
'Content-Type': type
|
||||
})
|
||||
}
|
||||
|
||||
const stats = await stat(fileName)
|
||||
|
||||
const attachedDoc = await findOrUpdateAttached<Attachment>(client, recruit.space.CandidatesPublic, attachment.class.Attachment, attachId, {
|
||||
name: f,
|
||||
file: attachId,
|
||||
type: type,
|
||||
size: stats.size,
|
||||
lastModified: stats.mtime.getTime()
|
||||
}, {
|
||||
attachedTo: candId,
|
||||
attachedClass: recruit.class.Candidate,
|
||||
collection: 'attachments'
|
||||
})
|
||||
|
||||
await tool.indexAttachmentDoc(attachedDoc, data)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create/Update Vacancy and Applicants.
|
||||
|
||||
if (withStatus.length > 0) {
|
||||
const { states, vacancyId } = await createUpdateVacancy(client, statuses)
|
||||
|
||||
// Applicant num sequence.
|
||||
const sequence = await client.findOne(task.class.Sequence, { attachedTo: recruit.class.Applicant })
|
||||
if (sequence === undefined) {
|
||||
throw new Error('sequence object not found')
|
||||
}
|
||||
|
||||
const rankGen = genRanks(withStatus.length)
|
||||
const firstState = Array.from(states.values())[0]
|
||||
for (const { candidateId, status } of withStatus) {
|
||||
const incResult = await client.updateDoc(
|
||||
task.class.Sequence,
|
||||
task.space.Sequence,
|
||||
sequence._id,
|
||||
{
|
||||
$inc: { sequence: 1 }
|
||||
},
|
||||
true
|
||||
)
|
||||
|
||||
const rank = rankGen.next().value
|
||||
const state = states.get(status) ?? firstState
|
||||
|
||||
if (rank === undefined) {
|
||||
throw Error('Failed to generate rank')
|
||||
}
|
||||
|
||||
const lastOne = await client.findOne(
|
||||
recruit.class.Applicant,
|
||||
{ state },
|
||||
{ sort: { rank: SortingOrder.Descending } }
|
||||
)
|
||||
await createApplicant(vacancyId, candidateId, incResult, state, lastOne, client)
|
||||
}
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.log(err)
|
||||
} finally {
|
||||
await done()
|
||||
console.log('manual closing connection')
|
||||
await connection.close()
|
||||
}
|
||||
}
|
||||
const onlyUniq = (value: any, index: number, self: any[]): boolean => self.indexOf(value) === index
|
||||
|
||||
async function createApplicant (vacancyId: Ref<Vacancy>, candidateId: Ref<Candidate>, incResult: TxResult, state: Ref<State>, lastOne: Applicant | undefined, client: TxOperations): Promise<void> {
|
||||
const applicantId = `vacancy-${vacancyId}-${candidateId}` as Ref<Applicant>
|
||||
|
||||
const applicant: AttachedData<Applicant> = {
|
||||
number: (incResult as any).object.sequence,
|
||||
assignee: null,
|
||||
state,
|
||||
doneState: null,
|
||||
rank: calcRank(lastOne, undefined)
|
||||
}
|
||||
|
||||
// Update or create candidate
|
||||
await findOrUpdateAttached(client, vacancyId, recruit.class.Applicant, applicantId, applicant, { attachedTo: candidateId, attachedClass: recruit.class.Candidate, collection: 'applications' })
|
||||
}
|
||||
|
||||
async function createUpdateVacancy (client: TxOperations, statuses: any): Promise<{states: Map<string, Ref<State>>, vacancyId: Ref<Vacancy>}> {
|
||||
const vacancy: Data<Vacancy> = {
|
||||
name: 'Imported Vacancy',
|
||||
description: '',
|
||||
fullDescription: '',
|
||||
location: '',
|
||||
company: '',
|
||||
members: [],
|
||||
archived: false,
|
||||
private: false
|
||||
}
|
||||
const vacancyId = 'imported-vacancy' as Ref<Vacancy>
|
||||
|
||||
console.log('Creating vacancy', vacancy.name)
|
||||
// Update or create candidate
|
||||
await findOrUpdate(client, core.space.Model, recruit.class.Vacancy, vacancyId, vacancy)
|
||||
|
||||
const states = await createUpdateSpaceKanban(vacancyId, client, statuses)
|
||||
|
||||
console.log('States generated', vacancy.name)
|
||||
return { states, vacancyId }
|
||||
}
|
||||
|
||||
async function createCandidate (_name: string, pos: number, len: number, c: any, client: TxOperations, candId: Ref<Candidate>): Promise<void> {
|
||||
const names = _name.trim().split(' ')
|
||||
console.log(`(${pos} pf ${len})`, names[0] + ',', names.slice(1).join(' '))
|
||||
|
||||
const { sourceFields, telegram, linkedin, github } = parseSocials(c)
|
||||
|
||||
const data: Data<Candidate> = {
|
||||
name: names.slice(1).join(' ') + ', ' + names[0],
|
||||
city: get(c, _.city) ?? '',
|
||||
title: [
|
||||
get(c, _.vacancyKind),
|
||||
get(c, _.area)
|
||||
].filter(p => p !== undefined && p.trim().length > 0).filter(onlyUniq).join('/'),
|
||||
source: [
|
||||
get(c, _.socialContacted),
|
||||
get(c, _.socialChannel),
|
||||
sourceFields.filter(onlyUniq).join(', ')
|
||||
].filter(p => p !== undefined && p.trim().length > 0).filter(onlyUniq).join('/'),
|
||||
channels: []
|
||||
}
|
||||
|
||||
pushChannel(c, data, _.email, contact.channelProvider.Email)
|
||||
pushChannel(c, data, _.phone, contact.channelProvider.Phone)
|
||||
|
||||
const commentData: string[] = []
|
||||
|
||||
addComment(commentData, c, _.socialContacted)
|
||||
addComment(commentData, c, _.recruiter)
|
||||
addComment(commentData, c, _.requiredPosition)
|
||||
addComment(commentData, c, _.area2)
|
||||
addComment(commentData, c, _.area3)
|
||||
addComment(commentData, c, _.framework)
|
||||
addComment(commentData, c, _.comment)
|
||||
|
||||
if (telegram !== undefined) {
|
||||
data.channels.push({ provider: contact.channelProvider.Telegram, value: telegram })
|
||||
}
|
||||
if (linkedin !== undefined) {
|
||||
data.channels.push({ provider: contact.channelProvider.LinkedIn, value: linkedin })
|
||||
}
|
||||
if (github !== undefined) {
|
||||
data.channels.push({ provider: contact.channelProvider.GitHub, value: github })
|
||||
}
|
||||
await findOrUpdate(client, recruit.space.CandidatesPublic, recruit.class.Candidate, candId, data)
|
||||
|
||||
const commentId = (candId + '.description.comment') as Ref<Comment>
|
||||
|
||||
if (commentData.length > 0) {
|
||||
await findOrUpdateAttached(client, recruit.space.CandidatesPublic, chunter.class.Comment, commentId, {
|
||||
message: commentData.join('\n<br/>')
|
||||
}, { attachedTo: candId, attachedClass: recruit.class.Candidate, collection: 'comments' })
|
||||
}
|
||||
}
|
||||
|
||||
function addComment (data: string[], c: any, key: string): void {
|
||||
const val = get(c, key)
|
||||
if (val !== undefined) {
|
||||
data.push(`${key}: ${val}`.replace('\n', '\n<br/>'))
|
||||
}
|
||||
}
|
||||
|
||||
function parseSocials (c: any): { sourceFields: string[], telegram: string | undefined, linkedin: string | undefined, github: string | undefined } {
|
||||
let telegram: string | undefined
|
||||
let linkedin: string | undefined
|
||||
let github: string | undefined
|
||||
|
||||
const sourceFields = ([
|
||||
get(c, _.socialLink),
|
||||
get(c, _.socialLink2),
|
||||
get(c, _.socialLink3),
|
||||
get(c, _.otherContact),
|
||||
get(c, _.area2),
|
||||
get(c, _.area3)
|
||||
].filter(p => p !== undefined && p.trim().length > 0) as string[]).filter(t => {
|
||||
const lc = t.toLocaleLowerCase()
|
||||
if (lc.startsWith('telegram')) {
|
||||
telegram = t.substring(8).replace(':', '').trim()
|
||||
return false
|
||||
}
|
||||
if (lc.includes('linkedin.')) {
|
||||
linkedin = t.trim()
|
||||
return false
|
||||
}
|
||||
if (lc.includes('github.com')) {
|
||||
github = t.trim()
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
return { sourceFields, telegram, linkedin, github }
|
||||
}
|
||||
|
||||
function pushChannel (c: any, data: Data<Candidate>, key: string, provider: Ref<ChannelProvider>): void {
|
||||
const value = get(c, key)
|
||||
if (value !== undefined) {
|
||||
data.channels.push({ provider, value })
|
||||
}
|
||||
}
|
||||
export async function findOrUpdate<T extends Doc> (client: TxOperations, space: Ref<Space>, _class: Ref<Class<T>>, objectId: Ref<T>, data: Data<T>): Promise<void> {
|
||||
const existingObj = await client.findOne<Doc>(_class, { _id: objectId, space })
|
||||
if (existingObj !== undefined) {
|
||||
// Check some field changes
|
||||
const { _id, _class, modifiedOn, modifiedBy, space, ...dta } = existingObj
|
||||
if (!deepEqual(dta, data)) {
|
||||
await client.updateDoc(_class, space, objectId, data)
|
||||
}
|
||||
} else {
|
||||
await client.createDoc(_class, space, data, objectId)
|
||||
}
|
||||
}
|
||||
|
||||
function randColor (): string {
|
||||
const letters = '0123456789ABCDEF'
|
||||
let color = '#'
|
||||
for (let i = 0; i < 6; i++) {
|
||||
color += letters[Math.floor(Math.random() * 16)]
|
||||
}
|
||||
return color
|
||||
}
|
||||
|
||||
async function createUpdateSpaceKanban (spaceId: Ref<Vacancy>, client: TxOperations, stateNames: string[]): Promise<Map<string, Ref<State>>> {
|
||||
const states: Map<string, Ref<State>> = new Map()
|
||||
const stateRanks = genRanks(stateNames.length)
|
||||
for (const st of stateNames) {
|
||||
const rank = stateRanks.next().value
|
||||
|
||||
if (rank === undefined) {
|
||||
console.error('Failed to generate rank')
|
||||
break
|
||||
}
|
||||
|
||||
const sid = ('generated-' + spaceId + '.state.' + st.toLowerCase().replace(' ', '_')) as Ref<State>
|
||||
await findOrUpdate(client, spaceId, task.class.State,
|
||||
sid,
|
||||
{
|
||||
title: st,
|
||||
color: randColor(),
|
||||
rank
|
||||
}
|
||||
)
|
||||
states.set(st, sid)
|
||||
}
|
||||
|
||||
const doneStates = [
|
||||
{ class: task.class.WonState, title: 'Won' },
|
||||
{ class: task.class.LostState, title: 'Lost' }
|
||||
]
|
||||
const doneStateRanks = genRanks(doneStates.length)
|
||||
for (const st of doneStates) {
|
||||
const rank = doneStateRanks.next().value
|
||||
|
||||
if (rank === undefined) {
|
||||
console.error('Failed to generate rank')
|
||||
break
|
||||
}
|
||||
|
||||
const sid = ('generated-' + spaceId + '.done-state.' + st.title.toLowerCase().replace(' ', '_')) as Ref<DoneState>
|
||||
await findOrUpdate(client, spaceId, st.class,
|
||||
sid,
|
||||
{
|
||||
title: st.title,
|
||||
rank
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
await findOrUpdate(client, spaceId,
|
||||
task.class.Kanban,
|
||||
('generated-' + spaceId + '.kanban') as Ref<Kanban>,
|
||||
{
|
||||
attachedTo: spaceId
|
||||
}
|
||||
)
|
||||
return states
|
||||
}
|
||||
async function findOrUpdateAttached<T extends AttachedDoc> (client: TxOperations, space: Ref<Space>, _class: Ref<Class<T>>, objectId: Ref<T>, data: AttachedData<T>, attached: {attachedTo: Ref<Doc>, attachedClass: Ref<Class<Doc>>, collection: string}): Promise<T> {
|
||||
let existingObj = await client.findOne<Doc>(_class, { _id: objectId, space }) as T
|
||||
if (existingObj !== undefined) {
|
||||
await client.updateCollection(_class, space, objectId, attached.attachedTo, attached.attachedClass, attached.collection, data as unknown as DocumentUpdate<T>)
|
||||
} else {
|
||||
await client.addCollection(_class, space, attached.attachedTo, attached.attachedClass, attached.collection, data, objectId)
|
||||
existingObj = { _id: objectId, _class, space, ...data, ...attached } as unknown as T
|
||||
}
|
||||
return existingObj
|
||||
}
|
@ -31,6 +31,7 @@ import { Client } from 'minio'
|
||||
import { Db, MongoClient } from 'mongodb'
|
||||
import { connect } from './connect'
|
||||
import { rebuildElastic } from './elastic'
|
||||
import { importXml } from './importer'
|
||||
import { clearTelegramHistory } from './telegram'
|
||||
import { diffWorkspace, dumpWorkspace, initWorkspace, restoreWorkspace, upgradeWorkspace } from './workspace'
|
||||
|
||||
@ -247,4 +248,11 @@ program
|
||||
console.log('rebuild end')
|
||||
})
|
||||
|
||||
program
|
||||
.command('import-xml <workspace> <fileName>')
|
||||
.description('dump workspace transactions and minio resources')
|
||||
.action(async (workspace, fileName, cmd) => {
|
||||
return await importXml(transactorUrl, workspace, minio, fileName, mongodbUri, elasticUrl)
|
||||
})
|
||||
|
||||
program.parse(process.argv)
|
||||
|
@ -43,7 +43,7 @@
|
||||
(result) => {
|
||||
objects = result
|
||||
},
|
||||
{ sort: { [sortKey]: sortOrder }, ...options }
|
||||
{ sort: { [sortKey]: sortOrder }, ...options, limit: 500 }
|
||||
)
|
||||
|
||||
function getValue (doc: Doc, key: string): any {
|
||||
|
@ -26,6 +26,10 @@ class ElasticAdapter implements FullTextAdapter {
|
||||
) {
|
||||
}
|
||||
|
||||
async close (): Promise<void> {
|
||||
await this.client.close()
|
||||
}
|
||||
|
||||
async search (
|
||||
search: string
|
||||
): Promise<IndexedDoc[]> {
|
||||
@ -110,7 +114,7 @@ class ElasticAdapter implements FullTextAdapter {
|
||||
/**
|
||||
* @public
|
||||
*/
|
||||
export async function createElasticAdapter (url: string, dbName: string): Promise<FullTextAdapter> {
|
||||
export async function createElasticAdapter (url: string, dbName: string): Promise<FullTextAdapter & {close: () => Promise<void>}> {
|
||||
const client = new Client({
|
||||
node: url
|
||||
})
|
||||
|
Loading…
Reference in New Issue
Block a user