Xml Importer (#701)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev 2021-12-23 16:02:14 +07:00 committed by GitHub
parent 07223ce5ae
commit edbf42c435
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 564 additions and 61 deletions

1
.vscode/launch.json vendored
View File

@ -51,7 +51,6 @@
"name": "Debug tool",
"type": "node",
"request": "launch",
// "args": ["src/index.ts", "import-xml", "ws1", "/Users/haiodo/Develop/private/hardware/suho/Кандидаты/Кандидаты.xml"],
"args": ["src/index.ts", "restore-workspace", "ws1", "../../temp/ws1/"],
"env": {
"MINIO_ACCESS_KEY":"minioadmin",

View File

@ -116,6 +116,7 @@ specifiers:
'@types/pdfkit': ~0.12.3
'@types/toposort': ^2.0.3
'@types/uuid': ^8.3.1
'@types/xml2js': ~0.4.9
'@typescript-eslint/eslint-plugin': ^5.4.0
autoprefixer: ^10.2.6
commander: ^8.1.0
@ -144,6 +145,7 @@ specifiers:
koa-bodyparser: ^4.3.0
koa-router: ^10.1.1
lexorank: ~1.0.4
mime-types: ~2.1.34
mini-css-extract-plugin: ^2.2.0
minio: ^7.0.19
node-html-parser: ^4.1.3
@ -168,6 +170,7 @@ specifiers:
webpack-bundle-analyzer: ^4.4.1
webpack-cli: ^4.6.0
webpack-dev-server: ^3.11.2
xml2js: ~0.4.23
dependencies:
'@elastic/elasticsearch': 7.16.0
@ -285,6 +288,7 @@ dependencies:
'@types/pdfkit': 0.12.3
'@types/toposort': 2.0.3
'@types/uuid': 8.3.3
'@types/xml2js': 0.4.9
'@typescript-eslint/eslint-plugin': 5.7.0_eslint@7.32.0+typescript@4.5.4
autoprefixer: 10.4.0_postcss@8.4.5
commander: 8.3.0
@ -313,6 +317,7 @@ dependencies:
koa-bodyparser: 4.3.0
koa-router: 10.1.1
lexorank: 1.0.4
mime-types: 2.1.34
mini-css-extract-plugin: 2.4.5_webpack@5.65.0
minio: 7.0.25
node-html-parser: 4.1.5
@ -337,6 +342,7 @@ dependencies:
webpack-bundle-analyzer: 4.5.0
webpack-cli: 4.9.1_1cfd1380a4e9b8401fb780accef05e9c
webpack-dev-server: 3.11.3_webpack-cli@4.9.1+webpack@5.65.0
xml2js: 0.4.23
packages:
@ -1713,6 +1719,10 @@ packages:
'@types/koa': 2.13.4
dev: false
/@types/mime-types/2.1.1:
resolution: {integrity: sha512-vXOTGVSLR2jMw440moWTC7H19iUyLtP3Z1YTj7cSsubOICinjMxFeb/V57v9QdyyPGbbWolUFSSmSiRSn94tFw==}
dev: false
/@types/mime/1.3.2:
resolution: {integrity: sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==}
dev: false
@ -1902,6 +1912,12 @@ packages:
'@types/node': 17.0.0
dev: false
/@types/xml2js/0.4.9:
resolution: {integrity: sha512-CHiCKIihl1pychwR2RNX5mAYmJDACgFVCMT5OArMaO3erzwXVcBqPcusr+Vl8yeeXukxZqtF8mZioqX+mpjjdw==}
dependencies:
'@types/node': 17.0.0
dev: false
/@types/yargs-parser/20.2.1:
resolution: {integrity: sha512-7tFImggNeNBVMsn0vLrpn1H1uPrUBdnARPTpZoitY37ZrdJREzf7I16tMrlK3hen349gr1NYh8CmZQa7CTG6Aw==}
dev: false
@ -11019,7 +11035,7 @@ packages:
dev: false
file:projects/dev-client-resources.tgz:
resolution: {integrity: sha512-d1gACtHmA2R8CbItLx5oJYfVQV7FvM4tQthnwsDeoIM9OUIlXaRVFhd/KlJf1MkqLIsOA4O+Gjvhih/+EtediA==, tarball: file:projects/dev-client-resources.tgz}
resolution: {integrity: sha512-hcwOdV5TgwA6yTNeZDtMwAaGEOYFO5VzqWTXk+sK+SPSiEN2yy6rb2Do1cfgL0wvSXLH/F2VT+nr1WAlI909lQ==, tarball: file:projects/dev-client-resources.tgz}
name: '@rush-temp/dev-client-resources'
version: 0.0.0
dependencies:
@ -12511,16 +12527,18 @@ packages:
dev: false
file:projects/tool.tgz:
resolution: {integrity: sha512-FQTXToKTNRIcen67zjbh/ZeibEPSskFtKhFZgTtUgB0TXhzJ0V+ZRcpnsXK+nND2DBBRYhtylQJmqWsoASRtFg==, tarball: file:projects/tool.tgz}
resolution: {integrity: sha512-kxAczrVzjVh47GTnScTkjjXIpSAkovYmCIxJSznbPqT4XOdtQ0F8eadl3h2mhqjKzgE7VoFOdTB5JJ3oy4+TUg==, tarball: file:projects/tool.tgz}
name: '@rush-temp/tool'
version: 0.0.0
dependencies:
'@elastic/elasticsearch': 7.16.0
'@rushstack/heft': 0.41.8
'@types/heft-jest': 1.0.2
'@types/mime-types': 2.1.1
'@types/minio': 7.0.11
'@types/node': 16.11.14
'@types/ws': 8.2.2
'@types/xml2js': 0.4.9
'@typescript-eslint/eslint-plugin': 5.7.0_c25e8c1f4f4f7aaed27aa6f9ce042237
'@typescript-eslint/parser': 5.7.0_eslint@7.32.0+typescript@4.5.4
commander: 8.3.0
@ -12533,12 +12551,14 @@ packages:
eslint-plugin-promise: 5.2.0_eslint@7.32.0
fast-equals: 2.0.4
jwt-simple: 0.5.6
mime-types: 2.1.34
minio: 7.0.25
mongodb: 4.2.2
prettier: 2.5.1
ts-node: 10.4.0_5d12c2add188ff0e728b4ade3dacd39b
typescript: 4.5.4
ws: 8.3.0
xml2js: 0.4.23
transitivePeerDependencies:
- '@swc/core'
- '@swc/wasm'

View File

@ -34,7 +34,9 @@
"prettier": "^2.4.1",
"@rushstack/heft": "^0.41.1",
"typescript": "^4.3.5",
"@types/ws": "^8.2.1"
"@types/ws": "^8.2.1",
"@types/xml2js": "~0.4.9",
"@types/mime-types": "~2.1.1"
},
"dependencies": {
"mongodb": "^4.1.1",
@ -62,6 +64,13 @@
"@anticrm/server-chunter": "~0.6.1",
"@anticrm/server-chunter-resources": "~0.6.0",
"@anticrm/server-recruit": "~0.6.0",
"@anticrm/server-recruit-resources": "~0.6.0"
"@anticrm/server-recruit-resources": "~0.6.0",
"xml2js": "~0.4.23",
"@anticrm/model-recruit": "~0.6.0",
"@anticrm/recruit": "~0.6.2",
"@anticrm/task": "~0.6.0",
"@anticrm/chunter": "~0.6.1",
"mime-types": "~2.1.34",
"@anticrm/attachment": "~0.6.1"
}
}

View File

@ -34,6 +34,7 @@ import core, {
} from '@anticrm/core'
import { createElasticAdapter } from '@anticrm/elastic'
import { DOMAIN_ATTACHMENT } from '@anticrm/model-attachment'
import { Attachment } from '@anticrm/attachment'
import { createMongoAdapter, createMongoTxAdapter } from '@anticrm/mongo'
import { addLocation } from '@anticrm/platform'
import { serverChunterId } from '@anticrm/server-chunter'
@ -91,29 +92,94 @@ async function dropElastic (elasticUrl: string, dbName: string): Promise<void> {
await client.close()
}
export class ElasticTool {
mongoClient: MongoClient
elastic!: FullTextAdapter & {close: () => Promise<void>}
storage!: ServerStorage
db!: Db
constructor (readonly mongoUrl: string, readonly dbName: string, readonly minio: Client, readonly elasticUrl: string) {
addLocation(serverChunterId, () => import('@anticrm/server-chunter-resources'))
addLocation(serverRecruitId, () => import('@anticrm/server-recruit-resources'))
this.mongoClient = new MongoClient(mongoUrl)
}
async connect (): Promise<() => Promise<void>> {
await this.mongoClient.connect()
this.db = this.mongoClient.db(this.dbName)
this.elastic = await createElasticAdapter(this.elasticUrl, this.dbName)
this.storage = await createStorage(this.mongoUrl, this.elasticUrl, this.dbName)
return async () => {
await this.mongoClient.close()
await this.elastic.close()
}
}
async indexAttachment (
name: string
): Promise<void> {
const doc: Attachment | null = await this.db.collection<Attachment>(DOMAIN_ATTACHMENT).findOne({ file: name })
if (doc == null) return
const buffer = await this.readMinioObject(name)
await this.indexAttachmentDoc(doc, buffer)
}
async indexAttachmentDoc (doc: Attachment, buffer: Buffer): Promise<void> {
const id: Ref<Doc> = (generateId() + '/attachments/') as Ref<Doc>
const indexedDoc: IndexedDoc = {
id: id,
_class: doc._class,
space: doc.space,
modifiedOn: doc.modifiedOn,
modifiedBy: 'core:account:System' as Ref<Account>,
attachedTo: doc.attachedTo,
data: buffer.toString('base64')
}
await this.elastic.index(indexedDoc)
}
private async readMinioObject (name: string): Promise<Buffer> {
const data = await this.minio.getObject(this.dbName, name)
const chunks: Buffer[] = []
await new Promise<void>((resolve) => {
data.on('readable', () => {
let chunk
while ((chunk = data.read()) !== null) {
const b = chunk as Buffer
chunks.push(b)
}
})
data.on('end', () => {
resolve()
})
})
return Buffer.concat(chunks)
}
}
async function restoreElastic (mongoUrl: string, dbName: string, minio: Client, elasticUrl: string): Promise<void> {
addLocation(serverChunterId, () => import('@anticrm/server-chunter-resources'))
addLocation(serverRecruitId, () => import('@anticrm/server-recruit-resources'))
const mongoClient = new MongoClient(mongoUrl)
const tool = new ElasticTool(mongoUrl, dbName, minio, elasticUrl)
const done = await tool.connect()
try {
await mongoClient.connect()
const db = mongoClient.db(dbName)
const elastic = await createElasticAdapter(elasticUrl, dbName)
const storage = await createStorage(mongoUrl, elasticUrl, dbName)
const txes = (await db.collection<Tx>(DOMAIN_TX).find().sort({ _id: 1 }).toArray())
const txes = (await tool.db.collection<Tx>(DOMAIN_TX).find().sort({ _id: 1 }).toArray())
const data = txes.filter((tx) => tx.objectSpace !== core.space.Model)
const metricsCtx = new MeasureMetricsContext('elastic', {})
for (const tx of data) {
await storage.tx(metricsCtx, tx)
await tool.storage.tx(metricsCtx, tx)
}
if (await minio.bucketExists(dbName)) {
const minioObjects = await listMinioObjects(minio, dbName)
for (const d of minioObjects) {
await indexAttachment(elastic, minio, db, dbName, d.name)
await tool.indexAttachment(d.name)
}
}
} finally {
await mongoClient.close()
await done()
}
}
@ -142,49 +208,6 @@ async function createStorage (mongoUrl: string, elasticUrl: string, workspace: s
return await createServerStorage(conf)
}
async function indexAttachment (
elastic: FullTextAdapter,
minio: Client,
db: Db,
dbName: string,
name: string
): Promise<void> {
const doc = await db.collection(DOMAIN_ATTACHMENT).findOne({
file: name
})
if (doc == null) return
const data = await minio.getObject(dbName, name)
const chunks: Buffer[] = []
await new Promise<void>((resolve) => {
data.on('readable', () => {
let chunk
while ((chunk = data.read()) !== null) {
const b = chunk as Buffer
chunks.push(b)
}
})
data.on('end', () => {
resolve()
})
})
const id: Ref<Doc> = (generateId() + '/attachments/') as Ref<Doc>
const indexedDoc: IndexedDoc = {
id: id,
_class: doc._class,
space: doc.space,
modifiedOn: doc.modifiedOn,
modifiedBy: 'core:account:System' as Ref<Account>,
attachedTo: doc.attachedTo,
data: Buffer.concat(chunks).toString('base64')
}
await elastic.index(indexedDoc)
}
async function createMongoReadOnlyAdapter (
hierarchy: Hierarchy,
url: string,

440
dev/tool/src/importer.ts Normal file
View File

@ -0,0 +1,440 @@
//
// Copyright © 2020, 2021 Anticrm Platform Contributors.
// Copyright © 2021 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import attachment, { Attachment } from '@anticrm/attachment'
import chunter, { Comment } from '@anticrm/chunter'
import contact, { ChannelProvider } from '@anticrm/contact'
import core, { AttachedData, AttachedDoc, Class, Data, Doc, DocumentUpdate, Ref, SortingOrder, Space, TxOperations, TxResult } from '@anticrm/core'
import recruit from '@anticrm/model-recruit'
import { Applicant, Candidate, Vacancy } from '@anticrm/recruit'
import task, { calcRank, DoneState, genRanks, Kanban, State } from '@anticrm/task'
import { deepEqual } from 'fast-equals'
import { existsSync } from 'fs'
import { readdir, readFile, stat } from 'fs/promises'
import mime from 'mime-types'
import { Client } from 'minio'
import { dirname, join } from 'path'
import { parseStringPromise } from 'xml2js'
import { connect } from './connect'
import { ElasticTool } from './elastic'
const _ = {
candidates: 'Кандидаты',
candidate: 'Кандидат',
delete: 'ПометкаУдаления', // -
id: 'Код', // +
fullName: 'Наименование', // +
recruiter: 'Рекрутер',
birthdate: 'ДатаРождения',
phone: 'НомерТелефона', // +
email: 'ЭлектроннаяПочта', // +
date: 'Дата',
notifyStatus: 'УведомлятьОСменеСтатуса',
requiredPosition: 'ЖелаемаяДолжность', // +
vacancyKind: 'ВидВакансии', // +
city: 'Проживание', // +
comment: 'Комментарий', // +
socialLink: 'СсылкиНаСоцСети', // +
socialLink2: 'СсылкиНаСоцСети1', // +
socialLink3: 'СсылкиНаСоцСети2', // +
otherContact: ругие_КонтактыMyMail',
socialChannel: 'КаналСвязи', // +
area: 'Отрасль', // +
area2: 'Отрасль1', // +
area3: 'Отрасль2', // +
status: 'Статус',
socialContacted: 'Откликнулся', // +
framework: 'ДополнительныйФреймворкТехнологияПлатформаЛибоВторойЯП' // +
}
function get (data: any, key: string): string | undefined {
const v = data[key]
if (Array.isArray(v) && v.length === 1) {
return v[0]
}
return v
}
export async function importXml (
transactorUrl: string,
dbName: string,
minio: Client,
xmlFile: string,
mongoUrl: string,
elasticUrl: string
): Promise<void> {
const connection = await connect(transactorUrl, dbName)
const tool = new ElasticTool(mongoUrl, dbName, minio, elasticUrl)
const done = await tool.connect()
try {
console.log('loading xml document...')
const xmlData = await readFile(xmlFile, 'utf-8')
const data = await parseStringPromise(xmlData)
const root = dirname(xmlFile)
const candidates = data[_.candidates][_.candidate]
console.log('Found candidates:', candidates.length)
// const attributes = new Set<string>()
const client = new TxOperations(connection, core.account.System)
const statuses = candidates.map((c: any) => get(c, _.status)).filter((c: any) => c !== undefined)
.filter(onlyUniq)
console.log(statuses)
const withStatus: {candidateId: Ref<Candidate>, status: string}[] = []
let pos = 0
const len = candidates.length as number
for (const c of candidates) {
pos++
const _id = get(c, _.id)
const _name = get(c, _.fullName)
if (_name === undefined || _id === undefined) {
console.log('error procesing', JSON.stringify(c, undefined, 2))
return
}
const candId = `tool-import-${_id}` as Ref<Candidate>
const _status = get(c, _.status)
if (_status !== undefined) {
withStatus.push({ candidateId: candId, status: _status })
}
await createCandidate(_name, pos, len, c, client, candId)
// Check and add attachments.
const candidateRoot = join(root, _id)
if (existsSync(candidateRoot)) {
const files = await readdir(candidateRoot)
for (const f of files) {
const attachId = (candId + f.toLowerCase()) as Ref<Attachment>
const type = mime.contentType(f)
if (typeof type === 'string') {
const fileName = join(candidateRoot, f)
const data = await readFile(fileName)
try {
await minio.statObject(dbName, attachId)
} catch (err: any) {
// No object, put new one.
await minio.putObject(dbName, attachId, data, data.length, {
'Content-Type': type
})
}
const stats = await stat(fileName)
const attachedDoc = await findOrUpdateAttached<Attachment>(client, recruit.space.CandidatesPublic, attachment.class.Attachment, attachId, {
name: f,
file: attachId,
type: type,
size: stats.size,
lastModified: stats.mtime.getTime()
}, {
attachedTo: candId,
attachedClass: recruit.class.Candidate,
collection: 'attachments'
})
await tool.indexAttachmentDoc(attachedDoc, data)
}
}
}
}
// Create/Update Vacancy and Applicants.
if (withStatus.length > 0) {
const { states, vacancyId } = await createUpdateVacancy(client, statuses)
// Applicant num sequence.
const sequence = await client.findOne(task.class.Sequence, { attachedTo: recruit.class.Applicant })
if (sequence === undefined) {
throw new Error('sequence object not found')
}
const rankGen = genRanks(withStatus.length)
const firstState = Array.from(states.values())[0]
for (const { candidateId, status } of withStatus) {
const incResult = await client.updateDoc(
task.class.Sequence,
task.space.Sequence,
sequence._id,
{
$inc: { sequence: 1 }
},
true
)
const rank = rankGen.next().value
const state = states.get(status) ?? firstState
if (rank === undefined) {
throw Error('Failed to generate rank')
}
const lastOne = await client.findOne(
recruit.class.Applicant,
{ state },
{ sort: { rank: SortingOrder.Descending } }
)
await createApplicant(vacancyId, candidateId, incResult, state, lastOne, client)
}
}
} catch (err: any) {
console.log(err)
} finally {
await done()
console.log('manual closing connection')
await connection.close()
}
}
const onlyUniq = (value: any, index: number, self: any[]): boolean => self.indexOf(value) === index
async function createApplicant (vacancyId: Ref<Vacancy>, candidateId: Ref<Candidate>, incResult: TxResult, state: Ref<State>, lastOne: Applicant | undefined, client: TxOperations): Promise<void> {
const applicantId = `vacancy-${vacancyId}-${candidateId}` as Ref<Applicant>
const applicant: AttachedData<Applicant> = {
number: (incResult as any).object.sequence,
assignee: null,
state,
doneState: null,
rank: calcRank(lastOne, undefined)
}
// Update or create candidate
await findOrUpdateAttached(client, vacancyId, recruit.class.Applicant, applicantId, applicant, { attachedTo: candidateId, attachedClass: recruit.class.Candidate, collection: 'applications' })
}
async function createUpdateVacancy (client: TxOperations, statuses: any): Promise<{states: Map<string, Ref<State>>, vacancyId: Ref<Vacancy>}> {
const vacancy: Data<Vacancy> = {
name: 'Imported Vacancy',
description: '',
fullDescription: '',
location: '',
company: '',
members: [],
archived: false,
private: false
}
const vacancyId = 'imported-vacancy' as Ref<Vacancy>
console.log('Creating vacancy', vacancy.name)
// Update or create candidate
await findOrUpdate(client, core.space.Model, recruit.class.Vacancy, vacancyId, vacancy)
const states = await createUpdateSpaceKanban(vacancyId, client, statuses)
console.log('States generated', vacancy.name)
return { states, vacancyId }
}
async function createCandidate (_name: string, pos: number, len: number, c: any, client: TxOperations, candId: Ref<Candidate>): Promise<void> {
const names = _name.trim().split(' ')
console.log(`(${pos} pf ${len})`, names[0] + ',', names.slice(1).join(' '))
const { sourceFields, telegram, linkedin, github } = parseSocials(c)
const data: Data<Candidate> = {
name: names.slice(1).join(' ') + ', ' + names[0],
city: get(c, _.city) ?? '',
title: [
get(c, _.vacancyKind),
get(c, _.area)
].filter(p => p !== undefined && p.trim().length > 0).filter(onlyUniq).join('/'),
source: [
get(c, _.socialContacted),
get(c, _.socialChannel),
sourceFields.filter(onlyUniq).join(', ')
].filter(p => p !== undefined && p.trim().length > 0).filter(onlyUniq).join('/'),
channels: []
}
pushChannel(c, data, _.email, contact.channelProvider.Email)
pushChannel(c, data, _.phone, contact.channelProvider.Phone)
const commentData: string[] = []
addComment(commentData, c, _.socialContacted)
addComment(commentData, c, _.recruiter)
addComment(commentData, c, _.requiredPosition)
addComment(commentData, c, _.area2)
addComment(commentData, c, _.area3)
addComment(commentData, c, _.framework)
addComment(commentData, c, _.comment)
if (telegram !== undefined) {
data.channels.push({ provider: contact.channelProvider.Telegram, value: telegram })
}
if (linkedin !== undefined) {
data.channels.push({ provider: contact.channelProvider.LinkedIn, value: linkedin })
}
if (github !== undefined) {
data.channels.push({ provider: contact.channelProvider.GitHub, value: github })
}
await findOrUpdate(client, recruit.space.CandidatesPublic, recruit.class.Candidate, candId, data)
const commentId = (candId + '.description.comment') as Ref<Comment>
if (commentData.length > 0) {
await findOrUpdateAttached(client, recruit.space.CandidatesPublic, chunter.class.Comment, commentId, {
message: commentData.join('\n<br/>')
}, { attachedTo: candId, attachedClass: recruit.class.Candidate, collection: 'comments' })
}
}
function addComment (data: string[], c: any, key: string): void {
const val = get(c, key)
if (val !== undefined) {
data.push(`${key}: ${val}`.replace('\n', '\n<br/>'))
}
}
function parseSocials (c: any): { sourceFields: string[], telegram: string | undefined, linkedin: string | undefined, github: string | undefined } {
let telegram: string | undefined
let linkedin: string | undefined
let github: string | undefined
const sourceFields = ([
get(c, _.socialLink),
get(c, _.socialLink2),
get(c, _.socialLink3),
get(c, _.otherContact),
get(c, _.area2),
get(c, _.area3)
].filter(p => p !== undefined && p.trim().length > 0) as string[]).filter(t => {
const lc = t.toLocaleLowerCase()
if (lc.startsWith('telegram')) {
telegram = t.substring(8).replace(':', '').trim()
return false
}
if (lc.includes('linkedin.')) {
linkedin = t.trim()
return false
}
if (lc.includes('github.com')) {
github = t.trim()
return false
}
return true
})
return { sourceFields, telegram, linkedin, github }
}
function pushChannel (c: any, data: Data<Candidate>, key: string, provider: Ref<ChannelProvider>): void {
const value = get(c, key)
if (value !== undefined) {
data.channels.push({ provider, value })
}
}
export async function findOrUpdate<T extends Doc> (client: TxOperations, space: Ref<Space>, _class: Ref<Class<T>>, objectId: Ref<T>, data: Data<T>): Promise<void> {
const existingObj = await client.findOne<Doc>(_class, { _id: objectId, space })
if (existingObj !== undefined) {
// Check some field changes
const { _id, _class, modifiedOn, modifiedBy, space, ...dta } = existingObj
if (!deepEqual(dta, data)) {
await client.updateDoc(_class, space, objectId, data)
}
} else {
await client.createDoc(_class, space, data, objectId)
}
}
function randColor (): string {
const letters = '0123456789ABCDEF'
let color = '#'
for (let i = 0; i < 6; i++) {
color += letters[Math.floor(Math.random() * 16)]
}
return color
}
async function createUpdateSpaceKanban (spaceId: Ref<Vacancy>, client: TxOperations, stateNames: string[]): Promise<Map<string, Ref<State>>> {
const states: Map<string, Ref<State>> = new Map()
const stateRanks = genRanks(stateNames.length)
for (const st of stateNames) {
const rank = stateRanks.next().value
if (rank === undefined) {
console.error('Failed to generate rank')
break
}
const sid = ('generated-' + spaceId + '.state.' + st.toLowerCase().replace(' ', '_')) as Ref<State>
await findOrUpdate(client, spaceId, task.class.State,
sid,
{
title: st,
color: randColor(),
rank
}
)
states.set(st, sid)
}
const doneStates = [
{ class: task.class.WonState, title: 'Won' },
{ class: task.class.LostState, title: 'Lost' }
]
const doneStateRanks = genRanks(doneStates.length)
for (const st of doneStates) {
const rank = doneStateRanks.next().value
if (rank === undefined) {
console.error('Failed to generate rank')
break
}
const sid = ('generated-' + spaceId + '.done-state.' + st.title.toLowerCase().replace(' ', '_')) as Ref<DoneState>
await findOrUpdate(client, spaceId, st.class,
sid,
{
title: st.title,
rank
}
)
}
await findOrUpdate(client, spaceId,
task.class.Kanban,
('generated-' + spaceId + '.kanban') as Ref<Kanban>,
{
attachedTo: spaceId
}
)
return states
}
async function findOrUpdateAttached<T extends AttachedDoc> (client: TxOperations, space: Ref<Space>, _class: Ref<Class<T>>, objectId: Ref<T>, data: AttachedData<T>, attached: {attachedTo: Ref<Doc>, attachedClass: Ref<Class<Doc>>, collection: string}): Promise<T> {
let existingObj = await client.findOne<Doc>(_class, { _id: objectId, space }) as T
if (existingObj !== undefined) {
await client.updateCollection(_class, space, objectId, attached.attachedTo, attached.attachedClass, attached.collection, data as unknown as DocumentUpdate<T>)
} else {
await client.addCollection(_class, space, attached.attachedTo, attached.attachedClass, attached.collection, data, objectId)
existingObj = { _id: objectId, _class, space, ...data, ...attached } as unknown as T
}
return existingObj
}

View File

@ -31,6 +31,7 @@ import { Client } from 'minio'
import { Db, MongoClient } from 'mongodb'
import { connect } from './connect'
import { rebuildElastic } from './elastic'
import { importXml } from './importer'
import { clearTelegramHistory } from './telegram'
import { diffWorkspace, dumpWorkspace, initWorkspace, restoreWorkspace, upgradeWorkspace } from './workspace'
@ -247,4 +248,11 @@ program
console.log('rebuild end')
})
program
.command('import-xml <workspace> <fileName>')
.description('dump workspace transactions and minio resources')
.action(async (workspace, fileName, cmd) => {
return await importXml(transactorUrl, workspace, minio, fileName, mongodbUri, elasticUrl)
})
program.parse(process.argv)

View File

@ -43,7 +43,7 @@
(result) => {
objects = result
},
{ sort: { [sortKey]: sortOrder }, ...options }
{ sort: { [sortKey]: sortOrder }, ...options, limit: 500 }
)
function getValue (doc: Doc, key: string): any {

View File

@ -26,6 +26,10 @@ class ElasticAdapter implements FullTextAdapter {
) {
}
async close (): Promise<void> {
await this.client.close()
}
async search (
search: string
): Promise<IndexedDoc[]> {
@ -110,7 +114,7 @@ class ElasticAdapter implements FullTextAdapter {
/**
* @public
*/
export async function createElasticAdapter (url: string, dbName: string): Promise<FullTextAdapter> {
export async function createElasticAdapter (url: string, dbName: string): Promise<FullTextAdapter & {close: () => Promise<void>}> {
const client = new Client({
node: url
})