From 3df1f1d6df4256ca88f3353d6e7e879745e5669a Mon Sep 17 00:00:00 2001 From: Maksim Karmatskikh Date: Thu, 7 Dec 2023 08:25:12 +0100 Subject: [PATCH] UBERF-4526: Elastic bulk error on re-indexing (#4155) Signed-off-by: Maxim Karmatskikh --- server/core/src/indexer/field.ts | 21 ++++++++++++++++++++- server/core/src/indexer/fulltextPush.ts | 11 +++++++++-- server/core/src/indexer/summary.ts | 18 +++++++++++++++++- server/core/src/indexer/types.ts | 2 +- server/core/src/indexer/utils.ts | 17 +++++++++++++++++ server/elastic/src/adapter.ts | 7 ++++++- 6 files changed, 70 insertions(+), 6 deletions(-) diff --git a/server/core/src/indexer/field.ts b/server/core/src/indexer/field.ts index a030e7e256..f5a2c264f2 100644 --- a/server/core/src/indexer/field.ts +++ b/server/core/src/indexer/field.ts @@ -37,7 +37,8 @@ import { getFullTextIndexableAttributes, getFullTextContext, isFullTextAttribute, - loadIndexStageStage + loadIndexStageStage, + getCustomAttrKeys } from './utils' /** @@ -146,7 +147,15 @@ export class IndexedFieldStage implements FullTextPipelineStage { } } + const customAttrValues: any = [] for (const [, v] of Object.entries(content)) { + if (v.attr.isCustom === true && v.value !== '' && v.value !== undefined) { + // No need to put localized text as attributes. We do not use it at all + // Just put all content for custom attribute inside one custom field + customAttrValues.push({ label: v.attr.label, value: v.value }) + continue + } + // Check for content changes and collect update const dKey = docKey(v.attr.name, { _class: v.attr.attributeOf }) const dUKey = docUpdKey(v.attr.name, { _class: v.attr.attributeOf }) @@ -159,6 +168,16 @@ export class IndexedFieldStage implements FullTextPipelineStage { } } } + + const { customAttrKey, customAttrUKey } = getCustomAttrKeys() + if ( + (docState.attributes[customAttrKey] !== undefined || customAttrValues.length > 0) && + !deepEqual(docState.attributes[customAttrKey], customAttrValues) + ) { + changes++ + ;(docUpdate as any)[customAttrUKey] = customAttrValues + } + if (docState.attachedTo != null && changes > 0) { const ctx = getFullTextContext(pipeline.hierarchy, objClass) if (ctx.parentPropagate ?? true) { diff --git a/server/core/src/indexer/fulltextPush.ts b/server/core/src/indexer/fulltextPush.ts index b018f49f63..7d92a016c9 100644 --- a/server/core/src/indexer/fulltextPush.ts +++ b/server/core/src/indexer/fulltextPush.ts @@ -40,7 +40,14 @@ import { FullTextPipelineStage, fullTextPushStageId } from './types' -import { collectPropagate, collectPropagateClasses, docKey, getFullTextContext, IndexKeyOptions } from './utils' +import { + collectPropagate, + collectPropagateClasses, + docKey, + getFullTextContext, + IndexKeyOptions, + isCustomAttr +} from './utils' import { updateDocWithPresenter } from '../mapper' /** @@ -283,7 +290,7 @@ function updateDoc2Elastic ( docId = docIdOverride ?? docId if (docId === undefined) { - if (typeof vv !== 'object') { + if (typeof vv !== 'object' || isCustomAttr(k)) { doc[k] = vv } continue diff --git a/server/core/src/indexer/summary.ts b/server/core/src/indexer/summary.ts index c01d8615bc..95864121e9 100644 --- a/server/core/src/indexer/summary.ts +++ b/server/core/src/indexer/summary.ts @@ -33,7 +33,13 @@ import { translate } from '@hcengineering/platform' import { convert } from 'html-to-text' import { IndexedDoc } from '../types' import { contentStageId, DocUpdateHandler, fieldStateId, FullTextPipeline, FullTextPipelineStage } from './types' -import { collectPropagate, collectPropagateClasses, getFullTextContext, loadIndexStageStage } from './utils' +import { + collectPropagate, + collectPropagateClasses, + getFullTextContext, + loadIndexStageStage, + isCustomAttr +} from './utils' /** * @public @@ -240,6 +246,16 @@ export async function extractIndexedValues ( continue } + if (isCustomAttr(attr)) { + const str = v + .map((pair: { label: string, value: string }) => { + return `${pair.label} is ${pair.value}` + }) + .join(' ') + const cl = doc.objectClass + attributes[cl] = { ...attributes[cl], [k]: str } + } + if (_class === undefined) { // Skip all helper fields. continue diff --git a/server/core/src/indexer/types.ts b/server/core/src/indexer/types.ts index 32f705130a..fd29195ed4 100644 --- a/server/core/src/indexer/types.ts +++ b/server/core/src/indexer/types.ts @@ -102,7 +102,7 @@ export const contentStageId = 'cnt-v2b' /** * @public */ -export const fieldStateId = 'fld-v9' +export const fieldStateId = 'fld-v10' /** * @public diff --git a/server/core/src/indexer/utils.ts b/server/core/src/indexer/utils.ts index e707dc21dd..22535e0e49 100644 --- a/server/core/src/indexer/utils.ts +++ b/server/core/src/indexer/utils.ts @@ -308,3 +308,20 @@ export function collectPropagateClasses (pipeline: FullTextPipeline, objectClass return Array.from(propagate.values()) } + +const CUSTOM_ATTR_KEY = 'customAttributes' +const CUSTOM_ATTR_UPDATE_KEY = 'attributes.customAttributes' + +/** + * @public + */ +export function getCustomAttrKeys (): { customAttrKey: string, customAttrUKey: string } { + return { customAttrKey: CUSTOM_ATTR_KEY, customAttrUKey: CUSTOM_ATTR_UPDATE_KEY } +} + +/** + * @public + */ +export function isCustomAttr (attr: string): boolean { + return attr === CUSTOM_ATTR_KEY +} diff --git a/server/elastic/src/adapter.ts b/server/elastic/src/adapter.ts index fde075f5a0..e8d3f352e1 100644 --- a/server/elastic/src/adapter.ts +++ b/server/elastic/src/adapter.ts @@ -416,7 +416,12 @@ class ElasticAdapter implements FullTextAdapter { const errorIds = new Set(errors.map((it: any) => it.index._id)) const erroDocs = docs.filter((it) => errorIds.has(it.id)) // Collect only errors - const errs = Array.from(errors.map((it: any) => it.index.error.reason as string)).join('\n') + const errs = Array.from( + errors.map((it: any) => { + return `${it.index.error.reason}: ${it.index.error.caused_by?.reason}` + }) + ).join('\n') + console.error(`Failed to process bulk request: ${errs} ${JSON.stringify(erroDocs)}`) } }