From 4328fdee9b1688d5924e83fb899a8cfa769aea3a Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Thu, 6 Jan 2022 18:09:45 +0700 Subject: [PATCH] Elastic fulltext search mixins support (#770) Signed-off-by: Andrey Sobolev --- dev/client-resources/src/connection.ts | 4 ++ dev/server/src/server.ts | 4 ++ server/core/src/fulltext.ts | 73 +++++++++++++--------- server/core/src/types.ts | 14 +---- server/elastic/src/adapter.ts | 22 +++---- server/mongo/src/__tests__/storage.test.ts | 4 ++ 6 files changed, 67 insertions(+), 54 deletions(-) diff --git a/dev/client-resources/src/connection.ts b/dev/client-resources/src/connection.ts index cd2c9e2543..e6c5a7ad0e 100644 --- a/dev/client-resources/src/connection.ts +++ b/dev/client-resources/src/connection.ts @@ -52,6 +52,10 @@ class NullFullTextAdapter implements FullTextAdapter { async search (query: any): Promise { return [] } + + async remove (id: Ref): Promise { + + } } async function createNullFullTextAdapter (): Promise { diff --git a/dev/server/src/server.ts b/dev/server/src/server.ts index 80248b8c14..3fb1eec25c 100644 --- a/dev/server/src/server.ts +++ b/dev/server/src/server.ts @@ -36,6 +36,10 @@ class NullFullTextAdapter implements FullTextAdapter { async search (query: any): Promise { return [] } + + async remove (id: Ref): Promise { + + } } async function createNullFullTextAdapter (): Promise { diff --git a/server/core/src/fulltext.ts b/server/core/src/fulltext.ts index a68456c80a..8b181622ad 100644 --- a/server/core/src/fulltext.ts +++ b/server/core/src/fulltext.ts @@ -14,8 +14,8 @@ // limitations under the License. // -import type { AttachedDoc, Class, Doc, Obj, Ref, TxCreateDoc, TxResult, TxUpdateDoc } from '@anticrm/core' import core, { + AttachedDoc, Class, ClassifierKind, Doc, Obj, Ref, TxCreateDoc, TxResult, TxUpdateDoc, AnyAttribute, Collection, DocumentQuery, @@ -56,12 +56,28 @@ export class FullTextIndex implements WithFind { protected async txRemoveDoc (ctx: MeasureContext, tx: TxRemoveDoc): Promise { // console.log('FullTextIndex.txRemoveDoc: Method not implemented.') + await this.adapter.remove(tx.objectId) return {} } protected async txMixin (ctx: MeasureContext, tx: TxMixin): Promise { - console.log('FullTextIndex.txMixin: Method not implemented') - return {} + const attributes = this.getFullTextAttributes(tx.mixin) + let result = {} + if (attributes === undefined) return result + const ops: any = tx.attributes + const update: any = {} + let shouldUpdate = false + for (const attr of attributes) { + if (ops[attr.name] !== undefined) { + update[(tx.mixin as string) + '.' + attr.name] = ops[attr.name] + shouldUpdate = true + } + } + if (shouldUpdate) { + result = await this.adapter.update(tx.objectId, update) + await this.updateAttachedDocs(ctx, tx, update) + } + return result } async tx (ctx: MeasureContext, tx: Tx): Promise { @@ -152,7 +168,7 @@ export class FullTextIndex implements WithFind { protected async txCreateDoc (ctx: MeasureContext, tx: TxCreateDoc): Promise { const attributes = this.getFullTextAttributes(tx.objectClass) const doc = TxProcessor.createDoc2Doc(tx) - let parentContent: any[] = [] + let parentContent: Record = {} if (this.hierarchy.isDerived(doc._class, core.class.AttachedDoc)) { const attachedDoc = doc as AttachedDoc const parentDoc = ( @@ -163,9 +179,10 @@ export class FullTextIndex implements WithFind { parentContent = this.getContent(parentAttributes, parentDoc) } } - if (attributes === undefined && parentContent.length === 0) return {} + if (attributes === undefined && Object.keys(parentContent).length === 0) return {} + let content = this.getContent(attributes, doc) - content = content.concat(parentContent) + content = { ...parentContent, ...content } const indexedDoc: IndexedDoc = { id: doc._id, _class: doc._class, @@ -173,16 +190,7 @@ export class FullTextIndex implements WithFind { modifiedOn: doc.modifiedOn, space: doc.space, attachedTo: (doc as AttachedDoc).attachedTo, - content0: content[0], - content1: content[1], - content2: content[2], - content3: content[3], - content4: content[4], - content5: content[5], - content6: content[6], - content7: content[7], - content8: content[8], - content9: content[9] + ...content } return await this.adapter.index(indexedDoc) } @@ -193,14 +201,12 @@ export class FullTextIndex implements WithFind { if (attributes === undefined) return result const ops: any = tx.operations const update: any = {} - let i = 0 let shouldUpdate = false for (const attr of attributes) { if (ops[attr.name] !== undefined) { - update[`content${i}`] = ops[attr.name] + update[attr.name] = ops[attr.name] shouldUpdate = true } - i++ } if (tx.operations.space !== undefined) { update.space = tx.operations.space @@ -213,28 +219,37 @@ export class FullTextIndex implements WithFind { return result } - private getContent (attributes: AnyAttribute[] | undefined, doc: Doc): any[] { - if (attributes === undefined) return [] - return attributes.map((attr) => (doc as any)[attr.name]?.toString() ?? '') + private getContent (attributes: AnyAttribute[] | undefined, doc: Doc): Record { + const attrs: Record = {} + + for (const attr of attributes ?? []) { + attrs[attr.name] = (doc as any)[attr.name]?.toString() ?? '' + } + return attrs } - private async updateAttachedDocs (ctx: MeasureContext, tx: TxUpdateDoc, update: any): Promise { + private async updateAttachedDocs (ctx: MeasureContext, tx: {objectId: Ref, objectClass: Ref>}, update: any): Promise { const doc = (await this.dbStorage.findAll(ctx, tx.objectClass, { _id: tx.objectId }, { limit: 1 }))[0] if (doc === undefined) return const attributes = this.hierarchy.getAllAttributes(doc._class) + + // Find all mixin atttibutes for document. + this.hierarchy.getDescendants(doc._class) + .filter((m) => this.hierarchy.getClass(m).kind === ClassifierKind.MIXIN && this.hierarchy.hasMixin(doc, m)) + .forEach((m) => { + for (const [k, v] of this.hierarchy.getAllAttributes(m, doc._class)) { + attributes.set(k, v) + } + }) + for (const attribute of attributes.values()) { if (this.hierarchy.isDerived(attribute.type._class, core.class.Collection)) { const collection = attribute.type as Collection const allAttached = await this.dbStorage.findAll(ctx, collection.of, { attachedTo: tx.objectId }) if (allAttached.length === 0) continue - const attributes = this.getFullTextAttributes(tx.objectClass) - const shift = attributes?.length ?? 0 const docUpdate: any = {} for (const key in update) { - const index = Number.parseInt(key.replace('content', '')) - if (!isNaN(index)) { - docUpdate[`content${index + shift}`] = update[key] - } + docUpdate[key] = update[key] } for (const attached of allAttached) { try { diff --git a/server/core/src/types.ts b/server/core/src/types.ts index 70d224c074..a8905fbbf7 100644 --- a/server/core/src/types.ts +++ b/server/core/src/types.ts @@ -45,17 +45,8 @@ export interface IndexedDoc { modifiedOn: Timestamp modifiedBy: Ref attachedTo?: Ref - content0?: string - content1?: string - content2?: string - content3?: string - content4?: string - content5?: string - content6?: string - content7?: string - content8?: string - content9?: string - data?: string + + [key: string]: any } /** @@ -64,6 +55,7 @@ export interface IndexedDoc { export interface FullTextAdapter { index: (doc: IndexedDoc) => Promise update: (id: Ref, update: Record) => Promise + remove: (id: Ref) => Promise search: (_class: Ref>, search: DocumentQuery, size: number | undefined) => Promise } diff --git a/server/elastic/src/adapter.ts b/server/elastic/src/adapter.ts index 290b40c89e..1aa491b4a4 100644 --- a/server/elastic/src/adapter.ts +++ b/server/elastic/src/adapter.ts @@ -35,20 +35,7 @@ class ElasticAdapter implements FullTextAdapter { must: [ { multi_match: { - query: search, - fields: [ - 'content0', - 'content1', - 'content2', - 'content3', - 'content4', - 'content5', - 'content6', - 'content7', - 'content8', - 'content9', - 'attachment.content' - ] + query: search } } ], @@ -124,6 +111,13 @@ class ElasticAdapter implements FullTextAdapter { return {} } + + async remove (id: Ref): Promise { + await this.client.delete({ + index: this.db, + id + }) + } } /** diff --git a/server/mongo/src/__tests__/storage.test.ts b/server/mongo/src/__tests__/storage.test.ts index 9ddc10f347..16e02965f5 100644 --- a/server/mongo/src/__tests__/storage.test.ts +++ b/server/mongo/src/__tests__/storage.test.ts @@ -78,6 +78,10 @@ class NullFullTextAdapter implements FullTextAdapter { async search (query: any): Promise { return [] } + + async remove (id: Ref): Promise { + + } } async function createNullFullTextAdapter (): Promise {