minor improve fulltext perfomance (#1429)

Signed-off-by: Denis Bykhov <80476319+BykhovDenis@users.noreply.github.com>
This commit is contained in:
Denis Bykhov 2022-04-18 13:55:19 +06:00 committed by GitHub
parent f6f59f04f5
commit ae51623556
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 66 additions and 18 deletions

View File

@ -149,21 +149,61 @@ export class FullTextIndex implements WithFind {
const classes = this.hierarchy.getDescendants(baseClass)
const fullTextLimit = 10000
const docs = await this.adapter.search(classes, query, fullTextLimit)
for (const doc of docs) {
if (this.hierarchy.isDerived(doc._class, baseClass)) {
ids.add(doc.id)
}
if (doc.attachedTo !== undefined) {
if (doc.attachedToClass !== undefined && this.hierarchy.isDerived(doc.attachedToClass, baseClass)) {
if (this.hierarchy.isDerived(doc.attachedToClass, baseClass)) {
ids.add(doc.attachedTo)
}
} else {
ids.add(doc.attachedTo)
}
}
const resultIds = getResultIds(ids, _id)
const { limit, ...otherOptions } = options ?? { }
const result = await this.dbStorage.findAll(ctx, _class, { _id: { $in: resultIds }, ...mainQuery }, otherOptions)
const total = result.total
result.sort((a, b) => resultIds.indexOf(a._id) - resultIds.indexOf(b._id))
if (limit !== undefined) {
const res = toFindResult(result.splice(0, limit), total)
return res
}
const resultIds = Array.from(getResultIds(ids, _id))
if (options?.limit === undefined) {
return await this.getResult(ctx, _class, resultIds, mainQuery as DocumentQuery<T>, options)
} else {
const result: T[] = []
const size = options.limit
while (true) {
const ids = resultIds.splice(0, size)
const res = await this.getResult(ctx, _class, ids, mainQuery as DocumentQuery<T>, options)
result.push(...res)
if (result.length >= size || res.length < size) {
break
}
}
if (result.length >= size) {
const total = await this.getResult(ctx, _class, resultIds, mainQuery as DocumentQuery<T>, { limit: 1 })
return toFindResult(result, total.total)
}
return toFindResult(result)
}
}
private async getResult <T extends Doc>(ctx: MeasureContext,
_class: Ref<Class<T>>,
ids: Ref<Doc>[],
mainQuery: DocumentQuery<T>,
options?: FindOptions<T>
): Promise<FindResult<T>> {
const orderMap = new Map<Ref<Doc>, number>()
for (let index = 0; index < ids.length; index++) {
orderMap.set(ids[index], index)
}
const result = await this.dbStorage.findAll(ctx, _class, { _id: { $in: ids }, ...mainQuery }, options)
const total = result.total
result.sort((a, b) => (orderMap.get(a._id) ?? 0) - (orderMap.get(b._id) ?? 0))
return toFindResult(result, total)
}
@ -222,6 +262,7 @@ export class FullTextIndex implements WithFind {
modifiedOn: doc.modifiedOn,
space: doc.space,
attachedTo: (doc as AttachedDoc).attachedTo,
attachedToClass: (doc as AttachedDoc).attachedToClass,
...content
}
return await this.adapter.index(indexedDoc)
@ -326,30 +367,30 @@ function isFullTextAttribute (attr: AnyAttribute): boolean {
)
}
function getResultIds (ids: Set<Ref<Doc>>, _id: ObjQueryType<Ref<Doc>> | undefined): Ref<Doc>[] {
let result = []
function getResultIds (ids: Set<Ref<Doc>>, _id: ObjQueryType<Ref<Doc>> | undefined): Set<Ref<Doc>> {
const result = new Set<Ref<Doc>>()
if (_id !== undefined) {
if (typeof _id === 'string') {
if (!ids.has(_id)) {
return []
return new Set()
} else {
result = [_id]
result.add(_id)
}
} else if (_id.$in !== undefined) {
for (const id of _id.$in) {
if (ids.has(id)) {
result.push(id)
result.add(id)
}
}
} else if (_id.$nin !== undefined) {
for (const id of Array.from(ids.values())) {
for (const id of ids) {
if (!_id.$nin.includes(id)) {
result.push(id)
result.add(id)
}
}
}
} else {
result = Array.from(ids)
return ids
}
return result
}

View File

@ -100,6 +100,7 @@ export interface IndexedDoc {
modifiedOn: Timestamp
modifiedBy: Ref<Account>
attachedTo?: Ref<Doc>
attachedToClass?: Ref<Class<Doc>>
[key: string]: any
}

View File

@ -43,6 +43,12 @@ class ElasticAdapter implements FullTextAdapter {
_class: _classes.map((c) => c.toLowerCase()),
boost: 10.0
}
},
{
terms: {
attachedToClass: _classes.map((c) => c.toLowerCase()),
boost: 5.0
}
}
]
}