From f44c9a59cae1e85aa9d04ba64d0a13deba912f0b Mon Sep 17 00:00:00 2001 From: Denis Bykhov <80476319+BykhovDenis@users.noreply.github.com> Date: Fri, 8 Apr 2022 09:06:38 +0600 Subject: [PATCH] FindResult {total: number} (#1320) Signed-off-by: Denis Bykhov <80476319+BykhovDenis@users.noreply.github.com> --- packages/core/src/client.ts | 17 ++- packages/core/src/memdb.ts | 20 +-- packages/core/src/storage.ts | 4 +- packages/core/src/utils.ts | 13 +- packages/query/src/index.ts | 52 +++++--- .../src/components/CreatePerson.svelte | 2 +- plugins/contact/src/index.ts | 53 +++++--- plugins/devmodel-resources/src/index.ts | 116 +++++++++++++----- .../src/components/CreateCandidate.svelte | 2 +- .../src/components/review/CreateReview.svelte | 2 +- plugins/task-resources/src/index.ts | 16 ++- server/core/src/fulltext.ts | 12 +- server/mongo/src/__tests__/storage.test.ts | 101 ++++++++++----- server/mongo/src/storage.ts | 101 +++++++++++---- server/server/src/server.ts | 108 ++++++++++------ server/ws/src/__tests__/server.test.ts | 50 +++++--- 16 files changed, 464 insertions(+), 205 deletions(-) diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index 6fc776429d..31f7430d42 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -22,6 +22,7 @@ import { ModelDb } from './memdb' import type { DocumentQuery, FindOptions, FindResult, Storage, TxResult, WithLookup } from './storage' import { SortingOrder } from './storage' import { Tx, TxCreateDoc, TxProcessor, TxUpdateDoc } from './tx' +import { toFindResult } from './utils' /** * @public @@ -73,7 +74,7 @@ class ClientImpl implements Client { options?: FindOptions ): Promise> { const domain = this.hierarchy.getDomain(_class) - let result = + const data = domain === DOMAIN_MODEL ? await this.model.findAll(_class, query, options) : await this.conn.findAll(_class, query, options) @@ -81,10 +82,10 @@ class ClientImpl implements Client { // In case of mixin we need to create mixin proxies. // Update mixins & lookups - result = result.map((v) => { + const result = data.map((v) => { return this.hierarchy.updateLookupMixin(_class, v, options) }) - return result + return toFindResult(result, data.total) } async findOne( @@ -162,9 +163,7 @@ export async function createClient ( if (t._class === core.class.TxCreateDoc) { const ct = t as TxCreateDoc if (ct.objectClass === core.class.PluginConfiguration) { - configs.set(ct.objectId as Ref, - TxProcessor.createDoc2Doc(ct) as PluginConfiguration - ) + configs.set(ct.objectId as Ref, TxProcessor.createDoc2Doc(ct) as PluginConfiguration) } } else if (t._class === core.class.TxUpdateDoc) { const ut = t as TxUpdateDoc @@ -177,7 +176,7 @@ export async function createClient ( } } - const excludedPlugins = Array.from(configs.values()).filter(it => !allowedPlugins.includes(it.pluginId as Plugin)) + const excludedPlugins = Array.from(configs.values()).filter((it) => !allowedPlugins.includes(it.pluginId as Plugin)) for (const a of excludedPlugins) { for (const c of configs.values()) { @@ -186,9 +185,9 @@ export async function createClient ( for (const id of c.transactions) { excluded.add(id as Ref) } - const exclude = systemTx.filter(t => excluded.has(t._id)) + const exclude = systemTx.filter((t) => excluded.has(t._id)) console.log('exclude plugin', c.pluginId, exclude.length) - systemTx = systemTx.filter(t => !excluded.has(t._id)) + systemTx = systemTx.filter((t) => !excluded.has(t._id)) } } } diff --git a/packages/core/src/memdb.ts b/packages/core/src/memdb.ts index 9ff4007917..cef6e08edb 100644 --- a/packages/core/src/memdb.ts +++ b/packages/core/src/memdb.ts @@ -23,6 +23,7 @@ import { matchQuery, resultSort } from './query' import type { DocumentQuery, FindOptions, FindResult, LookupData, Storage, TxResult, WithLookup } from './storage' import type { Tx, TxCreateDoc, TxMixin, TxPutBag, TxRemoveDoc, TxUpdateDoc } from './tx' import { TxProcessor } from './tx' +import { toFindResult } from './utils' /** * @public @@ -77,7 +78,7 @@ export abstract class MemDb extends TxProcessor { return doc as T } - private async getLookupValue (doc: T, lookup: Lookup, result: LookupData): Promise { + private async getLookupValue(doc: T, lookup: Lookup, result: LookupData): Promise { for (const key in lookup) { if (key === '_id') { await this.getReverseLookupValue(doc, lookup, result) @@ -101,7 +102,11 @@ export abstract class MemDb extends TxProcessor { } } - private async getReverseLookupValue (doc: T, lookup: ReverseLookups, result: LookupData): Promise { + private async getReverseLookupValue( + doc: T, + lookup: ReverseLookups, + result: LookupData + ): Promise { for (const key in lookup._id) { const value = lookup._id[key] if (Array.isArray(value)) { @@ -129,7 +134,7 @@ export abstract class MemDb extends TxProcessor { query: DocumentQuery, options?: FindOptions ): Promise> { - let result: Doc[] + let result: WithLookup[] const baseClass = this.hierarchy.getBaseClass(_class) if ( Object.prototype.hasOwnProperty.call(query, '_id') && @@ -144,16 +149,17 @@ export abstract class MemDb extends TxProcessor { if (baseClass !== _class) { // We need to filter instances without mixin was set - result = result.filter(r => (r as any)[_class] !== undefined) + result = result.filter((r) => (r as any)[_class] !== undefined) } if (options?.lookup !== undefined) result = await this.lookup(result as T[], options.lookup) if (options?.sort !== undefined) resultSort(result, options?.sort) - + const total = result.length result = result.slice(0, options?.limit) - const tresult = clone(result) as T[] - return tresult.map(it => this.hierarchy.updateLookupMixin(_class, it, options)) + const tresult = clone(result) as WithLookup[] + const res = tresult.map((it) => this.hierarchy.updateLookupMixin(_class, it, options)) + return toFindResult(res, total) } addDoc (doc: Doc): void { diff --git a/packages/core/src/storage.ts b/packages/core/src/storage.ts index 90c9a48949..58ea8d17a8 100644 --- a/packages/core/src/storage.ts +++ b/packages/core/src/storage.ts @@ -145,7 +145,9 @@ export type WithLookup = T & { /** * @public */ -export type FindResult = WithLookup[] +export type FindResult = WithLookup[] & { + total: number +} /** * @public diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index 3d14e823e5..a46a6256f4 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -14,6 +14,7 @@ // import type { Account, Doc, Ref } from './classes' +import { FindResult } from './storage' function toHex (value: number, chars: number): string { const result = value.toString(16) @@ -50,7 +51,9 @@ let currentAccount: Account * @public * @returns */ -export function getCurrentAccount (): Account { return currentAccount } +export function getCurrentAccount (): Account { + return currentAccount +} /** * @public @@ -65,3 +68,11 @@ export function setCurrentAccount (account: Account): void { export function escapeLikeForRegexp (value: string): string { return value.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&') } + +/** + * @public + */ +export function toFindResult (docs: T[], total?: number): FindResult { + const length = total ?? docs.length + return Object.assign(docs, { total: length }) +} diff --git a/packages/query/src/index.ts b/packages/query/src/index.ts index 34af407be5..95c0677946 100644 --- a/packages/query/src/index.ts +++ b/packages/query/src/index.ts @@ -41,7 +41,8 @@ import core, { TxRemoveDoc, TxResult, TxUpdateDoc, - WithLookup + WithLookup, + toFindResult } from '@anticrm/core' import justClone from 'just-clone' @@ -50,6 +51,7 @@ interface Query { query: DocumentQuery result: Doc[] | Promise options?: FindOptions + total: number callback: (result: FindResult) => void } @@ -124,12 +126,14 @@ export class LiveQuery extends TxProcessor implements Client { _class, query, result, + total: 0, options: options as FindOptions, callback: callback as (result: Doc[]) => void } this.queries.push(q) result .then((result) => { + q.total = result.total q.callback(result) }) .catch((err) => { @@ -155,7 +159,7 @@ export class LiveQuery extends TxProcessor implements Client { doc[tx.bag] = bag = {} } bag[tx.key] = tx.value - await this.callback(updatedDoc, q) + await this.updatedDocCallback(updatedDoc, q) } } return {} @@ -175,7 +179,7 @@ export class LiveQuery extends TxProcessor implements Client { if (updatedDoc !== undefined) { // Create or apply mixin value updatedDoc = TxProcessor.updateMixin4Doc(updatedDoc, tx.mixin, tx.attributes) - await this.callback(updatedDoc, q) + await this.updatedDocCallback(updatedDoc, q) } else { if (this.getHierarchy().isDerived(tx.mixin, q._class)) { // Mixin potentially added to object we doesn't have in out results @@ -241,6 +245,7 @@ export class LiveQuery extends TxProcessor implements Client { const match = await this.findOne(q._class, { $search: q.query.$search, _id: tx.objectId }, q.options) if (match === undefined) { q.result.splice(pos, 1) + q.total-- } else { q.result[pos] = match } @@ -253,18 +258,20 @@ export class LiveQuery extends TxProcessor implements Client { q.result[pos] = current } else { q.result.splice(pos, 1) + q.total-- } } else { await this.__updateDoc(q, updatedDoc, tx) if (!this.match(q, updatedDoc)) { q.result.splice(pos, 1) + q.total-- } else { q.result[pos] = updatedDoc } } } this.sort(q, tx) - await this.callback(q.result[pos], q) + await this.updatedDocCallback(q.result[pos], q) } else if (this.matchQuery(q, tx)) { return await this.refresh(q) } @@ -284,7 +291,7 @@ export class LiveQuery extends TxProcessor implements Client { if (q.options?.sort !== undefined) { resultSort(q.result, q.options?.sort) } - q.callback(this.clone(q.result)) + await this.callback(q) } } @@ -328,9 +335,10 @@ export class LiveQuery extends TxProcessor implements Client { } private async refresh (q: Query): Promise { - q.result = this.client.findAll(q._class, q.query, q.options) - q.result = await q.result - q.callback(this.clone(q.result)) + const res = await this.client.findAll(q._class, q.query, q.options) + q.result = res + q.total = res.total + await this.callback(q) } // Check if query is partially matched. @@ -442,6 +450,7 @@ export class LiveQuery extends TxProcessor implements Client { if (match === undefined) return } q.result.push(doc) + q.total++ if (q.options?.sort !== undefined) { resultSort(q.result, q.options?.sort) @@ -449,16 +458,24 @@ export class LiveQuery extends TxProcessor implements Client { if (q.options?.limit !== undefined && q.result.length > q.options.limit) { if (q.result.pop()?._id !== doc._id) { - q.callback(this.clone(q.result)) + await this.callback(q) } } else { - q.callback(this.clone(q.result)) + await this.callback(q) } } await this.handleDocAddLookup(q, doc) } + private async callback (q: Query): Promise { + if (q.result instanceof Promise) { + q.result = await q.result + } + const clone = this.clone(q.result) + q.callback(toFindResult(clone, q.total)) + } + private async handleDocAddLookup (q: Query, doc: Doc): Promise { if (q.options?.lookup === undefined) return const lookup = q.options.lookup @@ -472,7 +489,7 @@ export class LiveQuery extends TxProcessor implements Client { if (q.options?.sort !== undefined) { resultSort(q.result, q.options?.sort) } - q.callback(this.clone(q.result)) + await this.callback(q) } } @@ -529,7 +546,8 @@ export class LiveQuery extends TxProcessor implements Client { const index = q.result.findIndex((p) => p._id === tx.objectId) if (index > -1) { q.result.splice(index, 1) - q.callback(this.clone(q.result)) + q.total-- + await this.callback(q) } await this.handleDocRemoveLookup(q, tx) } @@ -568,7 +586,7 @@ export class LiveQuery extends TxProcessor implements Client { if (q.options?.sort !== undefined) { resultSort(q.result, q.options?.sort) } - q.callback(this.clone(q.result)) + await this.callback(q) } } @@ -717,16 +735,18 @@ export class LiveQuery extends TxProcessor implements Client { return false } - private async callback (updatedDoc: Doc, q: Query): Promise { + private async updatedDocCallback (updatedDoc: Doc, q: Query): Promise { q.result = q.result as Doc[] if (q.options?.limit !== undefined && q.result.length > q.options.limit) { if (q.result[q.options?.limit]._id === updatedDoc._id) { return await this.refresh(q) } - if (q.result.pop()?._id !== updatedDoc._id) q.callback(q.result) + if (q.result.pop()?._id !== updatedDoc._id) { + await this.callback(q) + } } else { - q.callback(this.clone(q.result)) + await this.callback(q) } } } diff --git a/plugins/contact-resources/src/components/CreatePerson.svelte b/plugins/contact-resources/src/components/CreatePerson.svelte index f90b5487c7..723b5371c7 100644 --- a/plugins/contact-resources/src/components/CreatePerson.svelte +++ b/plugins/contact-resources/src/components/CreatePerson.svelte @@ -78,7 +78,7 @@ let channels: AttachedData[] = [] - let matches: FindResult = [] + let matches: Person[] = [] $: findPerson(client, { ...object, name: combineName(firstName, lastName) }, channels).then((p) => { matches = p }) diff --git a/plugins/contact/src/index.ts b/plugins/contact/src/index.ts index 29cb4de4c2..db14ace64b 100644 --- a/plugins/contact/src/index.ts +++ b/plugins/contact/src/index.ts @@ -13,7 +13,19 @@ // limitations under the License. // -import type { Account, AttachedData, AttachedDoc, Class, Client, Data, Doc, FindResult, Ref, Space, UXObject } from '@anticrm/core' +import { + Account, + AttachedData, + AttachedDoc, + Class, + Client, + Data, + Doc, + FindResult, + Ref, + Space, + UXObject +} from '@anticrm/core' import type { Asset, Plugin } from '@anticrm/platform' import { IntlString, plugin } from '@anticrm/platform' import type { AnyComponent } from '@anticrm/ui' @@ -61,15 +73,12 @@ export interface Contact extends Doc { /** * @public */ -export interface Person extends Contact { -} +export interface Person extends Contact {} /** * @public */ -export interface Organization extends Contact { - -} +export interface Organization extends Contact {} /** * @public @@ -180,37 +189,47 @@ export default contactPlugin /** * @public */ -export async function findPerson (client: Client, person: Data, channels: AttachedData[]): Promise> { +export async function findPerson ( + client: Client, + person: Data, + channels: AttachedData[] +): Promise { if (channels.length === 0 || person.name.length === 0) { return [] } // Take only first part of first name for match. - const values = channels.map(it => it.value) + const values = channels.map((it) => it.value) // Same name persons const potentialChannels = await client.findAll(contactPlugin.class.Channel, { value: { $in: values } }) - let potentialPersonIds = Array.from(new Set(potentialChannels.map(it => it.attachedTo as Ref)).values()) + let potentialPersonIds = Array.from(new Set(potentialChannels.map((it) => it.attachedTo as Ref)).values()) if (potentialPersonIds.length === 0) { const firstName = getFirstName(person.name).split(' ').shift() ?? '' const lastName = getLastName(person.name) // try match using just first/last name - potentialPersonIds = (await client.findAll(contactPlugin.class.Person, { name: { $like: `${lastName}%${firstName}%` } })).map(it => it._id) + potentialPersonIds = ( + await client.findAll(contactPlugin.class.Person, { name: { $like: `${lastName}%${firstName}%` } }) + ).map((it) => it._id) if (potentialPersonIds.length === 0) { return [] } } - const potentialPersons: FindResult = await client.findAll(contactPlugin.class.Person, { _id: { $in: potentialPersonIds } }, { - lookup: { - _id: { - channels: contactPlugin.class.Channel + const potentialPersons: FindResult = await client.findAll( + contactPlugin.class.Person, + { _id: { $in: potentialPersonIds } }, + { + lookup: { + _id: { + channels: contactPlugin.class.Channel + } } } - }) + ) - const result: FindResult = [] + const result: Person[] = [] for (const c of potentialPersons) { let matches = 0 @@ -220,7 +239,7 @@ export async function findPerson (client: Client, person: Data, channels if (c.city === person.city) { matches++ } - for (const ch of c.$lookup?.channels as Channel[] ?? []) { + for (const ch of (c.$lookup?.channels as Channel[]) ?? []) { for (const chc of channels) { if (chc.provider === ch.provider && chc.value === ch.value.trim()) { // We have matched value diff --git a/plugins/devmodel-resources/src/index.ts b/plugins/devmodel-resources/src/index.ts index ecdbc22f15..1854084ffa 100644 --- a/plugins/devmodel-resources/src/index.ts +++ b/plugins/devmodel-resources/src/index.ts @@ -13,13 +13,27 @@ // limitations under the License. // +import core, { + Class, + Client, + Doc, + DocumentQuery, + FindOptions, + FindResult, + Hierarchy, + ModelDb, + Ref, + toFindResult, + Tx, + TxResult, + WithLookup +} from '@anticrm/core' +import { Builder } from '@anticrm/model' import { getMetadata, IntlString, Resources } from '@anticrm/platform' +import view from '@anticrm/view' +import workbench from '@anticrm/workbench' import ModelView from './components/ModelView.svelte' import QueryView from './components/QueryView.svelte' -import core, { Class, Client, Doc, DocumentQuery, FindOptions, Ref, FindResult, Hierarchy, ModelDb, Tx, TxResult, WithLookup, Metrics } from '@anticrm/core' -import { Builder } from '@anticrm/model' -import workbench from '@anticrm/workbench' -import view from '@anticrm/view' import devmodel from './plugin' export interface TxWitHResult { @@ -61,19 +75,53 @@ class ModelClient implements Client { return this.client.getModel() } - async findOne (_class: Ref>, query: DocumentQuery, options?: FindOptions): Promise | undefined> { + async findOne( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise | undefined> { const result = await this.client.findOne(_class, query, options) - console.info('devmodel# findOne=>', _class, query, options, 'result => ', result, ' =>model', this.client.getModel(), getMetadata(devmodel.metadata.DevModel)) - queries.push({ _class, query, options: options as FindOptions, result: result !== undefined ? [result] : [], findOne: true }) + console.info( + 'devmodel# findOne=>', + _class, + query, + options, + 'result => ', + result, + ' =>model', + this.client.getModel(), + getMetadata(devmodel.metadata.DevModel) + ) + queries.push({ + _class, + query, + options: options as FindOptions, + result: toFindResult(result !== undefined ? [result] : []), + findOne: true + }) if (queries.length > 100) { queries.shift() } return result } - async findAll(_class: Ref>, query: DocumentQuery, options?: FindOptions): Promise> { + async findAll( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise> { const result = await this.client.findAll(_class, query, options) - console.info('devmodel# findAll=>', _class, query, options, 'result => ', result, ' =>model', this.client.getModel(), getMetadata(devmodel.metadata.DevModel)) + console.info( + 'devmodel# findAll=>', + _class, + query, + options, + 'result => ', + result, + ' =>model', + this.client.getModel(), + getMetadata(devmodel.metadata.DevModel) + ) queries.push({ _class, query, options: options as FindOptions, result, findOne: false }) if (queries.length > 100) { queries.shift() @@ -101,29 +149,33 @@ export async function Hook (client: Client): Promise { // Client is alive here, we could hook with some model extensions special for DevModel plugin. const builder = new Builder() - builder.createDoc(workbench.class.Application, core.space.Model, { - label: 'DevModel' as IntlString, - icon: view.icon.Table, - hidden: false, - navigatorModel: { - spaces: [ - ], - specials: [ - { - label: 'Transactions' as IntlString, - icon: view.icon.Table, - id: 'transactions', - component: devmodel.component.ModelView - }, - { - label: 'Queries' as IntlString, - icon: view.icon.Table, - id: 'queries', - component: devmodel.component.QueryView - } - ] - } - }, devmodel.ids.DevModelApp) + builder.createDoc( + workbench.class.Application, + core.space.Model, + { + label: 'DevModel' as IntlString, + icon: view.icon.Table, + hidden: false, + navigatorModel: { + spaces: [], + specials: [ + { + label: 'Transactions' as IntlString, + icon: view.icon.Table, + id: 'transactions', + component: devmodel.component.ModelView + }, + { + label: 'Queries' as IntlString, + icon: view.icon.Table, + id: 'queries', + component: devmodel.component.QueryView + } + ] + } + }, + devmodel.ids.DevModelApp + ) const model = client.getModel() for (const tx of builder.getTxes()) { diff --git a/plugins/recruit-resources/src/components/CreateCandidate.svelte b/plugins/recruit-resources/src/components/CreateCandidate.svelte index 16c8392298..dc6cc890cb 100644 --- a/plugins/recruit-resources/src/components/CreateCandidate.svelte +++ b/plugins/recruit-resources/src/components/CreateCandidate.svelte @@ -378,7 +378,7 @@ ] } - let matches: FindResult = [] + let matches: Person[] = [] $: findPerson(client, { ...object, name: combineName(firstName, lastName) }, channels).then((p) => { matches = p }) diff --git a/plugins/recruit-resources/src/components/review/CreateReview.svelte b/plugins/recruit-resources/src/components/review/CreateReview.svelte index 08c4a348ec..cca0b67998 100644 --- a/plugins/recruit-resources/src/components/review/CreateReview.svelte +++ b/plugins/recruit-resources/src/components/review/CreateReview.svelte @@ -22,7 +22,7 @@ import type { Candidate, Review } from '@anticrm/recruit' import task, { SpaceWithStates } from '@anticrm/task' import { StyledTextBox } from '@anticrm/text-editor' - import ui, { DateRangePicker, Grid, Status as StatusControl, StylishEdit, EditBox, Row } from '@anticrm/ui' + import { DateRangePicker, Grid, Status as StatusControl, EditBox, Row } from '@anticrm/ui' import view from '@anticrm/view' import { createEventDispatcher } from 'svelte' import recruit from '../../plugin' diff --git a/plugins/task-resources/src/index.ts b/plugins/task-resources/src/index.ts index 1114623613..cf30a61cdb 100644 --- a/plugins/task-resources/src/index.ts +++ b/plugins/task-resources/src/index.ts @@ -93,15 +93,21 @@ async function UnarchiveSpace (object: SpaceWithStates): Promise { ) } -export async function queryTask (_class: Ref>, client: Client, search: string): Promise { +export async function queryTask ( + _class: Ref>, + client: Client, + search: string +): Promise { const cl = client.getHierarchy().getClass(_class) - const shortLabel = (await translate(cl.shortLabel ?? '' as IntlString, {})).toUpperCase() + const shortLabel = (await translate(cl.shortLabel ?? ('' as IntlString), {})).toUpperCase() // Check number pattern const sequence = (await client.findOne(task.class.Sequence, { attachedTo: _class }))?.sequence ?? 0 - const named = new Map((await client.findAll(_class, { name: { $like: `%${search}%` } }, { limit: 200 })).map(e => [e._id, e])) + const named = new Map( + (await client.findAll(_class, { name: { $like: `%${search}%` } }, { limit: 200 })).map((e) => [e._id, e]) + ) const nids: number[] = [] if (sequence > 0) { for (let n = 0; n < sequence; n++) { @@ -110,7 +116,7 @@ export async function queryTask (_class: Ref>, client: nids.push(n) } } - const numbered = await client.findAll(_class, { number: { $in: nids } }, { limit: 200 }) as D[] + const numbered = await client.findAll(_class, { number: { $in: nids } }, { limit: 200 }) for (const d of numbered) { if (!named.has(d._id)) { named.set(d._id, d) @@ -118,7 +124,7 @@ export async function queryTask (_class: Ref>, client: } } - return Array.from(named.values()).map(e => ({ + return Array.from(named.values()).map((e) => ({ doc: e, title: `${shortLabel}-${e.number}`, icon: task.icon.Task, diff --git a/server/core/src/fulltext.ts b/server/core/src/fulltext.ts index 37771a8bce..404d99e660 100644 --- a/server/core/src/fulltext.ts +++ b/server/core/src/fulltext.ts @@ -40,7 +40,8 @@ import core, { TxPutBag, TxRemoveDoc, TxResult, - TxUpdateDoc + TxUpdateDoc, + toFindResult } from '@anticrm/core' import type { FullTextAdapter, IndexedDoc, WithFind } from './types' @@ -141,13 +142,14 @@ export class FullTextIndex implements WithFind { ): Promise> { console.log('search', query) const { _id, $search, ...mainQuery } = query - if ($search === undefined) return [] + if ($search === undefined) return toFindResult([]) let skip = 0 - const result: FindResult = [] + const result: FindResult = toFindResult([]) while (true) { const docs = await this.adapter.search(_class, query, options?.limit, skip) if (docs.length === 0) { + result.total = result.length return result } skip += docs.length @@ -158,7 +160,9 @@ export class FullTextIndex implements WithFind { } } const resultIds = getResultIds(ids, _id) - result.push(...await this.dbStorage.findAll(ctx, _class, { _id: { $in: resultIds }, ...mainQuery }, options)) + const current = await this.dbStorage.findAll(ctx, _class, { _id: { $in: resultIds }, ...mainQuery }, options) + result.push(...current) + result.total += current.total if (result.length > 0 && result.length >= (options?.limit ?? 0)) { return result } diff --git a/server/mongo/src/__tests__/storage.test.ts b/server/mongo/src/__tests__/storage.test.ts index bbace83922..ea7cc86703 100644 --- a/server/mongo/src/__tests__/storage.test.ts +++ b/server/mongo/src/__tests__/storage.test.ts @@ -25,12 +25,16 @@ import core, { FindOptions, FindResult, generateId, - Hierarchy, MeasureMetricsContext, ModelDb, Ref, + Hierarchy, + MeasureMetricsContext, + ModelDb, + Ref, SortingOrder, Space, Tx, TxOperations, - TxResult + TxResult, + toFindResult } from '@anticrm/core' import { createServerStorage, DbAdapter, DbConfiguration, FullTextAdapter, IndexedDoc } from '@anticrm/server-core' import { MongoClient } from 'mongodb' @@ -50,7 +54,7 @@ class NullDbAdapter implements DbAdapter { query: DocumentQuery, options?: FindOptions | undefined ): Promise> { - return [] + return toFindResult([]) } async tx (tx: Tx): Promise { @@ -78,9 +82,7 @@ class NullFullTextAdapter implements FullTextAdapter { return [] } - async remove (id: Ref): Promise { - - } + async remove (id: Ref): Promise {} async close (): Promise {} } @@ -276,43 +278,76 @@ describe('mongo operations', () => { rate: 20 }) - const commentId = await operations.addCollection(taskPlugin.class.TaskComment, '' as Ref, docId, taskPlugin.class.Task, 'tasks', { - message: 'my-msg', - date: new Date() - }) - - await operations.addCollection(taskPlugin.class.TaskComment, '' as Ref, docId, taskPlugin.class.Task, 'tasks', { - message: 'my-msg2', - date: new Date() - }) - - const r2 = await client.findAll(taskPlugin.class.TaskComment, {}, { - lookup: { - attachedTo: taskPlugin.class.Task + const commentId = await operations.addCollection( + taskPlugin.class.TaskComment, + '' as Ref, + docId, + taskPlugin.class.Task, + 'tasks', + { + message: 'my-msg', + date: new Date() } - }) + ) + + await operations.addCollection( + taskPlugin.class.TaskComment, + '' as Ref, + docId, + taskPlugin.class.Task, + 'tasks', + { + message: 'my-msg2', + date: new Date() + } + ) + + const r2 = await client.findAll( + taskPlugin.class.TaskComment, + {}, + { + lookup: { + attachedTo: taskPlugin.class.Task + } + } + ) expect(r2.length).toEqual(2) expect((r2[0].$lookup?.attachedTo as Task)?._id).toEqual(docId) - const r3 = await client.findAll(taskPlugin.class.Task, {}, { - lookup: { - _id: { comment: taskPlugin.class.TaskComment } + const r3 = await client.findAll( + taskPlugin.class.Task, + {}, + { + lookup: { + _id: { comment: taskPlugin.class.TaskComment } + } } - }) + ) expect(r3).toHaveLength(1) expect((r3[0].$lookup as any).comment).toHaveLength(2) - const comment2Id = await operations.addCollection(taskPlugin.class.TaskComment, '' as Ref, commentId, taskPlugin.class.TaskComment, 'comments', { - message: 'my-msg3', - date: new Date() - }) + const comment2Id = await operations.addCollection( + taskPlugin.class.TaskComment, + '' as Ref, + commentId, + taskPlugin.class.TaskComment, + 'comments', + { + message: 'my-msg3', + date: new Date() + } + ) - const r4 = await client.findAll(taskPlugin.class.TaskComment, { - _id: comment2Id - }, { - lookup: { attachedTo: [taskPlugin.class.TaskComment, { attachedTo: taskPlugin.class.Task } as any] } - }) + const r4 = await client.findAll( + taskPlugin.class.TaskComment, + { + _id: comment2Id + }, + { + lookup: { attachedTo: [taskPlugin.class.TaskComment, { attachedTo: taskPlugin.class.Task } as any] } + } + ) expect((r4[0].$lookup?.attachedTo as TaskComment)?._id).toEqual(commentId) expect(((r4[0].$lookup?.attachedTo as any)?.$lookup.attachedTo as Task)?._id).toEqual(docId) }) diff --git a/server/mongo/src/storage.ts b/server/mongo/src/storage.ts index ad4660005c..71af7e2c85 100644 --- a/server/mongo/src/storage.ts +++ b/server/mongo/src/storage.ts @@ -16,12 +16,29 @@ import core, { Class, Doc, - DocumentQuery, DOMAIN_MODEL, DOMAIN_TX, escapeLikeForRegexp, FindOptions, FindResult, Hierarchy, isOperator, Lookup, Mixin, ModelDb, Ref, ReverseLookups, SortingOrder, Tx, + DocumentQuery, + DOMAIN_MODEL, + DOMAIN_TX, + escapeLikeForRegexp, + FindOptions, + FindResult, + Hierarchy, + isOperator, + Lookup, + Mixin, + ModelDb, + Ref, + ReverseLookups, + SortingOrder, + Tx, TxCreateDoc, - TxMixin, TxProcessor, TxPutBag, + TxMixin, + TxProcessor, + TxPutBag, TxRemoveDoc, TxResult, - TxUpdateDoc + TxUpdateDoc, + toFindResult } from '@anticrm/core' import type { DbAdapter, TxAdapter } from '@anticrm/server-core' import { Collection, Db, Document, Filter, MongoClient, Sort } from 'mongodb' @@ -39,7 +56,12 @@ interface LookupStep { } abstract class MongoAdapterBase extends TxProcessor { - constructor (protected readonly db: Db, protected readonly hierarchy: Hierarchy, protected readonly modelDb: ModelDb, protected readonly client: MongoClient) { + constructor ( + protected readonly db: Db, + protected readonly hierarchy: Hierarchy, + protected readonly modelDb: ModelDb, + protected readonly client: MongoClient + ) { super() } @@ -60,7 +82,10 @@ abstract class MongoAdapterBase extends TxProcessor { if (keys[0] === '$like') { const pattern = value.$like as string translated[tkey] = { - $regex: `^${pattern.split('%').map(it => escapeLikeForRegexp(it)).join('.*')}$`, + $regex: `^${pattern + .split('%') + .map((it) => escapeLikeForRegexp(it)) + .join('.*')}$`, $options: 'i' } continue @@ -84,7 +109,7 @@ abstract class MongoAdapterBase extends TxProcessor { return translated } - private async getLookupValue (lookup: Lookup, result: LookupStep[], parent?: string): Promise { + private async getLookupValue(lookup: Lookup, result: LookupStep[], parent?: string): Promise { for (const key in lookup) { if (key === '_id') { await this.getReverseLookupValue(lookup, result, parent) @@ -119,7 +144,11 @@ abstract class MongoAdapterBase extends TxProcessor { } } - private async getReverseLookupValue (lookup: ReverseLookups, result: LookupStep[], parent?: string): Promise { + private async getReverseLookupValue ( + lookup: ReverseLookups, + result: LookupStep[], + parent?: string + ): Promise { const fullKey = parent !== undefined ? parent + '.' + '_id' : '_id' for (const key in lookup._id) { const as = parent !== undefined ? parent + key : key @@ -147,14 +176,20 @@ abstract class MongoAdapterBase extends TxProcessor { } } - private async getLookups (lookup: Lookup | undefined, parent?: string): Promise { + private async getLookups(lookup: Lookup | undefined, parent?: string): Promise { if (lookup === undefined) return [] const result: [] = [] await this.getLookupValue(lookup, result, parent) return result } - private async fillLookup (_class: Ref>, object: any, key: string, fullKey: string, targetObject: any): Promise { + private async fillLookup( + _class: Ref>, + object: any, + key: string, + fullKey: string, + targetObject: any + ): Promise { if (targetObject.$lookup === undefined) { targetObject.$lookup = {} } @@ -173,7 +208,12 @@ abstract class MongoAdapterBase extends TxProcessor { } } - private async fillLookupValue (lookup: Lookup | undefined, object: any, parent?: string, parentObject?: any): Promise { + private async fillLookupValue( + lookup: Lookup | undefined, + object: any, + parent?: string, + parentObject?: any + ): Promise { if (lookup === undefined) return for (const key in lookup) { if (key === '_id') { @@ -193,7 +233,12 @@ abstract class MongoAdapterBase extends TxProcessor { } } - private async fillReverseLookup (lookup: ReverseLookups, object: any, parent?: string, parentObject?: any): Promise { + private async fillReverseLookup ( + lookup: ReverseLookups, + object: any, + parent?: string, + parentObject?: any + ): Promise { const targetObject = parentObject ?? object if (targetObject.$lookup === undefined) { targetObject.$lookup = {} @@ -316,7 +361,7 @@ abstract class MongoAdapterBase extends TxProcessor { if (options?.projection !== undefined) { cursor = cursor.project(options.projection) } - + let total: number | undefined if (options !== null && options !== undefined) { if (options.sort !== undefined) { const sort: Sort = {} @@ -328,10 +373,12 @@ abstract class MongoAdapterBase extends TxProcessor { cursor = cursor.sort(sort) } if (options.limit !== undefined) { + total = await cursor.count() cursor = cursor.limit(options.limit) } } - return await cursor.toArray() + const res = await cursor.toArray() + return toFindResult(res, total) } } @@ -400,18 +447,16 @@ class MongoAdapter extends MongoAdapterBase { ) } } else { - return await this.db - .collection(domain) - .updateOne( - { _id: tx.objectId }, - { - $set: { - ...this.translateMixinAttrs(tx.mixin, tx.attributes), - modifiedBy: tx.modifiedBy, - modifiedOn: tx.modifiedOn - } + return await this.db.collection(domain).updateOne( + { _id: tx.objectId }, + { + $set: { + ...this.translateMixinAttrs(tx.mixin, tx.attributes), + modifiedBy: tx.modifiedBy, + modifiedOn: tx.modifiedOn } - ) + } + ) } } @@ -569,12 +614,16 @@ class MongoTxAdapter extends MongoAdapterBase implements TxAdapter { } async getModel (): Promise { - const model = await this.db.collection(DOMAIN_TX).find({ objectSpace: core.space.Model }).sort({ _id: 1 }).toArray() + const model = await this.db + .collection(DOMAIN_TX) + .find({ objectSpace: core.space.Model }) + .sort({ _id: 1 }) + .toArray() // We need to put all core.account.System transactions first const systemTr: Tx[] = [] const userTx: Tx[] = [] - model.forEach(tx => ((tx.modifiedBy === core.account.System) ? systemTr : userTx).push(tx)) + model.forEach((tx) => (tx.modifiedBy === core.account.System ? systemTr : userTx).push(tx)) return systemTr.concat(userTx) } diff --git a/server/server/src/server.ts b/server/server/src/server.ts index 409240a7df..378e6f136b 100644 --- a/server/server/src/server.ts +++ b/server/server/src/server.ts @@ -15,7 +15,21 @@ // import { Client as MinioClient } from 'minio' -import { Class, Doc, DocumentQuery, DOMAIN_MODEL, DOMAIN_TX, FindOptions, FindResult, Hierarchy, ModelDb, Ref, Tx, TxResult } from '@anticrm/core' +import { + Class, + Doc, + DocumentQuery, + DOMAIN_MODEL, + DOMAIN_TX, + FindOptions, + FindResult, + Hierarchy, + ModelDb, + Ref, + Tx, + TxResult, + toFindResult +} from '@anticrm/core' import { createElasticAdapter } from '@anticrm/elastic' import { createMongoAdapter, createMongoTxAdapter } from '@anticrm/mongo' import type { DbAdapter, DbConfiguration } from '@anticrm/server-core' @@ -41,8 +55,18 @@ import { metricsContext } from './metrics' class NullDbAdapter implements DbAdapter { async init (model: Tx[]): Promise {} - async findAll (_class: Ref>, query: DocumentQuery, options?: FindOptions | undefined): Promise> { return [] } - async tx (tx: Tx): Promise { return {} } + async findAll( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions | undefined + ): Promise> { + return toFindResult([]) + } + + async tx (tx: Tx): Promise { + return {} + } + async close (): Promise {} } @@ -62,7 +86,13 @@ export interface MinioConfig { /** * @public */ -export function start (dbUrl: string, fullTextUrl: string, minioConf: MinioConfig, port: number, host?: string): () => void { +export function start ( + dbUrl: string, + fullTextUrl: string, + minioConf: MinioConfig, + port: number, + host?: string +): () => void { addLocation(serverAttachmentId, () => import('@anticrm/server-attachment-resources')) addLocation(serverContactId, () => import('@anticrm/server-contact-resources')) addLocation(serverNotificationId, () => import('@anticrm/server-notification-resources')) @@ -77,38 +107,44 @@ export function start (dbUrl: string, fullTextUrl: string, minioConf: MinioConfi addLocation(serverGmailId, () => import('@anticrm/server-gmail-resources')) addLocation(serverTelegramId, () => import('@anticrm/server-telegram-resources')) - return startJsonRpc(metricsContext, (workspace: string) => { - const conf: DbConfiguration = { - domains: { - [DOMAIN_TX]: 'MongoTx', - [DOMAIN_MODEL]: 'Null' - }, - defaultAdapter: 'Mongo', - adapters: { - MongoTx: { - factory: createMongoTxAdapter, - url: dbUrl + return startJsonRpc( + metricsContext, + (workspace: string) => { + const conf: DbConfiguration = { + domains: { + [DOMAIN_TX]: 'MongoTx', + [DOMAIN_MODEL]: 'Null' }, - Mongo: { - factory: createMongoAdapter, - url: dbUrl + defaultAdapter: 'Mongo', + adapters: { + MongoTx: { + factory: createMongoTxAdapter, + url: dbUrl + }, + Mongo: { + factory: createMongoAdapter, + url: dbUrl + }, + Null: { + factory: createNullAdapter, + url: '' + } }, - Null: { - factory: createNullAdapter, - url: '' - } - }, - fulltextAdapter: { - factory: createElasticAdapter, - url: fullTextUrl - }, - storageFactory: () => new MinioClient({ - ...minioConf, - port: 9000, - useSSL: false - }), - workspace - } - return createServerStorage(conf) - }, port, host) + fulltextAdapter: { + factory: createElasticAdapter, + url: fullTextUrl + }, + storageFactory: () => + new MinioClient({ + ...minioConf, + port: 9000, + useSSL: false + }), + workspace + } + return createServerStorage(conf) + }, + port, + host + ) } diff --git a/server/ws/src/__tests__/server.test.ts b/server/ws/src/__tests__/server.test.ts index e8523dd7a4..4cc0e23a0f 100644 --- a/server/ws/src/__tests__/server.test.ts +++ b/server/ws/src/__tests__/server.test.ts @@ -19,22 +19,36 @@ import { start, disableLogging } from '../server' import { generateToken } from '@anticrm/server-token' import WebSocket from 'ws' -import type { Doc, Ref, Class, DocumentQuery, FindOptions, FindResult, Tx, TxResult, MeasureContext } from '@anticrm/core' -import { MeasureMetricsContext } from '@anticrm/core' +import type { + Doc, + Ref, + Class, + DocumentQuery, + FindOptions, + FindResult, + Tx, + TxResult, + MeasureContext +} from '@anticrm/core' +import { MeasureMetricsContext, toFindResult } from '@anticrm/core' describe('server', () => { disableLogging() - start(new MeasureMetricsContext('test', {}), async () => ({ - findAll: async ( - ctx: MeasureContext, - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ): Promise> => ([]), - tx: async (ctx: MeasureContext, tx: Tx): Promise<[TxResult, Tx[]]> => ([{}, []]), - close: async () => {} - }), 3333) + start( + new MeasureMetricsContext('test', {}), + async () => ({ + findAll: async ( + ctx: MeasureContext, + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise> => toFindResult([]), + tx: async (ctx: MeasureContext, tx: Tx): Promise<[TxResult, Tx[]]> => [{}, []], + close: async () => {} + }), + 3333 + ) function connect (): WebSocket { const token: string = generateToken('', 'latest') @@ -46,7 +60,9 @@ describe('server', () => { conn.on('open', () => { conn.close() }) - conn.on('close', () => { done() }) + conn.on('close', () => { + done() + }) }) it('should not connect to server without token', (done) => { @@ -54,7 +70,9 @@ describe('server', () => { conn.on('error', () => { conn.close() }) - conn.on('close', () => { done() }) + conn.on('close', () => { + done() + }) }) it('should send many requests', (done) => { @@ -74,6 +92,8 @@ describe('server', () => { conn.close() } }) - conn.on('close', () => { done() }) + conn.on('close', () => { + done() + }) }) })