From 4eac1927f0d154ac9f7633e0ebcbb8b682458d44 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Tue, 9 Jul 2024 00:04:05 +0700 Subject: [PATCH] UBERF-7532: Bulk operations for triggers (#6023) Signed-off-by: Andrey Sobolev --- dev/prod/src/platform-dev.ts | 10 +- packages/core/src/__tests__/client.test.ts | 3 - packages/core/src/__tests__/connection.ts | 1 - packages/core/src/client.ts | 17 +- packages/core/src/operations.ts | 48 +++-- packages/core/src/server.ts | 9 +- packages/core/src/tx.ts | 8 +- packages/presentation/src/pipeline.ts | 12 +- packages/presentation/src/plugin.ts | 51 +++++- packages/presentation/src/utils.ts | 90 ++++----- packages/query/src/__tests__/connection.ts | 1 - .../src/components/DirectIcon.svelte | 13 +- .../chat-message/ChatMessageInput.svelte | 30 +-- .../chat/navigator/ChatNavGroup.svelte | 127 +++++++------ .../chat/navigator/ChatNavSection.svelte | 78 ++++---- .../chat/navigator/ChatNavigator.svelte | 2 +- .../src/components/chat/utils.ts | 12 +- plugins/chunter-resources/src/utils.ts | 23 ++- plugins/client-resources/src/connection.ts | 21 --- plugins/client-resources/src/index.ts | 31 +--- plugins/client/src/index.ts | 6 - plugins/controlled-documents/src/docutils.ts | 4 +- plugins/devmodel-resources/src/index.ts | 135 +++++--------- plugins/devmodel/src/index.ts | 6 +- .../src/components/inbox/Inbox.svelte | 29 ++- .../src/inboxNotificationsClient.ts | 12 +- plugins/notification-resources/src/utils.ts | 28 +-- .../src/components/CreateIssue.svelte | 11 +- .../components/projects/CreateProject.svelte | 2 +- plugins/tracker-resources/src/issues.ts | 4 +- .../activity-resources/src/references.ts | 6 +- server-plugins/chunter-resources/src/index.ts | 37 ++-- .../src/index.ts | 6 +- server-plugins/love-resources/src/index.ts | 7 +- .../notification-resources/src/index.ts | 32 +++- server-plugins/request-resources/src/index.ts | 2 +- server-plugins/time-resources/src/index.ts | 10 +- server/core/src/server/storage.ts | 173 ++++++++++++------ server/core/src/triggers.ts | 82 +++++---- server/core/src/types.ts | 13 +- server/core/src/utils.ts | 6 +- server/middleware/src/private.ts | 9 +- server/middleware/src/spacePermissions.ts | 10 +- server/middleware/src/spaceSecurity.ts | 25 ++- server/middleware/src/utils.ts | 12 -- server/mongo/src/__tests__/storage.test.ts | 6 +- server/ws/src/client.ts | 52 ++++-- server/ws/src/server.ts | 35 +--- server/ws/src/types.ts | 2 - 49 files changed, 656 insertions(+), 693 deletions(-) diff --git a/dev/prod/src/platform-dev.ts b/dev/prod/src/platform-dev.ts index 3c89fbbea3..2805e90d85 100644 --- a/dev/prod/src/platform-dev.ts +++ b/dev/prod/src/platform-dev.ts @@ -13,11 +13,11 @@ // limitations under the License. // -import { addLocation } from '@hcengineering/platform' +import { devModelId } from '@hcengineering/devmodel' +import { PresentationClientHook } from '@hcengineering/devmodel-resources' import login from '@hcengineering/login' -import { setMetadata } from '@hcengineering/platform' -import devmodel, { devModelId } from '@hcengineering/devmodel' -import client from '@hcengineering/client' +import { addLocation, setMetadata } from '@hcengineering/platform' +import presentation from '@hcengineering/presentation' export function configurePlatformDevServer() { console.log('Use Endpoint override:', process.env.LOGIN_ENDPOINT) @@ -28,6 +28,6 @@ export function configurePlatformDevServer() { } function enableDevModel() { - setMetadata(client.metadata.ClientHook, devmodel.hook.Hook) + setMetadata(presentation.metadata.ClientHook, new PresentationClientHook()) addLocation(devModelId, () => import(/* webpackChunkName: "devmodel" */ '@hcengineering/devmodel-resources')) } diff --git a/packages/core/src/__tests__/client.test.ts b/packages/core/src/__tests__/client.test.ts index 850a3dd6f1..f6cf6d466d 100644 --- a/packages/core/src/__tests__/client.test.ts +++ b/packages/core/src/__tests__/client.test.ts @@ -122,9 +122,6 @@ describe('client', () => { clean: async (domain: Domain, docs: Ref[]) => {}, loadModel: async (last: Timestamp) => clone(txes), getAccount: async () => null as unknown as Account, - measure: async () => { - return async () => ({ time: 0, serverTime: 0 }) - }, sendForceClose: async () => {} } } diff --git a/packages/core/src/__tests__/connection.ts b/packages/core/src/__tests__/connection.ts index a79354f6bb..e38a65cae6 100644 --- a/packages/core/src/__tests__/connection.ts +++ b/packages/core/src/__tests__/connection.ts @@ -73,7 +73,6 @@ export async function connect (handler: (tx: Tx) => void): Promise[]) => {}, loadModel: async (last: Timestamp) => txes, getAccount: async () => null as unknown as Account, - measure: async () => async () => ({ time: 0, serverTime: 0 }), sendForceClose: async () => {} } } diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index ebe72fd441..148b33ddc0 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -47,17 +47,10 @@ export interface Client extends Storage, FulltextStorage { close: () => Promise } -export type MeasureDoneOperation = () => Promise<{ time: number, serverTime: number }> - -export interface MeasureClient extends Client { - // Will perform on server operation measure and will return a local client time and on server time - measure: (operationName: string) => Promise -} - /** * @public */ -export interface AccountClient extends MeasureClient { +export interface AccountClient extends Client { getAccount: () => Promise } @@ -97,11 +90,9 @@ export interface ClientConnection extends Storage, FulltextStorage, BackupClient // If hash is passed, will return LoadModelResponse loadModel: (last: Timestamp, hash?: string) => Promise getAccount: () => Promise - - measure: (operationName: string) => Promise } -class ClientImpl implements AccountClient, BackupClient, MeasureClient { +class ClientImpl implements AccountClient, BackupClient { notify?: (...tx: Tx[]) => void hierarchy!: Hierarchy model!: ModelDb @@ -163,10 +154,6 @@ class ClientImpl implements AccountClient, BackupClient, MeasureClient { return result } - async measure (operationName: string): Promise { - return await this.conn.measure(operationName) - } - async updateFromRemote (...tx: Tx[]): Promise { for (const t of tx) { try { diff --git a/packages/core/src/operations.ts b/packages/core/src/operations.ts index ff8e7406f5..f6a0b7e258 100644 --- a/packages/core/src/operations.ts +++ b/packages/core/src/operations.ts @@ -312,8 +312,8 @@ export class TxOperations implements Omit { return this.removeDoc(doc._class, doc.space, doc._id) } - apply (scope: string): ApplyOperations { - return new ApplyOperations(this, scope) + apply (scope: string, measure?: string): ApplyOperations { + return new ApplyOperations(this, scope, measure) } async diffUpdate( @@ -423,6 +423,12 @@ export class TxOperations implements Omit { } } +export interface CommitResult { + result: boolean + time: number + serverTime: number +} + /** * @public * @@ -436,7 +442,8 @@ export class ApplyOperations extends TxOperations { notMatches: DocumentClassQuery[] = [] constructor ( readonly ops: TxOperations, - readonly scope: string + readonly scope: string, + readonly measureName?: string ) { const txClient: Client = { getHierarchy: () => ops.client.getHierarchy(), @@ -465,23 +472,28 @@ export class ApplyOperations extends TxOperations { return this } - async commit (notify: boolean = true, extraNotify: Ref>[] = []): Promise { + async commit (notify: boolean = true, extraNotify: Ref>[] = []): Promise { if (this.txes.length > 0) { - return ( - await ((await this.ops.tx( - this.ops.txFactory.createTxApplyIf( - core.space.Tx, - this.scope, - this.matches, - this.notMatches, - this.txes, - notify, - extraNotify - ) - )) as Promise) - ).success + const st = Date.now() + const result = await ((await this.ops.tx( + this.ops.txFactory.createTxApplyIf( + core.space.Tx, + this.scope, + this.matches, + this.notMatches, + this.txes, + this.measureName, + notify, + extraNotify + ) + )) as Promise) + return { + result: result.success, + time: Date.now() - st, + serverTime: result.serverTime + } } - return true + return { result: true, time: 0, serverTime: 0 } } } diff --git a/packages/core/src/server.ts b/packages/core/src/server.ts index f6456af0a5..1aea148b20 100644 --- a/packages/core/src/server.ts +++ b/packages/core/src/server.ts @@ -33,14 +33,15 @@ export interface StorageIterator { close: (ctx: MeasureContext) => Promise } +export type BroadcastTargets = Record string[] | undefined> + export interface SessionOperationContext { ctx: MeasureContext // A parts of derived data to deal with after operation will be complete derived: { - derived: Tx[] - target?: string[] - }[] - + txes: Tx[] + targets: BroadcastTargets // A set of broadcast filters if required + } with: ( name: string, params: ParamsType, diff --git a/packages/core/src/tx.ts b/packages/core/src/tx.ts index f0802bd0a0..d14d85704c 100644 --- a/packages/core/src/tx.ts +++ b/packages/core/src/tx.ts @@ -28,13 +28,13 @@ import type { Space, Timestamp } from './classes' +import { clone } from './clone' import core from './component' import { setObjectValue } from './objvalue' import { _getOperator } from './operator' import { _toDoc } from './proxy' import type { DocumentQuery, TxResult } from './storage' import { generateId } from './utils' -import { clone } from './clone' /** * @public @@ -137,10 +137,14 @@ export interface TxApplyIf extends Tx { // If passed, will send WorkspaceEvent.BulkUpdate event with list of classes to update extraNotify?: Ref>[] + + // If defined will go into a separate measure section + measureName?: string } export interface TxApplyResult { success: boolean + serverTime: number } /** @@ -618,6 +622,7 @@ export class TxFactory { match: DocumentClassQuery[], notMatch: DocumentClassQuery[], txes: TxCUD[], + measureName: string | undefined, notify: boolean = true, extraNotify: Ref>[] = [], modifiedOn?: Timestamp, @@ -634,6 +639,7 @@ export class TxFactory { match, notMatch, txes, + measureName, notify, extraNotify } diff --git a/packages/presentation/src/pipeline.ts b/packages/presentation/src/pipeline.ts index 13292fea32..79f3fe6663 100644 --- a/packages/presentation/src/pipeline.ts +++ b/packages/presentation/src/pipeline.ts @@ -8,8 +8,6 @@ import { type FindOptions, type FindResult, type Hierarchy, - type MeasureClient, - type MeasureDoneOperation, type ModelDb, type Ref, type SearchOptions, @@ -65,7 +63,7 @@ export type PresentationMiddlewareCreator = (client: Client, next?: Presentation /** * @public */ -export interface PresentationPipeline extends MeasureClient, Exclude { +export interface PresentationPipeline extends Client, Exclude { close: () => Promise } @@ -75,7 +73,7 @@ export interface PresentationPipeline extends MeasureClient, Exclude { - return await this.client.measure(operationName) - } - - static create (client: MeasureClient, constructors: PresentationMiddlewareCreator[]): PresentationPipeline { + static create (client: Client, constructors: PresentationMiddlewareCreator[]): PresentationPipeline { const pipeline = new PresentationPipelineImpl(client) pipeline.head = pipeline.buildChain(constructors) return pipeline diff --git a/packages/presentation/src/plugin.ts b/packages/presentation/src/plugin.ts index 3dd6d63552..d4a7fd5f8b 100644 --- a/packages/presentation/src/plugin.ts +++ b/packages/presentation/src/plugin.ts @@ -14,26 +14,64 @@ // limitations under the License. // -import { type Mixin, type Class, type Ref } from '@hcengineering/core' +import { + type Class, + type Client, + type Doc, + type DocumentQuery, + type FindOptions, + type FindResult, + type Mixin, + type Ref, + type SearchOptions, + type SearchQuery, + type SearchResult, + type Tx, + type TxResult, + type WithLookup +} from '@hcengineering/core' import type { Asset, IntlString, Metadata, Plugin, StatusCode } from '@hcengineering/platform' import { plugin } from '@hcengineering/platform' import { type ComponentExtensionId } from '@hcengineering/ui' import { type PresentationMiddlewareFactory } from './pipeline' +import type { PreviewConfig } from './preview' import { type ComponentPointExtension, - type DocRules, type DocCreateExtension, + type DocRules, type FilePreviewExtension, - type ObjectSearchCategory, - type InstantTransactions + type InstantTransactions, + type ObjectSearchCategory } from './types' -import type { PreviewConfig } from './preview' /** * @public */ export const presentationId = 'presentation' as Plugin +/** + * @public + */ +export interface ClientHook { + findAll: ( + client: Client, + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ) => Promise> + + findOne: ( + client: Client, + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ) => Promise | undefined> + + tx: (client: Client, tx: Tx) => Promise + + searchFulltext: (client: Client, query: SearchQuery, options: SearchOptions) => Promise +} + export default plugin(presentationId, { class: { ObjectSearchCategory: '' as Ref>, @@ -95,7 +133,8 @@ export default plugin(presentationId, { CollaboratorApiUrl: '' as Metadata, Token: '' as Metadata, FrontUrl: '' as Asset, - PreviewConfig: '' as Metadata + PreviewConfig: '' as Metadata, + ClientHook: '' as Metadata }, status: { FileTooLarge: '' as StatusCode diff --git a/packages/presentation/src/utils.ts b/packages/presentation/src/utils.ts index a454bf12b0..37e195e6d3 100644 --- a/packages/presentation/src/utils.ts +++ b/packages/presentation/src/utils.ts @@ -33,8 +33,6 @@ import core, { type FindOptions, type FindResult, type Hierarchy, - type MeasureClient, - type MeasureDoneOperation, type Mixin, type Obj, type Blob as PlatformBlob, @@ -65,7 +63,7 @@ export { reduceCalls } from '@hcengineering/core' let liveQuery: LQ let rawLiveQuery: LQ -let client: TxOperations & MeasureClient & OptimisticTxes +let client: TxOperations & Client & OptimisticTxes let pipeline: PresentationPipeline const txListeners: Array<(...tx: Tx[]) => void> = [] @@ -95,16 +93,15 @@ export interface OptimisticTxes { pendingCreatedDocs: Writable, boolean>> } -class UIClient extends TxOperations implements Client, MeasureClient, OptimisticTxes { +class UIClient extends TxOperations implements Client, OptimisticTxes { + hook = getMetadata(plugin.metadata.ClientHook) constructor ( - client: MeasureClient, + client: Client, private readonly liveQuery: Client ) { super(client, getCurrentAccount()._id) } - afterMeasure: Tx[] = [] - measureOp?: MeasureDoneOperation protected pendingTxes = new Set>() protected _pendingCreatedDocs = writable, boolean>>({}) @@ -113,34 +110,30 @@ class UIClient extends TxOperations implements Client, MeasureClient, Optimistic } async doNotify (...tx: Tx[]): Promise { - if (this.measureOp !== undefined) { - this.afterMeasure.push(...tx) - } else { - const pending = get(this._pendingCreatedDocs) - let pendingUpdated = false - tx.forEach((t) => { - if (this.pendingTxes.has(t._id)) { - this.pendingTxes.delete(t._id) + const pending = get(this._pendingCreatedDocs) + let pendingUpdated = false + tx.forEach((t) => { + if (this.pendingTxes.has(t._id)) { + this.pendingTxes.delete(t._id) - // Only CUD tx can be pending now - const innerTx = TxProcessor.extractTx(t) as TxCUD + // Only CUD tx can be pending now + const innerTx = TxProcessor.extractTx(t) as TxCUD - if (innerTx._class === core.class.TxCreateDoc) { - // eslint-disable-next-line @typescript-eslint/no-dynamic-delete - delete pending[innerTx.objectId] - pendingUpdated = true - } + if (innerTx._class === core.class.TxCreateDoc) { + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete pending[innerTx.objectId] + pendingUpdated = true } - }) - if (pendingUpdated) { - this._pendingCreatedDocs.set(pending) } - - // We still want to notify about all transactions because there might be queries created after - // the early applied transaction - // For old queries there's a check anyway that prevents the same document from being added twice - await this.provideNotify(...tx) + }) + if (pendingUpdated) { + this._pendingCreatedDocs.set(pending) } + + // We still want to notify about all transactions because there might be queries created after + // the early applied transaction + // For old queries there's a check anyway that prevents the same document from being added twice + await this.provideNotify(...tx) } private async provideNotify (...tx: Tx[]): Promise { @@ -165,6 +158,9 @@ class UIClient extends TxOperations implements Client, MeasureClient, Optimistic query: DocumentQuery, options?: FindOptions ): Promise> { + if (this.hook !== undefined) { + return await this.hook.findAll(this.liveQuery, _class, query, options) + } return await this.liveQuery.findAll(_class, query, options) } @@ -173,12 +169,17 @@ class UIClient extends TxOperations implements Client, MeasureClient, Optimistic query: DocumentQuery, options?: FindOptions ): Promise | undefined> { + if (this.hook !== undefined) { + return await this.hook.findOne(this.liveQuery, _class, query, options) + } return await this.liveQuery.findOne(_class, query, options) } override async tx (tx: Tx): Promise { void this.notifyEarly(tx) - + if (this.hook !== undefined) { + return await this.hook.tx(this.client, tx) + } return await this.client.tx(tx) } @@ -221,39 +222,24 @@ class UIClient extends TxOperations implements Client, MeasureClient, Optimistic } async searchFulltext (query: SearchQuery, options: SearchOptions): Promise { - return await this.client.searchFulltext(query, options) - } - - async measure (operationName: string): Promise { - // return await (this.client as MeasureClient).measure(operationName) - const mop = await (this.client as MeasureClient).measure(operationName) - this.measureOp = mop - return async () => { - const result = await mop() - this.measureOp = undefined - if (this.afterMeasure.length > 0) { - const txes = this.afterMeasure - this.afterMeasure = [] - for (const tx of txes) { - await this.doNotify(tx) - } - } - return result + if (this.hook !== undefined) { + return await this.hook.searchFulltext(this.client, query, options) } + return await this.client.searchFulltext(query, options) } } /** * @public */ -export function getClient (): TxOperations & MeasureClient & OptimisticTxes { +export function getClient (): TxOperations & Client & OptimisticTxes { return client } /** * @public */ -export async function setClient (_client: MeasureClient): Promise { +export async function setClient (_client: Client): Promise { if (liveQuery !== undefined) { await liveQuery.close() } @@ -276,6 +262,7 @@ export async function setClient (_client: MeasureClient): Promise { liveQuery = new LQ(pipeline) const uiClient = new UIClient(pipeline, liveQuery) + client = uiClient _client.notify = (...tx: Tx[]) => { @@ -285,7 +272,6 @@ export async function setClient (_client: MeasureClient): Promise { await refreshClient(true) } } - /** * @public */ diff --git a/packages/query/src/__tests__/connection.ts b/packages/query/src/__tests__/connection.ts index 8686246ea1..8ce3b5da53 100644 --- a/packages/query/src/__tests__/connection.ts +++ b/packages/query/src/__tests__/connection.ts @@ -98,7 +98,6 @@ FulltextStorage & { searchFulltext: async (query: SearchQuery, options: SearchOptions): Promise => { return { docs: [] } }, - measure: async () => async () => ({ time: 0, serverTime: 0 }), sendForceClose: async () => {} } } diff --git a/plugins/chunter-resources/src/components/DirectIcon.svelte b/plugins/chunter-resources/src/components/DirectIcon.svelte index b632445d6e..fb09084b17 100644 --- a/plugins/chunter-resources/src/components/DirectIcon.svelte +++ b/plugins/chunter-resources/src/components/DirectIcon.svelte @@ -14,12 +14,12 @@ -->