UBERF-8608: Rework connection management (#7248)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

packages/core/src/measurements/context.ts

@@ -114,7 +114,6 @@ export class MeasureMetricsContext implements MeasureContext {
       this.logParams
     )
     result.id = this.id
-    result.onEnd = this.onEnd.bind(this)
     result.contextData = this.contextData
     return result
   }
@@ -190,8 +189,6 @@ export class MeasureMetricsContext implements MeasureContext {
   end (): void {
     this.done()
   }
-
-  async onEnd (ctx: MeasureContext): Promise<void> {}
 }
 
 /**
@@ -220,7 +217,10 @@ export function registerOperationLog (ctx: MeasureContext): { opLogMetrics?: Met
   }
   const op: OperationLog = { start: Date.now(), ops: [], end: -1 }
   let opLogMetrics: Metrics | undefined
-  ctx.id = generateId()
+
+  if (ctx.id === undefined) {
+    ctx.id = 'op_' + generateId()
+  }
   if (ctx.metrics !== undefined) {
     if (ctx.metrics.opLog === undefined) {
       ctx.metrics.opLog = {}

packages/core/src/measurements/types.ts

@@ -111,6 +111,4 @@ export interface MeasureContext<Q = any> {
   // Mark current context as complete
   // If no value is passed, time difference will be used.
   end: (value?: number) => void
-
-  onEnd?: (ctx: MeasureContext) => Promise<void>
 }

packages/core/src/server.ts

@@ -59,6 +59,8 @@ export interface SessionData {
   branding: Branding | null
 
   fulltextUpdates?: Map<Ref<DocIndexState>, DocIndexState>
+
+  asyncRequests?: (() => Promise<void>)[]
 }
 
 /**

pods/fulltext/src/server.ts

@@ -144,9 +144,6 @@ class WorkspaceIndexer {
             await helper.checkDomain(ctx, DOMAIN_DOC_INDEX_STATE, 10000, dhelper)
           }
         }
-      },
-      async (ctx: MeasureContext) => {
-        await result.pipeline.context.adapterManager?.closeContext?.(ctx)
       }
     )
     await result.fulltext.startIndexing(() => {

server/core/src/adapter.ts

@@ -69,8 +69,7 @@ export interface DbAdapter extends LowLevelStorage {
 
   helper?: () => DomainHelperOperations
 
-  closeContext?: (ctx: MeasureContext) => Promise<void>
-
+  reserveContext?: (id: string) => () => void
   close: () => Promise<void>
   findAll: <T extends Doc>(
     ctx: MeasureContext,

server/core/src/dbAdapterManager.ts

@@ -42,16 +42,22 @@ export class DbAdapterManagerImpl implements DBAdapterManager {
     private readonly adapters: Map<string, DbAdapter>
   ) {}
 
-  async closeContext (ctx: MeasureContext): Promise<void> {
+  reserveContext (id: string): () => void {
+    const ops: (() => void)[] = []
     for (const adapter of this.adapters.values()) {
       try {
-        if (adapter.closeContext !== undefined) {
-          await adapter.closeContext(ctx)
+        if (adapter.reserveContext !== undefined) {
+          ops.push(adapter.reserveContext(id))
         }
       } catch (err: any) {
         Analytics.handleError(err)
       }
     }
+    return () => {
+      for (const op of ops) {
+        op()
+      }
+    }
   }
 
   getDefaultAdapter (): DbAdapter {

server/core/src/triggers.ts

@@ -17,7 +17,6 @@
 import core, {
   TxFactory,
   TxProcessor,
-  generateId,
   groupByArray,
   matchQuery,
   type Class,
@@ -145,13 +144,11 @@ export class Triggers {
           trigger.resource,
           {},
           async (ctx) => {
-            if (mode === 'async') {
-              ctx.id = generateId()
-            }
-            const tresult = await this.applyTrigger(ctx, ctrl, matches, { trigger })
-            result.push(...tresult)
-            if (ctx.onEnd !== undefined && mode === 'async') {
-              await ctx.onEnd(ctx)
+            try {
+              const tresult = await this.applyTrigger(ctx, ctrl, matches, { trigger })
+              result.push(...tresult)
+            } catch (err: any) {
+              ctx.error('error during async processing', { err })
             }
           },
           { count: matches.length }

server/core/src/types.ts

@@ -152,7 +152,7 @@ export interface DBAdapterManager {
 
   initAdapters: (ctx: MeasureContext) => Promise<void>
 
-  closeContext: (ctx: MeasureContext) => Promise<void>
+  reserveContext: (id: string) => () => void
 
   domainHelper?: DomainHelper
 }

server/indexer/src/indexer/indexer.ts

@@ -127,8 +127,7 @@ export class FullTextIndexPipeline implements FullTextPipeline {
     readonly storageAdapter: StorageAdapter,
     readonly contentAdapter: ContentTextAdapter,
     readonly broadcastUpdate: (ctx: MeasureContext, classes: Ref<Class<Doc>>[]) => void,
-    readonly checkIndexes: () => Promise<void>,
-    readonly closeContext: (ctx: MeasureContext) => Promise<void>
+    readonly checkIndexes: () => Promise<void>
   ) {
     this.contexts = new Map(model.findAllSync(core.class.FullTextSearchContext, {}).map((it) => [it.toClass, it]))
   }
@@ -406,7 +405,6 @@ export class FullTextIndexPipeline implements FullTextPipeline {
     await rateLimiter.exec(async () => {
       let st = Date.now()
 
-      ctx.id = generateId()
       let groupBy = await this.storage.groupBy(ctx, DOMAIN_DOC_INDEX_STATE, 'objectClass', { needIndex: true })
       const total = Array.from(groupBy.values()).reduce((a, b) => a + b, 0)
       while (true) {
@@ -524,7 +522,6 @@ export class FullTextIndexPipeline implements FullTextPipeline {
           this.metrics.error('error during index', { error: err })
         }
       }
-      await this.closeContext(ctx)
     })
     return { classUpdate: Array.from(_classUpdate.values()), processed }
   }

server/middleware/src/connectionMgr.ts

@@ -1,35 +0,0 @@
-import { generateId, type MeasureContext, type Tx } from '@hcengineering/core'
-import {
-  BaseMiddleware,
-  type Middleware,
-  type PipelineContext,
-  type TxMiddlewareResult
-} from '@hcengineering/server-core'
-
-/**
- * Will support apply tx
- * @public
- */
-export class ConnectionMgrMiddleware extends BaseMiddleware implements Middleware {
-  static async create (
-    ctx: MeasureContext,
-    context: PipelineContext,
-    next?: Middleware
-  ): Promise<Middleware | undefined> {
-    return new ConnectionMgrMiddleware(context, next)
-  }
-
-  async tx (ctx: MeasureContext, tx: Tx[]): Promise<TxMiddlewareResult> {
-    if (ctx.id === undefined) {
-      ctx.id = generateId()
-    }
-    ctx.onEnd = async (_ctx: MeasureContext) => {
-      await this.context.adapterManager?.closeContext?.(_ctx)
-    }
-    const result = await this.provideTx(ctx, tx)
-    if (ctx.onEnd !== undefined) {
-      await ctx.onEnd(ctx)
-    }
-    return result
-  }
-}

server/middleware/src/index.ts

@@ -16,7 +16,6 @@
 export * from './applyTx'
 export * from './broadcast'
 export * from './configuration'
-export * from './connectionMgr'
 export * from './contextName'
 export * from './dbAdapter'
 export * from './dbAdapterHelper'

server/middleware/src/triggers.ts

@@ -33,7 +33,6 @@ import core, {
   type TxRemoveDoc,
   type TxUpdateDoc,
   addOperation,
-  generateId,
   toFindResult,
   withContext
 } from '@hcengineering/core'
@@ -170,15 +169,18 @@ export class TriggersMiddleware extends BaseMiddleware implements Middleware {
       await this.processDerivedTxes(ctx, derived)
     }
 
-    const asyncProcess = this.processAsyncTriggers(ctx, triggerControl, findAll, txes, triggers)
-    // In case of async context, we execute both async and sync triggers as sync
+    const performSync = (ctx as MeasureContext<SessionDataImpl>).contextData.isAsyncContext ?? false
 
-    if ((ctx as MeasureContext<SessionDataImpl>).contextData.isAsyncContext ?? false) {
-      await asyncProcess
+    if (performSync) {
+      await this.processAsyncTriggers(ctx, triggerControl, findAll, txes, triggers)
     } else {
-      asyncProcess.catch((err) => {
-        ctx.error('error during processing async triggers', { err })
-      })
+      ctx.contextData.asyncRequests = [
+        ...(ctx.contextData.asyncRequests ?? []),
+        async () => {
+          // In case of async context, we execute both async and sync triggers as sync
+          await this.processAsyncTriggers(ctx, triggerControl, findAll, txes, triggers)
+        }
+      ]
     }
   }
 
@@ -224,10 +226,6 @@ export class TriggersMiddleware extends BaseMiddleware implements Middleware {
       this.context.modelDb
     )
     ctx.contextData = asyncContextData
-
-    if ((ctx as MeasureContext<SessionDataImpl>).contextData.isAsyncContext ?? false) {
-      ctx.id = 'async_tr' + generateId()
-    }
     const aresult = await this.triggers.apply(
       ctx,
       txes,
@@ -247,9 +245,6 @@ export class TriggersMiddleware extends BaseMiddleware implements Middleware {
         await this.context.head?.handleBroadcast(ctx)
       })
     }
-    if (ctx.onEnd !== undefined) {
-      await ctx.onEnd(ctx)
-    }
   }
 
   private async processDerivedTxes (ctx: MeasureContext<SessionData>, derived: Tx[]): Promise<void> {

server/postgres/src/utils.ts

@@ -272,8 +272,9 @@ export function convertDoc<T extends Doc> (
     createdBy: doc.createdBy,
     modifiedBy: doc.modifiedBy,
     modifiedOn: doc.modifiedOn,
-    createdOn: doc.createdOn,
-    _class: doc._class
+    createdOn: doc.createdOn ?? doc.modifiedOn,
+    _class: doc._class,
+    '%hash%': (doc as any)['%hash%'] ?? null
   }
   const remainingData: Partial<T> = {}
 
@@ -326,8 +327,8 @@ export function inferType (val: any): string {
 }
 
 export function parseUpdate<T extends Doc> (
-  domain: string,
-  ops: DocumentUpdate<T> | MixinUpdate<Doc, T>
+  ops: DocumentUpdate<T> | MixinUpdate<Doc, T>,
+  fields: Set<string>
 ): {
     extractedFields: Partial<T>
     remainingData: Partial<T>
@@ -339,14 +340,14 @@ export function parseUpdate<T extends Doc> (
     const val = (ops as any)[key]
     if (key.startsWith('$')) {
       for (const k in val) {
-        if (getDocFieldsByDomains(domain).includes(k)) {
+        if (fields.has(k)) {
           ;(extractedFields as any)[k] = val[key]
         } else {
           ;(remainingData as any)[k] = val[key]
         }
       }
     } else {
-      if (getDocFieldsByDomains(domain).includes(key)) {
+      if (fields.has(key)) {
         ;(extractedFields as any)[key] = val
       } else {
         ;(remainingData as any)[key] = val

server/server-pipeline/src/pipeline.ts

@@ -17,7 +17,6 @@ import {
   ApplyTxMiddleware,
   BroadcastMiddleware,
   ConfigurationMiddleware,
-  ConnectionMgrMiddleware,
   ContextNameMiddleware,
   DBAdapterInitMiddleware,
   DBAdapterMiddleware,
@@ -114,7 +113,6 @@ export function createServerPipeline (
       SpacePermissionsMiddleware.create,
       ConfigurationMiddleware.create,
       ContextNameMiddleware.create,
-      ConnectionMgrMiddleware.create,
       MarkDerivedEntryMiddleware.create,
       ApplyTxMiddleware.create, // Extract apply
       TxMiddleware.create, // Store tx into transaction domain

server/server/src/client.ts

@@ -15,6 +15,7 @@
 
 import core, {
   AccountRole,
+  generateId,
   TxFactory,
   TxProcessor,
   type Account,
@@ -29,6 +30,7 @@ import core, {
   type Ref,
   type SearchOptions,
   type SearchQuery,
+  type SessionData,
   type Timestamp,
   type Tx,
   type TxCUD,
@@ -49,6 +51,8 @@ import {
 import { type Token } from '@hcengineering/server-token'
 import { handleSend } from './utils'
 
+const useReserveContext = (process.env.USE_RESERVE_CTX ?? 'true') === 'true'
+
 /**
  * @public
  */
@@ -192,13 +196,35 @@ export class ClientSession implements Session {
     this.current.tx++
     this.includeSessionContext(ctx.ctx)
 
-    const result = await this._pipeline.tx(ctx.ctx, [tx])
+    let cid = 'client_' + generateId()
+    ctx.ctx.id = cid
+    let onEnd = useReserveContext ? this._pipeline.context.adapterManager?.reserveContext?.(cid) : undefined
+    try {
+      const result = await this._pipeline.tx(ctx.ctx, [tx])
 
-    // Send result immideately
-    await ctx.sendResponse(result)
+      // Send result immideately
+      await ctx.sendResponse(result)
 
-    // We need to broadcast all collected transactions
-    await this._pipeline.handleBroadcast(ctx.ctx)
+      // We need to broadcast all collected transactions
+      await this._pipeline.handleBroadcast(ctx.ctx)
+    } finally {
+      onEnd?.()
+    }
+
+    // ok we could perform async requests if any
+    const asyncs = (ctx.ctx.contextData as SessionData).asyncRequests ?? []
+    if (asyncs.length > 0) {
+      cid = 'client_async_' + generateId()
+      ctx.ctx.id = cid
+      onEnd = useReserveContext ? this._pipeline.context.adapterManager?.reserveContext?.(cid) : undefined
+      try {
+        for (const r of (ctx.ctx.contextData as SessionData).asyncRequests ?? []) {
+          await r()
+        }
+      } finally {
+        onEnd?.()
+      }
+    }
   }
 
   broadcast (ctx: MeasureContext, socket: ConnectionSocket, tx: Tx[]): void {