UBERF-6202: Use only one mongo pull per configuration (#5073)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

plugins/workbench-resources/src/components/SelectWorkspaceMenu.svelte

@@ -134,8 +134,13 @@
   <div class="antiPopup" on:keydown={keyDown}>
     <div class="ap-space x2" />
     {#if isAdmin}
-      <div class="ml-2 mr-2 mb-2 flex-grow">
+      <div class="ml-2 mr-2 mb-2 flex-grow flex-row-center">
         <SearchEdit bind:value={search} width={'100%'} />
+        {#if isAdminUser()}
+          <div class="p-1">
+            {$workspacesStore.length}
+          </div>
+        {/if}
       </div>
     {/if}
     <div class="ap-scroll">

plugins/workbench-resources/src/components/ServerManager.svelte

@@ -1,11 +1,12 @@
 <script lang="ts">
   import contact, { PersonAccount } from '@hcengineering/contact'
-  import { Metrics } from '@hcengineering/core'
+  import { Metrics, systemAccountEmail } from '@hcengineering/core'
   import login from '@hcengineering/login'
   import { getEmbeddedLabel, getMetadata } from '@hcengineering/platform'
   import presentation, { createQuery } from '@hcengineering/presentation'
   import {
     Button,
+    CheckBox,
     IconArrowRight,
     Loading,
     Panel,
@@ -117,6 +118,8 @@
     },
     { find: 0, tx: 0 }
   )
+
+  let realUsers: boolean
 </script>
 
 <Panel on:close isFullSize useMaxWidth={true}>
@@ -180,73 +183,84 @@
         </div>
       {/if}
     {:else if selectedTab === 'users'}
+      <div class="ml-4 p-1 flex-row-center">
+        <CheckBox bind:checked={realUsers} />
+        <div class="ml-1">Show only users</div>
+      </div>
       <div class="flex-column p-3 h-full" style:overflow="auto">
         {#each Object.entries(activeSessions) as act}
           {@const wsInstance = $workspacesStore.find((it) => it.workspaceId === act[0])}
           {@const totalFind = act[1].reduce((it, itm) => itm.current.find + it, 0)}
           {@const totalTx = act[1].reduce((it, itm) => itm.current.tx + it, 0)}
-          {@const employeeGroups = Array.from(new Set(act[1].map((it) => it.userId)))}
-          <span class="flex-col">
-            <Expandable contentColor expanded={false} expandable={true} bordered>
-              <svelte:fragment slot="title">
-                <div class="fs-title">
-                  Workspace: {wsInstance?.workspaceName ?? act[0]}: {act[1].length} current 5 mins => {totalFind}/{totalTx}
-                </div>
-              </svelte:fragment>
-              <div class="flex-col">
-                {#each employeeGroups as employeeId}
-                  {@const employee = employees.get(employeeId)}
-                  {@const connections = act[1].filter((it) => it.userId === employeeId)}
-
-                  {@const find = connections.reduce((it, itm) => itm.current.find + it, 0)}
-                  {@const txes = connections.reduce((it, itm) => itm.current.tx + it, 0)}
-                  <div class="p-1 flex-col ml-4">
-                    <Expandable>
-                      <svelte:fragment slot="title">
-                        <div class="flex-row-center p-1">
-                          {#if employee}
-                            <ObjectPresenter
-                              _class={contact.mixin.Employee}
-                              objectId={employee.person}
-                              props={{ shouldShowAvatar: true, disabled: true }}
-                            />
-                          {:else}
-                            {employeeId}
-                          {/if}
-                          : {connections.length}
-                          <div class="ml-4">
-                            <div class="ml-1">{find}/{txes}</div>
-                          </div>
-                        </div>
-                      </svelte:fragment>
-                      {#each connections as user, i}
-                        <div class="flex-row-center ml-10">
-                          #{i}
-                          {user.userId}
-                          <div class="p-1">
-                            Total: {user.total.find}/{user.total.tx}
-                          </div>
-                          <div class="p-1">
-                            Previous 5 mins: {user.mins5.find}/{user.mins5.tx}
-                          </div>
-                          <div class="p-1">
-                            Current 5 mins: {user.current.find}/{user.current.tx}
-                          </div>
-                        </div>
-                        <div class="p-1 flex-col ml-10">
-                          {#each Object.entries(user.data ?? {}) as [k, v]}
-                            <div class="p-1">
-                              {k}: {JSON.stringify(v)}
-                            </div>
-                          {/each}
-                        </div>
-                      {/each}
-                    </Expandable>
+          {@const employeeGroups = Array.from(new Set(act[1].map((it) => it.userId))).filter(
+            (it) => systemAccountEmail !== it || !realUsers
+          )}
+          {@const realGroup = Array.from(new Set(act[1].map((it) => it.userId))).filter(
+            (it) => systemAccountEmail !== it
+          )}
+          {#if employeeGroups.length > 0}
+            <span class="flex-col">
+              <Expandable contentColor expanded={false} expandable={true} bordered>
+                <svelte:fragment slot="title">
+                  <div class="fs-title" class:greyed={realGroup.length === 0}>
+                    Workspace: {wsInstance?.workspaceName ?? act[0]}: {employeeGroups.length} current 5 mins => {totalFind}/{totalTx}
                   </div>
-                {/each}
-              </div>
-            </Expandable>
-          </span>
+                </svelte:fragment>
+                <div class="flex-col">
+                  {#each employeeGroups as employeeId}
+                    {@const employee = employees.get(employeeId)}
+                    {@const connections = act[1].filter((it) => it.userId === employeeId)}
+
+                    {@const find = connections.reduce((it, itm) => itm.current.find + it, 0)}
+                    {@const txes = connections.reduce((it, itm) => itm.current.tx + it, 0)}
+                    <div class="p-1 flex-col ml-4">
+                      <Expandable>
+                        <svelte:fragment slot="title">
+                          <div class="flex-row-center p-1">
+                            {#if employee}
+                              <ObjectPresenter
+                                _class={contact.mixin.Employee}
+                                objectId={employee.person}
+                                props={{ shouldShowAvatar: true, disabled: true }}
+                              />
+                            {:else}
+                              {employeeId}
+                            {/if}
+                            : {connections.length}
+                            <div class="ml-4">
+                              <div class="ml-1">{find}/{txes}</div>
+                            </div>
+                          </div>
+                        </svelte:fragment>
+                        {#each connections as user, i}
+                          <div class="flex-row-center ml-10">
+                            #{i}
+                            {user.userId}
+                            <div class="p-1">
+                              Total: {user.total.find}/{user.total.tx}
+                            </div>
+                            <div class="p-1">
+                              Previous 5 mins: {user.mins5.find}/{user.mins5.tx}
+                            </div>
+                            <div class="p-1">
+                              Current 5 mins: {user.current.find}/{user.current.tx}
+                            </div>
+                          </div>
+                          <div class="p-1 flex-col ml-10">
+                            {#each Object.entries(user.data ?? {}) as [k, v]}
+                              <div class="p-1">
+                                {k}: {JSON.stringify(v)}
+                              </div>
+                            {/each}
+                          </div>
+                        {/each}
+                      </Expandable>
+                    </div>
+                  {/each}
+                </div>
+              </Expandable>
+            </span>
+          {/if}
         {/each}
       </div>
     {:else if selectedTab === 'statistics'}
@@ -266,3 +280,9 @@
     <Loading />
   {/if}
 </Panel>
+
+<style lang="scss">
+  .greyed {
+    color: rgba(black, 0.5);
+  }
+</style>

server/mongo/src/__tests__/storage.test.ts

@@ -44,9 +44,8 @@ import {
   DummyFullTextAdapter,
   type FullTextAdapter
 } from '@hcengineering/server-core'
-import { type MongoClient } from 'mongodb'
 import { createMongoAdapter, createMongoTxAdapter } from '..'
-import { getMongoClient, shutdown } from '../utils'
+import { getMongoClient, type MongoClientReference, shutdown } from '../utils'
 import { genMinModel } from './minmodel'
 import { createTaskModel, type Task, type TaskComment, taskPlugin } from './tasks'
 
@@ -80,7 +79,7 @@ async function createNullContentTextAdapter (): Promise<ContentTextAdapter> {
 
 describe('mongo operations', () => {
   const mongodbUri: string = process.env.MONGO_URL ?? 'mongodb://localhost:27017'
-  let mongoClient!: MongoClient
+  let mongoClient!: MongoClientReference
   let dbId: string = generateId()
   let hierarchy: Hierarchy
   let model: ModelDb
@@ -88,7 +87,7 @@ describe('mongo operations', () => {
   let operations: TxOperations
 
   beforeAll(async () => {
-    mongoClient = await getMongoClient(mongodbUri)
+    mongoClient = getMongoClient(mongodbUri)
   })
 
   afterAll(async () => {
@@ -101,7 +100,7 @@ describe('mongo operations', () => {
 
   afterEach(async () => {
     try {
-      await mongoClient.db(dbId).dropDatabase()
+      await (await mongoClient.getClient()).db(dbId).dropDatabase()
     } catch (eee) {}
   })
 

server/mongo/src/storage.ts

@@ -67,11 +67,10 @@ import {
   type Db,
   type Document,
   type Filter,
-  type MongoClient,
   type Sort,
   type UpdateFilter
 } from 'mongodb'
-import { getMongoClient, getWorkspaceDB } from './utils'
+import { getMongoClient, getWorkspaceDB, type MongoClientReference } from './utils'
 
 function translateDoc (doc: Doc): Document {
   return { ...doc, '%hash%': null }
@@ -106,7 +105,7 @@ abstract class MongoAdapterBase implements DbAdapter {
     protected readonly db: Db,
     protected readonly hierarchy: Hierarchy,
     protected readonly modelDb: ModelDb,
-    protected readonly client: MongoClient
+    protected readonly client: MongoClientReference
   ) {}
 
   async init (): Promise<void> {}
@@ -1408,8 +1407,8 @@ export async function createMongoAdapter (
   workspaceId: WorkspaceId,
   modelDb: ModelDb
 ): Promise<DbAdapter> {
-  const client = await getMongoClient(url)
-  const db = getWorkspaceDB(client, workspaceId)
+  const client = getMongoClient(url)
+  const db = getWorkspaceDB(await client.getClient(), workspaceId)
 
   return new MongoAdapter(db, hierarchy, modelDb, client)
 }
@@ -1423,7 +1422,7 @@ export async function createMongoTxAdapter (
   workspaceId: WorkspaceId,
   modelDb: ModelDb
 ): Promise<TxAdapter> {
-  const client = await getMongoClient(url)
-  const db = getWorkspaceDB(client, workspaceId)
+  const client = getMongoClient(url)
+  const db = getWorkspaceDB(await client.getClient(), workspaceId)
   return new MongoTxAdapter(db, hierarchy, modelDb, client)
 }

server/mongo/src/utils.ts

@@ -16,7 +16,7 @@
 import { toWorkspaceString, type WorkspaceId } from '@hcengineering/core'
 import { type Db, MongoClient, type MongoClientOptions } from 'mongodb'
 
-let connections: MongoClient[] = []
+const connections = new Map<string, MongoClientReference>()
 
 // Register mongo close on process exit.
 process.on('exit', () => {
@@ -30,24 +30,66 @@ process.on('exit', () => {
  */
 export async function shutdown (): Promise<void> {
   for (const c of connections.values()) {
-    await c.close()
+    await c.close(true)
   }
-  connections = []
+  connections.clear()
 }
+
+export class MongoClientReference {
+  count: number
+  client: MongoClient | Promise<MongoClient>
+
+  constructor (client: MongoClient | Promise<MongoClient>) {
+    this.count = 1
+    this.client = client
+  }
+
+  async getClient (): Promise<MongoClient> {
+    if (this.client instanceof Promise) {
+      this.client = await this.client
+    }
+    return this.client
+  }
+
+  async close (force: boolean = false): Promise<void> {
+    this.count--
+    if (this.count === 0 || force) {
+      if (force) {
+        this.count = 0
+      }
+      await (await this.client).close()
+    }
+  }
+
+  addRef (): void {
+    this.count++
+  }
+}
+
 /**
  * Initialize a workspace connection to DB
  * @public
  */
-export async function getMongoClient (uri: string, options?: MongoClientOptions): Promise<MongoClient> {
+export function getMongoClient (uri: string, options?: MongoClientOptions): MongoClientReference {
   const extraOptions = JSON.parse(process.env.MONGO_OPTIONS ?? '{}')
-  const client = await MongoClient.connect(uri, {
-    ...options,
-    enableUtf8Validation: false,
-    maxConnecting: 1024,
-    ...extraOptions
-  })
-  connections.push(client)
-  return client
+  const key = `${uri}${process.env.MONGO_OPTIONS}`
+  let existing = connections.get(key)
+
+  // If not created or closed
+  if (existing === undefined || existing.count === 0) {
+    existing = new MongoClientReference(
+      MongoClient.connect(uri, {
+        ...options,
+        enableUtf8Validation: false,
+        maxConnecting: 1024,
+        ...extraOptions
+      })
+    )
+    connections.set(key, existing)
+  } else {
+    existing.addRef()
+  }
+  return existing
 }
 
 /**