UBER-1198: Upgrade to mongo 7 (#4472)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

dev/docker-compose.yaml

@@ -1,7 +1,7 @@
 version: "3"
 services:
   mongodb:
-    image: mongo
+    image: 'mongo:7-jammy'
     container_name: mongodb
     environment:
       - PUID=1000

dev/tool/package.json

@@ -121,7 +121,7 @@
     "got": "^11.8.3",
     "libphonenumber-js": "^1.9.46",
     "mime-types": "~2.1.34",
-    "mongodb": "^4.11.0",
+    "mongodb": "^6.3.0",
     "ws": "^8.10.0",
     "xml2js": "~0.4.23"
   }

dev/tool/src/clean.ts

@@ -566,8 +566,8 @@ export async function fixSkills (
             tag: t._id
           })) as TagReference[]
           const ids = references.map((r) => r._id)
-          await db.collection(DOMAIN_TAGS).deleteMany({ _id: { $in: ids } })
+          await db.collection<Doc>(DOMAIN_TAGS).deleteMany({ _id: { $in: ids } })
-          await db.collection(DOMAIN_TAGS).deleteOne({ _id: t._id })
+          await db.collection<Doc>(DOMAIN_TAGS).deleteOne({ _id: t._id })
         }
       }
       await fixCount()
@@ -588,8 +588,8 @@ export async function fixSkills (
           tag: t._id
         })) as TagReference[]
         const ids = references.map((r) => r._id)
-        await db.collection(DOMAIN_TAGS).deleteMany({ _id: { $in: ids } })
+        await db.collection<Doc>(DOMAIN_TAGS).deleteMany({ _id: { $in: ids } })
-        await db.collection(DOMAIN_TAGS).deleteOne({ _id: t._id })
+        await db.collection<Doc>(DOMAIN_TAGS).deleteOne({ _id: t._id })
       }
       console.log('DONE 6 STEP')
     }
@@ -610,8 +610,8 @@ export async function fixSkills (
             tag: t._id
           })) as TagReference[]
           const ids = references.map((r) => r._id)
-          await db.collection(DOMAIN_TAGS).deleteMany({ _id: { $in: ids } })
+          await db.collection<Doc>(DOMAIN_TAGS).deleteMany({ _id: { $in: ids } })
-          await db.collection(DOMAIN_TAGS).deleteOne({ _id: t._id })
+          await db.collection<Doc>(DOMAIN_TAGS).deleteOne({ _id: t._id })
         }
       }
       await fixCount()

plugins/activity-resources/src/activityMessagesUtils.ts

@@ -46,7 +46,7 @@ import activity, {
 // Use 5 minutes to combine similar messages
 const combineThresholdMs = 5 * 60 * 1000
 // Use 10 seconds to combine update messages after creation.
-const createCombineThreshold = 10 * 1000
+const createCombineThreshold = parseInt(localStorage.getItem('platform.activity.threshold') ?? '10 * 1000')
 
 const valueTypes: ReadonlyArray<Ref<Class<Doc>>> = [
   core.class.TypeString,

pods/account/package.json

@@ -46,7 +46,7 @@
     "@hcengineering/account": "^0.6.0",
     "@hcengineering/platform": "^0.6.9",
     "@hcengineering/core": "^0.6.28",
-    "mongodb": "^4.11.0",
+    "mongodb": "^6.3.0",
     "koa": "^2.13.1",
     "koa-router": "^12.0.1",
     "koa-bodyparser": "^4.3.0",

pods/backup/package.json

@@ -40,7 +40,7 @@
   },
   "dependencies": {
     "@hcengineering/platform": "^0.6.9",
-    "mongodb": "^4.11.0",
+    "mongodb": "^6.3.0",
     "@hcengineering/server-tool": "^0.6.0",
     "@hcengineering/server-token": "^0.6.7",
     "@hcengineering/client": "^0.6.14",

server/account/package.json

@@ -31,7 +31,7 @@
     "prettier-plugin-svelte": "^3.1.0"
   },
   "dependencies": {
-    "mongodb": "^4.11.0",
+    "mongodb": "^6.3.0",
     "@hcengineering/platform": "^0.6.9",
     "@hcengineering/core": "^0.6.28",
     "@hcengineering/contact": "^0.6.20",

server/account/src/index.ts

@@ -182,7 +182,7 @@ function withProductId (productId: string, query: Filter<Workspace>): Filter<Wor
  * @returns
  */
 export async function getWorkspace (db: Db, productId: string, workspace: string): Promise<Workspace | null> {
-  return await db.collection(WORKSPACE_COLLECTION).findOne<Workspace>(withProductId(productId, { workspace }))
+  return await db.collection<Workspace>(WORKSPACE_COLLECTION).findOne(withProductId(productId, { workspace }))
 }
 
 function toAccountInfo (account: Account): AccountInfo {
@@ -196,7 +196,7 @@ async function getAccountInfo (db: Db, email: string, password: string): Promise
   if (account === null) {
     throw new PlatformError(new Status(Severity.ERROR, platform.status.AccountNotFound, { account: email }))
   }
-  if (!verifyPassword(password, account.hash.buffer, account.salt.buffer)) {
+  if (!verifyPassword(password, Buffer.from(account.hash.buffer), Buffer.from(account.salt.buffer))) {
     throw new PlatformError(new Status(Severity.ERROR, platform.status.InvalidPassword, { account: email }))
   }
   return toAccountInfo(account)

server/collaborator/package.json

@@ -58,7 +58,7 @@
     "@hocuspocus/transformer": "^2.9.0",
     "@tiptap/core": "^2.1.12",
     "@tiptap/html": "^2.1.12",
-    "mongodb": "^4.11.0",
+    "mongodb": "^6.3.0",
     "yjs": "^13.5.52",
     "y-prosemirror": "^1.2.1",
     "express": "^4.17.1",

server/collaborator/src/storage/mongodb.ts

@@ -13,7 +13,7 @@
 // limitations under the License.
 //
 
-import { MeasureContext, toWorkspaceString } from '@hcengineering/core'
+import { Doc, MeasureContext, Ref, toWorkspaceString } from '@hcengineering/core'
 import { Transformer } from '@hocuspocus/transformer'
 import { MongoClient } from 'mongodb'
 import { Doc as YDoc } from 'yjs'
@@ -66,10 +66,12 @@ export class MongodbStorageAdapter implements StorageAdapter {
     return await this.ctx.with('load-document', {}, async (ctx) => {
       const doc = await ctx.with('query', {}, async () => {
         const db = this.mongodb.db(toWorkspaceString(context.workspaceId))
-        return await db.collection(objectDomain).findOne({ _id: objectId }, { projection: { [objectAttr]: 1 } })
+        return await db
+          .collection<Doc>(objectDomain)
+          .findOne({ _id: objectId as Ref<Doc> }, { projection: { [objectAttr]: 1 } })
       })
 
-      const content = doc !== null && objectAttr in doc ? (doc[objectAttr] as string) : ''
+      const content = doc !== null && objectAttr in doc ? ((doc as any)[objectAttr] as string) : ''
 
       return await ctx.with('transform', {}, () => {
         return this.transformer.toYdoc(content, objectAttr)

server/mongo/package.json

@@ -34,6 +34,6 @@
     "@hcengineering/core": "^0.6.28",
     "@hcengineering/platform": "^0.6.9",
     "@hcengineering/server-core": "^0.6.1",
-    "mongodb": "^4.11.0"
+    "mongodb": "^6.3.0"
   }
 }

server/mongo/src/storage.ts

@@ -18,6 +18,7 @@ import core, {
   DOMAIN_TX,
   SortingOrder,
   TxProcessor,
+  cutObjectArray,
   escapeLikeForRegexp,
   getTypeOf,
   isOperator,
@@ -57,8 +58,8 @@ import core, {
   type WorkspaceId
 } from '@hcengineering/core'
 import type { DbAdapter, TxAdapter } from '@hcengineering/server-core'
-import serverCore from '@hcengineering/server-core'
 import {
+  type AbstractCursor,
   type AnyBulkWriteOperation,
   type Collection,
   type Db,
@@ -70,8 +71,6 @@ import {
 } from 'mongodb'
 import { createHash } from 'node:crypto'
 import { getMongoClient, getWorkspaceDB } from './utils'
-import { cutObjectArray } from '@hcengineering/core'
-import { getMetadata } from '@hcengineering/platform'
 
 function translateDoc (doc: Doc): Document {
   return { ...doc, '%hash%': null }
@@ -111,12 +110,21 @@ abstract class MongoAdapterBase implements DbAdapter {
 
   async init (): Promise<void> {}
 
+  async toArray<T>(cursor: AbstractCursor<T>): Promise<T[]> {
+    const data: T[] = []
+    for await (const r of cursor.stream()) {
+      data.push(r)
+    }
+    await cursor.close()
+    return data
+  }
+
   async createIndexes (domain: Domain, config: Pick<IndexingConfiguration<Doc>, 'indexes'>): Promise<void> {
     for (const vv of config.indexes) {
       try {
         await this.db.collection(domain).createIndex(vv)
       } catch (err: any) {
-        console.error(err)
+        console.error('failed to create index', domain, vv, err)
       }
     }
   }
@@ -451,16 +459,18 @@ abstract class MongoAdapterBase implements DbAdapter {
       checkKeys: false,
       enableUtf8Validation: false
     })
-    cursor.maxTimeMS(parseInt(getMetadata(serverCore.metadata.CursorMaxTimeMS) ?? '30000'))
+    const result: WithLookup<T>[] = []
-    let res: Document = []
+    let total = options?.total === true ? 0 : -1
     try {
-      res = (await cursor.toArray())[0]
+      const rres = await this.toArray(cursor)
+      for (const r of rres) {
+        result.push(...r.results)
+        total = options?.total === true ? r.totalCount?.shift()?.count ?? 0 : -1
+      }
     } catch (e) {
       console.error('error during executing cursor in findWithPipeline', clazz, cutObjectArray(query), options, e)
       throw e
     }
-    const result = res.results as WithLookup<T>[]
-    const total = options?.total === true ? res.totalCount?.shift()?.count ?? 0 : -1
     for (const row of result) {
       await this.fillLookupValue(clazz, options?.lookup, row)
       this.clearExtraLookups(row)
@@ -596,11 +606,8 @@ abstract class MongoAdapterBase implements DbAdapter {
     }
 
     // Error in case of timeout
-    cursor.maxTimeMS(parseInt(getMetadata(serverCore.metadata.CursorMaxTimeMS) ?? '30000'))
-    cursor.maxAwaitTimeMS(30000)
-    let res: T[] = []
     try {
-      res = await cursor.toArray()
+      const res: T[] = await this.toArray(cursor)
       if (options?.total === true && options?.limit === undefined) {
         total = res.length
       }
@@ -712,14 +719,9 @@ abstract class MongoAdapterBase implements DbAdapter {
   }
 
   async load (domain: Domain, docs: Ref<Doc>[]): Promise<Doc[]> {
-    return this.stripHash(
+    const cursor = this.db.collection<Doc>(domain).find<Doc>({ _id: { $in: docs } })
-      this.stripHash(
+    const result = await this.toArray(cursor)
-        await this.db
+    return this.stripHash(this.stripHash(result))
-          .collection(domain)
-          .find<Doc>({ _id: { $in: docs } })
-          .toArray()
-      )
-    )
   }
 
   async upload (domain: Domain, docs: Doc[]): Promise<void> {
@@ -783,7 +785,7 @@ abstract class MongoAdapterBase implements DbAdapter {
   }
 
   async clean (domain: Domain, docs: Ref<Doc>[]): Promise<void> {
-    await this.db.collection(domain).deleteMany({ _id: { $in: docs } })
+    await this.db.collection<Doc>(domain).deleteMany({ _id: { $in: docs } })
   }
 }
 
@@ -1087,7 +1089,7 @@ class MongoAdapter extends MongoAdapterBase {
                   '%hash%': null
                 }
               } as unknown as UpdateFilter<Document>,
-              { returnDocument: 'after' }
+              { returnDocument: 'after', includeResultMetadata: true }
             )
             return { object: result.value }
           }
@@ -1128,7 +1130,7 @@ class MongoAdapter extends MongoAdapterBase {
           ? async (): Promise<TxResult> => {
             const result = await this.db
               .collection(domain)
-              .findOneAndUpdate(filter, update, { returnDocument: 'after' })
+              .findOneAndUpdate(filter, update, { returnDocument: 'after', includeResultMetadata: true })
             return { object: result.value }
           }
           : async () => await this.db.collection(domain).updateOne(filter, update)
@@ -1163,11 +1165,11 @@ class MongoTxAdapter extends MongoAdapterBase implements TxAdapter {
   }
 
   async getModel (): Promise<Tx[]> {
-    const model = await this.db
+    const cursor = this.db
       .collection(DOMAIN_TX)
       .find<Tx>({ objectSpace: core.space.Model })
       .sort({ _id: 1, modifiedOn: 1 })
-      .toArray()
+    const model = await this.toArray(cursor)
     // We need to put all core.account.System transactions first
     const systemTx: Tx[] = []
     const userTx: Tx[] = []

server/tool/package.json

@@ -30,7 +30,7 @@
     "prettier-plugin-svelte": "^3.1.0"
   },
   "dependencies": {
-    "mongodb": "^4.11.0",
+    "mongodb": "^6.3.0",
     "@hcengineering/platform": "^0.6.9",
     "@hcengineering/core": "^0.6.28",
     "@hcengineering/contact": "^0.6.20",

server/tool/src/index.ts

@@ -275,6 +275,10 @@ async function createUpdateIndexes (connection: CoreClient, db: Db, logger: Mode
   }
 
   for (const [d, v] of domains.entries()) {
+    const collInfo = await db.listCollections({ name: d }).next()
+    if (collInfo === null) {
+      await db.createCollection(d)
+    }
     const collection = db.collection(d)
     const bb: (string | FieldIndex<Doc>)[] = []
     for (const vv of v.values()) {
@@ -286,7 +290,7 @@ async function createUpdateIndexes (connection: CoreClient, db: Db, logger: Mode
           await collection.createIndex(vv)
         }
       } catch (err: any) {
-        logger.log('error', JSON.stringify(err))
+        logger.log('error: failed to create index', d, vv, JSON.stringify(err))
       }
       bb.push(vv)
     }

server/tool/src/upgrade.ts

@@ -194,6 +194,6 @@ export class MigrateClientImpl implements MigrationClient {
   }
 
   async deleteMany<T extends Doc>(domain: Domain, query: DocumentQuery<T>): Promise<void> {
-    await this.db.collection(domain).deleteMany(query)
+    await this.db.collection<Doc>(domain).deleteMany(query as any)
   }
 }

tests/docker-compose.yaml

@@ -1,7 +1,7 @@
 version: "3"
 services:
   mongodb:
-    image: mongo
+    image: 'mongo:7-jammy'
     command: mongod --port 27018
     environment:
       - PUID=1000

tests/sanity/storage-dev.json

@@ -20,6 +20,7 @@
           "name": "#platform.notification.timeout",
           "value": "0"
         },
+       
         {
           "name": "#platform.testing.enabled",
           "value": "true"

tests/sanity/storage.json

@@ -16,14 +16,19 @@
           "name": "login:metadata:LoginEndpoint",
           "value": "ws://localhost:3334"
         },
-        {
-          "name": "#platform.notification.logging",
-          "value": "false"
-        },
         {
           "name": "#platform.notification.timeout",
           "value": "0"
         },
+       
+        {
+          "name": "#platform.testing.enabled",
+          "value": "true"
+        },
+        {
+          "name": "#platform.notification.logging",
+          "value": "false"
+        },
         {
           "name": "#platform.lazy.loading",
           "value": "false"

tests/sanity/storageSecond-dev.json

@@ -20,10 +20,7 @@
           "name": "#platform.notification.timeout",
           "value": "0"
         },
-        {
+       
-          "name": "#platform.notification.timeout",
-          "value": "0"
-        },
         {
           "name": "#platform.testing.enabled",
           "value": "true"

tests/sanity/storageSecond.json

@@ -16,6 +16,15 @@
           "name": "login:metadata:LoginEndpoint",
           "value": "ws://localhost:3334"
         },
+        {
+          "name": "#platform.notification.timeout",
+          "value": "0"
+        },
+       
+        {
+          "name": "#platform.testing.enabled",
+          "value": "true"
+        },
         {
           "name": "#platform.notification.logging",
           "value": "false"

tests/sanity/tests/collaborative/issues.spec.ts

@@ -50,6 +50,9 @@ test.describe('Collaborative test for issue', () => {
 
     // check created issued by second user
     const issuesPageSecond = new IssuesPage(userSecondPage)
+    await userSecondPage.evaluate(() => {
+      localStorage.setItem('platform.activity.threshold', '0')
+    })
     await issuesPageSecond.linkSidebarAll.click()
     await issuesPageSecond.modelSelectorAll.click()
     await issuesPageSecond.searchIssueByName(newIssue.title)
@@ -99,6 +102,9 @@ test.describe('Collaborative test for issue', () => {
     await issuesPageSecond.openIssueByName(issue.title)
 
     const issuesDetailsPageSecond = new IssuesDetailsPage(userSecondPage)
+    await userSecondPage.evaluate(() => {
+      localStorage.setItem('platform.activity.threshold', '0')
+    })
     await issuesDetailsPageSecond.checkIssue({
       ...issue,
       status: 'In Progress'

tests/sanity/tests/model/recruiting/common-recruiting-page.ts

@@ -9,7 +9,7 @@ export class CommonRecruitingPage extends CalendarPage {
   readonly buttonSendComment: Locator
   readonly textComment: Locator
   readonly inputAddAttachment: Locator
-  readonly textAttachmentName: Locator
+  textAttachmentName: Locator
   readonly buttonCreateFirstReview: Locator
   readonly buttonMoreActions: Locator
   readonly buttonDelete: Locator
@@ -60,7 +60,7 @@ export class CommonRecruitingPage extends CalendarPage {
 
   async addAttachments (filePath: string): Promise<void> {
     await this.inputAddAttachment.setInputFiles(path.join(__dirname, `../../files/${filePath}`))
-    await expect(this.textAttachmentName.filter({ hasText: filePath })).toBeVisible()
+    await expect(this.textAttachmentName.filter({ hasText: filePath }).first()).toBeVisible()
   }
 
   async addFirstReview (reviewTitle: string, reviewDescription: string): Promise<void> {

tests/sanity/tests/model/recruiting/talent-details-page.ts

@@ -45,7 +45,7 @@ export class TalentDetailsPage extends CommonRecruitingPage {
   }
 
   async checkSkill (skillTag: string): Promise<void> {
-    await expect(this.textTagItem).toContainText(skillTag)
+    await expect(this.textTagItem.first()).toContainText(skillTag)
   }
 
   async addTitle (title: string): Promise<void> {

tests/sanity/tests/model/recruiting/vacancy-details-page.ts

@@ -33,7 +33,7 @@ export class VacancyDetailsPage extends CommonRecruitingPage {
 
   async addAttachments (filePath: string): Promise<void> {
     await this.inputAttachFile.setInputFiles(path.join(__dirname, `../../files/${filePath}`))
-    await expect(this.textAttachmentName).toHaveAttribute('download', filePath)
+    await expect(this.textAttachmentName.first()).toHaveAttribute('download', filePath)
   }
 
   async addDescription (description: string): Promise<void> {

tests/sanity/tests/recruiting/talents.spec.ts

@@ -178,6 +178,6 @@ test.describe('candidate/talents tests', () => {
     await talentsPage.createNewTalentWithName(talentName.firstName, talentName.lastName)
 
     await talentsPage.rightClickAction(talentName, 'Match to vacancy')
-    await talentsPage.checkMatchVacancy(`${talentName.lastName} ${talentName.firstName}`, '0.')
+    await talentsPage.checkMatchVacancy(`${talentName.lastName} ${talentName.firstName}`, '0')
   })
 })