hcengineering/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2024-12-22 11:01:54 +03:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

UBER-1198: Upgrade to mongo 7 (#4472)

Browse Source 
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev 2024-01-30 14:47:54 +07:00 committed by GitHub
parent 54f167805e
commit 6996945e48
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 197 additions and 1115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1162
common/config/rush/pnpm-lock.yaml
View File

File diff suppressed because it is too large Load Diff

2
dev/docker-compose.yaml
View File

@ -1,7 +1,7 @@
version: "3"
services:
mongodb:
image: mongo
image: 'mongo:7-jammy'
container_name: mongodb
environment:
- PUID=1000

2
dev/tool/package.json
View File

@ -121,7 +121,7 @@
"got": "^11.8.3",
"libphonenumber-js": "^1.9.46",
"mime-types": "~2.1.34",
"mongodb": "^4.11.0",
"mongodb": "^6.3.0",
"ws": "^8.10.0",
"xml2js": "~0.4.23"
}

12
dev/tool/src/clean.ts
View File

@ -566,8 +566,8 @@ export async function fixSkills (
tag: t._id
})) as TagReference[]
const ids = references.map((r) => r._id)
await db.collection(DOMAIN_TAGS).deleteMany({ _id: { $in: ids } })
await db.collection(DOMAIN_TAGS).deleteOne({ _id: t._id })
await db.collection<Doc>(DOMAIN_TAGS).deleteMany({ _id: { $in: ids } })
await db.collection<Doc>(DOMAIN_TAGS).deleteOne({ _id: t._id })
}
}
await fixCount()
@ -588,8 +588,8 @@ export async function fixSkills (
tag: t._id
})) as TagReference[]
const ids = references.map((r) => r._id)
await db.collection(DOMAIN_TAGS).deleteMany({ _id: { $in: ids } })
await db.collection(DOMAIN_TAGS).deleteOne({ _id: t._id })
await db.collection<Doc>(DOMAIN_TAGS).deleteMany({ _id: { $in: ids } })
await db.collection<Doc>(DOMAIN_TAGS).deleteOne({ _id: t._id })
}
console.log('DONE 6 STEP')
}
@ -610,8 +610,8 @@ export async function fixSkills (
tag: t._id
})) as TagReference[]
const ids = references.map((r) => r._id)
await db.collection(DOMAIN_TAGS).deleteMany({ _id: { $in: ids } })
await db.collection(DOMAIN_TAGS).deleteOne({ _id: t._id })
await db.collection<Doc>(DOMAIN_TAGS).deleteMany({ _id: { $in: ids } })
await db.collection<Doc>(DOMAIN_TAGS).deleteOne({ _id: t._id })
}
}
await fixCount()

2
plugins/activity-resources/src/activityMessagesUtils.ts
View File

@ -46,7 +46,7 @@ import activity, {
// Use 5 minutes to combine similar messages
const combineThresholdMs = 5 * 60 * 1000
// Use 10 seconds to combine update messages after creation.
const createCombineThreshold = 10 * 1000
const createCombineThreshold = parseInt(localStorage.getItem('platform.activity.threshold') ?? '10 * 1000')
const valueTypes: ReadonlyArray<Ref<Class<Doc>>> = [
core.class.TypeString,

2
pods/account/package.json
View File

@ -46,7 +46,7 @@
"@hcengineering/account": "^0.6.0",
"@hcengineering/platform": "^0.6.9",
"@hcengineering/core": "^0.6.28",
"mongodb": "^4.11.0",
"mongodb": "^6.3.0",
"koa": "^2.13.1",
"koa-router": "^12.0.1",
"koa-bodyparser": "^4.3.0",

2
pods/backup/package.json
View File

@ -40,7 +40,7 @@
},
"dependencies": {
"@hcengineering/platform": "^0.6.9",
"mongodb": "^4.11.0",
"mongodb": "^6.3.0",
"@hcengineering/server-tool": "^0.6.0",
"@hcengineering/server-token": "^0.6.7",
"@hcengineering/client": "^0.6.14",

2
server/account/package.json
View File

@ -31,7 +31,7 @@
"prettier-plugin-svelte": "^3.1.0"
},
"dependencies": {
"mongodb": "^4.11.0",
"mongodb": "^6.3.0",
"@hcengineering/platform": "^0.6.9",
"@hcengineering/core": "^0.6.28",
"@hcengineering/contact": "^0.6.20",

4
server/account/src/index.ts
View File

@ -182,7 +182,7 @@ function withProductId (productId: string, query: Filter<Workspace>): Filter<Wor
* @returns
*/
export async function getWorkspace (db: Db, productId: string, workspace: string): Promise<Workspace | null> {
return await db.collection(WORKSPACE_COLLECTION).findOne<Workspace>(withProductId(productId, { workspace }))
return await db.collection<Workspace>(WORKSPACE_COLLECTION).findOne(withProductId(productId, { workspace }))
}
function toAccountInfo (account: Account): AccountInfo {
@ -196,7 +196,7 @@ async function getAccountInfo (db: Db, email: string, password: string): Promise
if (account === null) {
throw new PlatformError(new Status(Severity.ERROR, platform.status.AccountNotFound, { account: email }))
}
if (!verifyPassword(password, account.hash.buffer, account.salt.buffer)) {
if (!verifyPassword(password, Buffer.from(account.hash.buffer), Buffer.from(account.salt.buffer))) {
throw new PlatformError(new Status(Severity.ERROR, platform.status.InvalidPassword, { account: email }))
}
return toAccountInfo(account)

2
server/collaborator/package.json
View File

@ -58,7 +58,7 @@
"@hocuspocus/transformer": "^2.9.0",
"@tiptap/core": "^2.1.12",
"@tiptap/html": "^2.1.12",
"mongodb": "^4.11.0",
"mongodb": "^6.3.0",
"yjs": "^13.5.52",
"y-prosemirror": "^1.2.1",
"express": "^4.17.1",

8
server/collaborator/src/storage/mongodb.ts
View File

@ -13,7 +13,7 @@
// limitations under the License.
//
import { MeasureContext, toWorkspaceString } from '@hcengineering/core'
import { Doc, MeasureContext, Ref, toWorkspaceString } from '@hcengineering/core'
import { Transformer } from '@hocuspocus/transformer'
import { MongoClient } from 'mongodb'
import { Doc as YDoc } from 'yjs'
@ -66,10 +66,12 @@ export class MongodbStorageAdapter implements StorageAdapter {
return await this.ctx.with('load-document', {}, async (ctx) => {
const doc = await ctx.with('query', {}, async () => {
const db = this.mongodb.db(toWorkspaceString(context.workspaceId))
return await db.collection(objectDomain).findOne({ _id: objectId }, { projection: { [objectAttr]: 1 } })
return await db
.collection<Doc>(objectDomain)
.findOne({ _id: objectId as Ref<Doc> }, { projection: { [objectAttr]: 1 } })
})
const content = doc !== null && objectAttr in doc ? (doc[objectAttr] as string) : ''
const content = doc !== null && objectAttr in doc ? ((doc as any)[objectAttr] as string) : ''
return await ctx.with('transform', {}, () => {
return this.transformer.toYdoc(content, objectAttr)

2
server/mongo/package.json
View File

@ -34,6 +34,6 @@
"@hcengineering/core": "^0.6.28",
"@hcengineering/platform": "^0.6.9",
"@hcengineering/server-core": "^0.6.1",
"mongodb": "^4.11.0"
"mongodb": "^6.3.0"
}
}

54
server/mongo/src/storage.ts
View File

@ -18,6 +18,7 @@ import core, {
DOMAIN_TX,
SortingOrder,
TxProcessor,
cutObjectArray,
escapeLikeForRegexp,
getTypeOf,
isOperator,
@ -57,8 +58,8 @@ import core, {
type WorkspaceId
} from '@hcengineering/core'
import type { DbAdapter, TxAdapter } from '@hcengineering/server-core'
import serverCore from '@hcengineering/server-core'
import {
type AbstractCursor,
type AnyBulkWriteOperation,
type Collection,
type Db,
@ -70,8 +71,6 @@ import {
} from 'mongodb'
import { createHash } from 'node:crypto'
import { getMongoClient, getWorkspaceDB } from './utils'
import { cutObjectArray } from '@hcengineering/core'
import { getMetadata } from '@hcengineering/platform'
function translateDoc (doc: Doc): Document {
return { ...doc, '%hash%': null }
@ -111,12 +110,21 @@ abstract class MongoAdapterBase implements DbAdapter {
async init (): Promise<void> {}
async toArray<T>(cursor: AbstractCursor<T>): Promise<T[]> {
const data: T[] = []
for await (const r of cursor.stream()) {
data.push(r)
}
await cursor.close()
return data
}
async createIndexes (domain: Domain, config: Pick<IndexingConfiguration<Doc>, 'indexes'>): Promise<void> {
for (const vv of config.indexes) {
try {
await this.db.collection(domain).createIndex(vv)
} catch (err: any) {
console.error(err)
console.error('failed to create index', domain, vv, err)
}
}
}
@ -451,16 +459,18 @@ abstract class MongoAdapterBase implements DbAdapter {
checkKeys: false,
enableUtf8Validation: false
})
cursor.maxTimeMS(parseInt(getMetadata(serverCore.metadata.CursorMaxTimeMS) ?? '30000'))
let res: Document = []
const result: WithLookup<T>[] = []
let total = options?.total === true ? 0 : -1
try {
res = (await cursor.toArray())[0]
const rres = await this.toArray(cursor)
for (const r of rres) {
result.push(...r.results)
total = options?.total === true ? r.totalCount?.shift()?.count ?? 0 : -1
}
} catch (e) {
console.error('error during executing cursor in findWithPipeline', clazz, cutObjectArray(query), options, e)
throw e
}
const result = res.results as WithLookup<T>[]
const total = options?.total === true ? res.totalCount?.shift()?.count ?? 0 : -1
for (const row of result) {
await this.fillLookupValue(clazz, options?.lookup, row)
this.clearExtraLookups(row)
@ -596,11 +606,8 @@ abstract class MongoAdapterBase implements DbAdapter {
}
// Error in case of timeout
cursor.maxTimeMS(parseInt(getMetadata(serverCore.metadata.CursorMaxTimeMS) ?? '30000'))
cursor.maxAwaitTimeMS(30000)
let res: T[] = []
try {
res = await cursor.toArray()
const res: T[] = await this.toArray(cursor)
if (options?.total === true && options?.limit === undefined) {
total = res.length
}
@ -712,14 +719,9 @@ abstract class MongoAdapterBase implements DbAdapter {
}
async load (domain: Domain, docs: Ref<Doc>[]): Promise<Doc[]> {
return this.stripHash(
this.stripHash(
await this.db
.collection(domain)
.find<Doc>({ _id: { $in: docs } })
.toArray()
)
)
const cursor = this.db.collection<Doc>(domain).find<Doc>({ _id: { $in: docs } })
const result = await this.toArray(cursor)
return this.stripHash(this.stripHash(result))
}
async upload (domain: Domain, docs: Doc[]): Promise<void> {
@ -783,7 +785,7 @@ abstract class MongoAdapterBase implements DbAdapter {
}
async clean (domain: Domain, docs: Ref<Doc>[]): Promise<void> {
await this.db.collection(domain).deleteMany({ _id: { $in: docs } })
await this.db.collection<Doc>(domain).deleteMany({ _id: { $in: docs } })
}
}
@ -1087,7 +1089,7 @@ class MongoAdapter extends MongoAdapterBase {
'%hash%': null
}
} as unknown as UpdateFilter<Document>,
{ returnDocument: 'after' }
{ returnDocument: 'after', includeResultMetadata: true }
)
return { object: result.value }
}
@ -1128,7 +1130,7 @@ class MongoAdapter extends MongoAdapterBase {
? async (): Promise<TxResult> => {
const result = await this.db
.collection(domain)
.findOneAndUpdate(filter, update, { returnDocument: 'after' })
.findOneAndUpdate(filter, update, { returnDocument: 'after', includeResultMetadata: true })
return { object: result.value }
}
: async () => await this.db.collection(domain).updateOne(filter, update)
@ -1163,11 +1165,11 @@ class MongoTxAdapter extends MongoAdapterBase implements TxAdapter {
}
async getModel (): Promise<Tx[]> {
const model = await this.db
const cursor = this.db
.collection(DOMAIN_TX)
.find<Tx>({ objectSpace: core.space.Model })
.sort({ _id: 1, modifiedOn: 1 })
.toArray()
const model = await this.toArray(cursor)
// We need to put all core.account.System transactions first
const systemTx: Tx[] = []
const userTx: Tx[] = []

2
server/tool/package.json
View File

@ -30,7 +30,7 @@
"prettier-plugin-svelte": "^3.1.0"
},
"dependencies": {
"mongodb": "^4.11.0",
"mongodb": "^6.3.0",
"@hcengineering/platform": "^0.6.9",
"@hcengineering/core": "^0.6.28",
"@hcengineering/contact": "^0.6.20",

6
server/tool/src/index.ts
View File

@ -275,6 +275,10 @@ async function createUpdateIndexes (connection: CoreClient, db: Db, logger: Mode
}
for (const [d, v] of domains.entries()) {
const collInfo = await db.listCollections({ name: d }).next()
if (collInfo === null) {
await db.createCollection(d)
}
const collection = db.collection(d)
const bb: (string | FieldIndex<Doc>)[] = []
for (const vv of v.values()) {
@ -286,7 +290,7 @@ async function createUpdateIndexes (connection: CoreClient, db: Db, logger: Mode
await collection.createIndex(vv)
}
} catch (err: any) {
logger.log('error', JSON.stringify(err))
logger.log('error: failed to create index', d, vv, JSON.stringify(err))
}
bb.push(vv)
}

2
server/tool/src/upgrade.ts
View File

@ -194,6 +194,6 @@ export class MigrateClientImpl implements MigrationClient {
}
async deleteMany<T extends Doc>(domain: Domain, query: DocumentQuery<T>): Promise<void> {
await this.db.collection(domain).deleteMany(query)
await this.db.collection<Doc>(domain).deleteMany(query as any)
}
}

2
tests/docker-compose.yaml
View File

@ -1,7 +1,7 @@
version: "3"
services:
mongodb:
image: mongo
image: 'mongo:7-jammy'
command: mongod --port 27018
environment:
- PUID=1000

1
tests/sanity/storage-dev.json
View File

@ -20,6 +20,7 @@
"name": "#platform.notification.timeout",
"value": "0"
},
{
"name": "#platform.testing.enabled",
"value": "true"

13
tests/sanity/storage.json
View File

@ -16,14 +16,19 @@
"name": "login:metadata:LoginEndpoint",
"value": "ws://localhost:3334"
},
{
"name": "#platform.notification.logging",
"value": "false"
},
{
"name": "#platform.notification.timeout",
"value": "0"
},
{
"name": "#platform.testing.enabled",
"value": "true"
},
{
"name": "#platform.notification.logging",
"value": "false"
},
{
"name": "#platform.lazy.loading",
"value": "false"

5
tests/sanity/storageSecond-dev.json
View File

@ -20,10 +20,7 @@
"name": "#platform.notification.timeout",
"value": "0"
},
{
"name": "#platform.notification.timeout",
"value": "0"
},
{
"name": "#platform.testing.enabled",
"value": "true"

9
tests/sanity/storageSecond.json
View File

@ -16,6 +16,15 @@
"name": "login:metadata:LoginEndpoint",
"value": "ws://localhost:3334"
},
{
"name": "#platform.notification.timeout",
"value": "0"
},
{
"name": "#platform.testing.enabled",
"value": "true"
},
{
"name": "#platform.notification.logging",
"value": "false"

6
tests/sanity/tests/collaborative/issues.spec.ts
View File

@ -50,6 +50,9 @@ test.describe('Collaborative test for issue', () => {
// check created issued by second user
const issuesPageSecond = new IssuesPage(userSecondPage)
await userSecondPage.evaluate(() => {
localStorage.setItem('platform.activity.threshold', '0')
})
await issuesPageSecond.linkSidebarAll.click()
await issuesPageSecond.modelSelectorAll.click()
await issuesPageSecond.searchIssueByName(newIssue.title)
@ -99,6 +102,9 @@ test.describe('Collaborative test for issue', () => {
await issuesPageSecond.openIssueByName(issue.title)
const issuesDetailsPageSecond = new IssuesDetailsPage(userSecondPage)
await userSecondPage.evaluate(() => {
localStorage.setItem('platform.activity.threshold', '0')
})
await issuesDetailsPageSecond.checkIssue({
...issue,
status: 'In Progress'

4
tests/sanity/tests/model/recruiting/common-recruiting-page.ts
View File

@ -9,7 +9,7 @@ export class CommonRecruitingPage extends CalendarPage {
readonly buttonSendComment: Locator
readonly textComment: Locator
readonly inputAddAttachment: Locator
readonly textAttachmentName: Locator
textAttachmentName: Locator
readonly buttonCreateFirstReview: Locator
readonly buttonMoreActions: Locator
readonly buttonDelete: Locator
@ -60,7 +60,7 @@ export class CommonRecruitingPage extends CalendarPage {
async addAttachments (filePath: string): Promise<void> {
await this.inputAddAttachment.setInputFiles(path.join(__dirname, `../../files/${filePath}`))
await expect(this.textAttachmentName.filter({ hasText: filePath })).toBeVisible()
await expect(this.textAttachmentName.filter({ hasText: filePath }).first()).toBeVisible()
}
async addFirstReview (reviewTitle: string, reviewDescription: string): Promise<void> {

2
tests/sanity/tests/model/recruiting/talent-details-page.ts
View File

@ -45,7 +45,7 @@ export class TalentDetailsPage extends CommonRecruitingPage {
}
async checkSkill (skillTag: string): Promise<void> {
await expect(this.textTagItem).toContainText(skillTag)
await expect(this.textTagItem.first()).toContainText(skillTag)
}
async addTitle (title: string): Promise<void> {

2
tests/sanity/tests/model/recruiting/vacancy-details-page.ts
View File

@ -33,7 +33,7 @@ export class VacancyDetailsPage extends CommonRecruitingPage {
async addAttachments (filePath: string): Promise<void> {
await this.inputAttachFile.setInputFiles(path.join(__dirname, `../../files/${filePath}`))
await expect(this.textAttachmentName).toHaveAttribute('download', filePath)
await expect(this.textAttachmentName.first()).toHaveAttribute('download', filePath)
}
async addDescription (description: string): Promise<void> {

2
tests/sanity/tests/recruiting/talents.spec.ts
View File

@ -178,6 +178,6 @@ test.describe('candidate/talents tests', () => {
await talentsPage.createNewTalentWithName(talentName.firstName, talentName.lastName)
await talentsPage.rightClickAction(talentName, 'Match to vacancy')
await talentsPage.checkMatchVacancy(`${talentName.lastName} ${talentName.firstName}`, '0.')
await talentsPage.checkMatchVacancy(`${talentName.lastName} ${talentName.firstName}`, '0')
})
})