hcengineering/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2024-11-22 11:42:30 +03:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Backup hash store experimental (#3890)

Browse Source 
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev 2023-10-26 16:47:37 +07:00 committed by GitHub
parent b312a416c9
commit 28e881b78a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 104 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/core/src/hierarchy.ts
View File

@ -20,7 +20,7 @@ import core from './component'
import { _createMixinProxy, _mixinClass, _toDoc } from './proxy'
import type { Tx, TxCreateDoc, TxMixin, TxRemoveDoc, TxUpdateDoc } from './tx'
import { TxProcessor } from './tx'
import getTypeOf from './typeof'
import { getTypeOf } from './typeof'
/**
* @public

1
packages/core/src/index.ts
View File

@ -29,3 +29,4 @@ export * from './tx'
export * from './utils'
export * from './backup'
export * from './status'
export * from './typeof'

2
packages/core/src/typeof.ts
View File

@ -1,7 +1,7 @@
const se = typeof Symbol !== 'undefined'
const ste = se && typeof Symbol.toStringTag !== 'undefined'
export default function getTypeOf (obj: any): string {
export function getTypeOf (obj: any): string {
const typeofObj = typeof obj
if (typeofObj !== 'object') {
return typeofObj

1
plugins/tracker-resources/src/components/issues/IssuePresenter.svelte
View File

@ -74,7 +74,6 @@
{#if presenters.length > 0}
<div class="flex-row-center">
{#each presenters as mixinPresenter}
{mixinPresenter.presenter}
<Component is={mixinPresenter.presenter} props={{ value }} />
{/each}
</div>

105
server/mongo/src/storage.ts
View File

@ -53,7 +53,8 @@ import core, {
TxResult,
TxUpdateDoc,
WithLookup,
WorkspaceId
WorkspaceId,
getTypeOf
} from '@hcengineering/core'
import type { DbAdapter, TxAdapter } from '@hcengineering/server-core'
import { AnyBulkWriteOperation, Collection, Db, Document, Filter, MongoClient, Sort, UpdateFilter } from 'mongodb'
@ -61,7 +62,7 @@ import { createHash } from 'node:crypto'
import { getMongoClient, getWorkspaceDB } from './utils'
function translateDoc (doc: Doc): Document {
return doc as Document
return { ...doc, '%hash%': null }
}
function isLookupQuery<T extends Doc> (query: DocumentQuery<T>): boolean {
@ -409,6 +410,8 @@ abstract class MongoAdapterBase implements DbAdapter {
projection[ckey] = options.projection[key]
}
resultPipeline.push({ $project: projection })
} else {
resultPipeline.push({ $project: { '%hash%': 0 } })
}
pipeline.push({
$facet: {
@ -429,7 +432,7 @@ abstract class MongoAdapterBase implements DbAdapter {
await this.fillLookupValue(clazz, options?.lookup, row)
this.clearExtraLookups(row)
}
return toFindResult(result, total)
return toFindResult(this.stripHash(result), total)
}
private translateKey<T extends Doc>(key: string, clazz: Ref<Class<T>>): string {
@ -537,6 +540,8 @@ abstract class MongoAdapterBase implements DbAdapter {
projection[ckey] = options.projection[key]
}
cursor = cursor.project(projection)
} else {
cursor = cursor.project({ '%hash%': 0 })
}
let total: number = -1
if (options !== null && options !== undefined) {
@ -565,7 +570,17 @@ abstract class MongoAdapterBase implements DbAdapter {
if (options?.total === true && options?.limit === undefined) {
total = res.length
}
return toFindResult(res, total)
return toFindResult(this.stripHash(res), total)
}
stripHash<T extends Doc>(docs: T[]): T[] {
docs.forEach((it) => {
if ('%hash%' in it) {
delete it['%hash%']
}
return it
})
return docs
}
find (domain: Domain): StorageIterator {
@ -578,14 +593,21 @@ abstract class MongoAdapterBase implements DbAdapter {
if (d === null) {
return undefined
}
const doc = JSON.stringify(d)
const hash = createHash('sha256')
hash.update(doc)
const digest = hash.digest('base64')
let digest = (d as any)['%hash%']
if (digest == null) {
if ('%hash%' in d) {
delete d['%hash%']
}
const doc = JSON.stringify(d)
const hash = createHash('sha256')
hash.update(doc)
digest = hash.digest('base64')
await coll.updateOne({ _id: d._id }, { $set: { '%hash%': digest } })
}
return {
id: d._id,
hash: digest,
size: doc.length // Some approx size for document.
size: this.calcSize(d) // Some approx size for document.
}
},
close: async () => {
@ -594,11 +616,41 @@ abstract class MongoAdapterBase implements DbAdapter {
}
}
calcSize (obj: any): number {
if (typeof obj === 'undefined') {
return 0
}
if (typeof obj === 'function') {
return 0
}
let result = 0
for (const key in obj) {
// include prototype properties
const value = obj[key]
const type = getTypeOf(value)
if (type === 'Array') {
result += this.calcSize(value)
} else if (type === 'Object') {
result += this.calcSize(value)
} else if (type === 'Date') {
result += new Date(value.getTime()).toString().length
}
if (type === 'string') {
result += (value as string).length
} else {
result += JSON.stringify(value).length
}
}
return result
}
async load (domain: Domain, docs: Ref<Doc>[]): Promise<Doc[]> {
return await this.db
.collection(domain)
.find<Doc>({ _id: { $in: docs } })
.toArray()
return this.stripHash(
await this.db
.collection(domain)
.find<Doc>({ _id: { $in: docs } })
.toArray()
)
}
async upload (domain: Domain, docs: Doc[]): Promise<void> {
@ -883,6 +935,9 @@ class MongoAdapter extends MongoAdapterBase {
updateOne: {
filter: { _id: tx.objectId },
update: {
$set: {
'%hash%': null
},
$pull: {
[arr]: desc.$value
}
@ -895,7 +950,8 @@ class MongoAdapter extends MongoAdapterBase {
update: {
$set: {
modifiedBy: tx.modifiedBy,
modifiedOn: tx.modifiedOn
modifiedOn: tx.modifiedOn,
'%hash%': null
},
$push: {
[arr]: {
@ -925,7 +981,8 @@ class MongoAdapter extends MongoAdapterBase {
},
update: {
$set: {
...Object.fromEntries(Object.entries(desc.$update).map((it) => [arr + '.$.' + it[0], it[1]]))
...Object.fromEntries(Object.entries(desc.$update).map((it) => [arr + '.$.' + it[0], it[1]])),
'%hash%': null
}
}
}
@ -936,7 +993,8 @@ class MongoAdapter extends MongoAdapterBase {
update: {
$set: {
modifiedBy: tx.modifiedBy,
modifiedOn: tx.modifiedOn
modifiedOn: tx.modifiedOn,
'%hash%': null
}
}
}
@ -956,7 +1014,8 @@ class MongoAdapter extends MongoAdapterBase {
...tx.operations,
$set: {
modifiedBy: tx.modifiedBy,
modifiedOn: tx.modifiedOn
modifiedOn: tx.modifiedOn,
'%hash%': null
}
} as unknown as UpdateFilter<Document>,
{ returnDocument: 'after' }
@ -974,7 +1033,8 @@ class MongoAdapter extends MongoAdapterBase {
...tx.operations,
$set: {
modifiedBy: tx.modifiedBy,
modifiedOn: tx.modifiedOn
modifiedOn: tx.modifiedOn,
'%hash%': null
}
}
return {
@ -986,7 +1046,14 @@ class MongoAdapter extends MongoAdapterBase {
}
} else {
const filter = { _id: tx.objectId }
const update = { $set: { ...tx.operations, modifiedBy: tx.modifiedBy, modifiedOn: tx.modifiedOn } }
const update = {
$set: {
...tx.operations,
modifiedBy: tx.modifiedBy,
modifiedOn: tx.modifiedOn,
'%hash%': null
}
}
const raw =
tx.retrieve === true
? async (): Promise<TxResult> => {

3
server/tool/src/connect.ts
View File

@ -34,6 +34,9 @@ export async function connect (
// eslint-disable-next-line
const WebSocket = require('ws')
setMetadata(client.metadata.UseBinaryProtocol, true)
setMetadata(client.metadata.UseProtocolCompression, true)
setMetadata(client.metadata.ClientSocketFactory, (url) => new WebSocket(url))
addLocation(clientId, () => import('@hcengineering/client-resources'))

14
server/tool/src/upgrade.ts
View File

@ -68,13 +68,20 @@ export class MigrateClientImpl implements MigrationClient {
const t = Date.now()
try {
if (isOperator(operations)) {
if (operations?.$set !== undefined) {
operations.$set['%hash%'] = null
} else {
operations = { ...operations, $set: { '%hash%': null } }
}
const result = await this.db
.collection(domain)
.updateMany(this.translateQuery(query), { ...operations } as unknown as UpdateFilter<Document>)
return { matched: result.matchedCount, updated: result.modifiedCount }
} else {
const result = await this.db.collection(domain).updateMany(this.translateQuery(query), { $set: operations })
const result = await this.db
.collection(domain)
.updateMany(this.translateQuery(query), { $set: { ...operations, '%hash%': null } })
return { matched: result.matchedCount, updated: result.modifiedCount }
}
} finally {
@ -92,7 +99,7 @@ export class MigrateClientImpl implements MigrationClient {
operations.map((it) => ({
updateOne: {
filter: this.translateQuery(it.filter),
update: { $set: it.update }
update: { $set: { ...it.update, '%hash%': null } }
}
}))
)
@ -115,6 +122,9 @@ export class MigrateClientImpl implements MigrationClient {
}
let doc: Document | null
while ((doc = await cursor.next()) != null) {
if ('%hash%' in doc) {
delete doc['%hash%']
}
await target.insertOne(doc)
result.matched++
result.updated++