hcengineering/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2024-12-22 11:01:54 +03:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Fix backup check tool (#6997)

Browse Source 
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev 2024-10-21 12:58:12 +07:00 committed by GitHub
parent 8bd62ecede
commit 453be085fc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 38 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
common/config/rush/pnpm-lock.yaml
View File

@ -1332,8 +1332,8 @@ dependencies:
specifier: ~0.32.0
version: 0.32.0
'@types/tar-stream':
specifier: ^2.2.2
version: 2.2.3
specifier: ^3.1.3
version: 3.1.3
'@types/toposort':
specifier: ^2.0.3
version: 2.0.7
@ -1617,7 +1617,7 @@ dependencies:
specifier: ^9.7.1
version: 9.13.0
itty-router:
specifier: ^5.0.17
specifier: ^5.0.18
version: 5.0.18
jest:
specifier: ^29.7.0
@ -1839,8 +1839,8 @@ dependencies:
specifier: ^3.0.0
version: 3.0.3
tar-stream:
specifier: ^2.2.0
version: 2.2.0
specifier: ^3.1.7
version: 3.1.7
telegraf:
specifier: ^4.16.3
version: 4.16.3
@ -9833,8 +9833,8 @@ packages:
resolution: {integrity: sha512-7NQmHra/JILCd1QqpSzl8+mJRc8ZHz3uDm8YV1Ks9IhK0epEiTw8aIErbvH9PI+6XbqhyIQy3462nEsn7UVzjQ==}
dev: false
/@types/tar-stream@2.2.3:
resolution: {integrity: sha512-if3mugZfjVkXOMZdFjIHySxY13r6GXPpyOlsDmLffvyI7tLz9wXE8BFjNivXsvUeyJ1KNlOpfLnag+ISmxgxPw==}
/@types/tar-stream@3.1.3:
resolution: {integrity: sha512-Zbnx4wpkWBMBSu5CytMbrT5ZpMiF55qgM+EpHzR4yIDu7mv52cej8hTkOc6K+LzpkOAbxwn/m7j3iO+/l42YkQ==}
dependencies:
'@types/node': 20.11.19
dev: false
@ -26087,14 +26087,14 @@ packages:
dev: false
file:projects/backup-service.tgz(esbuild@0.20.1)(ts-node@10.9.2):
resolution: {integrity: sha512-yVHrvyvHvtRFu894lGDl0o19qrJv3scuxI80Dg+rQDzjb4nWXAPGAj477N6TR7vrZSmYAmsR5qEaHkXxc35ipg==, tarball: file:projects/backup-service.tgz}
resolution: {integrity: sha512-q/p/O9tTXiKT+qik2nsPL+zmBUwyvvfwBIH3RN4Z7vItg7gV/mI38jal1ppQU/Ddnwvfifn/h7tFc/sSTpjG2A==, tarball: file:projects/backup-service.tgz}
id: file:projects/backup-service.tgz
name: '@rush-temp/backup-service'
version: 0.0.0
dependencies:
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@types/tar-stream': 2.2.3
'@types/tar-stream': 3.1.3
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
eslint: 8.56.0
@ -26104,7 +26104,7 @@ packages:
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
prettier: 3.2.5
tar-stream: 2.2.0
tar-stream: 3.1.7
ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
typescript: 5.3.3
transitivePeerDependencies:
@ -26623,7 +26623,7 @@ packages:
dev: false
file:projects/cloud-branding.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.4):
resolution: {integrity: sha512-LuePDPK46tYQhJ9inFPje2LSd/RqlwYSW0k0UDsNWKNyJn4WwWGngaNblMp22YGBw8wMDgJpbaIjS/BPvZ7RQg==, tarball: file:projects/cloud-branding.tgz}
resolution: {integrity: sha512-f1NgjqZw48X0+O4cblNzjU8SeW4RYnvaMvYNFyN+eKbqTDHI1N97+q5f2OoDgRonvaZa+CXMETNFXMSkniWwyA==, tarball: file:projects/cloud-branding.tgz}
id: file:projects/cloud-branding.tgz
name: '@rush-temp/cloud-branding'
version: 0.0.0
@ -26658,7 +26658,7 @@ packages:
dev: false
file:projects/cloud-datalake.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.4):
resolution: {integrity: sha512-AA2lTsmPKPeYA1MTwIscZFRO40m9Ctc59Er2x8VRLNBBt4mQ01b1CCay4VFVPWYxAzh+Ru9RoUIB7lS+m8sj9Q==, tarball: file:projects/cloud-datalake.tgz}
resolution: {integrity: sha512-KdKIEaVTjeWtCEUYfkvyrIYxglGcfM8iICOnTxRzPiVmOIP3kswu7PaUKeH/xeFZQcvRBvhtNI7XT0dBr5mODA==, tarball: file:projects/cloud-datalake.tgz}
id: file:projects/cloud-datalake.tgz
name: '@rush-temp/cloud-datalake'
version: 0.0.0
@ -32916,14 +32916,14 @@ packages:
dev: false
file:projects/server-backup.tgz(esbuild@0.20.1)(ts-node@10.9.2):
resolution: {integrity: sha512-RCYMutijYbZbV1jzpWwwzQL9bZS8uzTOYX0eZgxmh3aYyU1Ub4v4TixQpdiiH72ujj0V2hPn9bM2hpYaxzbD1w==, tarball: file:projects/server-backup.tgz}
resolution: {integrity: sha512-oDyT5+30r1kpzb+Cx/HTsY1f4fRjjtSy7n13JjzYPmkn37nVcHYb1X1RImHxhoiPehYWdftTsE5v5l2on0mOPg==, tarball: file:projects/server-backup.tgz}
id: file:projects/server-backup.tgz
name: '@rush-temp/server-backup'
version: 0.0.0
dependencies:
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@types/tar-stream': 2.2.3
'@types/tar-stream': 3.1.3
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
eslint: 8.56.0
@ -32934,7 +32934,7 @@ packages:
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
prettier: 3.2.5
prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11)
tar-stream: 2.2.0
tar-stream: 3.1.7
ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
typescript: 5.3.3
transitivePeerDependencies:

4
server/backup-service/package.json
View File

@ -33,7 +33,7 @@
"eslint-config-standard-with-typescript": "^40.0.0",
"prettier": "^3.1.0",
"typescript": "^5.3.3",
"@types/tar-stream": "^2.2.2",
"@types/tar-stream": "^3.1.3",
"@types/node": "~20.11.16",
"jest": "^29.7.0",
"ts-jest": "^29.1.1",
@ -46,7 +46,7 @@
"@hcengineering/client-resources": "^0.6.27",
"@hcengineering/client": "^0.6.18",
"@hcengineering/model": "^0.6.11",
"tar-stream": "^2.2.0",
"tar-stream": "^3.1.7",
"@hcengineering/server-tool": "^0.6.0",
"@hcengineering/server-core": "^0.6.1",
"@hcengineering/server-storage": "^0.6.0",

4
server/backup/package.json
View File

@ -33,7 +33,7 @@
"eslint-config-standard-with-typescript": "^40.0.0",
"prettier": "^3.1.0",
"typescript": "^5.3.3",
"@types/tar-stream": "^2.2.2",
"@types/tar-stream": "^3.1.3",
"@types/node": "~20.11.16",
"jest": "^29.7.0",
"ts-jest": "^29.1.1",
@ -47,7 +47,7 @@
"@hcengineering/client": "^0.6.18",
"@hcengineering/model": "^0.6.11",
"@hcengineering/analytics": "^0.6.0",
"tar-stream": "^2.2.0",
"tar-stream": "^3.1.7",
"@hcengineering/server-tool": "^0.6.0",
"@hcengineering/server-client": "^0.6.0",
"@hcengineering/server-token": "^0.6.11",

35
server/backup/src/backup.ts
View File

@ -45,7 +45,7 @@ import { type StorageAdapter } from '@hcengineering/server-core'
import { fullTextPushStagePrefix } from '@hcengineering/server-indexer'
import { generateToken } from '@hcengineering/server-token'
import { connect } from '@hcengineering/server-tool'
import { createReadStream, createWriteStream, existsSync, mkdirSync } from 'node:fs'
import { createReadStream, createWriteStream, existsSync, mkdirSync, statSync } from 'node:fs'
import { rm } from 'node:fs/promises'
import { basename, dirname } from 'node:path'
import { PassThrough } from 'node:stream'
@ -178,7 +178,7 @@ async function loadDigest (
result.delete(k as Ref<Doc>)
}
} catch (err: any) {
ctx.error('digest is broken, will do full backup for', { domain })
ctx.error('digest is broken, will do full backup for', { domain, err: err.message, snapshot })
}
}
// Stop if stop date is matched and provided
@ -236,14 +236,10 @@ async function verifyDigest (
blobs.set(bname, { doc, buffer: undefined })
} else {
blobs.delete(bname)
const blob = doc as Blob
if (blob.size === bf.length) {
validDocs.add(name as Ref<Doc>)
}
validDocs.add(bname as Ref<Doc>)
}
} else {
validDocs.add(name as Ref<Doc>)
validDocs.add(bname as Ref<Doc>)
}
next()
})
@ -265,10 +261,7 @@ async function verifyDigest (
sz = bf.length
}
// If blob size matches doc size, remove from requiredDocs
if (sz === bf.length) {
validDocs.add(name as Ref<Doc>)
}
validDocs.add(name as Ref<Doc>)
}
next()
})
@ -364,7 +357,7 @@ async function verifyDigest (
}
} catch (err: any) {
digestToRemove.add(snapshot)
ctx.error('digest is broken, will do full backup for', { domain })
ctx.error('digest is broken, will do full backup for', { domain, err: err.message, snapshot })
modified = true
}
}
@ -1490,6 +1483,7 @@ export async function backupSize (storage: BackupStorage): Promise<void> {
*/
export async function backupDownload (storage: BackupStorage, storeIn: string): Promise<void> {
const infoFile = 'backup.json.gz'
const sizeFile = 'backup.size.gz'
if (!(await storage.exists(infoFile))) {
throw new Error(`${infoFile} should present to restore`)
@ -1499,6 +1493,12 @@ export async function backupDownload (storage: BackupStorage, storeIn: string):
const backupInfo: BackupInfo = JSON.parse(gunzipSync(await storage.loadFile(infoFile)).toString())
console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version)
let sizeInfo: Record<string, number> = {}
if (await storage.exists(sizeFile)) {
sizeInfo = JSON.parse(gunzipSync(await storage.loadFile(sizeFile)).toString())
}
console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version)
const addFileSize = async (file: string | undefined | null, force: boolean = false): Promise<void> => {
if (file != null) {
const target = join(storeIn, file)
@ -1506,8 +1506,11 @@ export async function backupDownload (storage: BackupStorage, storeIn: string):
if (!existsSync(dir)) {
mkdirSync(dir, { recursive: true })
}
if (!existsSync(target) || force) {
const fileSize = await storage.stat(file)
const serverSize: number | undefined = sizeInfo[file]
if (!existsSync(target) || force || (serverSize !== undefined && serverSize !== statSync(target).size)) {
const fileSize = serverSize ?? (await storage.stat(file))
console.log('downloading', file, fileSize)
const readStream = await storage.load(file)
const outp = createWriteStream(target)
@ -1781,7 +1784,7 @@ export async function restore (
if (sendSize > dataUploadSize || (doc === undefined && docs.length > 0)) {
totalSend += docs.length
ctx.info('upload', {
ctx.info('upload-' + c, {
docs: docs.length,
totalSend,
from: docsToAdd.size + totalSend,