mirror of
https://github.com/hcengineering/platform.git
synced 2025-01-05 10:29:51 +03:00
Fix backup check tool (#6997)
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
parent
8bd62ecede
commit
453be085fc
@ -1332,8 +1332,8 @@ dependencies:
|
|||||||
specifier: ~0.32.0
|
specifier: ~0.32.0
|
||||||
version: 0.32.0
|
version: 0.32.0
|
||||||
'@types/tar-stream':
|
'@types/tar-stream':
|
||||||
specifier: ^2.2.2
|
specifier: ^3.1.3
|
||||||
version: 2.2.3
|
version: 3.1.3
|
||||||
'@types/toposort':
|
'@types/toposort':
|
||||||
specifier: ^2.0.3
|
specifier: ^2.0.3
|
||||||
version: 2.0.7
|
version: 2.0.7
|
||||||
@ -1617,7 +1617,7 @@ dependencies:
|
|||||||
specifier: ^9.7.1
|
specifier: ^9.7.1
|
||||||
version: 9.13.0
|
version: 9.13.0
|
||||||
itty-router:
|
itty-router:
|
||||||
specifier: ^5.0.17
|
specifier: ^5.0.18
|
||||||
version: 5.0.18
|
version: 5.0.18
|
||||||
jest:
|
jest:
|
||||||
specifier: ^29.7.0
|
specifier: ^29.7.0
|
||||||
@ -1839,8 +1839,8 @@ dependencies:
|
|||||||
specifier: ^3.0.0
|
specifier: ^3.0.0
|
||||||
version: 3.0.3
|
version: 3.0.3
|
||||||
tar-stream:
|
tar-stream:
|
||||||
specifier: ^2.2.0
|
specifier: ^3.1.7
|
||||||
version: 2.2.0
|
version: 3.1.7
|
||||||
telegraf:
|
telegraf:
|
||||||
specifier: ^4.16.3
|
specifier: ^4.16.3
|
||||||
version: 4.16.3
|
version: 4.16.3
|
||||||
@ -9833,8 +9833,8 @@ packages:
|
|||||||
resolution: {integrity: sha512-7NQmHra/JILCd1QqpSzl8+mJRc8ZHz3uDm8YV1Ks9IhK0epEiTw8aIErbvH9PI+6XbqhyIQy3462nEsn7UVzjQ==}
|
resolution: {integrity: sha512-7NQmHra/JILCd1QqpSzl8+mJRc8ZHz3uDm8YV1Ks9IhK0epEiTw8aIErbvH9PI+6XbqhyIQy3462nEsn7UVzjQ==}
|
||||||
dev: false
|
dev: false
|
||||||
|
|
||||||
/@types/tar-stream@2.2.3:
|
/@types/tar-stream@3.1.3:
|
||||||
resolution: {integrity: sha512-if3mugZfjVkXOMZdFjIHySxY13r6GXPpyOlsDmLffvyI7tLz9wXE8BFjNivXsvUeyJ1KNlOpfLnag+ISmxgxPw==}
|
resolution: {integrity: sha512-Zbnx4wpkWBMBSu5CytMbrT5ZpMiF55qgM+EpHzR4yIDu7mv52cej8hTkOc6K+LzpkOAbxwn/m7j3iO+/l42YkQ==}
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 20.11.19
|
'@types/node': 20.11.19
|
||||||
dev: false
|
dev: false
|
||||||
@ -26087,14 +26087,14 @@ packages:
|
|||||||
dev: false
|
dev: false
|
||||||
|
|
||||||
file:projects/backup-service.tgz(esbuild@0.20.1)(ts-node@10.9.2):
|
file:projects/backup-service.tgz(esbuild@0.20.1)(ts-node@10.9.2):
|
||||||
resolution: {integrity: sha512-yVHrvyvHvtRFu894lGDl0o19qrJv3scuxI80Dg+rQDzjb4nWXAPGAj477N6TR7vrZSmYAmsR5qEaHkXxc35ipg==, tarball: file:projects/backup-service.tgz}
|
resolution: {integrity: sha512-q/p/O9tTXiKT+qik2nsPL+zmBUwyvvfwBIH3RN4Z7vItg7gV/mI38jal1ppQU/Ddnwvfifn/h7tFc/sSTpjG2A==, tarball: file:projects/backup-service.tgz}
|
||||||
id: file:projects/backup-service.tgz
|
id: file:projects/backup-service.tgz
|
||||||
name: '@rush-temp/backup-service'
|
name: '@rush-temp/backup-service'
|
||||||
version: 0.0.0
|
version: 0.0.0
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/jest': 29.5.12
|
'@types/jest': 29.5.12
|
||||||
'@types/node': 20.11.19
|
'@types/node': 20.11.19
|
||||||
'@types/tar-stream': 2.2.3
|
'@types/tar-stream': 3.1.3
|
||||||
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
|
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
|
||||||
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
|
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
|
||||||
eslint: 8.56.0
|
eslint: 8.56.0
|
||||||
@ -26104,7 +26104,7 @@ packages:
|
|||||||
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
|
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
|
||||||
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
|
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
|
||||||
prettier: 3.2.5
|
prettier: 3.2.5
|
||||||
tar-stream: 2.2.0
|
tar-stream: 3.1.7
|
||||||
ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
|
ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
|
||||||
typescript: 5.3.3
|
typescript: 5.3.3
|
||||||
transitivePeerDependencies:
|
transitivePeerDependencies:
|
||||||
@ -26623,7 +26623,7 @@ packages:
|
|||||||
dev: false
|
dev: false
|
||||||
|
|
||||||
file:projects/cloud-branding.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.4):
|
file:projects/cloud-branding.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.4):
|
||||||
resolution: {integrity: sha512-LuePDPK46tYQhJ9inFPje2LSd/RqlwYSW0k0UDsNWKNyJn4WwWGngaNblMp22YGBw8wMDgJpbaIjS/BPvZ7RQg==, tarball: file:projects/cloud-branding.tgz}
|
resolution: {integrity: sha512-f1NgjqZw48X0+O4cblNzjU8SeW4RYnvaMvYNFyN+eKbqTDHI1N97+q5f2OoDgRonvaZa+CXMETNFXMSkniWwyA==, tarball: file:projects/cloud-branding.tgz}
|
||||||
id: file:projects/cloud-branding.tgz
|
id: file:projects/cloud-branding.tgz
|
||||||
name: '@rush-temp/cloud-branding'
|
name: '@rush-temp/cloud-branding'
|
||||||
version: 0.0.0
|
version: 0.0.0
|
||||||
@ -26658,7 +26658,7 @@ packages:
|
|||||||
dev: false
|
dev: false
|
||||||
|
|
||||||
file:projects/cloud-datalake.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.4):
|
file:projects/cloud-datalake.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.4):
|
||||||
resolution: {integrity: sha512-AA2lTsmPKPeYA1MTwIscZFRO40m9Ctc59Er2x8VRLNBBt4mQ01b1CCay4VFVPWYxAzh+Ru9RoUIB7lS+m8sj9Q==, tarball: file:projects/cloud-datalake.tgz}
|
resolution: {integrity: sha512-KdKIEaVTjeWtCEUYfkvyrIYxglGcfM8iICOnTxRzPiVmOIP3kswu7PaUKeH/xeFZQcvRBvhtNI7XT0dBr5mODA==, tarball: file:projects/cloud-datalake.tgz}
|
||||||
id: file:projects/cloud-datalake.tgz
|
id: file:projects/cloud-datalake.tgz
|
||||||
name: '@rush-temp/cloud-datalake'
|
name: '@rush-temp/cloud-datalake'
|
||||||
version: 0.0.0
|
version: 0.0.0
|
||||||
@ -32916,14 +32916,14 @@ packages:
|
|||||||
dev: false
|
dev: false
|
||||||
|
|
||||||
file:projects/server-backup.tgz(esbuild@0.20.1)(ts-node@10.9.2):
|
file:projects/server-backup.tgz(esbuild@0.20.1)(ts-node@10.9.2):
|
||||||
resolution: {integrity: sha512-RCYMutijYbZbV1jzpWwwzQL9bZS8uzTOYX0eZgxmh3aYyU1Ub4v4TixQpdiiH72ujj0V2hPn9bM2hpYaxzbD1w==, tarball: file:projects/server-backup.tgz}
|
resolution: {integrity: sha512-oDyT5+30r1kpzb+Cx/HTsY1f4fRjjtSy7n13JjzYPmkn37nVcHYb1X1RImHxhoiPehYWdftTsE5v5l2on0mOPg==, tarball: file:projects/server-backup.tgz}
|
||||||
id: file:projects/server-backup.tgz
|
id: file:projects/server-backup.tgz
|
||||||
name: '@rush-temp/server-backup'
|
name: '@rush-temp/server-backup'
|
||||||
version: 0.0.0
|
version: 0.0.0
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/jest': 29.5.12
|
'@types/jest': 29.5.12
|
||||||
'@types/node': 20.11.19
|
'@types/node': 20.11.19
|
||||||
'@types/tar-stream': 2.2.3
|
'@types/tar-stream': 3.1.3
|
||||||
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
|
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
|
||||||
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
|
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
|
||||||
eslint: 8.56.0
|
eslint: 8.56.0
|
||||||
@ -32934,7 +32934,7 @@ packages:
|
|||||||
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
|
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
|
||||||
prettier: 3.2.5
|
prettier: 3.2.5
|
||||||
prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11)
|
prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11)
|
||||||
tar-stream: 2.2.0
|
tar-stream: 3.1.7
|
||||||
ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
|
ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
|
||||||
typescript: 5.3.3
|
typescript: 5.3.3
|
||||||
transitivePeerDependencies:
|
transitivePeerDependencies:
|
||||||
|
@ -33,7 +33,7 @@
|
|||||||
"eslint-config-standard-with-typescript": "^40.0.0",
|
"eslint-config-standard-with-typescript": "^40.0.0",
|
||||||
"prettier": "^3.1.0",
|
"prettier": "^3.1.0",
|
||||||
"typescript": "^5.3.3",
|
"typescript": "^5.3.3",
|
||||||
"@types/tar-stream": "^2.2.2",
|
"@types/tar-stream": "^3.1.3",
|
||||||
"@types/node": "~20.11.16",
|
"@types/node": "~20.11.16",
|
||||||
"jest": "^29.7.0",
|
"jest": "^29.7.0",
|
||||||
"ts-jest": "^29.1.1",
|
"ts-jest": "^29.1.1",
|
||||||
@ -46,7 +46,7 @@
|
|||||||
"@hcengineering/client-resources": "^0.6.27",
|
"@hcengineering/client-resources": "^0.6.27",
|
||||||
"@hcengineering/client": "^0.6.18",
|
"@hcengineering/client": "^0.6.18",
|
||||||
"@hcengineering/model": "^0.6.11",
|
"@hcengineering/model": "^0.6.11",
|
||||||
"tar-stream": "^2.2.0",
|
"tar-stream": "^3.1.7",
|
||||||
"@hcengineering/server-tool": "^0.6.0",
|
"@hcengineering/server-tool": "^0.6.0",
|
||||||
"@hcengineering/server-core": "^0.6.1",
|
"@hcengineering/server-core": "^0.6.1",
|
||||||
"@hcengineering/server-storage": "^0.6.0",
|
"@hcengineering/server-storage": "^0.6.0",
|
||||||
|
@ -33,7 +33,7 @@
|
|||||||
"eslint-config-standard-with-typescript": "^40.0.0",
|
"eslint-config-standard-with-typescript": "^40.0.0",
|
||||||
"prettier": "^3.1.0",
|
"prettier": "^3.1.0",
|
||||||
"typescript": "^5.3.3",
|
"typescript": "^5.3.3",
|
||||||
"@types/tar-stream": "^2.2.2",
|
"@types/tar-stream": "^3.1.3",
|
||||||
"@types/node": "~20.11.16",
|
"@types/node": "~20.11.16",
|
||||||
"jest": "^29.7.0",
|
"jest": "^29.7.0",
|
||||||
"ts-jest": "^29.1.1",
|
"ts-jest": "^29.1.1",
|
||||||
@ -47,7 +47,7 @@
|
|||||||
"@hcengineering/client": "^0.6.18",
|
"@hcengineering/client": "^0.6.18",
|
||||||
"@hcengineering/model": "^0.6.11",
|
"@hcengineering/model": "^0.6.11",
|
||||||
"@hcengineering/analytics": "^0.6.0",
|
"@hcengineering/analytics": "^0.6.0",
|
||||||
"tar-stream": "^2.2.0",
|
"tar-stream": "^3.1.7",
|
||||||
"@hcengineering/server-tool": "^0.6.0",
|
"@hcengineering/server-tool": "^0.6.0",
|
||||||
"@hcengineering/server-client": "^0.6.0",
|
"@hcengineering/server-client": "^0.6.0",
|
||||||
"@hcengineering/server-token": "^0.6.11",
|
"@hcengineering/server-token": "^0.6.11",
|
||||||
|
@ -45,7 +45,7 @@ import { type StorageAdapter } from '@hcengineering/server-core'
|
|||||||
import { fullTextPushStagePrefix } from '@hcengineering/server-indexer'
|
import { fullTextPushStagePrefix } from '@hcengineering/server-indexer'
|
||||||
import { generateToken } from '@hcengineering/server-token'
|
import { generateToken } from '@hcengineering/server-token'
|
||||||
import { connect } from '@hcengineering/server-tool'
|
import { connect } from '@hcengineering/server-tool'
|
||||||
import { createReadStream, createWriteStream, existsSync, mkdirSync } from 'node:fs'
|
import { createReadStream, createWriteStream, existsSync, mkdirSync, statSync } from 'node:fs'
|
||||||
import { rm } from 'node:fs/promises'
|
import { rm } from 'node:fs/promises'
|
||||||
import { basename, dirname } from 'node:path'
|
import { basename, dirname } from 'node:path'
|
||||||
import { PassThrough } from 'node:stream'
|
import { PassThrough } from 'node:stream'
|
||||||
@ -178,7 +178,7 @@ async function loadDigest (
|
|||||||
result.delete(k as Ref<Doc>)
|
result.delete(k as Ref<Doc>)
|
||||||
}
|
}
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
ctx.error('digest is broken, will do full backup for', { domain })
|
ctx.error('digest is broken, will do full backup for', { domain, err: err.message, snapshot })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Stop if stop date is matched and provided
|
// Stop if stop date is matched and provided
|
||||||
@ -236,14 +236,10 @@ async function verifyDigest (
|
|||||||
blobs.set(bname, { doc, buffer: undefined })
|
blobs.set(bname, { doc, buffer: undefined })
|
||||||
} else {
|
} else {
|
||||||
blobs.delete(bname)
|
blobs.delete(bname)
|
||||||
const blob = doc as Blob
|
validDocs.add(bname as Ref<Doc>)
|
||||||
|
|
||||||
if (blob.size === bf.length) {
|
|
||||||
validDocs.add(name as Ref<Doc>)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
validDocs.add(name as Ref<Doc>)
|
validDocs.add(bname as Ref<Doc>)
|
||||||
}
|
}
|
||||||
next()
|
next()
|
||||||
})
|
})
|
||||||
@ -265,10 +261,7 @@ async function verifyDigest (
|
|||||||
sz = bf.length
|
sz = bf.length
|
||||||
}
|
}
|
||||||
|
|
||||||
// If blob size matches doc size, remove from requiredDocs
|
validDocs.add(name as Ref<Doc>)
|
||||||
if (sz === bf.length) {
|
|
||||||
validDocs.add(name as Ref<Doc>)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
next()
|
next()
|
||||||
})
|
})
|
||||||
@ -364,7 +357,7 @@ async function verifyDigest (
|
|||||||
}
|
}
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
digestToRemove.add(snapshot)
|
digestToRemove.add(snapshot)
|
||||||
ctx.error('digest is broken, will do full backup for', { domain })
|
ctx.error('digest is broken, will do full backup for', { domain, err: err.message, snapshot })
|
||||||
modified = true
|
modified = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1490,6 +1483,7 @@ export async function backupSize (storage: BackupStorage): Promise<void> {
|
|||||||
*/
|
*/
|
||||||
export async function backupDownload (storage: BackupStorage, storeIn: string): Promise<void> {
|
export async function backupDownload (storage: BackupStorage, storeIn: string): Promise<void> {
|
||||||
const infoFile = 'backup.json.gz'
|
const infoFile = 'backup.json.gz'
|
||||||
|
const sizeFile = 'backup.size.gz'
|
||||||
|
|
||||||
if (!(await storage.exists(infoFile))) {
|
if (!(await storage.exists(infoFile))) {
|
||||||
throw new Error(`${infoFile} should present to restore`)
|
throw new Error(`${infoFile} should present to restore`)
|
||||||
@ -1499,6 +1493,12 @@ export async function backupDownload (storage: BackupStorage, storeIn: string):
|
|||||||
const backupInfo: BackupInfo = JSON.parse(gunzipSync(await storage.loadFile(infoFile)).toString())
|
const backupInfo: BackupInfo = JSON.parse(gunzipSync(await storage.loadFile(infoFile)).toString())
|
||||||
console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version)
|
console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version)
|
||||||
|
|
||||||
|
let sizeInfo: Record<string, number> = {}
|
||||||
|
if (await storage.exists(sizeFile)) {
|
||||||
|
sizeInfo = JSON.parse(gunzipSync(await storage.loadFile(sizeFile)).toString())
|
||||||
|
}
|
||||||
|
console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version)
|
||||||
|
|
||||||
const addFileSize = async (file: string | undefined | null, force: boolean = false): Promise<void> => {
|
const addFileSize = async (file: string | undefined | null, force: boolean = false): Promise<void> => {
|
||||||
if (file != null) {
|
if (file != null) {
|
||||||
const target = join(storeIn, file)
|
const target = join(storeIn, file)
|
||||||
@ -1506,8 +1506,11 @@ export async function backupDownload (storage: BackupStorage, storeIn: string):
|
|||||||
if (!existsSync(dir)) {
|
if (!existsSync(dir)) {
|
||||||
mkdirSync(dir, { recursive: true })
|
mkdirSync(dir, { recursive: true })
|
||||||
}
|
}
|
||||||
if (!existsSync(target) || force) {
|
|
||||||
const fileSize = await storage.stat(file)
|
const serverSize: number | undefined = sizeInfo[file]
|
||||||
|
|
||||||
|
if (!existsSync(target) || force || (serverSize !== undefined && serverSize !== statSync(target).size)) {
|
||||||
|
const fileSize = serverSize ?? (await storage.stat(file))
|
||||||
console.log('downloading', file, fileSize)
|
console.log('downloading', file, fileSize)
|
||||||
const readStream = await storage.load(file)
|
const readStream = await storage.load(file)
|
||||||
const outp = createWriteStream(target)
|
const outp = createWriteStream(target)
|
||||||
@ -1781,7 +1784,7 @@ export async function restore (
|
|||||||
|
|
||||||
if (sendSize > dataUploadSize || (doc === undefined && docs.length > 0)) {
|
if (sendSize > dataUploadSize || (doc === undefined && docs.length > 0)) {
|
||||||
totalSend += docs.length
|
totalSend += docs.length
|
||||||
ctx.info('upload', {
|
ctx.info('upload-' + c, {
|
||||||
docs: docs.length,
|
docs: docs.length,
|
||||||
totalSend,
|
totalSend,
|
||||||
from: docsToAdd.size + totalSend,
|
from: docsToAdd.size + totalSend,
|
||||||
|
Loading…
Reference in New Issue
Block a user