UBERF-8310: Optimize backup service (#6763)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

server/backup-service/src/config.ts

@@ -21,6 +21,7 @@ interface Config extends Omit<BackupConfig, 'Token'> {
   Secret: string
 
   Interval: number // Timeout in seconds
+  CoolDown: number
   Timeout: number // Timeout in seconds
   BucketName: string
   Storage: string // A bucket storage config
@@ -37,6 +38,7 @@ const envMap: { [key in keyof Config]: string } = {
   Secret: 'SECRET',
   BucketName: 'BUCKET_NAME',
   Interval: 'INTERVAL',
+  CoolDown: 'COOL_DOWN',
   Timeout: 'TIMEOUT',
   MongoURL: 'MONGO_URL',
   SkipWorkspaces: 'SKIP_WORKSPACES',
@@ -62,6 +64,7 @@ const config: Config = (() => {
     ServiceID: process.env[envMap.ServiceID] ?? 'backup-service',
     Interval: parseInt(process.env[envMap.Interval] ?? '3600'),
     Timeout: parseInt(process.env[envMap.Timeout] ?? '3600'),
+    CoolDown: parseInt(process.env[envMap.CoolDown] ?? '300'),
     MongoURL: process.env[envMap.MongoURL],
     SkipWorkspaces: process.env[envMap.SkipWorkspaces] ?? '',
     WorkspaceStorage: process.env[envMap.WorkspaceStorage],

server/backup/src/backup.ts

@@ -1121,9 +1121,26 @@ export async function backup (
     }
     result.result = true
 
+    const sizeFile = 'backup.size.gz'
+
+    let sizeInfo: Record<string, number> = {}
+
+    if (await storage.exists(sizeFile)) {
+      sizeInfo = JSON.parse(gunzipSync(await storage.loadFile(sizeFile)).toString())
+    }
+    let processed = 0
+
     const addFileSize = async (file: string | undefined | null): Promise<void> => {
-      if (file != null && (await storage.exists(file))) {
-        const fileSize = await storage.stat(file)
+      if (file != null) {
+        const sz = sizeInfo[file]
+        const fileSize = sz ?? (await storage.stat(file))
+        if (sz === undefined) {
+          sizeInfo[file] = fileSize
+          processed++
+          if (processed % 10 === 0) {
+            ctx.info('Calculate size processed', { processed, size: Math.round(result.backupSize / (1024 * 1024)) })
+          }
+        }
         result.backupSize += fileSize
       }
     }
@@ -1142,6 +1159,8 @@ export async function backup (
     }
     await addFileSize(infoFile)
 
+    await storage.writeFile(sizeFile, gzipSync(JSON.stringify(sizeInfo, undefined, 2), { level: defaultLevel }))
+
     return result
   } catch (err: any) {
     ctx.error('backup error', { err, workspace: workspaceId.name })

server/backup/src/service.ts

@@ -47,6 +47,8 @@ export interface BackupConfig {
   Token: string
 
   Interval: number // Timeout in seconds
+
+  CoolDown: number // Cooldown in seconds
   Timeout: number // Timeout in seconds
   BucketName: string
   SkipWorkspaces: string
@@ -67,15 +69,10 @@ class BackupWorker {
   ) {}
 
   canceled = false
-  interval: any
-
   async close (): Promise<void> {
     this.canceled = true
-    clearTimeout(this.interval)
   }
 
-  backupPromise: Promise<void> | undefined
-
   printStats (
     ctx: MeasureContext,
     stats: { failedWorkspaces: BaseWorkspaceInfo[], processed: number, skipped: number }
@@ -91,27 +88,14 @@ class BackupWorker {
     )
   }
 
-  async triggerBackup (ctx: MeasureContext): Promise<void> {
-    const { failedWorkspaces } = await this.backup(ctx)
-    if (failedWorkspaces.length > 0) {
-      ctx.info('Failed to backup workspaces, Retry failed workspaces once.', { failed: failedWorkspaces.length })
-      this.printStats(ctx, await this.doBackup(ctx, failedWorkspaces))
-    }
-  }
-
   async schedule (ctx: MeasureContext): Promise<void> {
-    console.log('schedule timeout for', this.config.Interval, ' seconds')
-    this.interval = setTimeout(
-      () => {
-        if (this.backupPromise !== undefined) {
-          void this.backupPromise.then(() => {
-            void this.triggerBackup(ctx)
-          })
-        }
-        void this.triggerBackup(ctx)
-      },
-      5 * 60 * 1000
-    ) // Re-check every 5 minutes.
+    console.log('schedule backup with interval', this.config.Interval, 'seconds')
+    while (!this.canceled) {
+      const res = await this.backup(ctx)
+      this.printStats(ctx, res)
+      console.log('cool down', this.config.CoolDown, 'seconds')
+      await new Promise<void>((resolve) => setTimeout(resolve, this.config.CoolDown * 1000))
+    }
   }
 
   async backup (
@@ -171,11 +155,7 @@ class BackupWorker {
         index,
         total: workspaces.length
       })
-      const childLogger = rootCtx.logger.childLogger?.(ws.workspace, {
-        workspace: ws.workspace,
-        enableConsole: 'true'
-      })
-      const ctx = rootCtx.newChild(ws.workspace, { workspace: ws.workspace }, {}, childLogger)
+      const ctx = rootCtx.newChild(ws.workspace, { workspace: ws.workspace })
       let pipeline: Pipeline | undefined
       try {
         const storage = await createStorageBackupStorage(
@@ -245,6 +225,8 @@ class BackupWorker {
           }
           rootCtx.warn('BACKUP STATS', {
             workspace: ws.workspace,
+            workspaceUrl: ws.workspaceUrl,
+            workspaceName: ws.workspaceName,
             index,
             ...backupInfo,
             time: Math.round((Date.now() - st) / 1000),
@@ -259,7 +241,6 @@ class BackupWorker {
       } catch (err: any) {
         rootCtx.error('\n\nFAILED to BACKUP', { workspace: ws.workspace, err })
         failedWorkspaces.push(ws)
-        await childLogger?.close()
       } finally {
         if (pipeline !== undefined) {
           await pipeline.close()
@@ -351,9 +332,6 @@ export function backupService (
     void backupWorker.close()
   }
 
-  void backupWorker.backup(ctx).then((res) => {
-    backupWorker.printStats(ctx, res)
-    void backupWorker.schedule(ctx)
-  })
+  void backupWorker.schedule(ctx)
   return shutdown
 }