UBERF-8569: Backup service regions support (#7090)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

.vscode/launch.json

@@ -299,8 +299,10 @@
         "ACCOUNTS_URL": "http://localhost:3000",
         "STORAGE": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin",
         "WORKSPACE_STORAGE": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin",
-        "MONGO_URL": "mongodb://localhost:27017",
+        "DB_URL": "mongodb://localhost:27017",
+        "MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json",
         "SECRET": "secret",
+        "REGION": "pg",
         "BUCKET_NAME":"backups",
         "INTERVAL":"30",
       },

server/account/src/operations.ts

@@ -816,11 +816,14 @@ export async function listWorkspaces (
   ctx: MeasureContext,
   db: AccountDB,
   branding: Branding | null,
-  token: string
+  token: string,
+  region?: string | null
 ): Promise<WorkspaceInfo[]> {
   decodeToken(ctx, token) // Just verify token is valid
 
-  return (await db.workspace.find({})).filter((it) => it.disabled !== true).map(trimWorkspaceInfo)
+  return (await db.workspace.find(region != null ? { region } : {}))
+    .filter((it) => it.disabled !== true)
+    .map(trimWorkspaceInfo)
 }
 
 /**

server/backup-service/src/config.ts

@@ -29,8 +29,9 @@ interface Config extends Omit<BackupConfig, 'Token'> {
 
   SkipWorkspaces: string
 
-  MongoURL: string
   DbURL: string
+
+  Region: string
 }
 
 const envMap: { [key in keyof Config]: string } = {
@@ -41,11 +42,11 @@ const envMap: { [key in keyof Config]: string } = {
   Interval: 'INTERVAL',
   CoolDown: 'COOL_DOWN',
   Timeout: 'TIMEOUT',
-  MongoURL: 'MONGO_URL',
   DbURL: 'DB_URL',
   SkipWorkspaces: 'SKIP_WORKSPACES',
   Storage: 'STORAGE',
-  WorkspaceStorage: 'WORKSPACE_STORAGE'
+  WorkspaceStorage: 'WORKSPACE_STORAGE',
+  Region: 'REGION'
 }
 
 const required: Array<keyof Config> = [
@@ -53,7 +54,6 @@ const required: Array<keyof Config> = [
   'Secret',
   'ServiceID',
   'BucketName',
-  'MongoURL',
   'DbURL',
   'Storage',
   'WorkspaceStorage'
@@ -68,11 +68,11 @@ const config: Config = (() => {
     Interval: parseInt(process.env[envMap.Interval] ?? '3600'),
     Timeout: parseInt(process.env[envMap.Timeout] ?? '3600'),
     CoolDown: parseInt(process.env[envMap.CoolDown] ?? '300'),
-    MongoURL: process.env[envMap.MongoURL],
     DbURL: process.env[envMap.DbURL],
     SkipWorkspaces: process.env[envMap.SkipWorkspaces] ?? '',
     WorkspaceStorage: process.env[envMap.WorkspaceStorage],
-    Storage: process.env[envMap.Storage]
+    Storage: process.env[envMap.Storage],
+    Region: process.env[envMap.Region] ?? ''
   }
 
   const missingEnv = required.filter((key) => params[key] === undefined).map((key) => envMap[key])

server/backup-service/src/index.ts

@@ -58,7 +58,8 @@ export function startBackup (
     workspaceStorageAdapter,
     (ctx, workspace, branding, externalStorage) => {
       return getConfig(ctx, mainDbUrl, workspace, branding, externalStorage)
-    }
+    },
+    config.Region
   )
 
   process.on('SIGINT', shutdown)

server/backup/src/service.ts

@@ -66,7 +66,8 @@ class BackupWorker {
       workspace: WorkspaceIdWithUrl,
       branding: Branding | null,
       externalStorage: StorageAdapter
-    ) => DbConfiguration
+    ) => DbConfiguration,
+    readonly region: string
   ) {}
 
   canceled = false
@@ -113,7 +114,8 @@ class BackupWorker {
     const workspacesIgnore = new Set(this.config.SkipWorkspaces.split(';'))
     ctx.info('skipped workspaces', { workspacesIgnore })
     let skipped = 0
-    const workspaces = (await listAccountWorkspaces(this.config.Token)).filter((it) => {
+    const allWorkspaces = await listAccountWorkspaces(this.config.Token, this.region)
+    const workspaces = allWorkspaces.filter((it) => {
       const lastBackup = it.backupInfo?.lastBackup ?? 0
       if ((Date.now() - lastBackup) / 1000 < this.config.Interval) {
         // No backup required, interval not elapsed
@@ -337,9 +339,10 @@ export function backupService (
     workspace: WorkspaceIdWithUrl,
     branding: Branding | null,
     externalStorage: StorageAdapter
-  ) => DbConfiguration
+  ) => DbConfiguration,
+  region: string
 ): () => void {
-  const backupWorker = new BackupWorker(storage, config, pipelineFactory, workspaceStorageAdapter, getConfig)
+  const backupWorker = new BackupWorker(storage, config, pipelineFactory, workspaceStorageAdapter, getConfig, region)
 
   const shutdown = (): void => {
     void backupWorker.close()

server/client/src/account.ts

@@ -29,7 +29,7 @@ export interface LoginInfo {
   email: string
 }
 
-export async function listAccountWorkspaces (token: string): Promise<BaseWorkspaceInfo[]> {
+export async function listAccountWorkspaces (token: string, region: string | null = null): Promise<BaseWorkspaceInfo[]> {
   const accountsUrl = getAccoutsUrlOrFail()
   const workspaces = await (
     await fetch(accountsUrl, {
@@ -39,7 +39,7 @@ export async function listAccountWorkspaces (token: string): Promise<BaseWorkspa
       },
       body: JSON.stringify({
         method: 'listWorkspaces',
-        params: [token]
+        params: [token, region]
       })
     })
   ).json()