From 492225b2bfb9ce8139e96be09846e6b282290835 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Mon, 4 Nov 2024 14:35:50 +0700 Subject: [PATCH] UBERF-8569: Backup service regions support (#7090) Signed-off-by: Andrey Sobolev --- .vscode/launch.json | 4 +++- server/account/src/operations.ts | 7 +++++-- server/backup-service/src/config.ts | 12 ++++++------ server/backup-service/src/index.ts | 3 ++- server/backup/src/service.ts | 11 +++++++---- server/client/src/account.ts | 4 ++-- 6 files changed, 25 insertions(+), 16 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 79f808f1f5..7f379a08ba 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -299,8 +299,10 @@ "ACCOUNTS_URL": "http://localhost:3000", "STORAGE": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin", "WORKSPACE_STORAGE": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin", - "MONGO_URL": "mongodb://localhost:27017", + "DB_URL": "mongodb://localhost:27017", + "MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json", "SECRET": "secret", + "REGION": "pg", "BUCKET_NAME":"backups", "INTERVAL":"30", }, diff --git a/server/account/src/operations.ts b/server/account/src/operations.ts index eb5358e907..ab0aeeff84 100644 --- a/server/account/src/operations.ts +++ b/server/account/src/operations.ts @@ -816,11 +816,14 @@ export async function listWorkspaces ( ctx: MeasureContext, db: AccountDB, branding: Branding | null, - token: string + token: string, + region?: string | null ): Promise { decodeToken(ctx, token) // Just verify token is valid - return (await db.workspace.find({})).filter((it) => it.disabled !== true).map(trimWorkspaceInfo) + return (await db.workspace.find(region != null ? { region } : {})) + .filter((it) => it.disabled !== true) + .map(trimWorkspaceInfo) } /** diff --git a/server/backup-service/src/config.ts b/server/backup-service/src/config.ts index fab5913745..576a26cf93 100644 --- a/server/backup-service/src/config.ts +++ b/server/backup-service/src/config.ts @@ -29,8 +29,9 @@ interface Config extends Omit { SkipWorkspaces: string - MongoURL: string DbURL: string + + Region: string } const envMap: { [key in keyof Config]: string } = { @@ -41,11 +42,11 @@ const envMap: { [key in keyof Config]: string } = { Interval: 'INTERVAL', CoolDown: 'COOL_DOWN', Timeout: 'TIMEOUT', - MongoURL: 'MONGO_URL', DbURL: 'DB_URL', SkipWorkspaces: 'SKIP_WORKSPACES', Storage: 'STORAGE', - WorkspaceStorage: 'WORKSPACE_STORAGE' + WorkspaceStorage: 'WORKSPACE_STORAGE', + Region: 'REGION' } const required: Array = [ @@ -53,7 +54,6 @@ const required: Array = [ 'Secret', 'ServiceID', 'BucketName', - 'MongoURL', 'DbURL', 'Storage', 'WorkspaceStorage' @@ -68,11 +68,11 @@ const config: Config = (() => { Interval: parseInt(process.env[envMap.Interval] ?? '3600'), Timeout: parseInt(process.env[envMap.Timeout] ?? '3600'), CoolDown: parseInt(process.env[envMap.CoolDown] ?? '300'), - MongoURL: process.env[envMap.MongoURL], DbURL: process.env[envMap.DbURL], SkipWorkspaces: process.env[envMap.SkipWorkspaces] ?? '', WorkspaceStorage: process.env[envMap.WorkspaceStorage], - Storage: process.env[envMap.Storage] + Storage: process.env[envMap.Storage], + Region: process.env[envMap.Region] ?? '' } const missingEnv = required.filter((key) => params[key] === undefined).map((key) => envMap[key]) diff --git a/server/backup-service/src/index.ts b/server/backup-service/src/index.ts index 08a876a029..e332a09dab 100644 --- a/server/backup-service/src/index.ts +++ b/server/backup-service/src/index.ts @@ -58,7 +58,8 @@ export function startBackup ( workspaceStorageAdapter, (ctx, workspace, branding, externalStorage) => { return getConfig(ctx, mainDbUrl, workspace, branding, externalStorage) - } + }, + config.Region ) process.on('SIGINT', shutdown) diff --git a/server/backup/src/service.ts b/server/backup/src/service.ts index 144bbc7022..41284874ff 100644 --- a/server/backup/src/service.ts +++ b/server/backup/src/service.ts @@ -66,7 +66,8 @@ class BackupWorker { workspace: WorkspaceIdWithUrl, branding: Branding | null, externalStorage: StorageAdapter - ) => DbConfiguration + ) => DbConfiguration, + readonly region: string ) {} canceled = false @@ -113,7 +114,8 @@ class BackupWorker { const workspacesIgnore = new Set(this.config.SkipWorkspaces.split(';')) ctx.info('skipped workspaces', { workspacesIgnore }) let skipped = 0 - const workspaces = (await listAccountWorkspaces(this.config.Token)).filter((it) => { + const allWorkspaces = await listAccountWorkspaces(this.config.Token, this.region) + const workspaces = allWorkspaces.filter((it) => { const lastBackup = it.backupInfo?.lastBackup ?? 0 if ((Date.now() - lastBackup) / 1000 < this.config.Interval) { // No backup required, interval not elapsed @@ -337,9 +339,10 @@ export function backupService ( workspace: WorkspaceIdWithUrl, branding: Branding | null, externalStorage: StorageAdapter - ) => DbConfiguration + ) => DbConfiguration, + region: string ): () => void { - const backupWorker = new BackupWorker(storage, config, pipelineFactory, workspaceStorageAdapter, getConfig) + const backupWorker = new BackupWorker(storage, config, pipelineFactory, workspaceStorageAdapter, getConfig, region) const shutdown = (): void => { void backupWorker.close() diff --git a/server/client/src/account.ts b/server/client/src/account.ts index 9daf7c7c36..96136395c3 100644 --- a/server/client/src/account.ts +++ b/server/client/src/account.ts @@ -29,7 +29,7 @@ export interface LoginInfo { email: string } -export async function listAccountWorkspaces (token: string): Promise { +export async function listAccountWorkspaces (token: string, region: string | null = null): Promise { const accountsUrl = getAccoutsUrlOrFail() const workspaces = await ( await fetch(accountsUrl, { @@ -39,7 +39,7 @@ export async function listAccountWorkspaces (token: string): Promise