UBERF-8433: Support for archived workspaces (#6937)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

.vscode/launch.json

@@ -341,6 +341,28 @@
       "cwd": "${workspaceRoot}/pods/backup",
       "protocol": "inspector"
     },
+    {
+      "name": "Debug archive tool",
+      "type": "node",
+      "request": "launch",
+      "args": ["src/__start.ts", "archive-workspaces-mongo", "--timeout", "7", "--remove"],
+      "env": {
+        "ACCOUNTS_URL": "http://localhost:3000",
+        "STORAGE": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin",
+        "WORKSPACE_STORAGE": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin",
+        "MONGO_URL": "mongodb://localhost:27017",
+        "DB_URL": "mongodb://localhost:27017",
+        "SERVER_SECRET": "secret",
+        "SECRET": "secret",
+        "BUCKET_NAME":"backups"
+      },
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
+      "showAsyncStacks": true,
+      "sourceMaps": true,
+      "outputCapture": "std",
+      "cwd": "${workspaceRoot}/dev/tool",
+      "protocol": "inspector"
+    },
     {
       "name": "Debug Github integration",
       "type": "node",

dev/tool/package.json

@@ -92,6 +92,7 @@
     "@hcengineering/server-collaboration": "^0.6.0",
     "@hcengineering/server-collaboration-resources": "^0.6.0",
     "@hcengineering/server-backup": "^0.6.0",
+    "@hcengineering/backup-service": "^0.6.0",
     "@hcengineering/server-storage": "^0.6.0",
     "@hcengineering/server-calendar": "^0.6.0",
     "@hcengineering/server-calendar-resources": "^0.6.0",

dev/tool/src/index.ts

@@ -32,10 +32,12 @@ import accountPlugin, {
   replacePassword,
   setAccountAdmin,
   setRole,
+  updateArchiveInfo,
   updateWorkspace,
   type AccountDB,
   type Workspace
 } from '@hcengineering/account'
+import { backupWorkspace } from '@hcengineering/backup-service'
 import { setMetadata } from '@hcengineering/platform'
 import {
   backup,
@@ -56,7 +58,13 @@ import serverClientPlugin, {
   listAccountWorkspaces,
   updateBackupInfo
 } from '@hcengineering/server-client'
-import { getServerPipeline, registerServerPlugins, registerStringLoaders } from '@hcengineering/server-pipeline'
+import {
+  createBackupPipeline,
+  getConfig,
+  getServerPipeline,
+  registerServerPlugins,
+  registerStringLoaders
+} from '@hcengineering/server-pipeline'
 import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
 import toolPlugin, { FileModelLogger } from '@hcengineering/server-tool'
 import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service'
@@ -89,7 +97,7 @@ import contact from '@hcengineering/model-contact'
 import { getMongoClient, getWorkspaceMongoDB, shutdown } from '@hcengineering/mongo'
 import { backupDownload } from '@hcengineering/server-backup/src/backup'
 
-import type { StorageAdapter, StorageAdapterEx } from '@hcengineering/server-core'
+import type { PipelineFactory, StorageAdapter, StorageAdapterEx } from '@hcengineering/server-core'
 import { deepEqual } from 'fast-equals'
 import { createWriteStream, readFileSync } from 'fs'
 import { getMongoDBUrl } from './__start'
@@ -509,13 +517,10 @@ export function devTool (
     })
 
   program
-    .command('list-unused-workspaces-mongo')
-    .description(
-      'remove unused workspaces, please pass --remove to really delete them. Without it will only mark them disabled'
-    )
-    .option('-r|--remove [remove]', 'Force remove', false)
-    .option('-t|--timeout [timeout]', 'Timeout in days', '7')
-    .action(async (cmd: { remove: boolean, disable: boolean, exclude: string, timeout: string }) => {
+    .command('list-unused-workspaces')
+    .description('remove unused workspaces. Without it will only mark them disabled')
+    .option('-t|--timeout [timeout]', 'Timeout in days', '60')
+    .action(async (cmd: { disable: boolean, exclude: string, timeout: string }) => {
       const { dbUrl } = prepareTools()
       await withDatabase(dbUrl, async (db) => {
         const workspaces = new Map((await listWorkspacesPure(db)).map((p) => [p._id.toString(), p]))
@@ -524,12 +529,9 @@ export function devTool (
 
         const _timeout = parseInt(cmd.timeout) ?? 7
 
-        await withStorage(async (adapter) => {
-          // We need to update workspaces with missing workspaceUrl
-          const mongodbUri = getMongoDBUrl()
-          const client = getMongoClient(mongodbUri ?? dbUrl)
-          const _client = await client.getClient()
-          try {
+        let used = 0
+        let unused = 0
+
         for (const a of accounts) {
           const authored = a.workspaces
             .map((it) => workspaces.get(it.toString()))
@@ -547,15 +549,14 @@ export function devTool (
               const lastVisitDays = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24)
 
               if (lastVisitDays > _timeout) {
+                unused++
                 toolCtx.warn('  --- unused', {
                   url: ws.workspaceUrl,
                   id: ws.workspace,
                   lastVisitDays
                 })
-                    if (cmd.remove) {
-                      await dropWorkspaceFull(toolCtx, db, _client, null, ws.workspace, adapter)
-                    }
               } else {
+                used++
                 toolCtx.warn('  +++ used', {
                   url: ws.workspaceUrl,
                   id: ws.workspace,
@@ -566,10 +567,153 @@ export function devTool (
             }
           }
         }
+
+        console.log('Used: ', used, 'Unused: ', unused)
+      })
+    })
+  program
+    .command('archive-workspaces-mongo')
+    .description('Archive and delete non visited workspaces...')
+    .option('-r|--remove [remove]', 'Pass to remove all data', false)
+    .option('--region [region]', 'Pass to remove all data', '')
+    .option('-t|--timeout [timeout]', 'Timeout in days', '60')
+    .option('-w|--workspace [workspace]', 'Force backup of selected workspace', '')
+    .action(
+      async (cmd: {
+        disable: boolean
+        exclude: string
+        timeout: string
+        remove: boolean
+        workspace: string
+        region: string
+      }) => {
+        const { dbUrl, txes } = prepareTools()
+        const mongodbUri = getMongoDBUrl()
+        await withDatabase(dbUrl, async (db) => {
+          const workspaces = (await listWorkspacesPure(db))
+            .sort((a, b) => a.lastVisit - b.lastVisit)
+            .filter((it) => cmd.workspace === '' || cmd.workspace === it.workspace)
+
+          const _timeout = parseInt(cmd.timeout) ?? 7
+
+          let unused = 0
+
+          // We need to update workspaces with missing workspaceUrl
+          const client = getMongoClient(mongodbUri ?? dbUrl)
+          const mongoClient = await client.getClient()
+          try {
+            for (const ws of workspaces) {
+              const lastVisitDays = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24)
+
+              if (lastVisitDays > _timeout && ws.mode !== 'archived') {
+                unused++
+                toolCtx.warn('--- unused', {
+                  url: ws.workspaceUrl,
+                  id: ws.workspace,
+                  lastVisitDays,
+                  mode: ws.mode
+                })
+                try {
+                  await backupWorkspace(
+                    toolCtx,
+                    ws,
+                    (dbUrl, storageAdapter) => {
+                      const factory: PipelineFactory = createBackupPipeline(toolCtx, dbUrl, txes, {
+                        externalStorage: storageAdapter,
+                        usePassedCtx: true
+                      })
+                      return factory
+                    },
+                    (ctx, dbUrls, workspace, branding, externalStorage) => {
+                      return getConfig(ctx, dbUrls, ctx, {
+                        externalStorage,
+                        disableTriggers: true
+                      })
+                    },
+                    cmd.region,
+                    true,
+                    5000, // 5 gigabytes per blob
+                    async (storage, workspaceStorage) => {
+                      if (cmd.remove) {
+                        await updateArchiveInfo(toolCtx, db, ws.workspace, true)
+                        const files = await workspaceStorage.listStream(toolCtx, { name: ws.workspace })
+
+                        while (true) {
+                          const docs = await files.next()
+                          if (docs.length === 0) {
+                            break
+                          }
+                          await workspaceStorage.remove(
+                            toolCtx,
+                            { name: ws.workspace },
+                            docs.map((it) => it._id)
+                          )
+                        }
+
+                        const mongoDb = getWorkspaceMongoDB(mongoClient, { name: ws.workspace })
+                        await mongoDb.dropDatabase()
+                      }
+                    }
+                  )
+                } catch (err: any) {
+                  toolCtx.error('Failed to backup/archive workspace', { workspace: ws.workspace })
+                }
+              }
+            }
           } finally {
             client.close()
           }
+          console.log('Processed unused workspaces', unused)
         })
+      }
+    )
+
+  program
+    .command('backup-all')
+    .description('Backup all workspaces...')
+    .option('--region [region]', 'Force backup of selected workspace', '')
+    .option('-w|--workspace [workspace]', 'Force backup of selected workspace', '')
+    .action(async (cmd: { workspace: string, region: string }) => {
+      const { dbUrl, txes } = prepareTools()
+      await withDatabase(dbUrl, async (db) => {
+        const workspaces = (await listWorkspacesPure(db))
+          .sort((a, b) => a.lastVisit - b.lastVisit)
+          .filter((it) => cmd.workspace === '' || cmd.workspace === it.workspace)
+
+        let processed = 0
+
+        // We need to update workspaces with missing workspaceUrl
+        for (const ws of workspaces) {
+          try {
+            if (
+              await backupWorkspace(
+                toolCtx,
+                ws,
+                (dbUrl, storageAdapter) => {
+                  const factory: PipelineFactory = createBackupPipeline(toolCtx, dbUrl, txes, {
+                    externalStorage: storageAdapter,
+                    usePassedCtx: true
+                  })
+                  return factory
+                },
+                (ctx, dbUrls, workspace, branding, externalStorage) => {
+                  return getConfig(ctx, dbUrls, ctx, {
+                    externalStorage,
+                    disableTriggers: true
+                  })
+                },
+                cmd.region,
+                true,
+                100
+              )
+            ) {
+              processed++
+            }
+          } catch (err: any) {
+            toolCtx.error('Failed to backup workspace', { workspace: ws.workspace })
+          }
+        }
+        console.log('Processed workspaces', processed)
       })
     })
 
@@ -656,7 +800,7 @@ export function devTool (
           try {
             for (const ws of workspacesJSON) {
               const lastVisit = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24)
-              if (lastVisit > 30) {
+              if (lastVisit > 60) {
                 await dropWorkspaceFull(toolCtx, db, _client, null, ws.workspace, storageAdapter)
               }
             }
@@ -693,6 +837,7 @@ export function devTool (
             !deepEqual(ws.version, version) ? `upgrade to ${versionToString(version)} is required` : ''
           )
           console.log('disabled:', ws.disabled)
+          console.log('mode:', ws.mode)
           console.log('created by:', ws.createdBy)
           console.log('members:', (ws.accounts ?? []).length)
           if (Number.isNaN(lastVisit)) {
@@ -1130,12 +1275,18 @@ export function devTool (
         })
       })
     })
-  program.command('clean-empty-buckets').action(async (cmd: any) => {
+  program
+    .command('clean-empty-buckets')
+    .option('--prefix [prefix]', 'Prefix', '')
+    .action(async (cmd: { prefix: string }) => {
       await withStorage(async (adapter) => {
         const buckets = await adapter.listBuckets(toolCtx)
         for (const ws of buckets) {
+          if (ws.name.startsWith(cmd.prefix)) {
+            console.log('Checking', ws.name)
             const l = await ws.list()
-        if ((await l.next()) === undefined) {
+            const docs = await l.next()
+            if (docs.length === 0) {
               await l.close()
               // No data, we could delete it.
               console.log('Clean bucket', ws.name)
@@ -1144,6 +1295,7 @@ export function devTool (
               await l.close()
             }
           }
+        }
       })
     })
   program
@@ -1213,6 +1365,10 @@ export function devTool (
                 if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) {
                   continue
                 }
+                if (workspace.mode === 'archived') {
+                  console.log('ignore archived workspace', workspace.workspace)
+                  continue
+                }
                 if (workspace.disabled === true && !cmd.disabled) {
                   console.log('ignore disabled workspace', workspace.workspace)
                   continue
@@ -1252,6 +1408,10 @@ export function devTool (
             workspaces.sort((a, b) => b.lastVisit - a.lastVisit)
 
             for (const workspace of workspaces) {
+              if (workspace.mode === 'archived') {
+                console.log('ignore archived workspace', workspace.workspace)
+                continue
+              }
               if (workspace.disabled === true && !cmd.disabled) {
                 console.log('ignore disabled workspace', workspace.workspace)
                 continue

packages/core/src/classes.ts

@@ -642,7 +642,14 @@ export interface DomainIndexConfiguration extends Doc {
   skip?: string[]
 }
 
-export type WorkspaceMode = 'pending-creation' | 'creating' | 'upgrading' | 'pending-deletion' | 'deleting' | 'active'
+export type WorkspaceMode =
+  | 'pending-creation'
+  | 'creating'
+  | 'upgrading'
+  | 'pending-deletion'
+  | 'deleting'
+  | 'active'
+  | 'archived'
 
 export interface BackupStatus {
   dataSize: number
@@ -664,9 +671,7 @@ export interface BaseWorkspaceInfo {
   workspaceName?: string // An displayed workspace name
   createdOn: number
   lastVisit: number
-
   createdBy: string
-
   mode: WorkspaceMode
   progress?: number // Some progress
 

packages/platform/src/platform.ts

@@ -148,6 +148,7 @@ export default plugin(platformId, {
     AccountNotFound: '' as StatusCode<{ account: string }>,
     AccountNotConfirmed: '' as StatusCode<{ account: string }>,
     WorkspaceNotFound: '' as StatusCode<{ workspace: string }>,
+    WorkspaceArchived: '' as StatusCode<{ workspace: string }>,
     InvalidPassword: '' as StatusCode<{ account: string }>,
     AccountAlreadyExists: '' as StatusCode<{ account: string }>,
     AccountAlreadyConfirmed: '' as StatusCode<{ account: string }>,

plugins/client-resources/src/connection.ts

@@ -42,7 +42,13 @@ import core, {
   toFindResult,
   type MeasureContext
 } from '@hcengineering/core'
-import { PlatformError, UNAUTHORIZED, broadcastEvent, getMetadata, unknownError } from '@hcengineering/platform'
+import platform, {
+  PlatformError,
+  UNAUTHORIZED,
+  broadcastEvent,
+  getMetadata,
+  unknownError
+} from '@hcengineering/platform'
 
 import { HelloRequest, HelloResponse, RPCHandler, ReqId, type Response } from '@hcengineering/rpc'
 
@@ -241,8 +247,13 @@ class Connection implements ClientConnection {
         Analytics.handleError(new PlatformError(resp.error))
         this.closed = true
         this.websocket?.close()
+        if (resp.error?.code === UNAUTHORIZED.code) {
           this.opt?.onUnauthorized?.()
         }
+        if (resp.error?.code === platform.status.WorkspaceArchived) {
+          this.opt?.onArchived?.()
+        }
+      }
       console.error(resp.error)
       return
     }

plugins/client/src/index.ts

@@ -61,6 +61,7 @@ export interface ClientFactoryOptions {
   onHello?: (serverVersion?: string) => boolean
   onUpgrade?: () => void
   onUnauthorized?: () => void
+  onArchived?: () => void
   onConnect?: (event: ClientConnectEvent, data: any) => void
   ctx?: MeasureContext
   onDialTimeout?: () => void | Promise<void>

plugins/login-resources/src/components/SelectWorkspace.svelte

@@ -123,6 +123,9 @@
             <div class="flex flex-col flex-grow">
               <span class="label overflow-label flex-center">
                 {wsName}
+                {#if workspace.mode === 'archived'}
+                  - <Label label={presentation.string.Archived} />
+                {/if}
                 {#if workspace.mode === 'creating'}
                   ({workspace.progress}%)
                 {/if}

plugins/login/src/index.ts

@@ -35,9 +35,7 @@ export interface Workspace {
   progress?: number
 
   lastVisit: number
-
   backupInfo?: BackupStatus
-
   region?: string
 }
 

plugins/workbench-assets/lang/en.json

@@ -35,6 +35,7 @@
     "OpenInSidebar": "Open in sidebar",
     "OpenInSidebarNewTab": "Open in sidebar new tab",
     "ConfigureWidgets": "Configure widgets",
-    "Tab": "Tab"
+    "Tab": "Tab",
+    "WorkspaceIsArchived": "Workspace is archived because of being unused, Please contact us to restore..."
   }
 }

plugins/workbench-assets/lang/es.json

@@ -35,6 +35,7 @@
     "OpenInSidebar": "Abrir en la barra lateral",
     "OpenInSidebarNewTab": "Abrir en una nueva pestaña de la barra lateral",
     "ConfigureWidgets": "Configurar widgets",
-    "Tab": "Pestaña"
+    "Tab": "Pestaña",
+    "WorkspaceIsArchived": "El espacio de trabajo está archivado por no estar en uso, por favor contáctenos para restaurarlo..."
   }
 }

plugins/workbench-assets/lang/fr.json

@@ -35,6 +35,7 @@
     "OpenInSidebar": "Ouvrir dans la barre latérale",
     "OpenInSidebarNewTab": "Ouvrir dans un nouvel onglet de la barre latérale",
     "ConfigureWidgets": "Configurer les widgets",
-    "Tab": "Onglet"
+    "Tab": "Onglet",
+    "WorkspaceIsArchived": "L'espace de travail est archivé en raison de son inactivité, veuillez nous contacter pour le restaurer..."
   }
 }

plugins/workbench-assets/lang/pt.json

@@ -35,6 +35,7 @@
     "OpenInSidebar": "Abrir na barra lateral",
     "OpenInSidebarNewTab": "Abrir em uma nova aba da barra lateral",
     "ConfigureWidgets": "Configurar widgets",
-    "Tab": "Aba"
+    "Tab": "Aba",
+    "WorkspaceIsArchived": "O espaço de trabalho está arquivado por estar inativo, por favor, entre em contato conosco para restaurá-lo..."
   }
 }

plugins/workbench-assets/lang/ru.json

@@ -35,6 +35,7 @@
     "OpenInSidebar": "Открыть в боковой панели",
     "OpenInSidebarNewTab": "Открыть в новой вкладке боковой панели",
     "ConfigureWidgets": "Настроить виджеты",
-    "Tab": "Вкладка"
+    "Tab": "Вкладка",
+    "WorkspaceIsArchived": "Рабочее пространство архивировано из-за неиспользования, пожалуйста, свяжитесь с нами для восстановления..."
   }
 }

plugins/workbench-assets/lang/zh.json

@@ -35,6 +35,7 @@
     "OpenInSidebar": "在侧边栏中打开",
     "OpenInSidebarNewTab": "在侧边栏新标签页中打开",
     "ConfigureWidgets": "配置小部件",
-    "Tab": "选项卡"
+    "Tab": "选项卡",
+    "WorkspaceIsArchived": "工作区因未使用而归档，请与我们联系以恢复..."
   }
 }

plugins/workbench-resources/src/components/SelectWorkspaceMenu.svelte

@@ -20,6 +20,7 @@
   import {
     Icon,
     IconCheck,
+    Label,
     Loading,
     Location,
     SearchEdit,
@@ -178,6 +179,9 @@
               <div class="flex-col flex-grow">
                 <span class="label overflow-label flex flex-grow flex-between">
                   {wsName}
+                  {#if ws.mode === 'archived'}
+                    - <Label label={presentation.string.Archived} />
+                  {/if}
                   {#if ws.region != null && ws.region !== ''}
                     - ({ws.region})
                   {/if}

plugins/workbench-resources/src/components/WorkbenchApp.svelte

@@ -14,6 +14,7 @@
 -->
 <script lang="ts">
   import { getMetadata } from '@hcengineering/platform'
+  import { upgradeDownloadProgress } from '@hcengineering/presentation'
   import {
     Button,
     Component,
@@ -25,7 +26,6 @@
     location,
     setMetadataLocalStorage
   } from '@hcengineering/ui'
-  import { upgradeDownloadProgress } from '@hcengineering/presentation'
   import { connect, disconnect, versionError } from '../connect'
 
   import workbench, { workbenchId } from '@hcengineering/workbench'

plugins/workbench-resources/src/connect.ts

@@ -16,7 +16,7 @@ import core, {
   type Version
 } from '@hcengineering/core'
 import login, { loginId } from '@hcengineering/login'
-import { broadcastEvent, getMetadata, getResource, setMetadata } from '@hcengineering/platform'
+import { broadcastEvent, getMetadata, getResource, setMetadata, translateCB } from '@hcengineering/platform'
 import presentation, {
   closeClient,
   loadServerConfig,
@@ -33,7 +33,8 @@ import {
   getCurrentLocation,
   locationStorageKeyId,
   navigate,
-  setMetadataLocalStorage
+  setMetadataLocalStorage,
+  themeStore
 } from '@hcengineering/ui'
 import { writable, get } from 'svelte/store'
 
@@ -206,14 +207,23 @@ export async function connect (title: string): Promise<Client | undefined> {
             query: {}
           })
         },
+        onArchived: () => {
+          translateCB(plugin.string.WorkspaceIsArchived, {}, get(themeStore).language, (r) => {
+            versionError.set(r)
+          })
+        },
         // We need to refresh all active live queries and clear old queries.
         onConnect: (event: ClientConnectEvent, data: any) => {
           console.log('WorkbenchClient: onConnect', event)
           if (event === ClientConnectEvent.Maintenance) {
             if (data != null && data.total !== 0) {
-              versionError.set(`Maintenance ${Math.floor((100 / data.total) * (data.total - data.toProcess))}%`)
+              translateCB(plugin.string.ServerUnderMaintenance, {}, get(themeStore).language, (r) => {
+                versionError.set(`${r} ${Math.floor((100 / data.total) * (data.total - data.toProcess))}%`)
+              })
             } else {
-              versionError.set('Maintenance...')
+              translateCB(plugin.string.ServerUnderMaintenance, {}, get(themeStore).language, (r) => {
+                versionError.set(r)
+              })
             }
             return
           }

plugins/workbench/src/index.ts

@@ -238,7 +238,8 @@ export default plugin(workbenchId, {
     UpgradeDownloadProgress: '' as IntlString,
     OpenInSidebar: '' as IntlString,
     OpenInSidebarNewTab: '' as IntlString,
-    ConfigureWidgets: '' as IntlString
+    ConfigureWidgets: '' as IntlString,
+    WorkspaceIsArchived: '' as IntlString
   },
   icon: {
     Search: '' as Asset

server/account/src/operations.ts

@@ -465,6 +465,18 @@ export async function selectWorkspace (
   }
 
   if (workspaceInfo !== null) {
+    if (workspaceInfo.mode === 'archived') {
+      const result: WorkspaceLoginInfo = {
+        endpoint: '',
+        email,
+        token: '',
+        workspace: workspaceUrl,
+        workspaceId: workspaceInfo.workspace,
+        mode: workspaceInfo.mode,
+        progress: workspaceInfo.progress
+      }
+      return result
+    }
     if (workspaceInfo.disabled === true && workspaceInfo.mode === 'active') {
       ctx.error('workspace disabled', { workspaceUrl, email })
       throw new PlatformError(
@@ -1201,6 +1213,28 @@ export async function updateBackupInfo (
   )
 }
 
+/**
+ * @public
+ */
+export async function updateArchiveInfo (
+  ctx: MeasureContext,
+  db: AccountDB,
+  workspace: string,
+  value: boolean
+): Promise<void> {
+  const workspaceInfo = await getWorkspaceById(db, workspace)
+  if (workspaceInfo === null) {
+    throw new PlatformError(new Status(Severity.ERROR, platform.status.WorkspaceNotFound, { workspace }))
+  }
+
+  await db.workspace.updateOne(
+    { _id: workspaceInfo._id },
+    {
+      mode: 'archived'
+    }
+  )
+}
+
 async function postCreateUserWorkspace (
   ctx: MeasureContext,
   db: AccountDB,
@@ -1485,7 +1519,7 @@ export async function getWorkspaceInfo (
     ctx.error('no workspace', { workspace: workspace.name, email })
     throw new PlatformError(new Status(Severity.ERROR, platform.status.Forbidden, {}))
   }
-  if (_updateLastVisit && (isAccount(account) || email === systemAccountEmail)) {
+  if (ws.mode !== 'archived' && _updateLastVisit && (isAccount(account) || email === systemAccountEmail)) {
     void ctx.with('update-last-visit', {}, async () => {
       await updateLastVisit(db, ws, account as Account)
     })

server/backup-service/src/config.ts

@@ -59,7 +59,7 @@ const required: Array<keyof Config> = [
   'WorkspaceStorage'
 ]
 
-const config: Config = (() => {
+export const config: () => Config = () => {
   const params: Partial<Config> = {
     AccountsURL: process.env[envMap.AccountsURL],
     Secret: process.env[envMap.Secret],
@@ -82,6 +82,4 @@ const config: Config = (() => {
   }
 
   return params as Config
-})()
-
-export default config
+}

server/backup-service/src/index.ts

@@ -13,14 +13,20 @@
 // limitations under the License.
 //
 
-import { MeasureContext, systemAccountEmail, type Branding, type WorkspaceIdWithUrl } from '@hcengineering/core'
+import {
+  MeasureContext,
+  systemAccountEmail,
+  type BaseWorkspaceInfo,
+  type Branding,
+  type WorkspaceIdWithUrl
+} from '@hcengineering/core'
 import { setMetadata } from '@hcengineering/platform'
-import { backupService } from '@hcengineering/server-backup'
+import { backupService, doBackupWorkspace } from '@hcengineering/server-backup'
 import serverClientPlugin from '@hcengineering/server-client'
 import { type DbConfiguration, type PipelineFactory, type StorageAdapter } from '@hcengineering/server-core'
 import { buildStorageFromConfig, createStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
 import serverToken, { generateToken } from '@hcengineering/server-token'
-import config from './config'
+import { config as _config } from './config'
 
 export function startBackup (
   ctx: MeasureContext,
@@ -33,6 +39,7 @@ export function startBackup (
     externalStorage: StorageAdapter
   ) => DbConfiguration
 ): void {
+  const config = _config()
   setMetadata(serverToken.metadata.Secret, config.Secret)
   setMetadata(serverClientPlugin.metadata.Endpoint, config.AccountsURL)
   setMetadata(serverClientPlugin.metadata.UserAgent, config.ServiceID)
@@ -71,3 +78,63 @@ export function startBackup (
     ctx.error('unhandledRejection', { err: e })
   })
 }
+
+export async function backupWorkspace (
+  ctx: MeasureContext,
+  workspace: BaseWorkspaceInfo,
+  pipelineFactoryFactory: (mongoUrl: string, storage: StorageAdapter) => PipelineFactory,
+  getConfig: (
+    ctx: MeasureContext,
+    dbUrls: string,
+    workspace: WorkspaceIdWithUrl,
+    branding: Branding | null,
+    externalStorage: StorageAdapter
+  ) => DbConfiguration,
+  region: string,
+  recheck: boolean = false,
+  downloadLimit: number,
+
+  onFinish?: (backupStorage: StorageAdapter, workspaceStorage: StorageAdapter) => Promise<void>
+): Promise<boolean> {
+  const config = _config()
+  setMetadata(serverToken.metadata.Secret, config.Secret)
+  setMetadata(serverClientPlugin.metadata.Endpoint, config.AccountsURL)
+  setMetadata(serverClientPlugin.metadata.UserAgent, config.ServiceID)
+
+  const mainDbUrl = config.DbURL
+
+  const backupStorageConfig = storageConfigFromEnv(config.Storage)
+  const workspaceStorageConfig = storageConfigFromEnv(config.WorkspaceStorage)
+
+  const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0])
+  const workspaceStorageAdapter = buildStorageFromConfig(workspaceStorageConfig)
+
+  const pipelineFactory = pipelineFactoryFactory(mainDbUrl, workspaceStorageAdapter)
+
+  // A token to access account service
+  const token = generateToken(systemAccountEmail, { name: 'backup' })
+
+  try {
+    const result = await doBackupWorkspace(
+      ctx,
+      workspace,
+      storageAdapter,
+      { ...config, Token: token },
+      pipelineFactory,
+      workspaceStorageAdapter,
+      (ctx, workspace, branding, externalStorage) => {
+        return getConfig(ctx, mainDbUrl, workspace, branding, externalStorage)
+      },
+      region,
+      recheck,
+      downloadLimit
+    )
+    if (result && onFinish !== undefined) {
+      await onFinish(storageAdapter, workspaceStorageAdapter)
+    }
+    return result
+  } finally {
+    await storageAdapter.close()
+    await workspaceStorageAdapter.close()
+  }
+}

server/backup/src/service.ts

@@ -56,6 +56,7 @@ export interface BackupConfig {
 }
 
 class BackupWorker {
+  downloadLimit: number = 100
   constructor (
     readonly storageAdapter: StorageAdapter,
     readonly config: BackupConfig,
@@ -67,7 +68,8 @@ class BackupWorker {
       branding: Branding | null,
       externalStorage: StorageAdapter
     ) => DbConfiguration,
-    readonly region: string
+    readonly region: string,
+    readonly recheck: boolean = false
   ) {}
 
   canceled = false
@@ -188,10 +190,10 @@ class BackupWorker {
           backup(ctx, '', getWorkspaceId(ws.workspace), storage, {
             skipDomains: [],
             force: true,
-            recheck: false,
+            recheck: this.recheck,
             timeout: this.config.Timeout * 1000,
             connectTimeout: 5 * 60 * 1000, // 5 minutes to,
-            blobDownloadLimit: 100,
+            blobDownloadLimit: this.downloadLimit,
             skipBlobContentTypes: [],
             storageAdapter: this.workspaceStorageAdapter,
             getLastTx: async (): Promise<Tx | undefined> => {
@@ -340,7 +342,8 @@ export function backupService (
     branding: Branding | null,
     externalStorage: StorageAdapter
   ) => DbConfiguration,
-  region: string
+  region: string,
+  recheck?: boolean
 ): () => void {
   const backupWorker = new BackupWorker(storage, config, pipelineFactory, workspaceStorageAdapter, getConfig, region)
 
@@ -351,3 +354,35 @@ export function backupService (
   void backupWorker.schedule(ctx)
   return shutdown
 }
+
+export async function doBackupWorkspace (
+  ctx: MeasureContext,
+  workspace: BaseWorkspaceInfo,
+  storage: StorageAdapter,
+  config: BackupConfig,
+  pipelineFactory: PipelineFactory,
+  workspaceStorageAdapter: StorageAdapter,
+  getConfig: (
+    ctx: MeasureContext,
+    workspace: WorkspaceIdWithUrl,
+    branding: Branding | null,
+    externalStorage: StorageAdapter
+  ) => DbConfiguration,
+  region: string,
+  recheck: boolean,
+  downloadLimit: number
+): Promise<boolean> {
+  const backupWorker = new BackupWorker(
+    storage,
+    config,
+    pipelineFactory,
+    workspaceStorageAdapter,
+    getConfig,
+    region,
+    recheck
+  )
+  backupWorker.downloadLimit = downloadLimit
+  const { processed } = await backupWorker.doBackup(ctx, [workspace], Number.MAX_VALUE)
+  await backupWorker.close()
+  return processed === 1
+}

server/core/src/types.ts

@@ -595,7 +595,10 @@ export interface AddSessionActive {
   context: MeasureContext
   workspaceId: string
 }
-export type AddSessionResponse = AddSessionActive | { upgrade: true } | { error: any, terminate?: boolean }
+export type AddSessionResponse =
+  | AddSessionActive
+  | { upgrade: true }
+  | { error: any, terminate?: boolean, archived?: boolean }
 
 /**
  * @public

server/mongo/src/storage.ts

@@ -1052,12 +1052,11 @@ abstract class MongoAdapterBase implements DbAdapter {
     return {
       next: async () => {
         if (iterator === undefined) {
-          if (recheck === true) {
-            await coll.updateMany({ '%hash%': { $ne: null } }, { $set: { '%hash%': null } })
-          }
           iterator = coll.find(
-            { '%hash%': { $nin: ['', null] } },
-            {
+            recheck === true ? {} : { '%hash%': { $nin: ['', null] } },
+            recheck === true
+              ? {}
+              : {
                   projection: {
                     '%hash%': 1,
                     _id: 1
@@ -1066,7 +1065,7 @@ abstract class MongoAdapterBase implements DbAdapter {
           )
         }
         let d = await ctx.with('next', { mode }, async () => await iterator.next())
-        if (d == null && mode === 'hashed') {
+        if (d == null && mode === 'hashed' && recheck !== true) {
           mode = 'non-hashed'
           await iterator.close()
           iterator = coll.find({ '%hash%': { $in: ['', null] } })
@@ -1074,10 +1073,10 @@ abstract class MongoAdapterBase implements DbAdapter {
         }
         const result: DocInfo[] = []
         if (d != null) {
-          result.push(this.toDocInfo(d, bulkUpdate))
+          result.push(this.toDocInfo(d, bulkUpdate, recheck))
         }
         if (iterator.bufferedCount() > 0) {
-          result.push(...iterator.readBufferedDocuments().map((it) => this.toDocInfo(it, bulkUpdate)))
+          result.push(...iterator.readBufferedDocuments().map((it) => this.toDocInfo(it, bulkUpdate, recheck)))
         }
         await ctx.with('flush', {}, async () => {
           await flush()
@@ -1096,13 +1095,14 @@ abstract class MongoAdapterBase implements DbAdapter {
     }
   }
 
-  private toDocInfo (d: Doc, bulkUpdate: Map<Ref<Doc>, string>): DocInfo {
+  private toDocInfo (d: Doc, bulkUpdate: Map<Ref<Doc>, string>, recheck?: boolean): DocInfo {
     let digest: string | null = (d as any)['%hash%']
     if ('%hash%' in d) {
       delete d['%hash%']
     }
     const pos = (digest ?? '').indexOf('|')
-    if (digest == null || digest === '') {
+    const oldDigest = digest
+    if (digest == null || digest === '' || recheck === true) {
       let size = estimateDocSize(d)
 
       if (this.options?.calculateHash !== undefined) {
@@ -1112,8 +1112,11 @@ abstract class MongoAdapterBase implements DbAdapter {
         updateHashForDoc(hash, d)
         digest = hash.digest('base64')
       }
+      const newDigest = `${digest}|${size.toString(16)}`
 
+      if (recheck !== true || oldDigest !== newDigest) {
         bulkUpdate.set(d._id, `${digest}|${size.toString(16)}`)
+      }
       return {
         id: d._id,
         hash: digest,

server/server/src/sessionManager.ts

@@ -308,7 +308,7 @@ class TSessionManager implements SessionManager {
   ): Promise<
     | { session: Session, context: MeasureContext, workspaceId: string }
     | { upgrade: true, upgradeInfo?: WorkspaceLoginInfo['upgrade'] }
-    | { error: any, terminate?: boolean }
+    | { error: any, terminate?: boolean, archived?: boolean }
     > {
     const wsString = toWorkspaceString(token.workspace)
 
@@ -325,6 +325,11 @@ class TSessionManager implements SessionManager {
       return { upgrade: true }
     }
 
+    if (workspaceInfo.mode === 'archived') {
+      // No access to disabled workspaces for regular users
+      return { error: new Error('Workspace is archived'), terminate: true, archived: true }
+    }
+
     if (workspaceInfo.disabled === true && token.email !== systemAccountEmail && token.extra?.admin !== 'true') {
       // No access to disabled workspaces for regular users
       return { error: new Error('Workspace not found or not available'), terminate: true }

server/workspace-service/src/service.ts

@@ -214,7 +214,7 @@ export class WorkspaceWorker {
   }
 
   private async _upgradeWorkspace (ctx: MeasureContext, ws: BaseWorkspaceInfo, opt: WorkspaceOptions): Promise<void> {
-    if (ws.disabled === true || (opt.ignore ?? '').includes(ws.workspace)) {
+    if (ws.disabled === true || ws.mode === 'archived' || (opt.ignore ?? '').includes(ws.workspace)) {
       return
     }
     const t = Date.now()

server/ws/src/server_http.ts

@@ -15,7 +15,7 @@
 
 import { Analytics } from '@hcengineering/analytics'
 import { generateId, toWorkspaceString, type MeasureContext, type Tx } from '@hcengineering/core'
-import { UNAUTHORIZED, unknownStatus } from '@hcengineering/platform'
+import platform, { Severity, Status, UNAUTHORIZED, unknownStatus } from '@hcengineering/platform'
 import { RPCHandler, type Response } from '@hcengineering/rpc'
 import {
   doSessionOp,
@@ -376,12 +376,27 @@ export function startHttpServer (
     if (webSocketData.session instanceof Promise) {
       void webSocketData.session.then((s) => {
         if ('error' in s) {
+          if (s.archived === true) {
+            cs.send(
+              ctx,
+              {
+                id: -1,
+                error: new Status(Severity.ERROR, platform.status.WorkspaceArchived, {
+                  workspace: token.workspace.name
+                }),
+                terminate: s.terminate
+              },
+              false,
+              false
+            )
+          } else {
             cs.send(
               ctx,
               { id: -1, error: unknownStatus(s.error.message ?? 'Unknown error'), terminate: s.terminate },
               false,
               false
             )
+          }
           // No connection to account service, retry from client.
           setTimeout(() => {
             cs.close()

services/github/pod-github/src/platform.ts

@@ -750,8 +750,12 @@ export class PlatformWorker {
           errors++
           return
         }
+        if (workspaceInfo?.mode === 'archived') {
+          this.ctx.warn('Workspace is archived.', { workspace })
+          return
+        }
         if (workspaceInfo?.disabled === true) {
-          this.ctx.error('Workspace is disabled workspaceId', { workspace })
+          this.ctx.warn('Workspace is disabled', { workspace })
           return
         }
         try {