diff --git a/models/core/src/index.ts b/models/core/src/index.ts index f273a93208..09e6a96fb0 100644 --- a/models/core/src/index.ts +++ b/models/core/src/index.ts @@ -201,18 +201,12 @@ export function createModel (builder: Builder): void { { createdBy: 1 }, { createdBy: -1 }, { createdOn: -1 }, - { modifiedBy: 1 }, - { objectSpace: 1 } + { modifiedBy: 1 } ], indexes: [ { keys: { - objectSpace: 1, - _id: 1, - modifiedOn: 1 - }, - filter: { - objectSpace: core.space.Model + objectSpace: 1 } } ] diff --git a/packages/core/src/classes.ts b/packages/core/src/classes.ts index b81656e2b2..00ae9bf50e 100644 --- a/packages/core/src/classes.ts +++ b/packages/core/src/classes.ts @@ -659,6 +659,16 @@ export interface DomainIndexConfiguration extends Doc { export type WorkspaceMode = 'pending-creation' | 'creating' | 'upgrading' | 'pending-deletion' | 'deleting' | 'active' +export interface BackupStatus { + dataSize: number + blobsSize: number + + backupSize: number + + lastBackup: Timestamp + backups: number +} + export interface BaseWorkspaceInfo { workspace: string // An uniq workspace name, Database names disabled?: boolean @@ -676,4 +686,6 @@ export interface BaseWorkspaceInfo { progress?: number // Some progress endpoint: string + + backupInfo?: BackupStatus } diff --git a/plugins/login/src/index.ts b/plugins/login/src/index.ts index 78c97d0e20..fecbade587 100644 --- a/plugins/login/src/index.ts +++ b/plugins/login/src/index.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import { AccountRole, Doc, Ref, Timestamp, WorkspaceMode } from '@hcengineering/core' +import { AccountRole, Doc, Ref, Timestamp, WorkspaceMode, type BackupStatus } from '@hcengineering/core' import type { Asset, IntlString, Metadata, Plugin, Resource, Status } from '@hcengineering/platform' import { plugin } from '@hcengineering/platform' import type { AnyComponent } from '@hcengineering/ui' @@ -35,6 +35,8 @@ export interface Workspace { progress?: number lastVisit: number + + backupInfo?: BackupStatus } /** diff --git a/plugins/workbench-resources/src/components/SelectWorkspaceMenu.svelte b/plugins/workbench-resources/src/components/SelectWorkspaceMenu.svelte index 84eb7c1c7f..f1d46ca615 100644 --- a/plugins/workbench-resources/src/components/SelectWorkspaceMenu.svelte +++ b/plugins/workbench-resources/src/components/SelectWorkspaceMenu.svelte @@ -37,9 +37,23 @@ import { workspacesStore } from '../utils' // import Drag from './icons/Drag.svelte' + function getLastVisitDays (it: Workspace): number { + return Math.floor((Date.now() - it.lastVisit) / (1000 * 3600 * 24)) + } + onMount(() => { void getResource(login.function.GetWorkspaces).then(async (f) => { const workspaces = await f() + + workspaces.sort((a, b) => { + const adays = getLastVisitDays(a) + const bdays = getLastVisitDays(a) + if (adays === bdays) { + return (b.backupInfo?.backupSize ?? 0) - (a.backupInfo?.backupSize ?? 0) + } + return bdays - adays + }) + $workspacesStore = workspaces }) }) @@ -181,6 +195,9 @@ {wsName} {#if isAdmin && ws.lastVisit != null && ws.lastVisit !== 0}
+ {#if ws.backupInfo != null} + {ws.backupInfo.backupSize}Mb - + {/if} ({lastUsageDays} days)
{/if} diff --git a/server/account/src/operations.ts b/server/account/src/operations.ts index b27478d2fc..b9789ed40f 100644 --- a/server/account/src/operations.ts +++ b/server/account/src/operations.ts @@ -44,6 +44,7 @@ import core, { Version, versionToString, WorkspaceId, + type BackupStatus, type Branding, type WorkspaceMode } from '@hcengineering/core' @@ -1356,6 +1357,40 @@ export async function updateWorkspaceInfo ( ) } +/** + * @public + */ +export async function updateBackupInfo ( + ctx: MeasureContext, + db: Db, + branding: Branding | null, + token: string, + backupInfo: BackupStatus +): Promise { + const decodedToken = decodeToken(ctx, token) + if (decodedToken.extra?.service !== 'backup') { + throw new PlatformError(new Status(Severity.ERROR, platform.status.Forbidden, {})) + } + const workspaceInfo = await getWorkspaceById(db, decodedToken.workspace.name) + if (workspaceInfo === null) { + throw new PlatformError( + new Status(Severity.ERROR, platform.status.WorkspaceNotFound, { workspace: decodedToken.workspace.name }) + ) + } + + const wsCollection = db.collection>(WORKSPACE_COLLECTION) + + await wsCollection.updateOne( + { _id: workspaceInfo._id }, + { + $set: { + backupInfo, + lastProcessingTime: Date.now() + } + } + ) +} + async function postCreateUserWorkspace ( ctx: MeasureContext, db: Db, @@ -2747,6 +2782,7 @@ export function getMethods (): Record { // Workspace service methods getPendingWorkspace: wrap(getPendingWorkspace), updateWorkspaceInfo: wrap(updateWorkspaceInfo), + updateBackupInfo: wrap(updateBackupInfo), workerHandshake: wrap(workerHandshake) } } diff --git a/server/backup/src/backup.ts b/server/backup/src/backup.ts index 1f89dd1935..9345f5766d 100644 --- a/server/backup/src/backup.ts +++ b/server/backup/src/backup.ts @@ -49,6 +49,7 @@ import { Writable } from 'stream' import { extract, Pack, pack } from 'tar-stream' import { createGunzip, gunzipSync, gzipSync } from 'zlib' import { BackupStorage } from './storage' +import type { BackupStatus } from '@hcengineering/core/src/classes' export * from './storage' const dataBlobSize = 50 * 1024 * 1024 @@ -498,6 +499,10 @@ function doTrimHash (s: string | undefined): string { return s } +export interface BackupResult extends Omit { + result: boolean +} + /** * @public */ @@ -531,7 +536,13 @@ export async function backup ( skipBlobContentTypes: [], blobDownloadLimit: 15 } -): Promise { +): Promise { + const result: BackupResult = { + result: false, + dataSize: 0, + blobsSize: 0, + backupSize: 0 + } ctx = ctx.newChild('backup', { workspaceId: workspaceId.name, force: options.force, @@ -589,7 +600,8 @@ export async function backup ( if (lastTx._id === backupInfo.lastTxId && !options.force) { printEnd = false ctx.info('No transaction changes. Skipping backup.', { workspace: workspaceId.name }) - return false + result.result = false + return result } } lastTxChecked = true @@ -613,14 +625,19 @@ export async function backup ( )) as CoreClient & BackupClient) if (!lastTxChecked) { - lastTx = await connection.findOne(core.class.Tx, {}, { limit: 1, sort: { modifiedOn: SortingOrder.Descending } }) + lastTx = await connection.findOne( + core.class.Tx, + { objectSpace: { $ne: core.space.Model } }, + { limit: 1, sort: { modifiedOn: SortingOrder.Descending } } + ) if (lastTx !== undefined) { if (lastTx._id === backupInfo.lastTxId && !options.force) { ctx.info('No transaction changes. Skipping backup.', { workspace: workspaceId.name }) if (options.getConnection === undefined) { await connection.close() } - return false + result.result = false + return result } } } @@ -700,6 +717,11 @@ export async function backup ( let currentNeedRetrieveSize = 0 for (const { id, hash, size } of currentChunk.docs) { + if (domain === DOMAIN_BLOB) { + result.blobsSize += size + } else { + result.dataSize += size + } processed++ if (Date.now() - st > 2500) { ctx.info('processed', { @@ -1034,10 +1056,33 @@ export async function backup ( backupInfo.lastTxId = lastTx?._id ?? '0' // We could store last tx, since full backup is complete await storage.writeFile(infoFile, gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel })) } - return true + result.result = true + + const addFileSize = async (file: string | undefined | null): Promise => { + if (file != null && (await storage.exists(file))) { + const fileSize = await storage.stat(file) + result.backupSize += fileSize + } + } + + // Let's calculate data size for backup + for (const sn of backupInfo.snapshots) { + for (const [, d] of Object.entries(sn.domains)) { + await addFileSize(d.snapshot) + for (const snp of d.snapshots ?? []) { + await addFileSize(snp) + } + for (const snp of d.storage ?? []) { + await addFileSize(snp) + } + } + } + await addFileSize(infoFile) + + return result } catch (err: any) { ctx.error('backup error', { err, workspace: workspaceId.name }) - return false + return result } finally { if (printEnd) { ctx.info('end backup', { workspace: workspaceId.name, totalTime: Date.now() - st }) diff --git a/server/backup/src/service.ts b/server/backup/src/service.ts index 78e432189b..407a3950b2 100644 --- a/server/backup/src/service.ts +++ b/server/backup/src/service.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import { +import core, { BaseWorkspaceInfo, DOMAIN_TX, getWorkspaceId, @@ -22,6 +22,7 @@ import { SortingOrder, systemAccountEmail, type BackupClient, + type BackupStatus, type Branding, type Client, type MeasureContext, @@ -29,7 +30,7 @@ import { type WorkspaceIdWithUrl } from '@hcengineering/core' import { PlatformError, unknownError } from '@hcengineering/platform' -import { listAccountWorkspaces } from '@hcengineering/server-client' +import { listAccountWorkspaces, updateBackupInfo } from '@hcengineering/server-client' import { BackupClientOps, SessionDataImpl, @@ -38,6 +39,7 @@ import { type PipelineFactory, type StorageAdapter } from '@hcengineering/server-core' +import { generateToken } from '@hcengineering/server-token' import { backup } from '.' import { createStorageBackupStorage } from './storage' export interface BackupConfig { @@ -114,6 +116,23 @@ class BackupWorker { ): Promise<{ failedWorkspaces: BaseWorkspaceInfo[], processed: number, skipped: number }> { const workspacesIgnore = new Set(this.config.SkipWorkspaces.split(';')) const workspaces = (await listAccountWorkspaces(this.config.Token)).filter((it) => { + const lastBackup = it.backupInfo?.lastBackup ?? 0 + if ((Date.now() - lastBackup) / 1000 < this.config.Interval) { + // No backup required, interval not elapsed + ctx.info('Skip backup', { workspace: it.workspace, lastBackup: Math.round((Date.now() - lastBackup) / 1000) }) + return false + } + + const lastVisitSec = Math.floor((Date.now() - it.lastVisit) / 1000) + if (lastVisitSec > this.config.Interval) { + // No backup required, interval not elapsed + ctx.info('Skip backup, since not visited since last check', { + workspace: it.workspace, + days: Math.floor(lastVisitSec / 3600 / 24), + seconds: lastVisitSec + }) + return false + } return !workspacesIgnore.has(it.workspace) }) workspaces.sort((a, b) => b.lastVisit - a.lastVisit) @@ -133,6 +152,7 @@ class BackupWorker { return { failedWorkspaces, processed, skipped: workspaces.length - processed } } index++ + const st = Date.now() rootCtx.warn('\n\nBACKUP WORKSPACE ', { workspace: ws.workspace, index, @@ -156,56 +176,73 @@ class BackupWorker { workspaceName: ws.workspaceName ?? '', workspaceUrl: ws.workspaceUrl ?? '' } - processed += (await ctx.with( - 'backup', - { workspace: ws.workspace }, - async (ctx) => - await backup(ctx, '', getWorkspaceId(ws.workspace), storage, { - skipDomains: [], - force: false, - recheck: false, - timeout: this.config.Timeout * 1000, - connectTimeout: 5 * 60 * 1000, // 5 minutes to, - blobDownloadLimit: 100, - skipBlobContentTypes: [], - storageAdapter: this.workspaceStorageAdapter, - getLastTx: async (): Promise => { - const config = this.getConfig(ctx, wsUrl, null, this.workspaceStorageAdapter) - const adapterConf = config.adapters[config.domains[DOMAIN_TX]] - const hierarchy = new Hierarchy() - const modelDb = new ModelDb(hierarchy) - const txAdapter = await adapterConf.factory( - ctx, - hierarchy, - adapterConf.url, - wsUrl, - modelDb, - this.workspaceStorageAdapter - ) - try { - await txAdapter.init?.() + const result = await ctx.with('backup', { workspace: ws.workspace }, (ctx) => + backup(ctx, '', getWorkspaceId(ws.workspace), storage, { + skipDomains: [], + force: true, + recheck: false, + timeout: this.config.Timeout * 1000, + connectTimeout: 5 * 60 * 1000, // 5 minutes to, + blobDownloadLimit: 100, + skipBlobContentTypes: [], + storageAdapter: this.workspaceStorageAdapter, + getLastTx: async (): Promise => { + const config = this.getConfig(ctx, wsUrl, null, this.workspaceStorageAdapter) + const adapterConf = config.adapters[config.domains[DOMAIN_TX]] + const hierarchy = new Hierarchy() + const modelDb = new ModelDb(hierarchy) + const txAdapter = await adapterConf.factory( + ctx, + hierarchy, + adapterConf.url, + wsUrl, + modelDb, + this.workspaceStorageAdapter + ) + try { + await txAdapter.init?.() - return ( - await txAdapter.rawFindAll( - DOMAIN_TX, - {}, - { limit: 1, sort: { modifiedOn: SortingOrder.Descending } } - ) - ).shift() - } finally { - await txAdapter.close() - } - }, - getConnection: async () => { - if (pipeline === undefined) { - pipeline = await this.pipelineFactory(ctx, wsUrl, true, () => {}, null) - } - return this.wrapPipeline(ctx, pipeline, wsUrl) + return ( + await txAdapter.rawFindAll( + DOMAIN_TX, + { objectSpace: { $ne: core.space.Model } }, + { limit: 1, sort: { modifiedOn: SortingOrder.Descending } } + ) + ).shift() + } finally { + await txAdapter.close() } - }) - )) - ? 1 - : 0 + }, + getConnection: async () => { + if (pipeline === undefined) { + pipeline = await this.pipelineFactory(ctx, wsUrl, true, () => {}, null) + } + return this.wrapPipeline(ctx, pipeline, wsUrl) + } + }) + ) + + if (result.result) { + const backupInfo: BackupStatus = { + backups: (ws.backupInfo?.backups ?? 0) + 1, + lastBackup: Date.now(), + backupSize: Math.round((result.backupSize * 100) / (1024 * 1024)) / 100, + dataSize: Math.round((result.dataSize * 100) / (1024 * 1024)) / 100, + blobsSize: Math.round((result.blobsSize * 100) / (1024 * 1024)) / 100 + } + rootCtx.warn('\n\nBACKUP STATS ', { + workspace: ws.workspace, + index, + ...backupInfo, + time: Math.round((Date.now() - st) / 1000), + total: workspaces.length + }) + // We need to report update for stats to account service + processed += 1 + + const token = generateToken(systemAccountEmail, { name: ws.workspace }, { service: 'backup' }) + await updateBackupInfo(token, backupInfo) + } } catch (err: any) { rootCtx.error('\n\nFAILED to BACKUP', { workspace: ws.workspace, err }) failedWorkspaces.push(ws) diff --git a/server/backup/src/storage.ts b/server/backup/src/storage.ts index e127ab5ec7..a4070d1582 100644 --- a/server/backup/src/storage.ts +++ b/server/backup/src/storage.ts @@ -1,6 +1,6 @@ import { MeasureContext, WorkspaceId } from '@hcengineering/core' import { StorageAdapter } from '@hcengineering/server-core' -import { createReadStream, createWriteStream, existsSync } from 'fs' +import { createReadStream, createWriteStream, existsSync, statSync } from 'fs' import { mkdir, readFile, rm, writeFile } from 'fs/promises' import { dirname, join } from 'path' import { PassThrough, Readable, Writable } from 'stream' @@ -14,6 +14,8 @@ export interface BackupStorage { write: (name: string) => Promise writeFile: (name: string, data: string | Buffer) => Promise exists: (name: string) => Promise + + stat: (name: string) => Promise delete: (name: string) => Promise } @@ -41,6 +43,10 @@ class FileStorage implements BackupStorage { return existsSync(join(this.root, name)) } + async stat (name: string): Promise { + return statSync(join(this.root, name)).size + } + async delete (name: string): Promise { await rm(join(this.root, name)) } @@ -87,6 +93,15 @@ class AdapterStorage implements BackupStorage { } } + async stat (name: string): Promise { + try { + const st = await this.client.stat(this.ctx, this.workspaceId, join(this.root, name)) + return st?.size ?? 0 + } catch (err: any) { + return 0 + } + } + async delete (name: string): Promise { await this.client.remove(this.ctx, this.workspaceId, [join(this.root, name)]) } diff --git a/server/client/src/account.ts b/server/client/src/account.ts index 0d20f61a34..a1aa253fea 100644 --- a/server/client/src/account.ts +++ b/server/client/src/account.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import { type BaseWorkspaceInfo, type Data, type Version } from '@hcengineering/core' +import { type BaseWorkspaceInfo, type Data, type Version, BackupStatus } from '@hcengineering/core' import { getMetadata, PlatformError, unknownError } from '@hcengineering/platform' import plugin from './plugin' @@ -47,6 +47,24 @@ export async function listAccountWorkspaces (token: string): Promise { + const accountsUrl = getAccoutsUrlOrFail() + const workspaces = await ( + await fetch(accountsUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + method: 'updateBackupInfo', + params: [token, info] + }) + }) + ).json() + + return (workspaces.result as BaseWorkspaceInfo[]) ?? [] +} + export async function getTransactorEndpoint ( token: string, kind: 'internal' | 'external' = 'internal', @@ -149,7 +167,10 @@ export async function workerHandshake ( }) } -export async function getWorkspaceInfo (token: string): Promise { +export async function getWorkspaceInfo ( + token: string, + updateLastAccess = false +): Promise { const accountsUrl = getAccoutsUrlOrFail() const workspaceInfo = await ( await fetch(accountsUrl, { @@ -160,7 +181,7 @@ export async function getWorkspaceInfo (token: string): Promise { - for (const a of this.adapters.values()) { - if (!(await a.exists(ctx, workspaceId))) { - await a.make(ctx, workspaceId) + for (const [k, a] of this.adapters.entries()) { + try { + if (!(await a.exists(ctx, workspaceId))) { + await a.make(ctx, workspaceId) + } + } catch (err: any) { + ctx.error('failed to init adapter', { adapter: k, workspaceId, error: err }) + // Do not throw error in case default adapter is ok + Analytics.handleError(err) + if (k === this.defaultAdapter) { + // We should throw in case default one is not valid + throw err + } } } } diff --git a/server/mongo/src/storage.ts b/server/mongo/src/storage.ts index 0d858d3160..32e76d4b15 100644 --- a/server/mongo/src/storage.ts +++ b/server/mongo/src/storage.ts @@ -1664,9 +1664,8 @@ class MongoTxAdapter extends MongoAdapterBase implements TxAdapter { @withContext('get-model') async getModel (ctx: MeasureContext): Promise { const txCollection = this.db.collection(DOMAIN_TX) - const exists = await txCollection.indexExists('objectSpace_fi_1__id_fi_1_modifiedOn_fi_1') const cursor = await ctx.with('find', {}, async () => { - let c = txCollection.find( + const c = txCollection.find( { objectSpace: core.space.Model }, { sort: { @@ -1675,9 +1674,6 @@ class MongoTxAdapter extends MongoAdapterBase implements TxAdapter { } } ) - if (exists) { - c = c.hint({ objectSpace: 1, _id: 1, modifiedOn: 1 }) - } return c }) const model = await ctx.with('to-array', {}, async () => await toArray(cursor)) diff --git a/server/tool/src/index.ts b/server/tool/src/index.ts index 0f162b6bc3..2e9baefd29 100644 --- a/server/tool/src/index.ts +++ b/server/tool/src/index.ts @@ -365,7 +365,7 @@ export async function upgradeModel ( await tryMigrate(migrateClient, coreId, [ { - state: 'indexes-v4', + state: 'indexes-v5', func: upgradeIndexes }, { diff --git a/server/ws/src/server_http.ts b/server/ws/src/server_http.ts index 4a297dafa3..d98a085889 100644 --- a/server/ws/src/server_http.ts +++ b/server/ws/src/server_http.ts @@ -311,7 +311,6 @@ export function startHttpServer ( : false, skipUTF8Validation: true, maxPayload: 250 * 1024 * 1024, - backlog: 1000, clientTracking: false // We do not need to track clients inside clients. }) // eslint-disable-next-line @typescript-eslint/no-misused-promises diff --git a/services/github/pod-github/src/account.ts b/services/github/pod-github/src/account.ts index 24e8823744..cf013c2a71 100644 --- a/services/github/pod-github/src/account.ts +++ b/services/github/pod-github/src/account.ts @@ -4,7 +4,7 @@ import config from './config' /** * @public */ -export async function getWorkspaceInfo (token: string): Promise { +export async function getWorkspaceInfo (token: string, updateLastModified = false): Promise { const accountsUrl = config.AccountsURL const workspaceInfo = await ( await fetch(accountsUrl, { @@ -15,7 +15,7 @@ export async function getWorkspaceInfo (token: string): Promise