feat: show lost files tool (#6721)

This commit is contained in:
Alexander Onnikov 2024-09-25 19:05:11 +07:00 committed by GitHub
parent 33c7615201
commit 94005da7c6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 72 additions and 1 deletions

View File

@ -116,7 +116,7 @@ import { fixJsonMarkup, migrateMarkup } from './markup'
import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin' import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin'
import { importNotion } from './notion' import { importNotion } from './notion'
import { fixAccountEmails, renameAccount } from './renameAccount' import { fixAccountEmails, renameAccount } from './renameAccount'
import { moveFiles, syncFiles } from './storage' import { moveFiles, showLostFiles, syncFiles } from './storage'
const colorConstants = { const colorConstants = {
colorRed: '\u001b[31m', colorRed: '\u001b[31m',
@ -1230,6 +1230,49 @@ export function devTool (
}) })
}) })
program
.command('show-lost-files')
.option('-w, --workspace <workspace>', 'Selected workspace only', '')
.option('--disabled', 'Include disabled workspaces', false)
.option('--all', 'Show all files', false)
.action(async (cmd: { workspace: string, disabled: boolean, all: boolean }) => {
const { mongodbUri } = prepareTools()
await withDatabase(mongodbUri, async (db, client) => {
await withStorage(mongodbUri, async (adapter) => {
try {
let index = 1
const workspaces = await listWorkspacesPure(db)
workspaces.sort((a, b) => b.lastVisit - a.lastVisit)
for (const workspace of workspaces) {
if (workspace.disabled === true && !cmd.disabled) {
console.log('ignore disabled workspace', workspace.workspace)
continue
}
if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) {
continue
}
try {
console.log('start', workspace.workspace, index, '/', workspaces.length)
const workspaceId = getWorkspaceId(workspace.workspace)
const wsDb = getWorkspaceDB(client, { name: workspace.workspace })
await showLostFiles(toolCtx, workspaceId, wsDb, adapter, { showAll: cmd.all })
console.log('done', workspace.workspace)
} catch (err) {
console.error(err)
}
index += 1
}
} catch (err: any) {
console.error(err)
}
})
})
})
program.command('fix-bw-workspace <workspace>').action(async (workspace: string) => { program.command('fix-bw-workspace <workspace>').action(async (workspace: string) => {
const { mongodbUri } = prepareTools() const { mongodbUri } = prepareTools()
await withStorage(mongodbUri, async (adapter) => { await withStorage(mongodbUri, async (adapter) => {

View File

@ -13,8 +13,11 @@
// limitations under the License. // limitations under the License.
// //
import { type Attachment } from '@hcengineering/attachment'
import { type Blob, type MeasureContext, type WorkspaceId, RateLimiter } from '@hcengineering/core' import { type Blob, type MeasureContext, type WorkspaceId, RateLimiter } from '@hcengineering/core'
import { DOMAIN_ATTACHMENT } from '@hcengineering/model-attachment'
import { type StorageAdapter, type StorageAdapterEx } from '@hcengineering/server-core' import { type StorageAdapter, type StorageAdapterEx } from '@hcengineering/server-core'
import { type Db } from 'mongodb'
import { PassThrough } from 'stream' import { PassThrough } from 'stream'
export interface MoveFilesParams { export interface MoveFilesParams {
@ -93,6 +96,31 @@ export async function moveFiles (
} }
} }
export async function showLostFiles (
ctx: MeasureContext,
workspaceId: WorkspaceId,
db: Db,
storageAdapter: StorageAdapter,
{ showAll }: { showAll: boolean }
): Promise<void> {
const iterator = db.collection<Attachment>(DOMAIN_ATTACHMENT).find({})
while (true) {
const attachment = await iterator.next()
if (attachment === null) break
const { _id, _class, file, name, modifiedOn } = attachment
const date = new Date(modifiedOn).toISOString()
const stat = await storageAdapter.stat(ctx, workspaceId, file)
if (stat === undefined) {
console.warn('-', date, _class, _id, file, name)
} else if (showAll) {
console.log('+', date, _class, _id, file, name)
}
}
}
async function processAdapter ( async function processAdapter (
ctx: MeasureContext, ctx: MeasureContext,
exAdapter: StorageAdapterEx, exAdapter: StorageAdapterEx,