feat: show lost files tool (#6721)

dev/tool/src/index.ts

@@ -116,7 +116,7 @@ import { fixJsonMarkup, migrateMarkup } from './markup'
 import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin'
 import { importNotion } from './notion'
 import { fixAccountEmails, renameAccount } from './renameAccount'
-import { moveFiles, syncFiles } from './storage'
+import { moveFiles, showLostFiles, syncFiles } from './storage'
 
 const colorConstants = {
   colorRed: '\u001b[31m',
@@ -1230,6 +1230,49 @@ export function devTool (
       })
     })
 
+  program
+    .command('show-lost-files')
+    .option('-w, --workspace <workspace>', 'Selected workspace only', '')
+    .option('--disabled', 'Include disabled workspaces', false)
+    .option('--all', 'Show all files', false)
+    .action(async (cmd: { workspace: string, disabled: boolean, all: boolean }) => {
+      const { mongodbUri } = prepareTools()
+      await withDatabase(mongodbUri, async (db, client) => {
+        await withStorage(mongodbUri, async (adapter) => {
+          try {
+            let index = 1
+            const workspaces = await listWorkspacesPure(db)
+            workspaces.sort((a, b) => b.lastVisit - a.lastVisit)
+
+            for (const workspace of workspaces) {
+              if (workspace.disabled === true && !cmd.disabled) {
+                console.log('ignore disabled workspace', workspace.workspace)
+                continue
+              }
+
+              if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) {
+                continue
+              }
+
+              try {
+                console.log('start', workspace.workspace, index, '/', workspaces.length)
+                const workspaceId = getWorkspaceId(workspace.workspace)
+                const wsDb = getWorkspaceDB(client, { name: workspace.workspace })
+                await showLostFiles(toolCtx, workspaceId, wsDb, adapter, { showAll: cmd.all })
+                console.log('done', workspace.workspace)
+              } catch (err) {
+                console.error(err)
+              }
+
+              index += 1
+            }
+          } catch (err: any) {
+            console.error(err)
+          }
+        })
+      })
+    })
+
   program.command('fix-bw-workspace <workspace>').action(async (workspace: string) => {
     const { mongodbUri } = prepareTools()
     await withStorage(mongodbUri, async (adapter) => {

dev/tool/src/storage.ts

@@ -13,8 +13,11 @@
 // limitations under the License.
 //
 
+import { type Attachment } from '@hcengineering/attachment'
 import { type Blob, type MeasureContext, type WorkspaceId, RateLimiter } from '@hcengineering/core'
+import { DOMAIN_ATTACHMENT } from '@hcengineering/model-attachment'
 import { type StorageAdapter, type StorageAdapterEx } from '@hcengineering/server-core'
+import { type Db } from 'mongodb'
 import { PassThrough } from 'stream'
 
 export interface MoveFilesParams {
@@ -93,6 +96,31 @@ export async function moveFiles (
   }
 }
 
+export async function showLostFiles (
+  ctx: MeasureContext,
+  workspaceId: WorkspaceId,
+  db: Db,
+  storageAdapter: StorageAdapter,
+  { showAll }: { showAll: boolean }
+): Promise<void> {
+  const iterator = db.collection<Attachment>(DOMAIN_ATTACHMENT).find({})
+
+  while (true) {
+    const attachment = await iterator.next()
+    if (attachment === null) break
+
+    const { _id, _class, file, name, modifiedOn } = attachment
+    const date = new Date(modifiedOn).toISOString()
+
+    const stat = await storageAdapter.stat(ctx, workspaceId, file)
+    if (stat === undefined) {
+      console.warn('-', date, _class, _id, file, name)
+    } else if (showAll) {
+      console.log('+', date, _class, _id, file, name)
+    }
+  }
+}
+
 async function processAdapter (
   ctx: MeasureContext,
   exAdapter: StorageAdapterEx,