UBERF-6426: Fix stuck backup (#5258)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

server/account/src/index.ts

@@ -818,7 +818,7 @@ export async function createWorkspace (
     searchPromise = generateWorkspaceRecord(db, email, productId, version, workspaceName, workspace)
 
     const workspaceInfo = await searchPromise
-    let client: Client
+    let client: Client | undefined
     const childLogger = ctx.newChild(
       'createWorkspace',
       { workspace: workspaceInfo.workspace },
@@ -837,8 +837,9 @@ export async function createWorkspace (
       const initWS = getMetadata(toolPlugin.metadata.InitWorkspace)
       const wsId = getWorkspaceId(workspaceInfo.workspace, productId)
       if (initWS !== undefined && (await getWorkspaceById(db, productId, initWS)) !== null) {
-        client = await initModel(ctx, getTransactor(), wsId, txes, [], ctxModellogger)
-        await client.close()
+        // Just any valid model for transactor to be able to function
+        await initModel(ctx, getTransactor(), wsId, txes, [], ctxModellogger, true)
+        // Clone init workspace.
         await cloneWorkspace(
           getTransactor(),
           getWorkspaceId(initWS, productId),

server/elastic/src/backup.ts

@@ -166,7 +166,10 @@ class ElasticDataAdapter implements DbAdapter {
 
   async load (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<Doc[]> {
     const result: Doc[] = []
+    const toLoad = [...docs]
 
+    while (toLoad.length > 0) {
+      const part = toLoad.splice(0, 5000)
       const resp = await this.client.search({
         index: indexName,
         type: '_doc',
@@ -176,7 +179,7 @@ class ElasticDataAdapter implements DbAdapter {
               must: [
                 {
                   terms: {
-                  _id: docs,
+                    _id: part,
                     boost: 1.0
                   }
                 },
@@ -188,7 +191,7 @@ class ElasticDataAdapter implements DbAdapter {
               ]
             }
           },
-        size: docs.length
+          size: part.length
         }
       })
       const buffer = resp.body.hits.hits.map((hit: any) => ({ _id: hit._id, data: hit._source }))
@@ -204,6 +207,7 @@ class ElasticDataAdapter implements DbAdapter {
         }
         result.push(dta)
       }
+    }
     return result
   }
 

server/server/src/minio.ts

@@ -16,6 +16,7 @@
 import core, {
   BlobData,
   Class,
+  cutObjectArray,
   Doc,
   DocumentQuery,
   DocumentUpdate,
@@ -90,7 +91,8 @@ class StorageBlobAdapter implements DbAdapter {
     for (const item of docs) {
       const stat = await this.client.stat(this.ctx, this.workspaceId, item)
       if (stat === undefined) {
-        throw new Error(`Could not find blob ${item}`)
+        await ctx.error('Could not find blob', { domain, item, allDocs: cutObjectArray(docs) })
+        continue
       }
       const chunks: Buffer[] = await this.client.read(this.ctx, this.workspaceId, item)
       const final = Buffer.concat(chunks)

server/tool/src/index.ts

@@ -111,8 +111,9 @@ export async function initModel (
   workspaceId: WorkspaceId,
   rawTxes: Tx[],
   migrateOperations: [string, MigrateOperation][],
-  logger: ModelLogger = consoleModelLogger
-): Promise<CoreClient> {
+  logger: ModelLogger = consoleModelLogger,
+  skipOperations: boolean = false
+): Promise<CoreClient | undefined> {
   const { mongodbUri, storageAdapter: minio, txes } = prepareTools(rawTxes)
   if (txes.some((tx) => tx.objectSpace !== core.space.Model)) {
     throw Error('Model txes must target only core.space.Model')
@@ -131,6 +132,11 @@ export async function initModel (
     logger.log('creating data...', { transactorUrl })
     const { model } = await fetchModelFromMongo(ctx, mongodbUri, workspaceId)
 
+    logger.log('create minio bucket', { workspaceId })
+    if (!(await minio.exists(ctx, workspaceId))) {
+      await minio.make(ctx, workspaceId)
+    }
+    if (!skipOperations) {
       connection = (await connect(
         transactorUrl,
         workspaceId,
@@ -150,19 +156,15 @@ export async function initModel (
 
         // Create update indexes
         await createUpdateIndexes(ctx, connection, db, logger)
-
-      logger.log('create minio bucket', { workspaceId })
-      if (!(await minio.exists(ctx, workspaceId))) {
-        await minio.make(ctx, workspaceId)
-      }
       } catch (e: any) {
         logger.error('error', { error: e })
         throw e
       }
+      return connection
+    }
   } finally {
     _client.close()
   }
-  return connection
 }
 
 /**
@@ -174,7 +176,8 @@ export async function upgradeModel (
   workspaceId: WorkspaceId,
   rawTxes: Tx[],
   migrateOperations: [string, MigrateOperation][],
-  logger: ModelLogger = consoleModelLogger
+  logger: ModelLogger = consoleModelLogger,
+  skipTxUpdate: boolean = false
 ): Promise<CoreClient> {
   const { mongodbUri, txes } = prepareTools(rawTxes)
 
@@ -188,6 +191,7 @@ export async function upgradeModel (
   try {
     const db = getWorkspaceDB(client, workspaceId)
 
+    if (!skipTxUpdate) {
       logger.log('removing model...', { workspaceId: workspaceId.name })
       // we're preserving accounts (created by core.account.System).
       const result = await ctx.with(
@@ -210,6 +214,7 @@ export async function upgradeModel (
       )
 
       logger.log('model transactions inserted.', { workspaceId: workspaceId.name, count: insert.insertedCount })
+    }
 
     const { hierarchy, modelDb, model } = await fetchModelFromMongo(ctx, mongodbUri, workspaceId)
 

tests/sanity/package.json

@@ -1,12 +1,19 @@
 {
   "name": "@hcengineering/tests-sanity",
   "version": "0.6.1",
+  "main": "lib/index.js",
+  "svelte": "src/index.ts",
+  "types": "types/index.d.ts",
   "author": "Anticrm Platform Contributors",
   "template": "@hcengineering/default-package",
   "license": "EPL-2.0",
   "scripts": {
-    "build": "",
-    "build:watch": "",
+    "build": "compile",
+    "build:watch": "compile",
+    "_phase:build": "compile transpile tests",
+    "_phase:test": "",
+    "_phase:format": "format src",
+    "_phase:validate": "compile validate",
     "lint:fix": "eslint --fix tests",
     "lint": "eslint tests",
     "format": "format tests",

tests/sanity/tests/index.ts

@@ -0,0 +1,10 @@
+export * from './model/login-page'
+export * from './model/signin-page'
+export * from './model/common-page'
+export * from './model/common-types'
+export * from './model/left-side-menu-page'
+export * from './model/signup-page'
+export * from './model/select-workspace-page'
+export * from './utils'
+export * from './model/tracker/issues-page'
+export * from './model/tracker/issues-details-page'

tests/sanity/tests/workspace/create.spec.ts

@@ -16,7 +16,7 @@ test.describe('Workspace tests', () => {
     const newUser: SignUpData = {
       firstName: `FirstName-${generateId()}`,
       lastName: `LastName-${generateId()}`,
-      email: `email+${generateId()}@gmail.com`,
+      email: `sanity-email+${generateId()}@gmail.com`,
       password: '1234'
     }
     const newWorkspaceName = `New Workspace Name - ${generateId(2)}`
@@ -39,7 +39,7 @@ test.describe('Workspace tests', () => {
     const newUser: SignUpData = {
       firstName: `FirstName-${generateId()}`,
       lastName: `LastName-${generateId()}`,
-      email: `email+${generateId()}@gmail.com`,
+      email: `sanity-email+${generateId()}@gmail.com`,
       password: '1234'
     }
     const newIssue: NewIssue = {
@@ -92,7 +92,7 @@ test.describe('Workspace tests', () => {
     const newUser: SignUpData = {
       firstName: `FirstName-${generateId()}`,
       lastName: `LastName-${generateId()}`,
-      email: `email+${generateId()}@gmail.com`,
+      email: `sanity-email+${generateId()}@gmail.com`,
       password: '1234'
     }
     const newWorkspaceName = `New Workspace Name - ${generateId(2)}`
@@ -125,7 +125,7 @@ test.describe('Workspace tests', () => {
     const newUser: SignUpData = {
       firstName: `FirstName-${generateId()}`,
       lastName: `LastName-${generateId()}`,
-      email: `email+${generateId()}@gmail.com`,
+      email: `sanity-email+${generateId()}@gmail.com`,
       password: '1234'
     }
     const newWorkspaceName = `Some HULY #@$ WS - ${generateId(12)}`
@@ -158,7 +158,7 @@ test.describe('Workspace tests', () => {
     const newUser2: SignUpData = {
       firstName: `FirstName2-${generateId()}`,
       lastName: `LastName2-${generateId()}`,
-      email: `email+${generateId()}@gmail.com`,
+      email: `sanity-email+${generateId()}@gmail.com`,
       password: '1234'
     }
 
@@ -174,7 +174,7 @@ test.describe('Workspace tests', () => {
     const newUser: SignUpData = {
       firstName: `FirstName-${generateId()}`,
       lastName: `LastName-${generateId()}`,
-      email: `email+${generateId()}@gmail.com`,
+      email: `sanity-email+${generateId()}@gmail.com`,
       password: '1234'
     }
     const newWorkspaceName = `Some HULY #@$ WS - ${generateId(12)}`
@@ -209,7 +209,7 @@ test.describe('Workspace tests', () => {
     const newUser2: SignUpData = {
       firstName: `FirstName2-${generateId()}`,
       lastName: `LastName2-${generateId()}`,
-      email: `email+${generateId()}@gmail.com`,
+      email: `sanity-email+${generateId()}@gmail.com`,
       password: '1234'
     }
 

tests/sanity/tsconfig.json

@@ -4,6 +4,7 @@
   "compilerOptions": {
     "rootDir": "./tests",
     "outDir": "./lib",
-    "lib": ["esnext", "dom"]
+    "declarationDir": "./types",
+    "lib": ["esnext", "dom"],
   }
 }