UBERF-6426: Fix stuck backup (#5258)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

server/account/src/index.ts

@@ -818,7 +818,7 @@ export async function createWorkspace (
     searchPromise = generateWorkspaceRecord(db, email, productId, version, workspaceName, workspace)
 
     const workspaceInfo = await searchPromise
-    let client: Client
+    let client: Client | undefined
     const childLogger = ctx.newChild(
       'createWorkspace',
       { workspace: workspaceInfo.workspace },
@@ -837,8 +837,9 @@ export async function createWorkspace (
       const initWS = getMetadata(toolPlugin.metadata.InitWorkspace)
       const wsId = getWorkspaceId(workspaceInfo.workspace, productId)
       if (initWS !== undefined && (await getWorkspaceById(db, productId, initWS)) !== null) {
-        client = await initModel(ctx, getTransactor(), wsId, txes, [], ctxModellogger)
-        await client.close()
+        // Just any valid model for transactor to be able to function
+        await initModel(ctx, getTransactor(), wsId, txes, [], ctxModellogger, true)
+        // Clone init workspace.
         await cloneWorkspace(
           getTransactor(),
           getWorkspaceId(initWS, productId),

server/elastic/src/backup.ts

@@ -166,43 +166,47 @@ class ElasticDataAdapter implements DbAdapter {
 
   async load (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<Doc[]> {
     const result: Doc[] = []
+    const toLoad = [...docs]
 
-    const resp = await this.client.search({
-      index: indexName,
-      type: '_doc',
-      body: {
-        query: {
-          bool: {
-            must: [
-              {
-                terms: {
-                  _id: docs,
-                  boost: 1.0
+    while (toLoad.length > 0) {
+      const part = toLoad.splice(0, 5000)
+      const resp = await this.client.search({
+        index: indexName,
+        type: '_doc',
+        body: {
+          query: {
+            bool: {
+              must: [
+                {
+                  terms: {
+                    _id: part,
+                    boost: 1.0
+                  }
+                },
+                {
+                  match: {
+                    workspaceId: { query: toWorkspaceString(this.workspaceId), operator: 'and' }
+                  }
                 }
-              },
-              {
-                match: {
-                  workspaceId: { query: toWorkspaceString(this.workspaceId), operator: 'and' }
-                }
-              }
-            ]
-          }
-        },
-        size: docs.length
-      }
-    })
-    const buffer = resp.body.hits.hits.map((hit: any) => ({ _id: hit._id, data: hit._source }))
+              ]
+            }
+          },
+          size: part.length
+        }
+      })
+      const buffer = resp.body.hits.hits.map((hit: any) => ({ _id: hit._id, data: hit._source }))
 
-    for (const item of buffer) {
-      const dta: FullTextData = {
-        _id: item._id as Ref<FullTextData>,
-        _class: core.class.FulltextData,
-        space: 'fulltext-blob' as Ref<Space>,
-        modifiedOn: item.data.modifiedOn,
-        modifiedBy: item.data.modifiedBy,
-        data: item.data
+      for (const item of buffer) {
+        const dta: FullTextData = {
+          _id: item._id as Ref<FullTextData>,
+          _class: core.class.FulltextData,
+          space: 'fulltext-blob' as Ref<Space>,
+          modifiedOn: item.data.modifiedOn,
+          modifiedBy: item.data.modifiedBy,
+          data: item.data
+        }
+        result.push(dta)
       }
-      result.push(dta)
     }
     return result
   }

server/server/src/minio.ts

@@ -16,6 +16,7 @@
 import core, {
   BlobData,
   Class,
+  cutObjectArray,
   Doc,
   DocumentQuery,
   DocumentUpdate,
@@ -90,7 +91,8 @@ class StorageBlobAdapter implements DbAdapter {
     for (const item of docs) {
       const stat = await this.client.stat(this.ctx, this.workspaceId, item)
       if (stat === undefined) {
-        throw new Error(`Could not find blob ${item}`)
+        await ctx.error('Could not find blob', { domain, item, allDocs: cutObjectArray(docs) })
+        continue
       }
       const chunks: Buffer[] = await this.client.read(this.ctx, this.workspaceId, item)
       const final = Buffer.concat(chunks)

server/tool/src/index.ts

@@ -111,8 +111,9 @@ export async function initModel (
   workspaceId: WorkspaceId,
   rawTxes: Tx[],
   migrateOperations: [string, MigrateOperation][],
-  logger: ModelLogger = consoleModelLogger
-): Promise<CoreClient> {
+  logger: ModelLogger = consoleModelLogger,
+  skipOperations: boolean = false
+): Promise<CoreClient | undefined> {
   const { mongodbUri, storageAdapter: minio, txes } = prepareTools(rawTxes)
   if (txes.some((tx) => tx.objectSpace !== core.space.Model)) {
     throw Error('Model txes must target only core.space.Model')
@@ -131,38 +132,39 @@ export async function initModel (
     logger.log('creating data...', { transactorUrl })
     const { model } = await fetchModelFromMongo(ctx, mongodbUri, workspaceId)
 
-    connection = (await connect(
-      transactorUrl,
-      workspaceId,
-      undefined,
-      {
-        model: 'upgrade',
-        admin: 'true'
-      },
-      model
-    )) as unknown as CoreClient & BackupClient
+    logger.log('create minio bucket', { workspaceId })
+    if (!(await minio.exists(ctx, workspaceId))) {
+      await minio.make(ctx, workspaceId)
+    }
+    if (!skipOperations) {
+      connection = (await connect(
+        transactorUrl,
+        workspaceId,
+        undefined,
+        {
+          model: 'upgrade',
+          admin: 'true'
+        },
+        model
+      )) as unknown as CoreClient & BackupClient
 
-    try {
-      for (const op of migrateOperations) {
-        logger.log('Migrate', { name: op[0] })
-        await op[1].upgrade(connection, logger)
+      try {
+        for (const op of migrateOperations) {
+          logger.log('Migrate', { name: op[0] })
+          await op[1].upgrade(connection, logger)
+        }
+
+        // Create update indexes
+        await createUpdateIndexes(ctx, connection, db, logger)
+      } catch (e: any) {
+        logger.error('error', { error: e })
+        throw e
       }
-
-      // Create update indexes
-      await createUpdateIndexes(ctx, connection, db, logger)
-
-      logger.log('create minio bucket', { workspaceId })
-      if (!(await minio.exists(ctx, workspaceId))) {
-        await minio.make(ctx, workspaceId)
-      }
-    } catch (e: any) {
-      logger.error('error', { error: e })
-      throw e
+      return connection
     }
   } finally {
     _client.close()
   }
-  return connection
 }
 
 /**
@@ -174,7 +176,8 @@ export async function upgradeModel (
   workspaceId: WorkspaceId,
   rawTxes: Tx[],
   migrateOperations: [string, MigrateOperation][],
-  logger: ModelLogger = consoleModelLogger
+  logger: ModelLogger = consoleModelLogger,
+  skipTxUpdate: boolean = false
 ): Promise<CoreClient> {
   const { mongodbUri, txes } = prepareTools(rawTxes)
 
@@ -188,28 +191,30 @@ export async function upgradeModel (
   try {
     const db = getWorkspaceDB(client, workspaceId)
 
-    logger.log('removing model...', { workspaceId: workspaceId.name })
-    // we're preserving accounts (created by core.account.System).
-    const result = await ctx.with(
-      'mongo-delete',
-      {},
-      async () =>
-        await db.collection(DOMAIN_TX).deleteMany({
-          objectSpace: core.space.Model,
-          modifiedBy: core.account.System,
-          objectClass: { $nin: [contact.class.PersonAccount, 'contact:class:EmployeeAccount'] }
-        })
-    )
-    logger.log('transactions deleted.', { workspaceId: workspaceId.name, count: result.deletedCount })
+    if (!skipTxUpdate) {
+      logger.log('removing model...', { workspaceId: workspaceId.name })
+      // we're preserving accounts (created by core.account.System).
+      const result = await ctx.with(
+        'mongo-delete',
+        {},
+        async () =>
+          await db.collection(DOMAIN_TX).deleteMany({
+            objectSpace: core.space.Model,
+            modifiedBy: core.account.System,
+            objectClass: { $nin: [contact.class.PersonAccount, 'contact:class:EmployeeAccount'] }
+          })
+      )
+      logger.log('transactions deleted.', { workspaceId: workspaceId.name, count: result.deletedCount })
 
-    logger.log('creating model...', { workspaceId: workspaceId.name })
-    const insert = await ctx.with(
-      'mongo-insert',
-      {},
-      async () => await db.collection(DOMAIN_TX).insertMany(txes as Document[])
-    )
+      logger.log('creating model...', { workspaceId: workspaceId.name })
+      const insert = await ctx.with(
+        'mongo-insert',
+        {},
+        async () => await db.collection(DOMAIN_TX).insertMany(txes as Document[])
+      )
 
-    logger.log('model transactions inserted.', { workspaceId: workspaceId.name, count: insert.insertedCount })
+      logger.log('model transactions inserted.', { workspaceId: workspaceId.name, count: insert.insertedCount })
+    }
 
     const { hierarchy, modelDb, model } = await fetchModelFromMongo(ctx, mongodbUri, workspaceId)
 

tests/sanity/package.json

@@ -1,12 +1,19 @@
 {
   "name": "@hcengineering/tests-sanity",
   "version": "0.6.1",
+  "main": "lib/index.js",
+  "svelte": "src/index.ts",
+  "types": "types/index.d.ts",
   "author": "Anticrm Platform Contributors",
   "template": "@hcengineering/default-package",
   "license": "EPL-2.0",
   "scripts": {
-    "build": "",
-    "build:watch": "",
+    "build": "compile",
+    "build:watch": "compile",
+    "_phase:build": "compile transpile tests",
+    "_phase:test": "",
+    "_phase:format": "format src",
+    "_phase:validate": "compile validate",
     "lint:fix": "eslint --fix tests",
     "lint": "eslint tests",
     "format": "format tests",

tests/sanity/tests/index.ts

@@ -0,0 +1,10 @@
+export * from './model/login-page'
+export * from './model/signin-page'
+export * from './model/common-page'
+export * from './model/common-types'
+export * from './model/left-side-menu-page'
+export * from './model/signup-page'
+export * from './model/select-workspace-page'
+export * from './utils'
+export * from './model/tracker/issues-page'
+export * from './model/tracker/issues-details-page'

tests/sanity/tests/workspace/create.spec.ts

@@ -16,7 +16,7 @@ test.describe('Workspace tests', () => {
     const newUser: SignUpData = {
       firstName: `FirstName-${generateId()}`,
       lastName: `LastName-${generateId()}`,
-      email: `email+${generateId()}@gmail.com`,
+      email: `sanity-email+${generateId()}@gmail.com`,
       password: '1234'
     }
     const newWorkspaceName = `New Workspace Name - ${generateId(2)}`
@@ -39,7 +39,7 @@ test.describe('Workspace tests', () => {
     const newUser: SignUpData = {
       firstName: `FirstName-${generateId()}`,
       lastName: `LastName-${generateId()}`,
-      email: `email+${generateId()}@gmail.com`,
+      email: `sanity-email+${generateId()}@gmail.com`,
       password: '1234'
     }
     const newIssue: NewIssue = {
@@ -92,7 +92,7 @@ test.describe('Workspace tests', () => {
     const newUser: SignUpData = {
       firstName: `FirstName-${generateId()}`,
       lastName: `LastName-${generateId()}`,
-      email: `email+${generateId()}@gmail.com`,
+      email: `sanity-email+${generateId()}@gmail.com`,
       password: '1234'
     }
     const newWorkspaceName = `New Workspace Name - ${generateId(2)}`
@@ -125,7 +125,7 @@ test.describe('Workspace tests', () => {
     const newUser: SignUpData = {
       firstName: `FirstName-${generateId()}`,
       lastName: `LastName-${generateId()}`,
-      email: `email+${generateId()}@gmail.com`,
+      email: `sanity-email+${generateId()}@gmail.com`,
       password: '1234'
     }
     const newWorkspaceName = `Some HULY #@$ WS - ${generateId(12)}`
@@ -158,7 +158,7 @@ test.describe('Workspace tests', () => {
     const newUser2: SignUpData = {
       firstName: `FirstName2-${generateId()}`,
       lastName: `LastName2-${generateId()}`,
-      email: `email+${generateId()}@gmail.com`,
+      email: `sanity-email+${generateId()}@gmail.com`,
       password: '1234'
     }
 
@@ -174,7 +174,7 @@ test.describe('Workspace tests', () => {
     const newUser: SignUpData = {
       firstName: `FirstName-${generateId()}`,
       lastName: `LastName-${generateId()}`,
-      email: `email+${generateId()}@gmail.com`,
+      email: `sanity-email+${generateId()}@gmail.com`,
       password: '1234'
     }
     const newWorkspaceName = `Some HULY #@$ WS - ${generateId(12)}`
@@ -209,7 +209,7 @@ test.describe('Workspace tests', () => {
     const newUser2: SignUpData = {
       firstName: `FirstName2-${generateId()}`,
       lastName: `LastName2-${generateId()}`,
-      email: `email+${generateId()}@gmail.com`,
+      email: `sanity-email+${generateId()}@gmail.com`,
       password: '1234'
     }
 

tests/sanity/tsconfig.json

@@ -4,6 +4,7 @@
   "compilerOptions": {
     "rootDir": "./tests",
     "outDir": "./lib",
-    "lib": ["esnext", "dom"]
+    "declarationDir": "./types",
+    "lib": ["esnext", "dom"],
   }
 }