From 8b0627fc0f389c12284ec608ee46a166a72a88a6 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Tue, 9 Apr 2024 16:34:34 +0700 Subject: [PATCH] UBERF-6426: Fix stuck backup (#5258) Signed-off-by: Andrey Sobolev --- server/account/src/index.ts | 7 +- server/elastic/src/backup.ts | 70 ++++++------- server/server/src/minio.ts | 4 +- server/tool/src/index.ts | 103 ++++++++++---------- tests/sanity/package.json | 11 ++- tests/sanity/tests/index.ts | 10 ++ tests/sanity/tests/workspace/create.spec.ts | 14 +-- tests/sanity/tsconfig.json | 3 +- 8 files changed, 126 insertions(+), 96 deletions(-) create mode 100644 tests/sanity/tests/index.ts diff --git a/server/account/src/index.ts b/server/account/src/index.ts index fe895c2784..eb4317f82a 100644 --- a/server/account/src/index.ts +++ b/server/account/src/index.ts @@ -818,7 +818,7 @@ export async function createWorkspace ( searchPromise = generateWorkspaceRecord(db, email, productId, version, workspaceName, workspace) const workspaceInfo = await searchPromise - let client: Client + let client: Client | undefined const childLogger = ctx.newChild( 'createWorkspace', { workspace: workspaceInfo.workspace }, @@ -837,8 +837,9 @@ export async function createWorkspace ( const initWS = getMetadata(toolPlugin.metadata.InitWorkspace) const wsId = getWorkspaceId(workspaceInfo.workspace, productId) if (initWS !== undefined && (await getWorkspaceById(db, productId, initWS)) !== null) { - client = await initModel(ctx, getTransactor(), wsId, txes, [], ctxModellogger) - await client.close() + // Just any valid model for transactor to be able to function + await initModel(ctx, getTransactor(), wsId, txes, [], ctxModellogger, true) + // Clone init workspace. await cloneWorkspace( getTransactor(), getWorkspaceId(initWS, productId), diff --git a/server/elastic/src/backup.ts b/server/elastic/src/backup.ts index d4c257f386..52e92e38b9 100644 --- a/server/elastic/src/backup.ts +++ b/server/elastic/src/backup.ts @@ -166,43 +166,47 @@ class ElasticDataAdapter implements DbAdapter { async load (ctx: MeasureContext, domain: Domain, docs: Ref[]): Promise { const result: Doc[] = [] + const toLoad = [...docs] - const resp = await this.client.search({ - index: indexName, - type: '_doc', - body: { - query: { - bool: { - must: [ - { - terms: { - _id: docs, - boost: 1.0 + while (toLoad.length > 0) { + const part = toLoad.splice(0, 5000) + const resp = await this.client.search({ + index: indexName, + type: '_doc', + body: { + query: { + bool: { + must: [ + { + terms: { + _id: part, + boost: 1.0 + } + }, + { + match: { + workspaceId: { query: toWorkspaceString(this.workspaceId), operator: 'and' } + } } - }, - { - match: { - workspaceId: { query: toWorkspaceString(this.workspaceId), operator: 'and' } - } - } - ] - } - }, - size: docs.length - } - }) - const buffer = resp.body.hits.hits.map((hit: any) => ({ _id: hit._id, data: hit._source })) + ] + } + }, + size: part.length + } + }) + const buffer = resp.body.hits.hits.map((hit: any) => ({ _id: hit._id, data: hit._source })) - for (const item of buffer) { - const dta: FullTextData = { - _id: item._id as Ref, - _class: core.class.FulltextData, - space: 'fulltext-blob' as Ref, - modifiedOn: item.data.modifiedOn, - modifiedBy: item.data.modifiedBy, - data: item.data + for (const item of buffer) { + const dta: FullTextData = { + _id: item._id as Ref, + _class: core.class.FulltextData, + space: 'fulltext-blob' as Ref, + modifiedOn: item.data.modifiedOn, + modifiedBy: item.data.modifiedBy, + data: item.data + } + result.push(dta) } - result.push(dta) } return result } diff --git a/server/server/src/minio.ts b/server/server/src/minio.ts index 31e71d28b3..3e6786f1de 100644 --- a/server/server/src/minio.ts +++ b/server/server/src/minio.ts @@ -16,6 +16,7 @@ import core, { BlobData, Class, + cutObjectArray, Doc, DocumentQuery, DocumentUpdate, @@ -90,7 +91,8 @@ class StorageBlobAdapter implements DbAdapter { for (const item of docs) { const stat = await this.client.stat(this.ctx, this.workspaceId, item) if (stat === undefined) { - throw new Error(`Could not find blob ${item}`) + await ctx.error('Could not find blob', { domain, item, allDocs: cutObjectArray(docs) }) + continue } const chunks: Buffer[] = await this.client.read(this.ctx, this.workspaceId, item) const final = Buffer.concat(chunks) diff --git a/server/tool/src/index.ts b/server/tool/src/index.ts index 9cf60378f2..f6bbb81d63 100644 --- a/server/tool/src/index.ts +++ b/server/tool/src/index.ts @@ -111,8 +111,9 @@ export async function initModel ( workspaceId: WorkspaceId, rawTxes: Tx[], migrateOperations: [string, MigrateOperation][], - logger: ModelLogger = consoleModelLogger -): Promise { + logger: ModelLogger = consoleModelLogger, + skipOperations: boolean = false +): Promise { const { mongodbUri, storageAdapter: minio, txes } = prepareTools(rawTxes) if (txes.some((tx) => tx.objectSpace !== core.space.Model)) { throw Error('Model txes must target only core.space.Model') @@ -131,38 +132,39 @@ export async function initModel ( logger.log('creating data...', { transactorUrl }) const { model } = await fetchModelFromMongo(ctx, mongodbUri, workspaceId) - connection = (await connect( - transactorUrl, - workspaceId, - undefined, - { - model: 'upgrade', - admin: 'true' - }, - model - )) as unknown as CoreClient & BackupClient + logger.log('create minio bucket', { workspaceId }) + if (!(await minio.exists(ctx, workspaceId))) { + await minio.make(ctx, workspaceId) + } + if (!skipOperations) { + connection = (await connect( + transactorUrl, + workspaceId, + undefined, + { + model: 'upgrade', + admin: 'true' + }, + model + )) as unknown as CoreClient & BackupClient - try { - for (const op of migrateOperations) { - logger.log('Migrate', { name: op[0] }) - await op[1].upgrade(connection, logger) + try { + for (const op of migrateOperations) { + logger.log('Migrate', { name: op[0] }) + await op[1].upgrade(connection, logger) + } + + // Create update indexes + await createUpdateIndexes(ctx, connection, db, logger) + } catch (e: any) { + logger.error('error', { error: e }) + throw e } - - // Create update indexes - await createUpdateIndexes(ctx, connection, db, logger) - - logger.log('create minio bucket', { workspaceId }) - if (!(await minio.exists(ctx, workspaceId))) { - await minio.make(ctx, workspaceId) - } - } catch (e: any) { - logger.error('error', { error: e }) - throw e + return connection } } finally { _client.close() } - return connection } /** @@ -174,7 +176,8 @@ export async function upgradeModel ( workspaceId: WorkspaceId, rawTxes: Tx[], migrateOperations: [string, MigrateOperation][], - logger: ModelLogger = consoleModelLogger + logger: ModelLogger = consoleModelLogger, + skipTxUpdate: boolean = false ): Promise { const { mongodbUri, txes } = prepareTools(rawTxes) @@ -188,28 +191,30 @@ export async function upgradeModel ( try { const db = getWorkspaceDB(client, workspaceId) - logger.log('removing model...', { workspaceId: workspaceId.name }) - // we're preserving accounts (created by core.account.System). - const result = await ctx.with( - 'mongo-delete', - {}, - async () => - await db.collection(DOMAIN_TX).deleteMany({ - objectSpace: core.space.Model, - modifiedBy: core.account.System, - objectClass: { $nin: [contact.class.PersonAccount, 'contact:class:EmployeeAccount'] } - }) - ) - logger.log('transactions deleted.', { workspaceId: workspaceId.name, count: result.deletedCount }) + if (!skipTxUpdate) { + logger.log('removing model...', { workspaceId: workspaceId.name }) + // we're preserving accounts (created by core.account.System). + const result = await ctx.with( + 'mongo-delete', + {}, + async () => + await db.collection(DOMAIN_TX).deleteMany({ + objectSpace: core.space.Model, + modifiedBy: core.account.System, + objectClass: { $nin: [contact.class.PersonAccount, 'contact:class:EmployeeAccount'] } + }) + ) + logger.log('transactions deleted.', { workspaceId: workspaceId.name, count: result.deletedCount }) - logger.log('creating model...', { workspaceId: workspaceId.name }) - const insert = await ctx.with( - 'mongo-insert', - {}, - async () => await db.collection(DOMAIN_TX).insertMany(txes as Document[]) - ) + logger.log('creating model...', { workspaceId: workspaceId.name }) + const insert = await ctx.with( + 'mongo-insert', + {}, + async () => await db.collection(DOMAIN_TX).insertMany(txes as Document[]) + ) - logger.log('model transactions inserted.', { workspaceId: workspaceId.name, count: insert.insertedCount }) + logger.log('model transactions inserted.', { workspaceId: workspaceId.name, count: insert.insertedCount }) + } const { hierarchy, modelDb, model } = await fetchModelFromMongo(ctx, mongodbUri, workspaceId) diff --git a/tests/sanity/package.json b/tests/sanity/package.json index 669951ada5..7265d98257 100644 --- a/tests/sanity/package.json +++ b/tests/sanity/package.json @@ -1,12 +1,19 @@ { "name": "@hcengineering/tests-sanity", "version": "0.6.1", + "main": "lib/index.js", + "svelte": "src/index.ts", + "types": "types/index.d.ts", "author": "Anticrm Platform Contributors", "template": "@hcengineering/default-package", "license": "EPL-2.0", "scripts": { - "build": "", - "build:watch": "", + "build": "compile", + "build:watch": "compile", + "_phase:build": "compile transpile tests", + "_phase:test": "", + "_phase:format": "format src", + "_phase:validate": "compile validate", "lint:fix": "eslint --fix tests", "lint": "eslint tests", "format": "format tests", diff --git a/tests/sanity/tests/index.ts b/tests/sanity/tests/index.ts new file mode 100644 index 0000000000..14b0bce3d6 --- /dev/null +++ b/tests/sanity/tests/index.ts @@ -0,0 +1,10 @@ +export * from './model/login-page' +export * from './model/signin-page' +export * from './model/common-page' +export * from './model/common-types' +export * from './model/left-side-menu-page' +export * from './model/signup-page' +export * from './model/select-workspace-page' +export * from './utils' +export * from './model/tracker/issues-page' +export * from './model/tracker/issues-details-page' diff --git a/tests/sanity/tests/workspace/create.spec.ts b/tests/sanity/tests/workspace/create.spec.ts index 0cfc595fae..d076bf5a00 100644 --- a/tests/sanity/tests/workspace/create.spec.ts +++ b/tests/sanity/tests/workspace/create.spec.ts @@ -16,7 +16,7 @@ test.describe('Workspace tests', () => { const newUser: SignUpData = { firstName: `FirstName-${generateId()}`, lastName: `LastName-${generateId()}`, - email: `email+${generateId()}@gmail.com`, + email: `sanity-email+${generateId()}@gmail.com`, password: '1234' } const newWorkspaceName = `New Workspace Name - ${generateId(2)}` @@ -39,7 +39,7 @@ test.describe('Workspace tests', () => { const newUser: SignUpData = { firstName: `FirstName-${generateId()}`, lastName: `LastName-${generateId()}`, - email: `email+${generateId()}@gmail.com`, + email: `sanity-email+${generateId()}@gmail.com`, password: '1234' } const newIssue: NewIssue = { @@ -92,7 +92,7 @@ test.describe('Workspace tests', () => { const newUser: SignUpData = { firstName: `FirstName-${generateId()}`, lastName: `LastName-${generateId()}`, - email: `email+${generateId()}@gmail.com`, + email: `sanity-email+${generateId()}@gmail.com`, password: '1234' } const newWorkspaceName = `New Workspace Name - ${generateId(2)}` @@ -125,7 +125,7 @@ test.describe('Workspace tests', () => { const newUser: SignUpData = { firstName: `FirstName-${generateId()}`, lastName: `LastName-${generateId()}`, - email: `email+${generateId()}@gmail.com`, + email: `sanity-email+${generateId()}@gmail.com`, password: '1234' } const newWorkspaceName = `Some HULY #@$ WS - ${generateId(12)}` @@ -158,7 +158,7 @@ test.describe('Workspace tests', () => { const newUser2: SignUpData = { firstName: `FirstName2-${generateId()}`, lastName: `LastName2-${generateId()}`, - email: `email+${generateId()}@gmail.com`, + email: `sanity-email+${generateId()}@gmail.com`, password: '1234' } @@ -174,7 +174,7 @@ test.describe('Workspace tests', () => { const newUser: SignUpData = { firstName: `FirstName-${generateId()}`, lastName: `LastName-${generateId()}`, - email: `email+${generateId()}@gmail.com`, + email: `sanity-email+${generateId()}@gmail.com`, password: '1234' } const newWorkspaceName = `Some HULY #@$ WS - ${generateId(12)}` @@ -209,7 +209,7 @@ test.describe('Workspace tests', () => { const newUser2: SignUpData = { firstName: `FirstName2-${generateId()}`, lastName: `LastName2-${generateId()}`, - email: `email+${generateId()}@gmail.com`, + email: `sanity-email+${generateId()}@gmail.com`, password: '1234' } diff --git a/tests/sanity/tsconfig.json b/tests/sanity/tsconfig.json index e11f47d740..bbadfd674c 100644 --- a/tests/sanity/tsconfig.json +++ b/tests/sanity/tsconfig.json @@ -4,6 +4,7 @@ "compilerOptions": { "rootDir": "./tests", "outDir": "./lib", - "lib": ["esnext", "dom"] + "declarationDir": "./types", + "lib": ["esnext", "dom"], } } \ No newline at end of file