From f1bee10ebb62e0349679ebda5308573e7741e9d7 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Mon, 21 Oct 2024 11:05:42 +0700 Subject: [PATCH] UBERF-8517: Fix github external sync (#6999) Signed-off-by: Andrey Sobolev --- .../github/pod-github/src/sync/issueBase.ts | 36 +++++++++++++------ services/github/pod-github/src/sync/issues.ts | 9 ++--- .../pod-github/src/sync/pullrequests.ts | 9 ++--- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/services/github/pod-github/src/sync/issueBase.ts b/services/github/pod-github/src/sync/issueBase.ts index 0ff45f0f96..0b2365ce29 100644 --- a/services/github/pod-github/src/sync/issueBase.ts +++ b/services/github/pod-github/src/sync/issueBase.ts @@ -1068,17 +1068,20 @@ export abstract class IssueSyncManagerBase { _class: Ref>, repo: GithubIntegrationRepository, issues: IssueExternalData[], - derivedClient: TxOperations + derivedClient: TxOperations, + syncDocs?: DocSyncInfo[] ): Promise { if (repo.githubProject == null) { return } - const syncInfo = await this.client.findAll(github.class.DocSyncInfo, { - space: repo.githubProject, - repository: repo._id, - objectClass: _class, - url: { $in: issues.map((it) => (it.url ?? '').toLowerCase()) } - }) + const syncInfo = + syncDocs ?? + (await this.client.findAll(github.class.DocSyncInfo, { + space: repo.githubProject, + repository: repo._id, + objectClass: _class, + url: { $in: issues.map((it) => (it.url ?? '').toLowerCase()) } + })) const ops = derivedClient.apply() @@ -1088,8 +1091,10 @@ export abstract class IssueSyncManagerBase { this.ctx.info('Retrieve empty document', { repo: repo.name, workspace: this.provider.getWorkspaceId().name }) continue } - const existing = syncInfo.find((it) => it.url === issue.url.toLowerCase()) - if (existing === undefined) { + const existing = + syncInfo.find((it) => it.url.toLowerCase() === issue.url.toLowerCase()) ?? + syncInfo.find((it) => (it.external as IssueExternalData)?.id === issue.id) + if (existing === undefined && syncDocs === undefined) { this.ctx.info('Create sync doc', { url: issue.url, workspace: this.provider.getWorkspaceId().name }) await ops.createDoc(github.class.DocSyncInfo, repo.githubProject, { url: issue.url.toLowerCase(), @@ -1103,7 +1108,10 @@ export abstract class IssueSyncManagerBase { externalVersionSince: '', lastModified: new Date(issue.updatedAt).getTime() }) - } else { + } else if (existing !== undefined) { + if (syncDocs !== undefined) { + syncDocs = syncDocs.filter((it) => it._id !== existing._id) + } const externalEqual = deepEqual(existing.external, issue) if (!externalEqual || existing.externalVersion !== githubExternalSyncVersion) { this.ctx.info('Update sync doc', { url: issue.url, workspace: this.provider.getWorkspaceId().name }) @@ -1126,6 +1134,14 @@ export abstract class IssueSyncManagerBase { this.ctx.error(err) } } + // if no sync doc, mark it as synchronized + for (const sd of syncDocs ?? []) { + await ops.update(sd, { + needSync: githubSyncVersion, + externalVersion: githubExternalSyncVersion, + error: 'not found external doc' + }) + } await ops.commit(true) this.provider.sync() } diff --git a/services/github/pod-github/src/sync/issues.ts b/services/github/pod-github/src/sync/issues.ts index 290ce7ed15..818d97a69b 100644 --- a/services/github/pod-github/src/sync/issues.ts +++ b/services/github/pod-github/src/sync/issues.ts @@ -980,7 +980,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan // Wait global project sync await integration.syncLock.get(prj._id) - const ids = syncDocs.map((it) => (it.external as IssueExternalData).id).filter((it) => it !== undefined) + const allSyncDocs = [...syncDocs] // let partsize = 50 try { @@ -988,7 +988,8 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan if (this.provider.isClosing()) { break } - const idsPart = ids.splice(0, partsize) + const docsPart = allSyncDocs.splice(0, partsize) + const idsPart = docsPart.map((it) => (it.external as IssueExternalData).id).filter((it) => it !== undefined) if (idsPart.length === 0) { break } @@ -1023,11 +1024,11 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan }) } - await this.syncIssues(tracker.class.Issue, repo, issues, derivedClient) + await this.syncIssues(tracker.class.Issue, repo, issues, derivedClient, docsPart) } catch (err: any) { if (partsize > 1) { partsize = 1 - ids.push(...idsPart) + allSyncDocs.push(...docsPart) this.ctx.warn('issue external retrieval switch to one by one mode', { errors: err.errors, msg: err.message, diff --git a/services/github/pod-github/src/sync/pullrequests.ts b/services/github/pod-github/src/sync/pullrequests.ts index 51d3e78d54..5978c48de0 100644 --- a/services/github/pod-github/src/sync/pullrequests.ts +++ b/services/github/pod-github/src/sync/pullrequests.ts @@ -1334,12 +1334,13 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS ): Promise { await integration.syncLock.get(prj._id) - const ids = syncDocs.map((it) => (it.external as IssueExternalData).id).filter((it) => it !== undefined) + const allSyncDocs = [...syncDocs] let partsize = 50 try { while (true) { - const idsPart = ids.splice(0, partsize) + const docsPart = allSyncDocs.splice(0, partsize) + const idsPart = docsPart.map((it) => (it.external as IssueExternalData).id).filter((it) => it !== undefined) if (idsPart.length === 0) { break } @@ -1373,11 +1374,11 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS data: cutObjectArray(response) }) } - await this.syncIssues(github.class.GithubPullRequest, repo, issues, derivedClient) + await this.syncIssues(github.class.GithubPullRequest, repo, issues, derivedClient, docsPart) } catch (err: any) { if (partsize > 1) { partsize = 1 - ids.push(...idsPart) + allSyncDocs.push(...docsPart) this.ctx.warn('pull request external retrieval switch to one by one mode', { errors: err.errors, msg: err.message,