hcengineering/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2024-12-22 19:11:33 +03:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

UBERF-8517: Fix github external sync (#6999)

Browse Source 
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev 2024-10-21 11:05:42 +07:00 committed by GitHub
parent d65176fb84
commit f1bee10ebb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 36 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
services/github/pod-github/src/sync/issueBase.ts
View File

@ -1068,17 +1068,20 @@ export abstract class IssueSyncManagerBase {
_class: Ref<Class<Doc>>,
repo: GithubIntegrationRepository,
issues: IssueExternalData[],
derivedClient: TxOperations
derivedClient: TxOperations,
syncDocs?: DocSyncInfo[]
): Promise<void> {
if (repo.githubProject == null) {
return
}
const syncInfo = await this.client.findAll<DocSyncInfo>(github.class.DocSyncInfo, {
space: repo.githubProject,
repository: repo._id,
objectClass: _class,
url: { $in: issues.map((it) => (it.url ?? '').toLowerCase()) }
})
const syncInfo =
syncDocs ??
(await this.client.findAll<DocSyncInfo>(github.class.DocSyncInfo, {
space: repo.githubProject,
repository: repo._id,
objectClass: _class,
url: { $in: issues.map((it) => (it.url ?? '').toLowerCase()) }
}))
const ops = derivedClient.apply()
@ -1088,8 +1091,10 @@ export abstract class IssueSyncManagerBase {
this.ctx.info('Retrieve empty document', { repo: repo.name, workspace: this.provider.getWorkspaceId().name })
continue
}
const existing = syncInfo.find((it) => it.url === issue.url.toLowerCase())
if (existing === undefined) {
const existing =
syncInfo.find((it) => it.url.toLowerCase() === issue.url.toLowerCase()) ??
syncInfo.find((it) => (it.external as IssueExternalData)?.id === issue.id)
if (existing === undefined && syncDocs === undefined) {
this.ctx.info('Create sync doc', { url: issue.url, workspace: this.provider.getWorkspaceId().name })
await ops.createDoc<DocSyncInfo>(github.class.DocSyncInfo, repo.githubProject, {
url: issue.url.toLowerCase(),
@ -1103,7 +1108,10 @@ export abstract class IssueSyncManagerBase {
externalVersionSince: '',
lastModified: new Date(issue.updatedAt).getTime()
})
} else {
} else if (existing !== undefined) {
if (syncDocs !== undefined) {
syncDocs = syncDocs.filter((it) => it._id !== existing._id)
}
const externalEqual = deepEqual(existing.external, issue)
if (!externalEqual || existing.externalVersion !== githubExternalSyncVersion) {
this.ctx.info('Update sync doc', { url: issue.url, workspace: this.provider.getWorkspaceId().name })
@ -1126,6 +1134,14 @@ export abstract class IssueSyncManagerBase {
this.ctx.error(err)
}
}
// if no sync doc, mark it as synchronized
for (const sd of syncDocs ?? []) {
await ops.update(sd, {
needSync: githubSyncVersion,
externalVersion: githubExternalSyncVersion,
error: 'not found external doc'
})
}
await ops.commit(true)
this.provider.sync()
}

9
services/github/pod-github/src/sync/issues.ts
View File

@ -980,7 +980,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
// Wait global project sync
await integration.syncLock.get(prj._id)
const ids = syncDocs.map((it) => (it.external as IssueExternalData).id).filter((it) => it !== undefined)
const allSyncDocs = [...syncDocs]
//
let partsize = 50
try {
@ -988,7 +988,8 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
if (this.provider.isClosing()) {
break
}
const idsPart = ids.splice(0, partsize)
const docsPart = allSyncDocs.splice(0, partsize)
const idsPart = docsPart.map((it) => (it.external as IssueExternalData).id).filter((it) => it !== undefined)
if (idsPart.length === 0) {
break
}
@ -1023,11 +1024,11 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
})
}
await this.syncIssues(tracker.class.Issue, repo, issues, derivedClient)
await this.syncIssues(tracker.class.Issue, repo, issues, derivedClient, docsPart)
} catch (err: any) {
if (partsize > 1) {
partsize = 1
ids.push(...idsPart)
allSyncDocs.push(...docsPart)
this.ctx.warn('issue external retrieval switch to one by one mode', {
errors: err.errors,
msg: err.message,

9
services/github/pod-github/src/sync/pullrequests.ts
View File

@ -1334,12 +1334,13 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
): Promise<void> {
await integration.syncLock.get(prj._id)
const ids = syncDocs.map((it) => (it.external as IssueExternalData).id).filter((it) => it !== undefined)
const allSyncDocs = [...syncDocs]
let partsize = 50
try {
while (true) {
const idsPart = ids.splice(0, partsize)
const docsPart = allSyncDocs.splice(0, partsize)
const idsPart = docsPart.map((it) => (it.external as IssueExternalData).id).filter((it) => it !== undefined)
if (idsPart.length === 0) {
break
}
@ -1373,11 +1374,11 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
data: cutObjectArray(response)
})
}
await this.syncIssues(github.class.GithubPullRequest, repo, issues, derivedClient)
await this.syncIssues(github.class.GithubPullRequest, repo, issues, derivedClient, docsPart)
} catch (err: any) {
if (partsize > 1) {
partsize = 1
ids.push(...idsPart)
allSyncDocs.push(...docsPart)
this.ctx.warn('pull request external retrieval switch to one by one mode', {
errors: err.errors,
msg: err.message,