UBERF-8517: Fix github external sync (#6999)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

services/github/pod-github/src/sync/issueBase.ts

@@ -1068,17 +1068,20 @@ export abstract class IssueSyncManagerBase {
     _class: Ref<Class<Doc>>,
     repo: GithubIntegrationRepository,
     issues: IssueExternalData[],
-    derivedClient: TxOperations
+    derivedClient: TxOperations,
+    syncDocs?: DocSyncInfo[]
   ): Promise<void> {
     if (repo.githubProject == null) {
       return
     }
-    const syncInfo = await this.client.findAll<DocSyncInfo>(github.class.DocSyncInfo, {
+    const syncInfo =
-      space: repo.githubProject,
+      syncDocs ??
-      repository: repo._id,
+      (await this.client.findAll<DocSyncInfo>(github.class.DocSyncInfo, {
-      objectClass: _class,
+        space: repo.githubProject,
-      url: { $in: issues.map((it) => (it.url ?? '').toLowerCase()) }
+        repository: repo._id,
-    })
+        objectClass: _class,
+        url: { $in: issues.map((it) => (it.url ?? '').toLowerCase()) }
+      }))
 
     const ops = derivedClient.apply()
 
@@ -1088,8 +1091,10 @@ export abstract class IssueSyncManagerBase {
           this.ctx.info('Retrieve empty document', { repo: repo.name, workspace: this.provider.getWorkspaceId().name })
           continue
         }
-        const existing = syncInfo.find((it) => it.url === issue.url.toLowerCase())
+        const existing =
-        if (existing === undefined) {
+          syncInfo.find((it) => it.url.toLowerCase() === issue.url.toLowerCase()) ??
+          syncInfo.find((it) => (it.external as IssueExternalData)?.id === issue.id)
+        if (existing === undefined && syncDocs === undefined) {
           this.ctx.info('Create sync doc', { url: issue.url, workspace: this.provider.getWorkspaceId().name })
           await ops.createDoc<DocSyncInfo>(github.class.DocSyncInfo, repo.githubProject, {
             url: issue.url.toLowerCase(),
@@ -1103,7 +1108,10 @@ export abstract class IssueSyncManagerBase {
             externalVersionSince: '',
             lastModified: new Date(issue.updatedAt).getTime()
           })
-        } else {
+        } else if (existing !== undefined) {
+          if (syncDocs !== undefined) {
+            syncDocs = syncDocs.filter((it) => it._id !== existing._id)
+          }
           const externalEqual = deepEqual(existing.external, issue)
           if (!externalEqual || existing.externalVersion !== githubExternalSyncVersion) {
             this.ctx.info('Update sync doc', { url: issue.url, workspace: this.provider.getWorkspaceId().name })
@@ -1126,6 +1134,14 @@ export abstract class IssueSyncManagerBase {
         this.ctx.error(err)
       }
     }
+    // if no sync doc, mark it as synchronized
+    for (const sd of syncDocs ?? []) {
+      await ops.update(sd, {
+        needSync: githubSyncVersion,
+        externalVersion: githubExternalSyncVersion,
+        error: 'not found external doc'
+      })
+    }
     await ops.commit(true)
     this.provider.sync()
   }

services/github/pod-github/src/sync/issues.ts

@@ -980,7 +980,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
     // Wait global project sync
     await integration.syncLock.get(prj._id)
 
-    const ids = syncDocs.map((it) => (it.external as IssueExternalData).id).filter((it) => it !== undefined)
+    const allSyncDocs = [...syncDocs]
     //
     let partsize = 50
     try {
@@ -988,7 +988,8 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
         if (this.provider.isClosing()) {
           break
         }
-        const idsPart = ids.splice(0, partsize)
+        const docsPart = allSyncDocs.splice(0, partsize)
+        const idsPart = docsPart.map((it) => (it.external as IssueExternalData).id).filter((it) => it !== undefined)
         if (idsPart.length === 0) {
           break
         }
@@ -1023,11 +1024,11 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
             })
           }
 
-          await this.syncIssues(tracker.class.Issue, repo, issues, derivedClient)
+          await this.syncIssues(tracker.class.Issue, repo, issues, derivedClient, docsPart)
         } catch (err: any) {
           if (partsize > 1) {
             partsize = 1
-            ids.push(...idsPart)
+            allSyncDocs.push(...docsPart)
             this.ctx.warn('issue external retrieval switch to one by one mode', {
               errors: err.errors,
               msg: err.message,

services/github/pod-github/src/sync/pullrequests.ts

@@ -1334,12 +1334,13 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
   ): Promise<void> {
     await integration.syncLock.get(prj._id)
 
-    const ids = syncDocs.map((it) => (it.external as IssueExternalData).id).filter((it) => it !== undefined)
+    const allSyncDocs = [...syncDocs]
 
     let partsize = 50
     try {
       while (true) {
-        const idsPart = ids.splice(0, partsize)
+        const docsPart = allSyncDocs.splice(0, partsize)
+        const idsPart = docsPart.map((it) => (it.external as IssueExternalData).id).filter((it) => it !== undefined)
         if (idsPart.length === 0) {
           break
         }
@@ -1373,11 +1374,11 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
               data: cutObjectArray(response)
             })
           }
-          await this.syncIssues(github.class.GithubPullRequest, repo, issues, derivedClient)
+          await this.syncIssues(github.class.GithubPullRequest, repo, issues, derivedClient, docsPart)
         } catch (err: any) {
           if (partsize > 1) {
             partsize = 1
-            ids.push(...idsPart)
+            allSyncDocs.push(...docsPart)
             this.ctx.warn('pull request external retrieval switch to one by one mode', {
               errors: err.errors,
               msg: err.message,