mirror of
https://github.com/MichaelMure/git-bug.git
synced 2024-12-15 02:01:43 +03:00
0fd570171d
The Github bridge itself should not write anything. This commit removes code writing to stdout and itroduces an event `ImportEventRateLimiting` to `core.ImportResult` in order to inform about a rate limiting situation of the Github GraphQL API. Now the communication with the user is delegated to the various user interfaces.
589 lines
15 KiB
Go
589 lines
15 KiB
Go
package github
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/shurcooL/githubv4"
|
|
|
|
"github.com/MichaelMure/git-bug/bridge/core"
|
|
"github.com/MichaelMure/git-bug/bridge/core/auth"
|
|
"github.com/MichaelMure/git-bug/bug"
|
|
"github.com/MichaelMure/git-bug/cache"
|
|
"github.com/MichaelMure/git-bug/entity"
|
|
"github.com/MichaelMure/git-bug/util/text"
|
|
)
|
|
|
|
const EMPTY_TITLE_PLACEHOLDER = "<empty string>"
|
|
|
|
// githubImporter implement the Importer interface
|
|
type githubImporter struct {
|
|
conf core.Configuration
|
|
|
|
// mediator to access the Github API
|
|
mediator *importMediator
|
|
|
|
// send only channel
|
|
out chan<- core.ImportResult
|
|
}
|
|
|
|
func (gi *githubImporter) Init(_ context.Context, _ *cache.RepoCache, conf core.Configuration) error {
|
|
gi.conf = conf
|
|
return nil
|
|
}
|
|
|
|
// ImportAll iterate over all the configured repository issues and ensure the creation of the
|
|
// missing issues / timeline items / edits / label events ...
|
|
func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
|
|
creds, err := auth.List(repo,
|
|
auth.WithTarget(target),
|
|
auth.WithKind(auth.KindToken),
|
|
auth.WithMeta(auth.MetaKeyLogin, gi.conf[confKeyDefaultLogin]),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(creds) <= 0 {
|
|
return nil, ErrMissingIdentityToken
|
|
}
|
|
client := buildClient(creds[0].(*auth.Token))
|
|
gi.mediator = NewImportMediator(ctx, client, gi.conf[confKeyOwner], gi.conf[confKeyProject], since)
|
|
out := make(chan core.ImportResult)
|
|
gi.out = out
|
|
|
|
go func() {
|
|
defer close(gi.out)
|
|
var currBug *cache.BugCache
|
|
var currEvent ImportEvent
|
|
var nextEvent ImportEvent
|
|
var err error
|
|
for {
|
|
// An IssueEvent contains the issue in its most recent state. If an issue
|
|
// has at least one issue edit, then the history of the issue edits is
|
|
// represented by IssueEditEvents. That is, the unedited (original) issue
|
|
// might be saved only in the IssueEditEvent following the IssueEvent.
|
|
// Since we replicate the edit history we need to either use the IssueEvent
|
|
// (if there are no edits) or the IssueEvent together with its first
|
|
// IssueEditEvent (if there are edits).
|
|
// Exactly the same is true for comments and comment edits.
|
|
// As a consequence we need to look at the current event and one look ahead
|
|
// event.
|
|
currEvent = nextEvent
|
|
if currEvent == nil {
|
|
currEvent = gi.getEventHandleMsgs()
|
|
}
|
|
if currEvent == nil {
|
|
break
|
|
}
|
|
nextEvent = gi.getEventHandleMsgs()
|
|
|
|
switch event := currEvent.(type) {
|
|
case RateLimitingEvent:
|
|
out <- core.NewImportRateLimiting(event.msg)
|
|
case IssueEvent:
|
|
// first: commit what is being held in currBug
|
|
if err = gi.commit(currBug, out); err != nil {
|
|
out <- core.NewImportError(err, "")
|
|
return
|
|
}
|
|
// second: create new issue
|
|
switch next := nextEvent.(type) {
|
|
case IssueEditEvent:
|
|
// consuming and using next event
|
|
nextEvent = nil
|
|
currBug, err = gi.ensureIssue(ctx, repo, &event.issue, &next.userContentEdit)
|
|
default:
|
|
currBug, err = gi.ensureIssue(ctx, repo, &event.issue, nil)
|
|
}
|
|
if err != nil {
|
|
err := fmt.Errorf("issue creation: %v", err)
|
|
out <- core.NewImportError(err, "")
|
|
return
|
|
}
|
|
case IssueEditEvent:
|
|
err = gi.ensureIssueEdit(ctx, repo, currBug, event.issueId, &event.userContentEdit)
|
|
if err != nil {
|
|
err = fmt.Errorf("issue edit: %v", err)
|
|
out <- core.NewImportError(err, "")
|
|
return
|
|
}
|
|
case TimelineEvent:
|
|
if next, ok := nextEvent.(CommentEditEvent); ok && event.Typename == "IssueComment" {
|
|
// consuming and using next event
|
|
nextEvent = nil
|
|
err = gi.ensureComment(ctx, repo, currBug, &event.timelineItem.IssueComment, &next.userContentEdit)
|
|
} else {
|
|
err = gi.ensureTimelineItem(ctx, repo, currBug, &event.timelineItem)
|
|
}
|
|
if err != nil {
|
|
err = fmt.Errorf("timeline item creation: %v", err)
|
|
out <- core.NewImportError(err, "")
|
|
return
|
|
}
|
|
case CommentEditEvent:
|
|
err = gi.ensureCommentEdit(ctx, repo, currBug, event.commentId, &event.userContentEdit)
|
|
if err != nil {
|
|
err = fmt.Errorf("comment edit: %v", err)
|
|
out <- core.NewImportError(err, "")
|
|
return
|
|
}
|
|
default:
|
|
panic("Unknown event type")
|
|
}
|
|
}
|
|
// commit what is being held in currBug before returning
|
|
if err = gi.commit(currBug, out); err != nil {
|
|
out <- core.NewImportError(err, "")
|
|
}
|
|
if err = gi.mediator.Error(); err != nil {
|
|
gi.out <- core.NewImportError(err, "")
|
|
}
|
|
}()
|
|
|
|
return out, nil
|
|
}
|
|
|
|
func (gi *githubImporter) getEventHandleMsgs() ImportEvent {
|
|
for {
|
|
// read event from import mediator
|
|
event := gi.mediator.NextImportEvent()
|
|
// consume (and use) all rate limiting events
|
|
if e, ok := event.(RateLimitingEvent); ok {
|
|
gi.out <- core.NewImportRateLimiting(e.msg)
|
|
continue
|
|
}
|
|
return event
|
|
}
|
|
}
|
|
|
|
func (gi *githubImporter) commit(b *cache.BugCache, out chan<- core.ImportResult) error {
|
|
if b == nil {
|
|
return nil
|
|
}
|
|
if !b.NeedCommit() {
|
|
out <- core.NewImportNothing(b.Id(), "no imported operation")
|
|
return nil
|
|
} else if err := b.Commit(); err != nil {
|
|
// commit bug state
|
|
return fmt.Errorf("bug commit: %v", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (gi *githubImporter) ensureIssue(ctx context.Context, repo *cache.RepoCache, issue *issue, issueEdit *userContentEdit) (*cache.BugCache, error) {
|
|
author, err := gi.ensurePerson(ctx, repo, issue.Author)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// resolve bug
|
|
b, err := repo.ResolveBugMatcher(func(excerpt *cache.BugExcerpt) bool {
|
|
return excerpt.CreateMetadata[core.MetaKeyOrigin] == target &&
|
|
excerpt.CreateMetadata[metaKeyGithubId] == parseId(issue.Id)
|
|
})
|
|
if err == nil {
|
|
return b, nil
|
|
}
|
|
if err != bug.ErrBugNotExist {
|
|
return nil, err
|
|
}
|
|
|
|
// At Github there exist issues with seemingly empty titles. An example is
|
|
// https://github.com/NixOS/nixpkgs/issues/72730 .
|
|
// The title provided by the GraphQL API actually consists of a space followed by a
|
|
// zero width space (U+200B). This title would cause the NewBugRaw() function to
|
|
// return an error: empty title.
|
|
title := string(issue.Title)
|
|
if title == " \u200b" { // U+200B == zero width space
|
|
title = EMPTY_TITLE_PLACEHOLDER
|
|
}
|
|
|
|
var textInput string
|
|
if issueEdit != nil {
|
|
// use the first issue edit: it represents the bug creation itself
|
|
textInput = string(*issueEdit.Diff)
|
|
} else {
|
|
// if there are no issue edits then the issue struct holds the bug creation
|
|
textInput = string(issue.Body)
|
|
}
|
|
cleanText, err := text.Cleanup(textInput)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// create bug
|
|
b, _, err = repo.NewBugRaw(
|
|
author,
|
|
issue.CreatedAt.Unix(),
|
|
title, // TODO: this is the *current* title, not the original one
|
|
cleanText,
|
|
nil,
|
|
map[string]string{
|
|
core.MetaKeyOrigin: target,
|
|
metaKeyGithubId: parseId(issue.Id),
|
|
metaKeyGithubUrl: issue.Url.String(),
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// importing a new bug
|
|
gi.out <- core.NewImportBug(b.Id())
|
|
|
|
return b, nil
|
|
}
|
|
|
|
func (gi *githubImporter) ensureIssueEdit(ctx context.Context, repo *cache.RepoCache, bug *cache.BugCache, ghIssueId githubv4.ID, edit *userContentEdit) error {
|
|
return gi.ensureCommentEdit(ctx, repo, bug, ghIssueId, edit)
|
|
}
|
|
|
|
func (gi *githubImporter) ensureTimelineItem(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, item *timelineItem) error {
|
|
|
|
switch item.Typename {
|
|
case "IssueComment":
|
|
err := gi.ensureComment(ctx, repo, b, &item.IssueComment, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("timeline comment creation: %v", err)
|
|
}
|
|
return nil
|
|
|
|
case "LabeledEvent":
|
|
id := parseId(item.LabeledEvent.Id)
|
|
_, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
|
|
if err != cache.ErrNoMatchingOp {
|
|
return err
|
|
}
|
|
author, err := gi.ensurePerson(ctx, repo, item.LabeledEvent.Actor)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
op, err := b.ForceChangeLabelsRaw(
|
|
author,
|
|
item.LabeledEvent.CreatedAt.Unix(),
|
|
[]string{
|
|
string(item.LabeledEvent.Label.Name),
|
|
},
|
|
nil,
|
|
map[string]string{metaKeyGithubId: id},
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
gi.out <- core.NewImportLabelChange(op.Id())
|
|
return nil
|
|
|
|
case "UnlabeledEvent":
|
|
id := parseId(item.UnlabeledEvent.Id)
|
|
_, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
if err != cache.ErrNoMatchingOp {
|
|
return err
|
|
}
|
|
author, err := gi.ensurePerson(ctx, repo, item.UnlabeledEvent.Actor)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
op, err := b.ForceChangeLabelsRaw(
|
|
author,
|
|
item.UnlabeledEvent.CreatedAt.Unix(),
|
|
nil,
|
|
[]string{
|
|
string(item.UnlabeledEvent.Label.Name),
|
|
},
|
|
map[string]string{metaKeyGithubId: id},
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
gi.out <- core.NewImportLabelChange(op.Id())
|
|
return nil
|
|
|
|
case "ClosedEvent":
|
|
id := parseId(item.ClosedEvent.Id)
|
|
_, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
|
|
if err != cache.ErrNoMatchingOp {
|
|
return err
|
|
}
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
author, err := gi.ensurePerson(ctx, repo, item.ClosedEvent.Actor)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
op, err := b.CloseRaw(
|
|
author,
|
|
item.ClosedEvent.CreatedAt.Unix(),
|
|
map[string]string{metaKeyGithubId: id},
|
|
)
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
gi.out <- core.NewImportStatusChange(op.Id())
|
|
return nil
|
|
|
|
case "ReopenedEvent":
|
|
id := parseId(item.ReopenedEvent.Id)
|
|
_, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
|
|
if err != cache.ErrNoMatchingOp {
|
|
return err
|
|
}
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
author, err := gi.ensurePerson(ctx, repo, item.ReopenedEvent.Actor)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
op, err := b.OpenRaw(
|
|
author,
|
|
item.ReopenedEvent.CreatedAt.Unix(),
|
|
map[string]string{metaKeyGithubId: id},
|
|
)
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
gi.out <- core.NewImportStatusChange(op.Id())
|
|
return nil
|
|
|
|
case "RenamedTitleEvent":
|
|
id := parseId(item.RenamedTitleEvent.Id)
|
|
_, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
|
|
if err != cache.ErrNoMatchingOp {
|
|
return err
|
|
}
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
author, err := gi.ensurePerson(ctx, repo, item.RenamedTitleEvent.Actor)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// At Github there exist issues with seemingly empty titles. An example is
|
|
// https://github.com/NixOS/nixpkgs/issues/72730 .
|
|
// The title provided by the GraphQL API actually consists of a space followed
|
|
// by a zero width space (U+200B). This title would cause the NewBugRaw()
|
|
// function to return an error: empty title.
|
|
title := string(item.RenamedTitleEvent.CurrentTitle)
|
|
if title == " \u200b" { // U+200B == zero width space
|
|
title = EMPTY_TITLE_PLACEHOLDER
|
|
}
|
|
|
|
op, err := b.SetTitleRaw(
|
|
author,
|
|
item.RenamedTitleEvent.CreatedAt.Unix(),
|
|
title,
|
|
map[string]string{metaKeyGithubId: id},
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
gi.out <- core.NewImportTitleEdition(op.Id())
|
|
return nil
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (gi *githubImporter) ensureCommentEdit(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, ghTargetId githubv4.ID, edit *userContentEdit) error {
|
|
// find comment
|
|
target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(ghTargetId))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(edit.Id))
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
if err != cache.ErrNoMatchingOp {
|
|
// real error
|
|
return err
|
|
}
|
|
|
|
editor, err := gi.ensurePerson(ctx, repo, edit.Editor)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if edit.DeletedAt != nil {
|
|
// comment deletion, not supported yet
|
|
return nil
|
|
}
|
|
|
|
cleanText, err := text.Cleanup(string(*edit.Diff))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// comment edition
|
|
op, err := b.EditCommentRaw(
|
|
editor,
|
|
edit.CreatedAt.Unix(),
|
|
target,
|
|
cleanText,
|
|
map[string]string{
|
|
metaKeyGithubId: parseId(edit.Id),
|
|
},
|
|
)
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
gi.out <- core.NewImportCommentEdition(op.Id())
|
|
return nil
|
|
}
|
|
|
|
func (gi *githubImporter) ensureComment(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, comment *issueComment, firstEdit *userContentEdit) error {
|
|
author, err := gi.ensurePerson(ctx, repo, comment.Author)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
_, err = b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(comment.Id))
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
if err != cache.ErrNoMatchingOp {
|
|
// real error
|
|
return err
|
|
}
|
|
|
|
var textInput string
|
|
if firstEdit != nil {
|
|
// use the first comment edit: it represents the comment creation itself
|
|
textInput = string(*firstEdit.Diff)
|
|
} else {
|
|
// if there are not comment edits, then the comment struct holds the comment creation
|
|
textInput = string(comment.Body)
|
|
}
|
|
cleanText, err := text.Cleanup(textInput)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// add comment operation
|
|
op, err := b.AddCommentRaw(
|
|
author,
|
|
comment.CreatedAt.Unix(),
|
|
cleanText,
|
|
nil,
|
|
map[string]string{
|
|
metaKeyGithubId: parseId(comment.Id),
|
|
metaKeyGithubUrl: comment.Url.String(),
|
|
},
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
gi.out <- core.NewImportComment(op.Id())
|
|
return nil
|
|
}
|
|
|
|
// ensurePerson create a bug.Person from the Github data
|
|
func (gi *githubImporter) ensurePerson(ctx context.Context, repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
|
|
// When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
|
|
// in it's UI. So we need a special case to get it.
|
|
if actor == nil {
|
|
return gi.getGhost(ctx, repo)
|
|
}
|
|
|
|
// Look first in the cache
|
|
i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, string(actor.Login))
|
|
if err == nil {
|
|
return i, nil
|
|
}
|
|
if entity.IsErrMultipleMatch(err) {
|
|
return nil, err
|
|
}
|
|
|
|
// importing a new identity
|
|
var name string
|
|
var email string
|
|
|
|
switch actor.Typename {
|
|
case "User":
|
|
if actor.User.Name != nil {
|
|
name = string(*(actor.User.Name))
|
|
}
|
|
email = string(actor.User.Email)
|
|
case "Organization":
|
|
if actor.Organization.Name != nil {
|
|
name = string(*(actor.Organization.Name))
|
|
}
|
|
if actor.Organization.Email != nil {
|
|
email = string(*(actor.Organization.Email))
|
|
}
|
|
case "Bot":
|
|
}
|
|
|
|
// Name is not necessarily set, fallback to login as a name is required in the identity
|
|
if name == "" {
|
|
name = string(actor.Login)
|
|
}
|
|
|
|
i, err = repo.NewIdentityRaw(
|
|
name,
|
|
email,
|
|
string(actor.Login),
|
|
string(actor.AvatarUrl),
|
|
map[string]string{
|
|
metaKeyGithubLogin: string(actor.Login),
|
|
},
|
|
)
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
gi.out <- core.NewImportIdentity(i.Id())
|
|
return i, nil
|
|
}
|
|
|
|
func (gi *githubImporter) getGhost(ctx context.Context, repo *cache.RepoCache) (*cache.IdentityCache, error) {
|
|
loginName := "ghost"
|
|
// Look first in the cache
|
|
i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, loginName)
|
|
if err == nil {
|
|
return i, nil
|
|
}
|
|
if entity.IsErrMultipleMatch(err) {
|
|
return nil, err
|
|
}
|
|
user, err := gi.mediator.User(ctx, loginName)
|
|
userName := ""
|
|
if user.Name != nil {
|
|
userName = string(*user.Name)
|
|
}
|
|
return repo.NewIdentityRaw(
|
|
userName,
|
|
"",
|
|
string(user.Login),
|
|
string(user.AvatarUrl),
|
|
map[string]string{
|
|
metaKeyGithubLogin: string(user.Login),
|
|
},
|
|
)
|
|
}
|
|
|
|
// parseId converts the unusable githubv4.ID (an interface{}) into a string
|
|
func parseId(id githubv4.ID) string {
|
|
return fmt.Sprintf("%v", id)
|
|
}
|