mirror of
https://github.com/MichaelMure/git-bug.git
synced 2024-12-13 12:05:02 +03:00
gitlab: Add new iterator with state change events
Retrieving events is spread across various various Gitlab APIs. This makes importing and sorting Gitlab events by time quite complicated. This commit replaces the old iterators with a goroutine/channel-based iterator, which merges the individual Gitlab API streams into a single (sorted) event stream.
This commit is contained in:
Matthias Simon
parent
a8f3b55986
commit
aa4e225a80
184
bridge/gitlab/event.go
Normal file
184
bridge/gitlab/event.go
Normal file
@ -0,0 +1,184 @@
|
||||
package gitlab
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/MichaelMure/git-bug/util/text"
|
||||
"github.com/xanzy/go-gitlab"
|
||||
)
|
||||
|
||||
type EventKind int
|
||||
|
||||
const (
|
||||
EventUnknown EventKind = iota
|
||||
EventError
|
||||
EventComment
|
||||
EventTitleChanged
|
||||
EventDescriptionChanged
|
||||
EventClosed
|
||||
EventReopened
|
||||
EventLocked
|
||||
EventUnlocked
|
||||
EventChangedDuedate
|
||||
EventRemovedDuedate
|
||||
EventAssigned
|
||||
EventUnassigned
|
||||
EventChangedMilestone
|
||||
EventRemovedMilestone
|
||||
EventAddLabel
|
||||
EventRemoveLabel
|
||||
EventMentionedInIssue
|
||||
EventMentionedInMergeRequest
|
||||
)
|
||||
|
||||
type Event interface {
|
||||
ID() string
|
||||
UserID() int
|
||||
Kind() EventKind
|
||||
CreatedAt() time.Time
|
||||
}
|
||||
|
||||
type ErrorEvent struct {
|
||||
Err error
|
||||
Time time.Time
|
||||
}
|
||||
|
||||
func (e ErrorEvent) ID() string { return "" }
|
||||
func (e ErrorEvent) UserID() int { return -1 }
|
||||
func (e ErrorEvent) CreatedAt() time.Time { return e.Time }
|
||||
func (e ErrorEvent) Kind() EventKind { return EventError }
|
||||
|
||||
type NoteEvent struct{ gitlab.Note }
|
||||
|
||||
func (n NoteEvent) ID() string { return fmt.Sprintf("%d", n.Note.ID) }
|
||||
func (n NoteEvent) UserID() int { return n.Author.ID }
|
||||
func (n NoteEvent) CreatedAt() time.Time { return *n.Note.CreatedAt }
|
||||
func (n NoteEvent) Kind() EventKind {
|
||||
|
||||
switch {
|
||||
case !n.System:
|
||||
return EventComment
|
||||
|
||||
case n.Body == "closed":
|
||||
return EventClosed
|
||||
|
||||
case n.Body == "reopened":
|
||||
return EventReopened
|
||||
|
||||
case n.Body == "changed the description":
|
||||
return EventDescriptionChanged
|
||||
|
||||
case n.Body == "locked this issue":
|
||||
return EventLocked
|
||||
|
||||
case n.Body == "unlocked this issue":
|
||||
return EventUnlocked
|
||||
|
||||
case strings.HasPrefix(n.Body, "changed title from"):
|
||||
return EventTitleChanged
|
||||
|
||||
case strings.HasPrefix(n.Body, "changed due date to"):
|
||||
return EventChangedDuedate
|
||||
|
||||
case n.Body == "removed due date":
|
||||
return EventRemovedDuedate
|
||||
|
||||
case strings.HasPrefix(n.Body, "assigned to @"):
|
||||
return EventAssigned
|
||||
|
||||
case strings.HasPrefix(n.Body, "unassigned @"):
|
||||
return EventUnassigned
|
||||
|
||||
case strings.HasPrefix(n.Body, "changed milestone to %"):
|
||||
return EventChangedMilestone
|
||||
|
||||
case strings.HasPrefix(n.Body, "removed milestone"):
|
||||
return EventRemovedMilestone
|
||||
|
||||
case strings.HasPrefix(n.Body, "mentioned in issue"):
|
||||
return EventMentionedInIssue
|
||||
|
||||
case strings.HasPrefix(n.Body, "mentioned in merge request"):
|
||||
return EventMentionedInMergeRequest
|
||||
|
||||
default:
|
||||
return EventUnknown
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (n NoteEvent) Title() string {
|
||||
if n.Kind() == EventTitleChanged {
|
||||
return getNewTitle(n.Body)
|
||||
}
|
||||
return text.CleanupOneLine(n.Body)
|
||||
}
|
||||
|
||||
type LabelEvent struct{ gitlab.LabelEvent }
|
||||
|
||||
func (l LabelEvent) ID() string { return fmt.Sprintf("%d", l.LabelEvent.ID) }
|
||||
func (l LabelEvent) UserID() int { return l.User.ID }
|
||||
func (l LabelEvent) CreatedAt() time.Time { return *l.LabelEvent.CreatedAt }
|
||||
func (l LabelEvent) Kind() EventKind {
|
||||
switch l.Action {
|
||||
case "add":
|
||||
return EventAddLabel
|
||||
case "remove":
|
||||
return EventRemoveLabel
|
||||
default:
|
||||
return EventUnknown
|
||||
}
|
||||
}
|
||||
|
||||
type StateEvent struct{ gitlab.StateEvent }
|
||||
|
||||
func (s StateEvent) ID() string { return fmt.Sprintf("%d", s.StateEvent.ID) }
|
||||
func (s StateEvent) UserID() int { return s.User.ID }
|
||||
func (s StateEvent) CreatedAt() time.Time { return *s.StateEvent.CreatedAt }
|
||||
func (s StateEvent) Kind() EventKind {
|
||||
switch s.State {
|
||||
case "closed":
|
||||
return EventClosed
|
||||
case "opened", "reopened":
|
||||
return EventReopened
|
||||
default:
|
||||
return EventUnknown
|
||||
}
|
||||
}
|
||||
|
||||
func SortedEvents(c <-chan Event) []Event {
|
||||
var events []Event
|
||||
for e := range c {
|
||||
events = append(events, e)
|
||||
}
|
||||
sort.Sort(eventsByCreation(events))
|
||||
return events
|
||||
}
|
||||
|
||||
type eventsByCreation []Event
|
||||
|
||||
func (e eventsByCreation) Len() int {
|
||||
return len(e)
|
||||
}
|
||||
|
||||
func (e eventsByCreation) Less(i, j int) bool {
|
||||
return e[i].CreatedAt().Before(e[j].CreatedAt())
|
||||
}
|
||||
|
||||
func (e eventsByCreation) Swap(i, j int) {
|
||||
e[i], e[j] = e[j], e[i]
|
||||
}
|
||||
|
||||
// getNewTitle parses body diff given by gitlab api and return it final form
|
||||
// examples: "changed title from **fourth issue** to **fourth issue{+ changed+}**"
|
||||
// "changed title from **fourth issue{- changed-}** to **fourth issue**"
|
||||
// because Gitlab
|
||||
func getNewTitle(diff string) string {
|
||||
newTitle := strings.Split(diff, "** to **")[1]
|
||||
newTitle = strings.Replace(newTitle, "{+", "", -1)
|
||||
newTitle = strings.Replace(newTitle, "+}", "", -1)
|
||||
return strings.TrimSuffix(newTitle, "**")
|
||||
}
|
||||
171
bridge/gitlab/gitlab_api.go
Normal file
171
bridge/gitlab/gitlab_api.go
Normal file
@ -0,0 +1,171 @@
|
||||
package gitlab
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/MichaelMure/git-bug/util/text"
|
||||
"github.com/xanzy/go-gitlab"
|
||||
)
|
||||
|
||||
func Issues(ctx context.Context, client *gitlab.Client, pid string, since time.Time) <-chan *gitlab.Issue {
|
||||
|
||||
out := make(chan *gitlab.Issue)
|
||||
|
||||
go func() {
|
||||
defer close(out)
|
||||
|
||||
opts := gitlab.ListProjectIssuesOptions{
|
||||
UpdatedAfter: &since,
|
||||
Scope: gitlab.String("all"),
|
||||
Sort: gitlab.String("asc"),
|
||||
}
|
||||
|
||||
for {
|
||||
issues, resp, err := client.Issues.ListProjectIssues(pid, &opts)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, issue := range issues {
|
||||
out <- issue
|
||||
}
|
||||
|
||||
if resp.CurrentPage >= resp.TotalPages {
|
||||
break
|
||||
}
|
||||
|
||||
opts.Page = resp.NextPage
|
||||
}
|
||||
}()
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func IssueEvents(ctx context.Context, client *gitlab.Client, issue *gitlab.Issue) <-chan Event {
|
||||
cs := []<-chan Event{
|
||||
Notes(ctx, client, issue),
|
||||
LabelEvents(ctx, client, issue),
|
||||
StateEvents(ctx, client, issue),
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
out := make(chan Event)
|
||||
|
||||
output := func(c <-chan Event) {
|
||||
for n := range c {
|
||||
out <- n
|
||||
}
|
||||
wg.Done()
|
||||
}
|
||||
|
||||
wg.Add(len(cs))
|
||||
for _, c := range cs {
|
||||
go output(c)
|
||||
}
|
||||
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(out)
|
||||
}()
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func Notes(ctx context.Context, client *gitlab.Client, issue *gitlab.Issue) <-chan Event {
|
||||
|
||||
out := make(chan Event)
|
||||
|
||||
go func() {
|
||||
defer close(out)
|
||||
|
||||
opts := gitlab.ListIssueNotesOptions{
|
||||
OrderBy: gitlab.String("created_at"),
|
||||
Sort: gitlab.String("asc"),
|
||||
}
|
||||
|
||||
for {
|
||||
notes, resp, err := client.Notes.ListIssueNotes(issue.ProjectID, issue.IID, &opts)
|
||||
|
||||
if err != nil {
|
||||
out <- ErrorEvent{Err: err, Time: time.Now()}
|
||||
}
|
||||
|
||||
for _, note := range notes {
|
||||
out <- NoteEvent{*note}
|
||||
}
|
||||
|
||||
if resp.CurrentPage >= resp.TotalPages {
|
||||
break
|
||||
}
|
||||
|
||||
opts.Page = resp.NextPage
|
||||
}
|
||||
}()
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func LabelEvents(ctx context.Context, client *gitlab.Client, issue *gitlab.Issue) <-chan Event {
|
||||
|
||||
out := make(chan Event)
|
||||
|
||||
go func() {
|
||||
defer close(out)
|
||||
|
||||
opts := gitlab.ListLabelEventsOptions{}
|
||||
|
||||
for {
|
||||
events, resp, err := client.ResourceLabelEvents.ListIssueLabelEvents(issue.ProjectID, issue.IID, &opts)
|
||||
|
||||
if err != nil {
|
||||
out <- ErrorEvent{Err: err, Time: time.Now()}
|
||||
}
|
||||
|
||||
for _, e := range events {
|
||||
le := LabelEvent{*e}
|
||||
le.Label.Name = text.CleanupOneLine(le.Label.Name)
|
||||
out <- le
|
||||
}
|
||||
|
||||
if resp.CurrentPage >= resp.TotalPages {
|
||||
break
|
||||
}
|
||||
|
||||
opts.Page = resp.NextPage
|
||||
}
|
||||
}()
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func StateEvents(ctx context.Context, client *gitlab.Client, issue *gitlab.Issue) <-chan Event {
|
||||
|
||||
out := make(chan Event)
|
||||
|
||||
go func() {
|
||||
defer close(out)
|
||||
|
||||
opts := gitlab.ListStateEventsOptions{}
|
||||
|
||||
for {
|
||||
events, resp, err := client.ResourceStateEvents.ListIssueStateEvents(issue.ProjectID, issue.IID, &opts)
|
||||
if err != nil {
|
||||
out <- ErrorEvent{Err: err, Time: time.Now()}
|
||||
}
|
||||
|
||||
for _, e := range events {
|
||||
out <- StateEvent{*e}
|
||||
}
|
||||
|
||||
if resp.CurrentPage >= resp.TotalPages {
|
||||
break
|
||||
}
|
||||
|
||||
opts.Page = resp.NextPage
|
||||
}
|
||||
}()
|
||||
|
||||
return out
|
||||
}
|
||||
@ -10,7 +10,6 @@ import (
|
||||
|
||||
"github.com/MichaelMure/git-bug/bridge/core"
|
||||
"github.com/MichaelMure/git-bug/bridge/core/auth"
|
||||
"github.com/MichaelMure/git-bug/bridge/gitlab/iterator"
|
||||
"github.com/MichaelMure/git-bug/bug"
|
||||
"github.com/MichaelMure/git-bug/cache"
|
||||
"github.com/MichaelMure/git-bug/entity"
|
||||
@ -24,9 +23,6 @@ type gitlabImporter struct {
|
||||
// default client
|
||||
client *gitlab.Client
|
||||
|
||||
// iterator
|
||||
iterator *iterator.Iterator
|
||||
|
||||
// send only channel
|
||||
out chan<- core.ImportResult
|
||||
}
|
||||
@ -59,18 +55,15 @@ func (gi *gitlabImporter) Init(_ context.Context, repo *cache.RepoCache, conf co
|
||||
// ImportAll iterate over all the configured repository issues (notes) and ensure the creation
|
||||
// of the missing issues / comments / label events / title changes ...
|
||||
func (gi *gitlabImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
|
||||
gi.iterator = iterator.NewIterator(ctx, gi.client, 10, gi.conf[confKeyProjectID], since)
|
||||
|
||||
out := make(chan core.ImportResult)
|
||||
gi.out = out
|
||||
|
||||
go func() {
|
||||
defer close(gi.out)
|
||||
defer close(out)
|
||||
|
||||
// Loop over all matching issues
|
||||
for gi.iterator.NextIssue() {
|
||||
issue := gi.iterator.IssueValue()
|
||||
for issue := range Issues(ctx, gi.client, gi.conf[confKeyProjectID], since) {
|
||||
|
||||
// create issue
|
||||
b, err := gi.ensureIssue(repo, issue)
|
||||
if err != nil {
|
||||
err := fmt.Errorf("issue creation: %v", err)
|
||||
@ -78,23 +71,14 @@ func (gi *gitlabImporter) ImportAll(ctx context.Context, repo *cache.RepoCache,
|
||||
return
|
||||
}
|
||||
|
||||
// Loop over all notes
|
||||
for gi.iterator.NextNote() {
|
||||
note := gi.iterator.NoteValue()
|
||||
if err := gi.ensureNote(repo, b, note); err != nil {
|
||||
err := fmt.Errorf("note creation: %v", err)
|
||||
out <- core.NewImportError(err, entity.Id(strconv.Itoa(note.ID)))
|
||||
return
|
||||
for _, e := range SortedEvents(IssueEvents(ctx, gi.client, issue)) {
|
||||
if e, ok := e.(ErrorEvent); ok {
|
||||
out <- core.NewImportError(e.Err, "")
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Loop over all label events
|
||||
for gi.iterator.NextLabelEvent() {
|
||||
labelEvent := gi.iterator.LabelEventValue()
|
||||
if err := gi.ensureLabelEvent(repo, b, labelEvent); err != nil {
|
||||
err := fmt.Errorf("label event creation: %v", err)
|
||||
out <- core.NewImportError(err, entity.Id(strconv.Itoa(labelEvent.ID)))
|
||||
return
|
||||
if err := gi.ensureIssueEvent(repo, b, issue, e); err != nil {
|
||||
err := fmt.Errorf("issue event creation: %v", err)
|
||||
out <- core.NewImportError(err, entity.Id(e.ID()))
|
||||
}
|
||||
}
|
||||
|
||||
@ -107,10 +91,6 @@ func (gi *gitlabImporter) ImportAll(ctx context.Context, repo *cache.RepoCache,
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := gi.iterator.Error(); err != nil {
|
||||
out <- core.NewImportError(err, "")
|
||||
}
|
||||
}()
|
||||
|
||||
return out, nil
|
||||
@ -126,7 +106,7 @@ func (gi *gitlabImporter) ensureIssue(repo *cache.RepoCache, issue *gitlab.Issue
|
||||
// resolve bug
|
||||
b, err := repo.ResolveBugMatcher(func(excerpt *cache.BugExcerpt) bool {
|
||||
return excerpt.CreateMetadata[core.MetaKeyOrigin] == target &&
|
||||
excerpt.CreateMetadata[metaKeyGitlabId] == parseID(issue.IID) &&
|
||||
excerpt.CreateMetadata[metaKeyGitlabId] == fmt.Sprintf("%d", issue.IID) &&
|
||||
excerpt.CreateMetadata[metaKeyGitlabBaseUrl] == gi.conf[confKeyGitlabBaseUrl] &&
|
||||
excerpt.CreateMetadata[metaKeyGitlabProject] == gi.conf[confKeyProjectID]
|
||||
})
|
||||
@ -146,7 +126,7 @@ func (gi *gitlabImporter) ensureIssue(repo *cache.RepoCache, issue *gitlab.Issue
|
||||
nil,
|
||||
map[string]string{
|
||||
core.MetaKeyOrigin: target,
|
||||
metaKeyGitlabId: parseID(issue.IID),
|
||||
metaKeyGitlabId: fmt.Sprintf("%d", issue.IID),
|
||||
metaKeyGitlabUrl: issue.WebURL,
|
||||
metaKeyGitlabProject: gi.conf[confKeyProjectID],
|
||||
metaKeyGitlabBaseUrl: gi.conf[confKeyGitlabBaseUrl],
|
||||
@ -163,50 +143,49 @@ func (gi *gitlabImporter) ensureIssue(repo *cache.RepoCache, issue *gitlab.Issue
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, note *gitlab.Note) error {
|
||||
gitlabID := parseID(note.ID)
|
||||
func (gi *gitlabImporter) ensureIssueEvent(repo *cache.RepoCache, b *cache.BugCache, issue *gitlab.Issue, event Event) error {
|
||||
|
||||
id, errResolve := b.ResolveOperationWithMetadata(metaKeyGitlabId, gitlabID)
|
||||
id, errResolve := b.ResolveOperationWithMetadata(metaKeyGitlabId, event.ID())
|
||||
if errResolve != nil && errResolve != cache.ErrNoMatchingOp {
|
||||
return errResolve
|
||||
}
|
||||
|
||||
// ensure issue author
|
||||
author, err := gi.ensurePerson(repo, note.Author.ID)
|
||||
author, err := gi.ensurePerson(repo, event.UserID())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
noteType, body := GetNoteType(note)
|
||||
switch noteType {
|
||||
case NOTE_CLOSED:
|
||||
switch event.Kind() {
|
||||
case EventClosed:
|
||||
if errResolve == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
op, err := b.CloseRaw(
|
||||
author,
|
||||
note.CreatedAt.Unix(),
|
||||
event.CreatedAt().Unix(),
|
||||
map[string]string{
|
||||
metaKeyGitlabId: gitlabID,
|
||||
metaKeyGitlabId: event.ID(),
|
||||
},
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
gi.out <- core.NewImportStatusChange(op.Id())
|
||||
|
||||
case NOTE_REOPENED:
|
||||
case EventReopened:
|
||||
if errResolve == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
op, err := b.OpenRaw(
|
||||
author,
|
||||
note.CreatedAt.Unix(),
|
||||
event.CreatedAt().Unix(),
|
||||
map[string]string{
|
||||
metaKeyGitlabId: gitlabID,
|
||||
metaKeyGitlabId: event.ID(),
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
@ -215,9 +194,7 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n
|
||||
|
||||
gi.out <- core.NewImportStatusChange(op.Id())
|
||||
|
||||
case NOTE_DESCRIPTION_CHANGED:
|
||||
issue := gi.iterator.IssueValue()
|
||||
|
||||
case EventDescriptionChanged:
|
||||
firstComment := b.Snapshot().Comments[0]
|
||||
// since gitlab doesn't provide the issue history
|
||||
// we should check for "changed the description" notes and compare issue texts
|
||||
@ -226,11 +203,11 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n
|
||||
// comment edition
|
||||
op, err := b.EditCommentRaw(
|
||||
author,
|
||||
note.UpdatedAt.Unix(),
|
||||
event.(NoteEvent).UpdatedAt.Unix(),
|
||||
firstComment.Id(),
|
||||
text.Cleanup(issue.Description),
|
||||
map[string]string{
|
||||
metaKeyGitlabId: gitlabID,
|
||||
metaKeyGitlabId: event.ID(),
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
@ -240,8 +217,8 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n
|
||||
gi.out <- core.NewImportTitleEdition(op.Id())
|
||||
}
|
||||
|
||||
case NOTE_COMMENT:
|
||||
cleanText := text.Cleanup(body)
|
||||
case EventComment:
|
||||
cleanText := text.Cleanup(event.(NoteEvent).Body)
|
||||
|
||||
// if we didn't import the comment
|
||||
if errResolve == cache.ErrNoMatchingOp {
|
||||
@ -249,11 +226,11 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n
|
||||
// add comment operation
|
||||
op, err := b.AddCommentRaw(
|
||||
author,
|
||||
note.CreatedAt.Unix(),
|
||||
event.CreatedAt().Unix(),
|
||||
cleanText,
|
||||
nil,
|
||||
map[string]string{
|
||||
metaKeyGitlabId: gitlabID,
|
||||
metaKeyGitlabId: event.ID(),
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
@ -271,12 +248,12 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n
|
||||
return err
|
||||
}
|
||||
|
||||
// compare local bug comment with the new note body
|
||||
// compare local bug comment with the new event body
|
||||
if comment.Message != cleanText {
|
||||
// comment edition
|
||||
op, err := b.EditCommentRaw(
|
||||
author,
|
||||
note.UpdatedAt.Unix(),
|
||||
event.(NoteEvent).UpdatedAt.Unix(),
|
||||
comment.Id(),
|
||||
cleanText,
|
||||
nil,
|
||||
@ -290,7 +267,7 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n
|
||||
|
||||
return nil
|
||||
|
||||
case NOTE_TITLE_CHANGED:
|
||||
case EventTitleChanged:
|
||||
// title change events are given new notes
|
||||
if errResolve == nil {
|
||||
return nil
|
||||
@ -298,10 +275,10 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n
|
||||
|
||||
op, err := b.SetTitleRaw(
|
||||
author,
|
||||
note.CreatedAt.Unix(),
|
||||
text.CleanupOneLine(body),
|
||||
event.CreatedAt().Unix(),
|
||||
event.(NoteEvent).Title(),
|
||||
map[string]string{
|
||||
metaKeyGitlabId: gitlabID,
|
||||
metaKeyGitlabId: event.ID(),
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
@ -310,69 +287,50 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n
|
||||
|
||||
gi.out <- core.NewImportTitleEdition(op.Id())
|
||||
|
||||
case NOTE_UNKNOWN,
|
||||
NOTE_ASSIGNED,
|
||||
NOTE_UNASSIGNED,
|
||||
NOTE_CHANGED_MILESTONE,
|
||||
NOTE_REMOVED_MILESTONE,
|
||||
NOTE_CHANGED_DUEDATE,
|
||||
NOTE_REMOVED_DUEDATE,
|
||||
NOTE_LOCKED,
|
||||
NOTE_UNLOCKED,
|
||||
NOTE_MENTIONED_IN_ISSUE,
|
||||
NOTE_MENTIONED_IN_MERGE_REQUEST:
|
||||
case EventAddLabel:
|
||||
_, err = b.ForceChangeLabelsRaw(
|
||||
author,
|
||||
event.CreatedAt().Unix(),
|
||||
[]string{event.(LabelEvent).Label.Name},
|
||||
nil,
|
||||
map[string]string{
|
||||
metaKeyGitlabId: event.ID(),
|
||||
},
|
||||
)
|
||||
return err
|
||||
|
||||
case EventRemoveLabel:
|
||||
_, err = b.ForceChangeLabelsRaw(
|
||||
author,
|
||||
event.CreatedAt().Unix(),
|
||||
nil,
|
||||
[]string{event.(LabelEvent).Label.Name},
|
||||
map[string]string{
|
||||
metaKeyGitlabId: event.ID(),
|
||||
},
|
||||
)
|
||||
return err
|
||||
|
||||
case EventAssigned,
|
||||
EventUnassigned,
|
||||
EventChangedMilestone,
|
||||
EventRemovedMilestone,
|
||||
EventChangedDuedate,
|
||||
EventRemovedDuedate,
|
||||
EventLocked,
|
||||
EventUnlocked,
|
||||
EventMentionedInIssue,
|
||||
EventMentionedInMergeRequest:
|
||||
|
||||
return nil
|
||||
|
||||
default:
|
||||
panic("unhandled note type")
|
||||
return fmt.Errorf("unexpected event")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (gi *gitlabImporter) ensureLabelEvent(repo *cache.RepoCache, b *cache.BugCache, labelEvent *gitlab.LabelEvent) error {
|
||||
_, err := b.ResolveOperationWithMetadata(metaKeyGitlabId, parseID(labelEvent.ID))
|
||||
if err != cache.ErrNoMatchingOp {
|
||||
return err
|
||||
}
|
||||
|
||||
// ensure issue author
|
||||
author, err := gi.ensurePerson(repo, labelEvent.User.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
switch labelEvent.Action {
|
||||
case "add":
|
||||
_, err = b.ForceChangeLabelsRaw(
|
||||
author,
|
||||
labelEvent.CreatedAt.Unix(),
|
||||
[]string{text.CleanupOneLine(labelEvent.Label.Name)},
|
||||
nil,
|
||||
map[string]string{
|
||||
metaKeyGitlabId: parseID(labelEvent.ID),
|
||||
},
|
||||
)
|
||||
|
||||
case "remove":
|
||||
_, err = b.ForceChangeLabelsRaw(
|
||||
author,
|
||||
labelEvent.CreatedAt.Unix(),
|
||||
nil,
|
||||
[]string{text.CleanupOneLine(labelEvent.Label.Name)},
|
||||
map[string]string{
|
||||
metaKeyGitlabId: parseID(labelEvent.ID),
|
||||
},
|
||||
)
|
||||
|
||||
default:
|
||||
err = fmt.Errorf("unexpected label event action")
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (gi *gitlabImporter) ensurePerson(repo *cache.RepoCache, id int) (*cache.IdentityCache, error) {
|
||||
// Look first in the cache
|
||||
i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGitlabId, strconv.Itoa(id))
|
||||
@ -407,7 +365,3 @@ func (gi *gitlabImporter) ensurePerson(repo *cache.RepoCache, id int) (*cache.Id
|
||||
gi.out <- core.NewImportIdentity(i.Id())
|
||||
return i, nil
|
||||
}
|
||||
|
||||
func parseID(id int) string {
|
||||
return fmt.Sprintf("%d", id)
|
||||
}
|
||||
|
||||
@ -1,147 +0,0 @@
|
||||
package gitlab
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/xanzy/go-gitlab"
|
||||
)
|
||||
|
||||
type NoteType int
|
||||
|
||||
const (
|
||||
_ NoteType = iota
|
||||
NOTE_COMMENT
|
||||
NOTE_TITLE_CHANGED
|
||||
NOTE_DESCRIPTION_CHANGED
|
||||
NOTE_CLOSED
|
||||
NOTE_REOPENED
|
||||
NOTE_LOCKED
|
||||
NOTE_UNLOCKED
|
||||
NOTE_CHANGED_DUEDATE
|
||||
NOTE_REMOVED_DUEDATE
|
||||
NOTE_ASSIGNED
|
||||
NOTE_UNASSIGNED
|
||||
NOTE_CHANGED_MILESTONE
|
||||
NOTE_REMOVED_MILESTONE
|
||||
NOTE_MENTIONED_IN_ISSUE
|
||||
NOTE_MENTIONED_IN_MERGE_REQUEST
|
||||
NOTE_UNKNOWN
|
||||
)
|
||||
|
||||
func (nt NoteType) String() string {
|
||||
switch nt {
|
||||
case NOTE_COMMENT:
|
||||
return "note comment"
|
||||
case NOTE_TITLE_CHANGED:
|
||||
return "note title changed"
|
||||
case NOTE_DESCRIPTION_CHANGED:
|
||||
return "note description changed"
|
||||
case NOTE_CLOSED:
|
||||
return "note closed"
|
||||
case NOTE_REOPENED:
|
||||
return "note reopened"
|
||||
case NOTE_LOCKED:
|
||||
return "note locked"
|
||||
case NOTE_UNLOCKED:
|
||||
return "note unlocked"
|
||||
case NOTE_CHANGED_DUEDATE:
|
||||
return "note changed duedate"
|
||||
case NOTE_REMOVED_DUEDATE:
|
||||
return "note remove duedate"
|
||||
case NOTE_ASSIGNED:
|
||||
return "note assigned"
|
||||
case NOTE_UNASSIGNED:
|
||||
return "note unassigned"
|
||||
case NOTE_CHANGED_MILESTONE:
|
||||
return "note changed milestone"
|
||||
case NOTE_REMOVED_MILESTONE:
|
||||
return "note removed in milestone"
|
||||
case NOTE_MENTIONED_IN_ISSUE:
|
||||
return "note mentioned in issue"
|
||||
case NOTE_MENTIONED_IN_MERGE_REQUEST:
|
||||
return "note mentioned in merge request"
|
||||
case NOTE_UNKNOWN:
|
||||
return "note unknown"
|
||||
default:
|
||||
panic("unknown note type")
|
||||
}
|
||||
}
|
||||
|
||||
// GetNoteType parse a note system and body and return the note type and it content
|
||||
func GetNoteType(n *gitlab.Note) (NoteType, string) {
|
||||
// when a note is a comment system is set to false
|
||||
// when a note is a different event system is set to true
|
||||
// because Gitlab
|
||||
if !n.System {
|
||||
return NOTE_COMMENT, n.Body
|
||||
}
|
||||
|
||||
if n.Body == "closed" {
|
||||
return NOTE_CLOSED, ""
|
||||
}
|
||||
|
||||
if n.Body == "reopened" {
|
||||
return NOTE_REOPENED, ""
|
||||
}
|
||||
|
||||
if n.Body == "changed the description" {
|
||||
return NOTE_DESCRIPTION_CHANGED, ""
|
||||
}
|
||||
|
||||
if n.Body == "locked this issue" {
|
||||
return NOTE_LOCKED, ""
|
||||
}
|
||||
|
||||
if n.Body == "unlocked this issue" {
|
||||
return NOTE_UNLOCKED, ""
|
||||
}
|
||||
|
||||
if strings.HasPrefix(n.Body, "changed title from") {
|
||||
return NOTE_TITLE_CHANGED, getNewTitle(n.Body)
|
||||
}
|
||||
|
||||
if strings.HasPrefix(n.Body, "changed due date to") {
|
||||
return NOTE_CHANGED_DUEDATE, ""
|
||||
}
|
||||
|
||||
if n.Body == "removed due date" {
|
||||
return NOTE_REMOVED_DUEDATE, ""
|
||||
}
|
||||
|
||||
if strings.HasPrefix(n.Body, "assigned to @") {
|
||||
return NOTE_ASSIGNED, ""
|
||||
}
|
||||
|
||||
if strings.HasPrefix(n.Body, "unassigned @") {
|
||||
return NOTE_UNASSIGNED, ""
|
||||
}
|
||||
|
||||
if strings.HasPrefix(n.Body, "changed milestone to %") {
|
||||
return NOTE_CHANGED_MILESTONE, ""
|
||||
}
|
||||
|
||||
if strings.HasPrefix(n.Body, "removed milestone") {
|
||||
return NOTE_REMOVED_MILESTONE, ""
|
||||
}
|
||||
|
||||
if strings.HasPrefix(n.Body, "mentioned in issue") {
|
||||
return NOTE_MENTIONED_IN_ISSUE, ""
|
||||
}
|
||||
|
||||
if strings.HasPrefix(n.Body, "mentioned in merge request") {
|
||||
return NOTE_MENTIONED_IN_MERGE_REQUEST, ""
|
||||
}
|
||||
|
||||
return NOTE_UNKNOWN, ""
|
||||
}
|
||||
|
||||
// getNewTitle parses body diff given by gitlab api and return it final form
|
||||
// examples: "changed title from **fourth issue** to **fourth issue{+ changed+}**"
|
||||
// "changed title from **fourth issue{- changed-}** to **fourth issue**"
|
||||
// because Gitlab
|
||||
func getNewTitle(diff string) string {
|
||||
newTitle := strings.Split(diff, "** to **")[1]
|
||||
newTitle = strings.Replace(newTitle, "{+", "", -1)
|
||||
newTitle = strings.Replace(newTitle, "+}", "", -1)
|
||||
return strings.TrimSuffix(newTitle, "**")
|
||||
}
|
||||
@ -1,89 +0,0 @@
|
||||
package iterator
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/xanzy/go-gitlab"
|
||||
)
|
||||
|
||||
type issueIterator struct {
|
||||
page int
|
||||
lastPage bool
|
||||
index int
|
||||
cache []*gitlab.Issue
|
||||
}
|
||||
|
||||
func newIssueIterator() *issueIterator {
|
||||
ii := &issueIterator{}
|
||||
ii.Reset()
|
||||
return ii
|
||||
}
|
||||
|
||||
func (ii *issueIterator) Next(ctx context.Context, conf config) (bool, error) {
|
||||
// first query
|
||||
if ii.cache == nil {
|
||||
return ii.getNext(ctx, conf)
|
||||
}
|
||||
|
||||
// move cursor index
|
||||
if ii.index < len(ii.cache)-1 {
|
||||
ii.index++
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return ii.getNext(ctx, conf)
|
||||
}
|
||||
|
||||
func (ii *issueIterator) Value() *gitlab.Issue {
|
||||
return ii.cache[ii.index]
|
||||
}
|
||||
|
||||
func (ii *issueIterator) getNext(ctx context.Context, conf config) (bool, error) {
|
||||
if ii.lastPage {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, conf.timeout)
|
||||
defer cancel()
|
||||
|
||||
issues, resp, err := conf.gc.Issues.ListProjectIssues(
|
||||
conf.project,
|
||||
&gitlab.ListProjectIssuesOptions{
|
||||
ListOptions: gitlab.ListOptions{
|
||||
Page: ii.page,
|
||||
PerPage: conf.capacity,
|
||||
},
|
||||
Scope: gitlab.String("all"),
|
||||
UpdatedAfter: &conf.since,
|
||||
Sort: gitlab.String("asc"),
|
||||
},
|
||||
gitlab.WithContext(ctx),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
ii.Reset()
|
||||
return false, err
|
||||
}
|
||||
|
||||
if resp.TotalPages == ii.page {
|
||||
ii.lastPage = true
|
||||
}
|
||||
|
||||
// if repository doesn't have any issues
|
||||
if len(issues) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
ii.cache = issues
|
||||
ii.index = 0
|
||||
ii.page++
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (ii *issueIterator) Reset() {
|
||||
ii.index = -1
|
||||
ii.page = 1
|
||||
ii.lastPage = false
|
||||
ii.cache = nil
|
||||
}
|
||||
@ -1,138 +0,0 @@
|
||||
package iterator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/xanzy/go-gitlab"
|
||||
)
|
||||
|
||||
type Iterator struct {
|
||||
// shared context
|
||||
ctx context.Context
|
||||
|
||||
// to pass to sub-iterators
|
||||
conf config
|
||||
|
||||
// sticky error
|
||||
err error
|
||||
|
||||
// issues iterator
|
||||
issue *issueIterator
|
||||
|
||||
// notes iterator
|
||||
note *noteIterator
|
||||
|
||||
// labelEvent iterator
|
||||
labelEvent *labelEventIterator
|
||||
}
|
||||
|
||||
type config struct {
|
||||
// gitlab api v4 client
|
||||
gc *gitlab.Client
|
||||
|
||||
timeout time.Duration
|
||||
|
||||
// if since is given the iterator will query only the issues
|
||||
// updated after this date
|
||||
since time.Time
|
||||
|
||||
// project id
|
||||
project string
|
||||
|
||||
// number of issues and notes to query at once
|
||||
capacity int
|
||||
}
|
||||
|
||||
// NewIterator create a new iterator
|
||||
func NewIterator(ctx context.Context, client *gitlab.Client, capacity int, projectID string, since time.Time) *Iterator {
|
||||
return &Iterator{
|
||||
ctx: ctx,
|
||||
conf: config{
|
||||
gc: client,
|
||||
timeout: 60 * time.Second,
|
||||
since: since,
|
||||
project: projectID,
|
||||
capacity: capacity,
|
||||
},
|
||||
issue: newIssueIterator(),
|
||||
note: newNoteIterator(),
|
||||
labelEvent: newLabelEventIterator(),
|
||||
}
|
||||
}
|
||||
|
||||
// Error return last encountered error
|
||||
func (i *Iterator) Error() error {
|
||||
return i.err
|
||||
}
|
||||
|
||||
func (i *Iterator) NextIssue() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if i.ctx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
more, err := i.issue.Next(i.ctx, i.conf)
|
||||
if err != nil {
|
||||
i.err = err
|
||||
return false
|
||||
}
|
||||
|
||||
// Also reset the other sub iterators as they would
|
||||
// no longer be valid
|
||||
i.note.Reset(i.issue.Value().IID)
|
||||
i.labelEvent.Reset(i.issue.Value().IID)
|
||||
|
||||
return more
|
||||
}
|
||||
|
||||
func (i *Iterator) IssueValue() *gitlab.Issue {
|
||||
return i.issue.Value()
|
||||
}
|
||||
|
||||
func (i *Iterator) NextNote() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if i.ctx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
more, err := i.note.Next(i.ctx, i.conf)
|
||||
if err != nil {
|
||||
i.err = err
|
||||
return false
|
||||
}
|
||||
|
||||
return more
|
||||
}
|
||||
|
||||
func (i *Iterator) NoteValue() *gitlab.Note {
|
||||
return i.note.Value()
|
||||
}
|
||||
|
||||
func (i *Iterator) NextLabelEvent() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if i.ctx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
more, err := i.labelEvent.Next(i.ctx, i.conf)
|
||||
if err != nil {
|
||||
i.err = err
|
||||
return false
|
||||
}
|
||||
|
||||
return more
|
||||
}
|
||||
|
||||
func (i *Iterator) LabelEventValue() *gitlab.LabelEvent {
|
||||
return i.labelEvent.Value()
|
||||
}
|
||||
@ -1,105 +0,0 @@
|
||||
package iterator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
|
||||
"github.com/xanzy/go-gitlab"
|
||||
)
|
||||
|
||||
// Since Gitlab does not return the label events items in the correct order
|
||||
// we need to sort the list ourselves and stop relying on the pagination model
|
||||
// #BecauseGitlab
|
||||
type labelEventIterator struct {
|
||||
issue int
|
||||
index int
|
||||
cache []*gitlab.LabelEvent
|
||||
}
|
||||
|
||||
func newLabelEventIterator() *labelEventIterator {
|
||||
lei := &labelEventIterator{}
|
||||
lei.Reset(-1)
|
||||
return lei
|
||||
}
|
||||
|
||||
func (lei *labelEventIterator) Next(ctx context.Context, conf config) (bool, error) {
|
||||
// first query
|
||||
if lei.cache == nil {
|
||||
return lei.getNext(ctx, conf)
|
||||
}
|
||||
|
||||
// move cursor index
|
||||
if lei.index < len(lei.cache)-1 {
|
||||
lei.index++
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (lei *labelEventIterator) Value() *gitlab.LabelEvent {
|
||||
return lei.cache[lei.index]
|
||||
}
|
||||
|
||||
func (lei *labelEventIterator) getNext(ctx context.Context, conf config) (bool, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, conf.timeout)
|
||||
defer cancel()
|
||||
|
||||
// since order is not guaranteed we should query all label events
|
||||
// and sort them by ID
|
||||
page := 1
|
||||
for {
|
||||
labelEvents, resp, err := conf.gc.ResourceLabelEvents.ListIssueLabelEvents(
|
||||
conf.project,
|
||||
lei.issue,
|
||||
&gitlab.ListLabelEventsOptions{
|
||||
ListOptions: gitlab.ListOptions{
|
||||
Page: page,
|
||||
PerPage: conf.capacity,
|
||||
},
|
||||
},
|
||||
gitlab.WithContext(ctx),
|
||||
)
|
||||
if err != nil {
|
||||
lei.Reset(-1)
|
||||
return false, err
|
||||
}
|
||||
|
||||
if len(labelEvents) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
lei.cache = append(lei.cache, labelEvents...)
|
||||
|
||||
if resp.TotalPages == page {
|
||||
break
|
||||
}
|
||||
|
||||
page++
|
||||
}
|
||||
|
||||
sort.Sort(lei)
|
||||
lei.index = 0
|
||||
|
||||
return len(lei.cache) > 0, nil
|
||||
}
|
||||
|
||||
func (lei *labelEventIterator) Reset(issue int) {
|
||||
lei.issue = issue
|
||||
lei.index = -1
|
||||
lei.cache = nil
|
||||
}
|
||||
|
||||
// ORDERING
|
||||
|
||||
func (lei *labelEventIterator) Len() int {
|
||||
return len(lei.cache)
|
||||
}
|
||||
|
||||
func (lei *labelEventIterator) Swap(i, j int) {
|
||||
lei.cache[i], lei.cache[j] = lei.cache[j], lei.cache[i]
|
||||
}
|
||||
|
||||
func (lei *labelEventIterator) Less(i, j int) bool {
|
||||
return lei.cache[i].ID < lei.cache[j].ID
|
||||
}
|
||||
@ -1,90 +0,0 @@
|
||||
package iterator
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/xanzy/go-gitlab"
|
||||
)
|
||||
|
||||
type noteIterator struct {
|
||||
issue int
|
||||
page int
|
||||
lastPage bool
|
||||
index int
|
||||
cache []*gitlab.Note
|
||||
}
|
||||
|
||||
func newNoteIterator() *noteIterator {
|
||||
in := ¬eIterator{}
|
||||
in.Reset(-1)
|
||||
return in
|
||||
}
|
||||
|
||||
func (in *noteIterator) Next(ctx context.Context, conf config) (bool, error) {
|
||||
// first query
|
||||
if in.cache == nil {
|
||||
return in.getNext(ctx, conf)
|
||||
}
|
||||
|
||||
// move cursor index
|
||||
if in.index < len(in.cache)-1 {
|
||||
in.index++
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return in.getNext(ctx, conf)
|
||||
}
|
||||
|
||||
func (in *noteIterator) Value() *gitlab.Note {
|
||||
return in.cache[in.index]
|
||||
}
|
||||
|
||||
func (in *noteIterator) getNext(ctx context.Context, conf config) (bool, error) {
|
||||
if in.lastPage {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, conf.timeout)
|
||||
defer cancel()
|
||||
|
||||
notes, resp, err := conf.gc.Notes.ListIssueNotes(
|
||||
conf.project,
|
||||
in.issue,
|
||||
&gitlab.ListIssueNotesOptions{
|
||||
ListOptions: gitlab.ListOptions{
|
||||
Page: in.page,
|
||||
PerPage: conf.capacity,
|
||||
},
|
||||
Sort: gitlab.String("asc"),
|
||||
OrderBy: gitlab.String("created_at"),
|
||||
},
|
||||
gitlab.WithContext(ctx),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
in.Reset(-1)
|
||||
return false, err
|
||||
}
|
||||
|
||||
if resp.TotalPages == in.page {
|
||||
in.lastPage = true
|
||||
}
|
||||
|
||||
if len(notes) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
in.cache = notes
|
||||
in.index = 0
|
||||
in.page++
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (in *noteIterator) Reset(issue int) {
|
||||
in.issue = issue
|
||||
in.index = -1
|
||||
in.page = 1
|
||||
in.lastPage = false
|
||||
in.cache = nil
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user