Merge pull request #571 from 5nord/add-new-gitlab-iterator

[gitlab] Add new iterator with state change events

bridge/gitlab/event.go

@@ -0,0 +1,215 @@
+package gitlab
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/MichaelMure/git-bug/util/text"
+	"github.com/xanzy/go-gitlab"
+)
+
+// Event represents a unified GitLab event (note, label or state event).
+type Event interface {
+	ID() string
+	UserID() int
+	Kind() EventKind
+	CreatedAt() time.Time
+}
+
+type EventKind int
+
+const (
+	EventUnknown EventKind = iota
+	EventError
+	EventComment
+	EventTitleChanged
+	EventDescriptionChanged
+	EventClosed
+	EventReopened
+	EventLocked
+	EventUnlocked
+	EventChangedDuedate
+	EventRemovedDuedate
+	EventAssigned
+	EventUnassigned
+	EventChangedMilestone
+	EventRemovedMilestone
+	EventAddLabel
+	EventRemoveLabel
+	EventMentionedInIssue
+	EventMentionedInMergeRequest
+)
+
+var _ Event = &NoteEvent{}
+
+type NoteEvent struct{ gitlab.Note }
+
+func (n NoteEvent) ID() string           { return fmt.Sprintf("%d", n.Note.ID) }
+func (n NoteEvent) UserID() int          { return n.Author.ID }
+func (n NoteEvent) CreatedAt() time.Time { return *n.Note.CreatedAt }
+func (n NoteEvent) Kind() EventKind {
+
+	switch {
+	case !n.System:
+		return EventComment
+
+	case n.Body == "closed":
+		return EventClosed
+
+	case n.Body == "reopened":
+		return EventReopened
+
+	case n.Body == "changed the description":
+		return EventDescriptionChanged
+
+	case n.Body == "locked this issue":
+		return EventLocked
+
+	case n.Body == "unlocked this issue":
+		return EventUnlocked
+
+	case strings.HasPrefix(n.Body, "changed title from"):
+		return EventTitleChanged
+
+	case strings.HasPrefix(n.Body, "changed due date to"):
+		return EventChangedDuedate
+
+	case n.Body == "removed due date":
+		return EventRemovedDuedate
+
+	case strings.HasPrefix(n.Body, "assigned to @"):
+		return EventAssigned
+
+	case strings.HasPrefix(n.Body, "unassigned @"):
+		return EventUnassigned
+
+	case strings.HasPrefix(n.Body, "changed milestone to %"):
+		return EventChangedMilestone
+
+	case strings.HasPrefix(n.Body, "removed milestone"):
+		return EventRemovedMilestone
+
+	case strings.HasPrefix(n.Body, "mentioned in issue"):
+		return EventMentionedInIssue
+
+	case strings.HasPrefix(n.Body, "mentioned in merge request"):
+		return EventMentionedInMergeRequest
+
+	default:
+		return EventUnknown
+	}
+
+}
+
+func (n NoteEvent) Title() string {
+	if n.Kind() == EventTitleChanged {
+		return getNewTitle(n.Body)
+	}
+	return text.CleanupOneLine(n.Body)
+}
+
+var _ Event = &LabelEvent{}
+
+type LabelEvent struct{ gitlab.LabelEvent }
+
+func (l LabelEvent) ID() string           { return fmt.Sprintf("%d", l.LabelEvent.ID) }
+func (l LabelEvent) UserID() int          { return l.User.ID }
+func (l LabelEvent) CreatedAt() time.Time { return *l.LabelEvent.CreatedAt }
+func (l LabelEvent) Kind() EventKind {
+	switch l.Action {
+	case "add":
+		return EventAddLabel
+	case "remove":
+		return EventRemoveLabel
+	default:
+		return EventUnknown
+	}
+}
+
+var _ Event = &StateEvent{}
+
+type StateEvent struct{ gitlab.StateEvent }
+
+func (s StateEvent) ID() string           { return fmt.Sprintf("%d", s.StateEvent.ID) }
+func (s StateEvent) UserID() int          { return s.User.ID }
+func (s StateEvent) CreatedAt() time.Time { return *s.StateEvent.CreatedAt }
+func (s StateEvent) Kind() EventKind {
+	switch s.State {
+	case "closed":
+		return EventClosed
+	case "opened", "reopened":
+		return EventReopened
+	default:
+		return EventUnknown
+	}
+}
+
+var _ Event = &ErrorEvent{}
+
+type ErrorEvent struct {
+	Err  error
+	Time time.Time
+}
+
+func (e ErrorEvent) ID() string           { return "" }
+func (e ErrorEvent) UserID() int          { return -1 }
+func (e ErrorEvent) CreatedAt() time.Time { return e.Time }
+func (e ErrorEvent) Kind() EventKind      { return EventError }
+
+// SortedEvents fan-in some Event-channels into one, sorted by creation date, using CreatedAt-method.
+// This function assume that each channel is pre-ordered.
+func SortedEvents(inputs ...<-chan Event) chan Event {
+	out := make(chan Event)
+
+	go func() {
+		defer close(out)
+
+		heads := make([]Event, len(inputs))
+
+		// pre-fill the head view
+		for i, input := range inputs {
+			if event, ok := <-input; ok {
+				heads[i] = event
+			}
+		}
+
+		for {
+			var earliestEvent Event
+			var originChannel int
+
+			// pick the earliest event of the heads
+			for i, head := range heads {
+				if head != nil && (earliestEvent == nil || head.CreatedAt().Before(earliestEvent.CreatedAt())) {
+					earliestEvent = head
+					originChannel = i
+				}
+			}
+
+			if earliestEvent == nil {
+				// no event anymore, we are done
+				return
+			}
+
+			// we have an event: consume it and replace it if possible
+			heads[originChannel] = nil
+			if event, ok := <-inputs[originChannel]; ok {
+				heads[originChannel] = event
+			}
+			out <- earliestEvent
+		}
+	}()
+
+	return out
+}
+
+// getNewTitle parses body diff given by gitlab api and return it final form
+// examples: "changed title from **fourth issue** to **fourth issue{+ changed+}**"
+//           "changed title from **fourth issue{- changed-}** to **fourth issue**"
+// because Gitlab
+func getNewTitle(diff string) string {
+	newTitle := strings.Split(diff, "** to **")[1]
+	newTitle = strings.Replace(newTitle, "{+", "", -1)
+	newTitle = strings.Replace(newTitle, "+}", "", -1)
+	return strings.TrimSuffix(newTitle, "**")
+}

bridge/gitlab/event_test.go

@@ -2,8 +2,10 @@ package gitlab
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestGetNewTitle(t *testing.T) {
@@ -54,3 +56,40 @@ func TestGetNewTitle(t *testing.T) {
 		})
 	}
 }
+
+var _ Event = mockEvent(0)
+
+type mockEvent int64
+
+func (m mockEvent) ID() string           { panic("implement me") }
+func (m mockEvent) UserID() int          { panic("implement me") }
+func (m mockEvent) Kind() EventKind      { panic("implement me") }
+func (m mockEvent) CreatedAt() time.Time { return time.Unix(int64(m), 0) }
+
+func TestSortedEvents(t *testing.T) {
+	makeInput := func(times ...int64) chan Event {
+		out := make(chan Event)
+		go func() {
+			for _, t := range times {
+				out <- mockEvent(t)
+			}
+			close(out)
+		}()
+		return out
+	}
+
+	sorted := SortedEvents(
+		makeInput(),
+		makeInput(1, 7, 9, 19),
+		makeInput(2, 8, 23),
+		makeInput(35, 48, 59, 64, 721),
+	)
+
+	var previous Event
+	for event := range sorted {
+		if previous != nil {
+			require.True(t, previous.CreatedAt().Before(event.CreatedAt()))
+		}
+		previous = event
+	}
+}

bridge/gitlab/export_test.go

@@ -242,11 +242,6 @@ func TestGitlabPushPull(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if tt.name == "bug changed status" {
-				t.Skip("test known as broken, see https://github.com/MichaelMure/git-bug/issues/435 and complain to gitlab")
-				// TODO: fix, somehow, someday, or drop support.
-			}
-
 			// for each operation a SetMetadataOperation will be added
 			// so number of operations should double
 			require.Len(t, tt.bug.Snapshot().Operations, tt.numOpExp)

bridge/gitlab/gitlab_api.go

@@ -0,0 +1,140 @@
+package gitlab
+
+import (
+	"context"
+	"time"
+
+	"github.com/MichaelMure/git-bug/util/text"
+	"github.com/xanzy/go-gitlab"
+)
+
+// Issues returns a channel with gitlab project issues, ascending order.
+func Issues(ctx context.Context, client *gitlab.Client, pid string, since time.Time) <-chan *gitlab.Issue {
+	out := make(chan *gitlab.Issue)
+
+	go func() {
+		defer close(out)
+
+		opts := gitlab.ListProjectIssuesOptions{
+			UpdatedAfter: &since,
+			Scope:        gitlab.String("all"),
+			Sort:         gitlab.String("asc"),
+		}
+
+		for {
+			issues, resp, err := client.Issues.ListProjectIssues(pid, &opts, gitlab.WithContext(ctx))
+			if err != nil {
+				return
+			}
+
+			for _, issue := range issues {
+				out <- issue
+			}
+
+			if resp.CurrentPage >= resp.TotalPages {
+				break
+			}
+
+			opts.Page = resp.NextPage
+		}
+	}()
+
+	return out
+}
+
+// Notes returns a channel with note events
+func Notes(ctx context.Context, client *gitlab.Client, issue *gitlab.Issue) <-chan Event {
+	out := make(chan Event)
+
+	go func() {
+		defer close(out)
+
+		opts := gitlab.ListIssueNotesOptions{
+			OrderBy: gitlab.String("created_at"),
+			Sort:    gitlab.String("asc"),
+		}
+
+		for {
+			notes, resp, err := client.Notes.ListIssueNotes(issue.ProjectID, issue.IID, &opts, gitlab.WithContext(ctx))
+
+			if err != nil {
+				out <- ErrorEvent{Err: err, Time: time.Now()}
+			}
+
+			for _, note := range notes {
+				out <- NoteEvent{*note}
+			}
+
+			if resp.CurrentPage >= resp.TotalPages {
+				break
+			}
+
+			opts.Page = resp.NextPage
+		}
+	}()
+
+	return out
+}
+
+// LabelEvents returns a channel with label events.
+func LabelEvents(ctx context.Context, client *gitlab.Client, issue *gitlab.Issue) <-chan Event {
+	out := make(chan Event)
+
+	go func() {
+		defer close(out)
+
+		opts := gitlab.ListLabelEventsOptions{}
+
+		for {
+			events, resp, err := client.ResourceLabelEvents.ListIssueLabelEvents(issue.ProjectID, issue.IID, &opts, gitlab.WithContext(ctx))
+
+			if err != nil {
+				out <- ErrorEvent{Err: err, Time: time.Now()}
+			}
+
+			for _, e := range events {
+				le := LabelEvent{*e}
+				le.Label.Name = text.CleanupOneLine(le.Label.Name)
+				out <- le
+			}
+
+			if resp.CurrentPage >= resp.TotalPages {
+				break
+			}
+
+			opts.Page = resp.NextPage
+		}
+	}()
+
+	return out
+}
+
+// StateEvents returns a channel with state change events.
+func StateEvents(ctx context.Context, client *gitlab.Client, issue *gitlab.Issue) <-chan Event {
+	out := make(chan Event)
+
+	go func() {
+		defer close(out)
+
+		opts := gitlab.ListStateEventsOptions{}
+
+		for {
+			events, resp, err := client.ResourceStateEvents.ListIssueStateEvents(issue.ProjectID, issue.IID, &opts, gitlab.WithContext(ctx))
+			if err != nil {
+				out <- ErrorEvent{Err: err, Time: time.Now()}
+			}
+
+			for _, e := range events {
+				out <- StateEvent{*e}
+			}
+
+			if resp.CurrentPage >= resp.TotalPages {
+				break
+			}
+
+			opts.Page = resp.NextPage
+		}
+	}()
+
+	return out
+}

bridge/gitlab/import.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/MichaelMure/git-bug/bridge/core"
 	"github.com/MichaelMure/git-bug/bridge/core/auth"
-	"github.com/MichaelMure/git-bug/bridge/gitlab/iterator"
 	"github.com/MichaelMure/git-bug/bug"
 	"github.com/MichaelMure/git-bug/cache"
 	"github.com/MichaelMure/git-bug/entity"
@@ -24,9 +23,6 @@ type gitlabImporter struct {
 	// default client
 	client *gitlab.Client
 
-	// iterator
-	iterator *iterator.Iterator
-
 	// send only channel
 	out chan<- core.ImportResult
 }
@@ -59,18 +55,15 @@ func (gi *gitlabImporter) Init(_ context.Context, repo *cache.RepoCache, conf co
 // ImportAll iterate over all the configured repository issues (notes) and ensure the creation
 // of the missing issues / comments / label events / title changes ...
 func (gi *gitlabImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
-	gi.iterator = iterator.NewIterator(ctx, gi.client, 10, gi.conf[confKeyProjectID], since)
+
 	out := make(chan core.ImportResult)
 	gi.out = out
 
 	go func() {
-		defer close(gi.out)
+		defer close(out)
 
-		// Loop over all matching issues
+		for issue := range Issues(ctx, gi.client, gi.conf[confKeyProjectID], since) {
-		for gi.iterator.NextIssue() {
-			issue := gi.iterator.IssueValue()
 
-			// create issue
 			b, err := gi.ensureIssue(repo, issue)
 			if err != nil {
 				err := fmt.Errorf("issue creation: %v", err)
@@ -78,23 +71,20 @@ func (gi *gitlabImporter) ImportAll(ctx context.Context, repo *cache.RepoCache,
 				return
 			}
 
-			// Loop over all notes
+			issueEvents := SortedEvents(
-			for gi.iterator.NextNote() {
+				Notes(ctx, gi.client, issue),
-				note := gi.iterator.NoteValue()
+				LabelEvents(ctx, gi.client, issue),
-				if err := gi.ensureNote(repo, b, note); err != nil {
+				StateEvents(ctx, gi.client, issue),
-					err := fmt.Errorf("note creation: %v", err)
+			)
-					out <- core.NewImportError(err, entity.Id(strconv.Itoa(note.ID)))
-					return
-				}
-			}
 
-			// Loop over all label events
+			for e := range issueEvents {
-			for gi.iterator.NextLabelEvent() {
+				if e, ok := e.(ErrorEvent); ok {
-				labelEvent := gi.iterator.LabelEventValue()
+					out <- core.NewImportError(e.Err, "")
-				if err := gi.ensureLabelEvent(repo, b, labelEvent); err != nil {
+					continue
-					err := fmt.Errorf("label event creation: %v", err)
+				}
-					out <- core.NewImportError(err, entity.Id(strconv.Itoa(labelEvent.ID)))
+				if err := gi.ensureIssueEvent(repo, b, issue, e); err != nil {
-					return
+					err := fmt.Errorf("issue event creation: %v", err)
+					out <- core.NewImportError(err, entity.Id(e.ID()))
 				}
 			}
 
@@ -107,10 +97,6 @@ func (gi *gitlabImporter) ImportAll(ctx context.Context, repo *cache.RepoCache,
 				return
 			}
 		}
-
-		if err := gi.iterator.Error(); err != nil {
-			out <- core.NewImportError(err, "")
-		}
 	}()
 
 	return out, nil
@@ -126,7 +112,7 @@ func (gi *gitlabImporter) ensureIssue(repo *cache.RepoCache, issue *gitlab.Issue
 	// resolve bug
 	b, err := repo.ResolveBugMatcher(func(excerpt *cache.BugExcerpt) bool {
 		return excerpt.CreateMetadata[core.MetaKeyOrigin] == target &&
-			excerpt.CreateMetadata[metaKeyGitlabId] == parseID(issue.IID) &&
+			excerpt.CreateMetadata[metaKeyGitlabId] == fmt.Sprintf("%d", issue.IID) &&
 			excerpt.CreateMetadata[metaKeyGitlabBaseUrl] == gi.conf[confKeyGitlabBaseUrl] &&
 			excerpt.CreateMetadata[metaKeyGitlabProject] == gi.conf[confKeyProjectID]
 	})
@@ -146,7 +132,7 @@ func (gi *gitlabImporter) ensureIssue(repo *cache.RepoCache, issue *gitlab.Issue
 		nil,
 		map[string]string{
 			core.MetaKeyOrigin:   target,
-			metaKeyGitlabId:      parseID(issue.IID),
+			metaKeyGitlabId:      fmt.Sprintf("%d", issue.IID),
 			metaKeyGitlabUrl:     issue.WebURL,
 			metaKeyGitlabProject: gi.conf[confKeyProjectID],
 			metaKeyGitlabBaseUrl: gi.conf[confKeyGitlabBaseUrl],
@@ -163,50 +149,49 @@ func (gi *gitlabImporter) ensureIssue(repo *cache.RepoCache, issue *gitlab.Issue
 	return b, nil
 }
 
-func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, note *gitlab.Note) error {
+func (gi *gitlabImporter) ensureIssueEvent(repo *cache.RepoCache, b *cache.BugCache, issue *gitlab.Issue, event Event) error {
-	gitlabID := parseID(note.ID)
 
-	id, errResolve := b.ResolveOperationWithMetadata(metaKeyGitlabId, gitlabID)
+	id, errResolve := b.ResolveOperationWithMetadata(metaKeyGitlabId, event.ID())
 	if errResolve != nil && errResolve != cache.ErrNoMatchingOp {
 		return errResolve
 	}
 
 	// ensure issue author
-	author, err := gi.ensurePerson(repo, note.Author.ID)
+	author, err := gi.ensurePerson(repo, event.UserID())
 	if err != nil {
 		return err
 	}
 
-	noteType, body := GetNoteType(note)
+	switch event.Kind() {
-	switch noteType {
+	case EventClosed:
-	case NOTE_CLOSED:
 		if errResolve == nil {
 			return nil
 		}
 
 		op, err := b.CloseRaw(
 			author,
-			note.CreatedAt.Unix(),
+			event.CreatedAt().Unix(),
 			map[string]string{
-				metaKeyGitlabId: gitlabID,
+				metaKeyGitlabId: event.ID(),
 			},
 		)
+
 		if err != nil {
 			return err
 		}
 
 		gi.out <- core.NewImportStatusChange(op.Id())
 
-	case NOTE_REOPENED:
+	case EventReopened:
 		if errResolve == nil {
 			return nil
 		}
 
 		op, err := b.OpenRaw(
 			author,
-			note.CreatedAt.Unix(),
+			event.CreatedAt().Unix(),
 			map[string]string{
-				metaKeyGitlabId: gitlabID,
+				metaKeyGitlabId: event.ID(),
 			},
 		)
 		if err != nil {
@@ -215,9 +200,7 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n
 
 		gi.out <- core.NewImportStatusChange(op.Id())
 
-	case NOTE_DESCRIPTION_CHANGED:
+	case EventDescriptionChanged:
-		issue := gi.iterator.IssueValue()
-
 		firstComment := b.Snapshot().Comments[0]
 		// since gitlab doesn't provide the issue history
 		// we should check for "changed the description" notes and compare issue texts
@@ -226,11 +209,11 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n
 			// comment edition
 			op, err := b.EditCommentRaw(
 				author,
-				note.UpdatedAt.Unix(),
+				event.(NoteEvent).UpdatedAt.Unix(),
 				firstComment.Id(),
 				text.Cleanup(issue.Description),
 				map[string]string{
-					metaKeyGitlabId: gitlabID,
+					metaKeyGitlabId: event.ID(),
 				},
 			)
 			if err != nil {
@@ -240,8 +223,8 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n
 			gi.out <- core.NewImportTitleEdition(op.Id())
 		}
 
-	case NOTE_COMMENT:
+	case EventComment:
-		cleanText := text.Cleanup(body)
+		cleanText := text.Cleanup(event.(NoteEvent).Body)
 
 		// if we didn't import the comment
 		if errResolve == cache.ErrNoMatchingOp {
@@ -249,11 +232,11 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n
 			// add comment operation
 			op, err := b.AddCommentRaw(
 				author,
-				note.CreatedAt.Unix(),
+				event.CreatedAt().Unix(),
 				cleanText,
 				nil,
 				map[string]string{
-					metaKeyGitlabId: gitlabID,
+					metaKeyGitlabId: event.ID(),
 				},
 			)
 			if err != nil {
@@ -271,12 +254,12 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n
 			return err
 		}
 
-		// compare local bug comment with the new note body
+		// compare local bug comment with the new event body
 		if comment.Message != cleanText {
 			// comment edition
 			op, err := b.EditCommentRaw(
 				author,
-				note.UpdatedAt.Unix(),
+				event.(NoteEvent).UpdatedAt.Unix(),
 				comment.Id(),
 				cleanText,
 				nil,
@@ -290,7 +273,7 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n
 
 		return nil
 
-	case NOTE_TITLE_CHANGED:
+	case EventTitleChanged:
 		// title change events are given new notes
 		if errResolve == nil {
 			return nil
@@ -298,10 +281,10 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n
 
 		op, err := b.SetTitleRaw(
 			author,
-			note.CreatedAt.Unix(),
+			event.CreatedAt().Unix(),
-			text.CleanupOneLine(body),
+			event.(NoteEvent).Title(),
 			map[string]string{
-				metaKeyGitlabId: gitlabID,
+				metaKeyGitlabId: event.ID(),
 			},
 		)
 		if err != nil {
@@ -310,69 +293,50 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n
 
 		gi.out <- core.NewImportTitleEdition(op.Id())
 
-	case NOTE_UNKNOWN,
+	case EventAddLabel:
-		NOTE_ASSIGNED,
+		_, err = b.ForceChangeLabelsRaw(
-		NOTE_UNASSIGNED,
+			author,
-		NOTE_CHANGED_MILESTONE,
+			event.CreatedAt().Unix(),
-		NOTE_REMOVED_MILESTONE,
+			[]string{event.(LabelEvent).Label.Name},
-		NOTE_CHANGED_DUEDATE,
+			nil,
-		NOTE_REMOVED_DUEDATE,
+			map[string]string{
-		NOTE_LOCKED,
+				metaKeyGitlabId: event.ID(),
-		NOTE_UNLOCKED,
+			},
-		NOTE_MENTIONED_IN_ISSUE,
+		)
-		NOTE_MENTIONED_IN_MERGE_REQUEST:
+		return err
+
+	case EventRemoveLabel:
+		_, err = b.ForceChangeLabelsRaw(
+			author,
+			event.CreatedAt().Unix(),
+			nil,
+			[]string{event.(LabelEvent).Label.Name},
+			map[string]string{
+				metaKeyGitlabId: event.ID(),
+			},
+		)
+		return err
+
+	case EventAssigned,
+		EventUnassigned,
+		EventChangedMilestone,
+		EventRemovedMilestone,
+		EventChangedDuedate,
+		EventRemovedDuedate,
+		EventLocked,
+		EventUnlocked,
+		EventMentionedInIssue,
+		EventMentionedInMergeRequest:
 
 		return nil
 
 	default:
-		panic("unhandled note type")
+		return fmt.Errorf("unexpected event")
 	}
 
 	return nil
 }
 
-func (gi *gitlabImporter) ensureLabelEvent(repo *cache.RepoCache, b *cache.BugCache, labelEvent *gitlab.LabelEvent) error {
-	_, err := b.ResolveOperationWithMetadata(metaKeyGitlabId, parseID(labelEvent.ID))
-	if err != cache.ErrNoMatchingOp {
-		return err
-	}
-
-	// ensure issue author
-	author, err := gi.ensurePerson(repo, labelEvent.User.ID)
-	if err != nil {
-		return err
-	}
-
-	switch labelEvent.Action {
-	case "add":
-		_, err = b.ForceChangeLabelsRaw(
-			author,
-			labelEvent.CreatedAt.Unix(),
-			[]string{text.CleanupOneLine(labelEvent.Label.Name)},
-			nil,
-			map[string]string{
-				metaKeyGitlabId: parseID(labelEvent.ID),
-			},
-		)
-
-	case "remove":
-		_, err = b.ForceChangeLabelsRaw(
-			author,
-			labelEvent.CreatedAt.Unix(),
-			nil,
-			[]string{text.CleanupOneLine(labelEvent.Label.Name)},
-			map[string]string{
-				metaKeyGitlabId: parseID(labelEvent.ID),
-			},
-		)
-
-	default:
-		err = fmt.Errorf("unexpected label event action")
-	}
-
-	return err
-}
-
 func (gi *gitlabImporter) ensurePerson(repo *cache.RepoCache, id int) (*cache.IdentityCache, error) {
 	// Look first in the cache
 	i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGitlabId, strconv.Itoa(id))
@@ -407,7 +371,3 @@ func (gi *gitlabImporter) ensurePerson(repo *cache.RepoCache, id int) (*cache.Id
 	gi.out <- core.NewImportIdentity(i.Id())
 	return i, nil
 }
-
-func parseID(id int) string {
-	return fmt.Sprintf("%d", id)
-}

bridge/gitlab/import_notes.go

@@ -1,147 +0,0 @@
-package gitlab
-
-import (
-	"strings"
-
-	"github.com/xanzy/go-gitlab"
-)
-
-type NoteType int
-
-const (
-	_ NoteType = iota
-	NOTE_COMMENT
-	NOTE_TITLE_CHANGED
-	NOTE_DESCRIPTION_CHANGED
-	NOTE_CLOSED
-	NOTE_REOPENED
-	NOTE_LOCKED
-	NOTE_UNLOCKED
-	NOTE_CHANGED_DUEDATE
-	NOTE_REMOVED_DUEDATE
-	NOTE_ASSIGNED
-	NOTE_UNASSIGNED
-	NOTE_CHANGED_MILESTONE
-	NOTE_REMOVED_MILESTONE
-	NOTE_MENTIONED_IN_ISSUE
-	NOTE_MENTIONED_IN_MERGE_REQUEST
-	NOTE_UNKNOWN
-)
-
-func (nt NoteType) String() string {
-	switch nt {
-	case NOTE_COMMENT:
-		return "note comment"
-	case NOTE_TITLE_CHANGED:
-		return "note title changed"
-	case NOTE_DESCRIPTION_CHANGED:
-		return "note description changed"
-	case NOTE_CLOSED:
-		return "note closed"
-	case NOTE_REOPENED:
-		return "note reopened"
-	case NOTE_LOCKED:
-		return "note locked"
-	case NOTE_UNLOCKED:
-		return "note unlocked"
-	case NOTE_CHANGED_DUEDATE:
-		return "note changed duedate"
-	case NOTE_REMOVED_DUEDATE:
-		return "note remove duedate"
-	case NOTE_ASSIGNED:
-		return "note assigned"
-	case NOTE_UNASSIGNED:
-		return "note unassigned"
-	case NOTE_CHANGED_MILESTONE:
-		return "note changed milestone"
-	case NOTE_REMOVED_MILESTONE:
-		return "note removed in milestone"
-	case NOTE_MENTIONED_IN_ISSUE:
-		return "note mentioned in issue"
-	case NOTE_MENTIONED_IN_MERGE_REQUEST:
-		return "note mentioned in merge request"
-	case NOTE_UNKNOWN:
-		return "note unknown"
-	default:
-		panic("unknown note type")
-	}
-}
-
-// GetNoteType parse a note system and body and return the note type and it content
-func GetNoteType(n *gitlab.Note) (NoteType, string) {
-	// when a note is a comment system is set to false
-	// when a note is a different event system is set to true
-	// because Gitlab
-	if !n.System {
-		return NOTE_COMMENT, n.Body
-	}
-
-	if n.Body == "closed" {
-		return NOTE_CLOSED, ""
-	}
-
-	if n.Body == "reopened" {
-		return NOTE_REOPENED, ""
-	}
-
-	if n.Body == "changed the description" {
-		return NOTE_DESCRIPTION_CHANGED, ""
-	}
-
-	if n.Body == "locked this issue" {
-		return NOTE_LOCKED, ""
-	}
-
-	if n.Body == "unlocked this issue" {
-		return NOTE_UNLOCKED, ""
-	}
-
-	if strings.HasPrefix(n.Body, "changed title from") {
-		return NOTE_TITLE_CHANGED, getNewTitle(n.Body)
-	}
-
-	if strings.HasPrefix(n.Body, "changed due date to") {
-		return NOTE_CHANGED_DUEDATE, ""
-	}
-
-	if n.Body == "removed due date" {
-		return NOTE_REMOVED_DUEDATE, ""
-	}
-
-	if strings.HasPrefix(n.Body, "assigned to @") {
-		return NOTE_ASSIGNED, ""
-	}
-
-	if strings.HasPrefix(n.Body, "unassigned @") {
-		return NOTE_UNASSIGNED, ""
-	}
-
-	if strings.HasPrefix(n.Body, "changed milestone to %") {
-		return NOTE_CHANGED_MILESTONE, ""
-	}
-
-	if strings.HasPrefix(n.Body, "removed milestone") {
-		return NOTE_REMOVED_MILESTONE, ""
-	}
-
-	if strings.HasPrefix(n.Body, "mentioned in issue") {
-		return NOTE_MENTIONED_IN_ISSUE, ""
-	}
-
-	if strings.HasPrefix(n.Body, "mentioned in merge request") {
-		return NOTE_MENTIONED_IN_MERGE_REQUEST, ""
-	}
-
-	return NOTE_UNKNOWN, ""
-}
-
-// getNewTitle parses body diff given by gitlab api and return it final form
-// examples: "changed title from **fourth issue** to **fourth issue{+ changed+}**"
-//           "changed title from **fourth issue{- changed-}** to **fourth issue**"
-// because Gitlab
-func getNewTitle(diff string) string {
-	newTitle := strings.Split(diff, "** to **")[1]
-	newTitle = strings.Replace(newTitle, "{+", "", -1)
-	newTitle = strings.Replace(newTitle, "+}", "", -1)
-	return strings.TrimSuffix(newTitle, "**")
-}

bridge/gitlab/iterator/issue.go

@@ -1,89 +0,0 @@
-package iterator
-
-import (
-	"context"
-
-	"github.com/xanzy/go-gitlab"
-)
-
-type issueIterator struct {
-	page     int
-	lastPage bool
-	index    int
-	cache    []*gitlab.Issue
-}
-
-func newIssueIterator() *issueIterator {
-	ii := &issueIterator{}
-	ii.Reset()
-	return ii
-}
-
-func (ii *issueIterator) Next(ctx context.Context, conf config) (bool, error) {
-	// first query
-	if ii.cache == nil {
-		return ii.getNext(ctx, conf)
-	}
-
-	// move cursor index
-	if ii.index < len(ii.cache)-1 {
-		ii.index++
-		return true, nil
-	}
-
-	return ii.getNext(ctx, conf)
-}
-
-func (ii *issueIterator) Value() *gitlab.Issue {
-	return ii.cache[ii.index]
-}
-
-func (ii *issueIterator) getNext(ctx context.Context, conf config) (bool, error) {
-	if ii.lastPage {
-		return false, nil
-	}
-
-	ctx, cancel := context.WithTimeout(ctx, conf.timeout)
-	defer cancel()
-
-	issues, resp, err := conf.gc.Issues.ListProjectIssues(
-		conf.project,
-		&gitlab.ListProjectIssuesOptions{
-			ListOptions: gitlab.ListOptions{
-				Page:    ii.page,
-				PerPage: conf.capacity,
-			},
-			Scope:        gitlab.String("all"),
-			UpdatedAfter: &conf.since,
-			Sort:         gitlab.String("asc"),
-		},
-		gitlab.WithContext(ctx),
-	)
-
-	if err != nil {
-		ii.Reset()
-		return false, err
-	}
-
-	if resp.TotalPages == ii.page {
-		ii.lastPage = true
-	}
-
-	// if repository doesn't have any issues
-	if len(issues) == 0 {
-		return false, nil
-	}
-
-	ii.cache = issues
-	ii.index = 0
-	ii.page++
-
-	return true, nil
-}
-
-func (ii *issueIterator) Reset() {
-	ii.index = -1
-	ii.page = 1
-	ii.lastPage = false
-	ii.cache = nil
-}

bridge/gitlab/iterator/iterator.go

@@ -1,138 +0,0 @@
-package iterator
-
-import (
-	"context"
-	"time"
-
-	"github.com/xanzy/go-gitlab"
-)
-
-type Iterator struct {
-	// shared context
-	ctx context.Context
-
-	// to pass to sub-iterators
-	conf config
-
-	// sticky error
-	err error
-
-	// issues iterator
-	issue *issueIterator
-
-	// notes iterator
-	note *noteIterator
-
-	// labelEvent iterator
-	labelEvent *labelEventIterator
-}
-
-type config struct {
-	// gitlab api v4 client
-	gc *gitlab.Client
-
-	timeout time.Duration
-
-	// if since is given the iterator will query only the issues
-	// updated after this date
-	since time.Time
-
-	// project id
-	project string
-
-	// number of issues and notes to query at once
-	capacity int
-}
-
-// NewIterator create a new iterator
-func NewIterator(ctx context.Context, client *gitlab.Client, capacity int, projectID string, since time.Time) *Iterator {
-	return &Iterator{
-		ctx: ctx,
-		conf: config{
-			gc:       client,
-			timeout:  60 * time.Second,
-			since:    since,
-			project:  projectID,
-			capacity: capacity,
-		},
-		issue:      newIssueIterator(),
-		note:       newNoteIterator(),
-		labelEvent: newLabelEventIterator(),
-	}
-}
-
-// Error return last encountered error
-func (i *Iterator) Error() error {
-	return i.err
-}
-
-func (i *Iterator) NextIssue() bool {
-	if i.err != nil {
-		return false
-	}
-
-	if i.ctx.Err() != nil {
-		return false
-	}
-
-	more, err := i.issue.Next(i.ctx, i.conf)
-	if err != nil {
-		i.err = err
-		return false
-	}
-
-	// Also reset the other sub iterators as they would
-	// no longer be valid
-	i.note.Reset(i.issue.Value().IID)
-	i.labelEvent.Reset(i.issue.Value().IID)
-
-	return more
-}
-
-func (i *Iterator) IssueValue() *gitlab.Issue {
-	return i.issue.Value()
-}
-
-func (i *Iterator) NextNote() bool {
-	if i.err != nil {
-		return false
-	}
-
-	if i.ctx.Err() != nil {
-		return false
-	}
-
-	more, err := i.note.Next(i.ctx, i.conf)
-	if err != nil {
-		i.err = err
-		return false
-	}
-
-	return more
-}
-
-func (i *Iterator) NoteValue() *gitlab.Note {
-	return i.note.Value()
-}
-
-func (i *Iterator) NextLabelEvent() bool {
-	if i.err != nil {
-		return false
-	}
-
-	if i.ctx.Err() != nil {
-		return false
-	}
-
-	more, err := i.labelEvent.Next(i.ctx, i.conf)
-	if err != nil {
-		i.err = err
-		return false
-	}
-
-	return more
-}
-
-func (i *Iterator) LabelEventValue() *gitlab.LabelEvent {
-	return i.labelEvent.Value()
-}

bridge/gitlab/iterator/labelEvent.go

@@ -1,105 +0,0 @@
-package iterator
-
-import (
-	"context"
-	"sort"
-
-	"github.com/xanzy/go-gitlab"
-)
-
-// Since Gitlab does not return the label events items in the correct order
-// we need to sort the list ourselves and stop relying on the pagination model
-// #BecauseGitlab
-type labelEventIterator struct {
-	issue int
-	index int
-	cache []*gitlab.LabelEvent
-}
-
-func newLabelEventIterator() *labelEventIterator {
-	lei := &labelEventIterator{}
-	lei.Reset(-1)
-	return lei
-}
-
-func (lei *labelEventIterator) Next(ctx context.Context, conf config) (bool, error) {
-	// first query
-	if lei.cache == nil {
-		return lei.getNext(ctx, conf)
-	}
-
-	// move cursor index
-	if lei.index < len(lei.cache)-1 {
-		lei.index++
-		return true, nil
-	}
-
-	return false, nil
-}
-
-func (lei *labelEventIterator) Value() *gitlab.LabelEvent {
-	return lei.cache[lei.index]
-}
-
-func (lei *labelEventIterator) getNext(ctx context.Context, conf config) (bool, error) {
-	ctx, cancel := context.WithTimeout(ctx, conf.timeout)
-	defer cancel()
-
-	// since order is not guaranteed we should query all label events
-	// and sort them by ID
-	page := 1
-	for {
-		labelEvents, resp, err := conf.gc.ResourceLabelEvents.ListIssueLabelEvents(
-			conf.project,
-			lei.issue,
-			&gitlab.ListLabelEventsOptions{
-				ListOptions: gitlab.ListOptions{
-					Page:    page,
-					PerPage: conf.capacity,
-				},
-			},
-			gitlab.WithContext(ctx),
-		)
-		if err != nil {
-			lei.Reset(-1)
-			return false, err
-		}
-
-		if len(labelEvents) == 0 {
-			break
-		}
-
-		lei.cache = append(lei.cache, labelEvents...)
-
-		if resp.TotalPages == page {
-			break
-		}
-
-		page++
-	}
-
-	sort.Sort(lei)
-	lei.index = 0
-
-	return len(lei.cache) > 0, nil
-}
-
-func (lei *labelEventIterator) Reset(issue int) {
-	lei.issue = issue
-	lei.index = -1
-	lei.cache = nil
-}
-
-// ORDERING
-
-func (lei *labelEventIterator) Len() int {
-	return len(lei.cache)
-}
-
-func (lei *labelEventIterator) Swap(i, j int) {
-	lei.cache[i], lei.cache[j] = lei.cache[j], lei.cache[i]
-}
-
-func (lei *labelEventIterator) Less(i, j int) bool {
-	return lei.cache[i].ID < lei.cache[j].ID
-}

bridge/gitlab/iterator/note.go

@@ -1,90 +0,0 @@
-package iterator
-
-import (
-	"context"
-
-	"github.com/xanzy/go-gitlab"
-)
-
-type noteIterator struct {
-	issue    int
-	page     int
-	lastPage bool
-	index    int
-	cache    []*gitlab.Note
-}
-
-func newNoteIterator() *noteIterator {
-	in := &noteIterator{}
-	in.Reset(-1)
-	return in
-}
-
-func (in *noteIterator) Next(ctx context.Context, conf config) (bool, error) {
-	// first query
-	if in.cache == nil {
-		return in.getNext(ctx, conf)
-	}
-
-	// move cursor index
-	if in.index < len(in.cache)-1 {
-		in.index++
-		return true, nil
-	}
-
-	return in.getNext(ctx, conf)
-}
-
-func (in *noteIterator) Value() *gitlab.Note {
-	return in.cache[in.index]
-}
-
-func (in *noteIterator) getNext(ctx context.Context, conf config) (bool, error) {
-	if in.lastPage {
-		return false, nil
-	}
-
-	ctx, cancel := context.WithTimeout(ctx, conf.timeout)
-	defer cancel()
-
-	notes, resp, err := conf.gc.Notes.ListIssueNotes(
-		conf.project,
-		in.issue,
-		&gitlab.ListIssueNotesOptions{
-			ListOptions: gitlab.ListOptions{
-				Page:    in.page,
-				PerPage: conf.capacity,
-			},
-			Sort:    gitlab.String("asc"),
-			OrderBy: gitlab.String("created_at"),
-		},
-		gitlab.WithContext(ctx),
-	)
-
-	if err != nil {
-		in.Reset(-1)
-		return false, err
-	}
-
-	if resp.TotalPages == in.page {
-		in.lastPage = true
-	}
-
-	if len(notes) == 0 {
-		return false, nil
-	}
-
-	in.cache = notes
-	in.index = 0
-	in.page++
-
-	return true, nil
-}
-
-func (in *noteIterator) Reset(issue int) {
-	in.issue = issue
-	in.index = -1
-	in.page = 1
-	in.lastPage = false
-	in.cache = nil
-}