git-bug/bridge/github/export.go

package github

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/shurcooL/githubv4"
	"golang.org/x/sync/errgroup"

	"github.com/MichaelMure/git-bug/bridge/core"
	"github.com/MichaelMure/git-bug/bridge/core/auth"
	"github.com/MichaelMure/git-bug/cache"
	"github.com/MichaelMure/git-bug/entities/bug"
	"github.com/MichaelMure/git-bug/entities/common"
	"github.com/MichaelMure/git-bug/entity"
	"github.com/MichaelMure/git-bug/entity/dag"
)

var (
	ErrMissingIdentityToken = errors.New("missing identity token")
)

// githubExporter implement the Exporter interface
type githubExporter struct {
	conf core.Configuration

	// cache identities clients
	identityClient map[entity.Id]*rateLimitHandlerClient

	// the client to use for non user-specific queries
	// it's the client associated to the "default-login" config
	// used for the github V4 API (graphql)
	defaultClient *rateLimitHandlerClient

	// the token of the default user
	// it's the token associated to the "default-login" config
	// used for the github V3 API (REST)
	defaultToken *auth.Token

	// github repository ID
	repositoryID string

	// cache identifiers used to speed up exporting operations
	// cleared for each bug
	cachedOperationIDs map[entity.Id]string

	// cache labels used to speed up exporting labels events
	cachedLabels map[string]string

	// channel to send export results
	out chan<- core.ExportResult
}

// Init .
func (ge *githubExporter) Init(_ context.Context, repo *cache.RepoCache, conf core.Configuration) error {
	ge.conf = conf
	ge.identityClient = make(map[entity.Id]*rateLimitHandlerClient)
	ge.cachedOperationIDs = make(map[entity.Id]string)
	ge.cachedLabels = make(map[string]string)

	// preload all clients
	err := ge.cacheAllClient(repo)
	if err != nil {
		return err
	}

	return nil
}

func (ge *githubExporter) cacheAllClient(repo *cache.RepoCache) error {
	creds, err := auth.List(repo, auth.WithTarget(target), auth.WithKind(auth.KindToken))
	if err != nil {
		return err
	}

	for _, cred := range creds {
		login, ok := cred.GetMetadata(auth.MetaKeyLogin)
		if !ok {
			_, _ = fmt.Fprintf(os.Stderr, "credential %s is not tagged with a Github login\n", cred.ID().Human())
			continue
		}

		user, err := repo.Identities().ResolveIdentityImmutableMetadata(metaKeyGithubLogin, login)
		if entity.IsErrNotFound(err) {
			continue
		}
		if err != nil {
			return nil
		}

		if _, ok := ge.identityClient[user.Id()]; ok {
			continue
		}

		client := buildClient(creds[0].(*auth.Token))
		ge.identityClient[user.Id()] = client

		// assign the default client and token as well
		if ge.defaultClient == nil && login == ge.conf[confKeyDefaultLogin] {
			ge.defaultClient = client
			ge.defaultToken = creds[0].(*auth.Token)
		}
	}

	if ge.defaultClient == nil {
		return fmt.Errorf("no token found for the default login \"%s\"", ge.conf[confKeyDefaultLogin])
	}

	return nil
}

// getClientForIdentity return a githubv4 API client configured with the access token of the given identity.
func (ge *githubExporter) getClientForIdentity(userId entity.Id) (*rateLimitHandlerClient, error) {
	client, ok := ge.identityClient[userId]
	if ok {
		return client, nil
	}

	return nil, ErrMissingIdentityToken
}

// ExportAll export all event made by the current user to Github
func (ge *githubExporter) ExportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ExportResult, error) {
	out := make(chan core.ExportResult)
	ge.out = out

	var err error
	// get repository node id
	ge.repositoryID, err = getRepositoryNodeID(
		ctx,
		ge.defaultToken,
		ge.conf[confKeyOwner],
		ge.conf[confKeyProject],
	)
	if err != nil {
		return nil, err
	}

	go func() {
		defer close(out)

		// query all labels
		err = ge.cacheGithubLabels(ctx, ge.defaultClient)
		if err != nil {
			out <- core.NewExportError(errors.Wrap(err, "can't obtain Github labels"), "")
			return
		}

		allIdentitiesIds := make([]entity.Id, 0, len(ge.identityClient))
		for id := range ge.identityClient {
			allIdentitiesIds = append(allIdentitiesIds, id)
		}

		allBugsIds := repo.Bugs().AllIds()

		for _, id := range allBugsIds {
			b, err := repo.Bugs().Resolve(id)
			if err != nil {
				out <- core.NewExportError(errors.Wrap(err, "can't load bug"), id)
				return
			}

			select {

			case <-ctx.Done():
				// stop iterating if context cancel function is called
				return

			default:
				snapshot := b.Snapshot()

				// ignore issues created before since date
				// TODO: compare the Lamport time instead of using the unix time
				if snapshot.CreateTime.Before(since) {
					out <- core.NewExportNothing(b.Id(), "bug created before the since date")
					continue
				}

				if snapshot.HasAnyActor(allIdentitiesIds...) {
					// try to export the bug and it associated events
					ge.exportBug(ctx, b, out)
				}
			}
		}
	}()

	return out, nil
}

// exportBug publish bugs and related events
func (ge *githubExporter) exportBug(ctx context.Context, b *cache.BugCache, out chan<- core.ExportResult) {
	snapshot := b.Snapshot()
	var bugUpdated bool

	var bugGithubID string
	var bugGithubURL string

	// Special case:
	// if a user try to export a bug that is not already exported to Github (or imported
	// from Github) and we do not have the token of the bug author, there is nothing we can do.

	// first operation is always createOp
	createOp := snapshot.Operations[0].(*bug.CreateOperation)
	author := snapshot.Author

	// skip bug if origin is not allowed
	origin, ok := snapshot.GetCreateMetadata(core.MetaKeyOrigin)
	if ok && origin != target {
		out <- core.NewExportNothing(b.Id(), fmt.Sprintf("issue tagged with origin: %s", origin))
		return
	}

	// get github bug ID
	githubID, ok := snapshot.GetCreateMetadata(metaKeyGithubId)
	if ok {
		githubURL, ok := snapshot.GetCreateMetadata(metaKeyGithubUrl)
		if !ok {
			// if we find github ID, github URL must be found too
			err := fmt.Errorf("incomplete Github metadata: expected to find issue URL")
			out <- core.NewExportError(err, b.Id())
		}

		// extract owner and project
		owner, project, err := splitURL(githubURL)
		if err != nil {
			err := fmt.Errorf("bad project url: %v", err)
			out <- core.NewExportError(err, b.Id())
			return
		}

		// ignore issue coming from other repositories
		if owner != ge.conf[confKeyOwner] && project != ge.conf[confKeyProject] {
			out <- core.NewExportNothing(b.Id(), fmt.Sprintf("skipping issue from url:%s", githubURL))
			return
		}

		// will be used to mark operation related to a bug as exported
		bugGithubID = githubID
		bugGithubURL = githubURL

	} else {
		// check that we have a token for operation author
		client, err := ge.getClientForIdentity(author.Id())
		if err != nil {
			// if bug is still not exported and we do not have the author stop the execution
			out <- core.NewExportNothing(b.Id(), fmt.Sprintf("missing author token"))
			return
		}

		// create bug
		id, url, err := ge.createGithubIssue(ctx, client, ge.repositoryID, createOp.Title, createOp.Message)
		if err != nil {
			err := errors.Wrap(err, "exporting github issue")
			out <- core.NewExportError(err, b.Id())
			return
		}

		out <- core.NewExportBug(b.Id())

		// mark bug creation operation as exported
		if err := markOperationAsExported(b, createOp.Id(), id, url); err != nil {
			err := errors.Wrap(err, "marking operation as exported")
			out <- core.NewExportError(err, b.Id())
			return
		}

		// commit operation to avoid creating multiple issues with multiple pushes
		if err := b.CommitAsNeeded(); err != nil {
			err := errors.Wrap(err, "bug commit")
			out <- core.NewExportError(err, b.Id())
			return
		}

		// cache bug github ID and URL
		bugGithubID = id
		bugGithubURL = url
	}

	// cache operation github id
	ge.cachedOperationIDs[createOp.Id()] = bugGithubID

	for _, op := range snapshot.Operations[1:] {
		// ignore SetMetadata operations
		if _, ok := op.(dag.OperationDoesntChangeSnapshot); ok {
			continue
		}

		// ignore operations already existing in github (due to import or export)
		// cache the ID of already exported or imported issues and events from Github
		if id, ok := op.GetMetadata(metaKeyGithubId); ok {
			ge.cachedOperationIDs[op.Id()] = id
			continue
		}

		opAuthor := op.Author()
		client, err := ge.getClientForIdentity(opAuthor.Id())
		if err != nil {
			continue
		}

		var id, url string
		switch op := op.(type) {
		case *bug.AddCommentOperation:
			// send operation to github
			id, url, err = ge.addCommentGithubIssue(ctx, client, bugGithubID, op.Message)
			if err != nil {
				err := errors.Wrap(err, "adding comment")
				out <- core.NewExportError(err, b.Id())
				return
			}

			out <- core.NewExportComment(b.Id())

			// cache comment id
			ge.cachedOperationIDs[op.Id()] = id

		case *bug.EditCommentOperation:
			// Since github doesn't consider the issue body as a comment
			if op.Target == createOp.Id() {

				// case bug creation operation: we need to edit the Github issue
				if err := ge.updateGithubIssueBody(ctx, client, bugGithubID, op.Message); err != nil {
					err := errors.Wrap(err, "editing issue")
					out <- core.NewExportError(err, b.Id())
					return
				}

				out <- core.NewExportCommentEdition(b.Id())

				id = bugGithubID
				url = bugGithubURL

			} else {

				// case comment edition operation: we need to edit the Github comment
				commentID, ok := ge.cachedOperationIDs[op.Target]
				if !ok {
					panic("unexpected error: comment id not found")
				}

				eid, eurl, err := ge.editCommentGithubIssue(ctx, client, commentID, op.Message)
				if err != nil {
					err := errors.Wrap(err, "editing comment")
					out <- core.NewExportError(err, b.Id())
					return
				}

				out <- core.NewExportCommentEdition(b.Id())

				// use comment id/url instead of issue id/url
				id = eid
				url = eurl
			}

		case *bug.SetStatusOperation:
			if err := ge.updateGithubIssueStatus(ctx, client, bugGithubID, op.Status); err != nil {
				err := errors.Wrap(err, "editing status")
				out <- core.NewExportError(err, b.Id())
				return
			}

			out <- core.NewExportStatusChange(b.Id())

			id = bugGithubID
			url = bugGithubURL

		case *bug.SetTitleOperation:
			if err := ge.updateGithubIssueTitle(ctx, client, bugGithubID, op.Title); err != nil {
				err := errors.Wrap(err, "editing title")
				out <- core.NewExportError(err, b.Id())
				return
			}

			out <- core.NewExportTitleEdition(b.Id())

			id = bugGithubID
			url = bugGithubURL

		case *bug.LabelChangeOperation:
			if err := ge.updateGithubIssueLabels(ctx, client, bugGithubID, op.Added, op.Removed); err != nil {
				err := errors.Wrap(err, "updating labels")
				out <- core.NewExportError(err, b.Id())
				return
			}

			out <- core.NewExportLabelChange(b.Id())

			id = bugGithubID
			url = bugGithubURL

		default:
			panic("unhandled operation type case")
		}

		// mark operation as exported
		if err := markOperationAsExported(b, op.Id(), id, url); err != nil {
			err := errors.Wrap(err, "marking operation as exported")
			out <- core.NewExportError(err, b.Id())
			return
		}

		// commit at each operation export to avoid exporting same events multiple times
		if err := b.CommitAsNeeded(); err != nil {
			err := errors.Wrap(err, "bug commit")
			out <- core.NewExportError(err, b.Id())
			return
		}

		bugUpdated = true
	}

	if !bugUpdated {
		out <- core.NewExportNothing(b.Id(), "nothing has been exported")
	}
}

// getRepositoryNodeID request github api v3 to get repository node id
func getRepositoryNodeID(ctx context.Context, token *auth.Token, owner, project string) (string, error) {
	url := fmt.Sprintf("%s/repos/%s/%s", githubV3Url, owner, project)
	client := &http.Client{}

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return "", err
	}

	// need the token for private repositories
	req.Header.Set("Authorization", fmt.Sprintf("token %s", token.Value))

	ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
	defer cancel()
	req = req.WithContext(ctx)

	resp, err := client.Do(req)
	if err != nil {
		return "", err
	}

	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("HTTP error %v retrieving repository node id", resp.StatusCode)
	}

	aux := struct {
		NodeID string `json:"node_id"`
	}{}

	data, _ := ioutil.ReadAll(resp.Body)
	err = resp.Body.Close()
	if err != nil {
		return "", err
	}

	err = json.Unmarshal(data, &aux)
	if err != nil {
		return "", err
	}

	return aux.NodeID, nil
}

func markOperationAsExported(b *cache.BugCache, target entity.Id, githubID, githubURL string) error {
	_, err := b.SetMetadata(
		target,
		map[string]string{
			metaKeyGithubId:  githubID,
			metaKeyGithubUrl: githubURL,
		},
	)

	return err
}

func (ge *githubExporter) cacheGithubLabels(ctx context.Context, gc *rateLimitHandlerClient) error {
	variables := map[string]interface{}{
		"owner": githubv4.String(ge.conf[confKeyOwner]),
		"name":  githubv4.String(ge.conf[confKeyProject]),
		"first": githubv4.Int(10),
		"after": (*githubv4.String)(nil),
	}

	q := labelsQuery{}

	hasNextPage := true
	for hasNextPage {
		if err := gc.queryExport(ctx, &q, variables, ge.out); err != nil {
			return err
		}

		for _, label := range q.Repository.Labels.Nodes {
			ge.cachedLabels[label.Name] = label.ID
		}

		hasNextPage = q.Repository.Labels.PageInfo.HasNextPage
		variables["after"] = q.Repository.Labels.PageInfo.EndCursor
	}

	return nil
}

func (ge *githubExporter) getLabelID(label string) (string, error) {
	label = strings.ToLower(label)
	for cachedLabel, ID := range ge.cachedLabels {
		if label == strings.ToLower(cachedLabel) {
			return ID, nil
		}
	}

	return "", fmt.Errorf("didn't find label id in cache")
}

// create a new label and return it github id
// NOTE: since createLabel mutation is still in preview mode we use github api v3 to create labels
// see https://developer.github.com/v4/mutation/createlabel/ and https://developer.github.com/v4/previews/#labels-preview
func (ge *githubExporter) createGithubLabel(ctx context.Context, label, color string) (string, error) {
	url := fmt.Sprintf("%s/repos/%s/%s/labels", githubV3Url, ge.conf[confKeyOwner], ge.conf[confKeyProject])
	client := &http.Client{}

	params := struct {
		Name        string `json:"name"`
		Color       string `json:"color"`
		Description string `json:"description"`
	}{
		Name:  label,
		Color: color,
	}

	data, err := json.Marshal(params)
	if err != nil {
		return "", err
	}

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
	if err != nil {
		return "", err
	}

	ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
	defer cancel()
	req = req.WithContext(ctx)

	// need the token for private repositories
	req.Header.Set("Authorization", fmt.Sprintf("token %s", ge.defaultToken.Value))

	resp, err := client.Do(req)
	if err != nil {
		return "", err
	}

	if resp.StatusCode != http.StatusCreated {
		return "", fmt.Errorf("error creating label: response status %v", resp.StatusCode)
	}

	aux := struct {
		ID     int    `json:"id"`
		NodeID string `json:"node_id"`
		Color  string `json:"color"`
	}{}

	data, _ = ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()

	err = json.Unmarshal(data, &aux)
	if err != nil {
		return "", err
	}

	return aux.NodeID, nil
}

/**
// create github label using api v4
func (ge *githubExporter) createGithubLabelV4(gc *githubv4.Client, label, labelColor string) (string, error) {
	m := createLabelMutation{}
	input := createLabelInput{
		RepositoryID: ge.repositoryID,
		Name:         githubv4.String(label),
		Color:        githubv4.String(labelColor),
	}

	ctx := context.Background()

	if err := gc.mutate(ctx, &m, input, nil); err != nil {
		return "", err
	}

	return m.CreateLabel.Label.ID, nil
}
*/

func (ge *githubExporter) getOrCreateGithubLabelID(ctx context.Context, gc *rateLimitHandlerClient, repositoryID string, label bug.Label) (string, error) {
	// try to get label id from cache
	labelID, err := ge.getLabelID(string(label))
	if err == nil {
		return labelID, nil
	}

	// RGBA to hex color
	rgba := label.Color().RGBA()
	hexColor := fmt.Sprintf("%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B)

	ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
	defer cancel()

	labelID, err = ge.createGithubLabel(ctx, string(label), hexColor)
	if err != nil {
		return "", err
	}

	return labelID, nil
}

func (ge *githubExporter) getLabelsIDs(ctx context.Context, gc *rateLimitHandlerClient, repositoryID string, labels []bug.Label) ([]githubv4.ID, error) {
	ids := make([]githubv4.ID, 0, len(labels))
	var err error

	// check labels ids
	for _, label := range labels {
		id, ok := ge.cachedLabels[string(label)]
		if !ok {
			// try to query label id
			id, err = ge.getOrCreateGithubLabelID(ctx, gc, repositoryID, label)
			if err != nil {
				return nil, errors.Wrap(err, "get or create github label")
			}

			// cache label id
			ge.cachedLabels[string(label)] = id
		}

		ids = append(ids, githubv4.ID(id))
	}

	return ids, nil
}

// create a github issue and return it ID
func (ge *githubExporter) createGithubIssue(ctx context.Context, gc *rateLimitHandlerClient, repositoryID, title, body string) (string, string, error) {
	m := &createIssueMutation{}
	input := githubv4.CreateIssueInput{
		RepositoryID: repositoryID,
		Title:        githubv4.String(title),
		Body:         (*githubv4.String)(&body),
	}

	if err := gc.mutate(ctx, m, input, nil, ge.out); err != nil {
		return "", "", err
	}

	issue := m.CreateIssue.Issue
	return issue.ID, issue.URL, nil
}

// add a comment to an issue and return its ID
func (ge *githubExporter) addCommentGithubIssue(ctx context.Context, gc *rateLimitHandlerClient, subjectID string, body string) (string, string, error) {
	m := &addCommentToIssueMutation{}
	input := githubv4.AddCommentInput{
		SubjectID: subjectID,
		Body:      githubv4.String(body),
	}

	if err := gc.mutate(ctx, m, input, nil, ge.out); err != nil {
		return "", "", err
	}

	node := m.AddComment.CommentEdge.Node
	return node.ID, node.URL, nil
}

func (ge *githubExporter) editCommentGithubIssue(ctx context.Context, gc *rateLimitHandlerClient, commentID, body string) (string, string, error) {
	m := &updateIssueCommentMutation{}
	input := githubv4.UpdateIssueCommentInput{
		ID:   commentID,
		Body: githubv4.String(body),
	}

	if err := gc.mutate(ctx, m, input, nil, ge.out); err != nil {
		return "", "", err
	}

	return commentID, m.UpdateIssueComment.IssueComment.URL, nil
}

func (ge *githubExporter) updateGithubIssueStatus(ctx context.Context, gc *rateLimitHandlerClient, id string, status common.Status) error {
	m := &updateIssueMutation{}

	// set state
	var state githubv4.IssueState

	switch status {
	case common.OpenStatus:
		state = githubv4.IssueStateOpen
	case common.ClosedStatus:
		state = githubv4.IssueStateClosed
	default:
		panic("unknown bug state")
	}

	input := githubv4.UpdateIssueInput{
		ID:    id,
		State: &state,
	}

	if err := gc.mutate(ctx, m, input, nil, ge.out); err != nil {
		return err
	}

	return nil
}

func (ge *githubExporter) updateGithubIssueBody(ctx context.Context, gc *rateLimitHandlerClient, id string, body string) error {
	m := &updateIssueMutation{}
	input := githubv4.UpdateIssueInput{
		ID:   id,
		Body: (*githubv4.String)(&body),
	}

	if err := gc.mutate(ctx, m, input, nil, ge.out); err != nil {
		return err
	}

	return nil
}

func (ge *githubExporter) updateGithubIssueTitle(ctx context.Context, gc *rateLimitHandlerClient, id, title string) error {
	m := &updateIssueMutation{}
	input := githubv4.UpdateIssueInput{
		ID:    id,
		Title: (*githubv4.String)(&title),
	}

	if err := gc.mutate(ctx, m, input, nil, ge.out); err != nil {
		return err
	}

	return nil
}

// update github issue labels
func (ge *githubExporter) updateGithubIssueLabels(ctx context.Context, gc *rateLimitHandlerClient, labelableID string, added, removed []bug.Label) error {

	wg, ctx := errgroup.WithContext(ctx)
	if len(added) > 0 {
		wg.Go(func() error {
			addedIDs, err := ge.getLabelsIDs(ctx, gc, labelableID, added)
			if err != nil {
				return err
			}

			m := &addLabelsToLabelableMutation{}
			inputAdd := githubv4.AddLabelsToLabelableInput{
				LabelableID: labelableID,
				LabelIDs:    addedIDs,
			}

			// add labels
			if err := gc.mutate(ctx, m, inputAdd, nil, ge.out); err != nil {
				return err
			}
			return nil
		})
	}

	if len(removed) > 0 {
		wg.Go(func() error {
			removedIDs, err := ge.getLabelsIDs(ctx, gc, labelableID, removed)
			if err != nil {
				return err
			}

			m2 := &removeLabelsFromLabelableMutation{}
			inputRemove := githubv4.RemoveLabelsFromLabelableInput{
				LabelableID: labelableID,
				LabelIDs:    removedIDs,
			}

			// remove label labels
			if err := gc.mutate(ctx, m2, inputRemove, nil, ge.out); err != nil {
				return err
			}
			return nil
		})
	}

	return wg.Wait()
}