🌱 scdiff: Add workflow to run scdiff against PRs on demand (#3640)

* wip Signed-off-by: Spencer Schrock <sschrock@google.com> * try to use jq without quotes Signed-off-by: Spencer Schrock <sschrock@google.com> * try to make file another way. Signed-off-by: Spencer Schrock <sschrock@google.com> * try using homedir Signed-off-by: Spencer Schrock <sschrock@google.com> * add github token to env Signed-off-by: Spencer Schrock <sschrock@google.com> * add link to workflow run Signed-off-by: Spencer Schrock <sschrock@google.com> * make comment its own job Signed-off-by: Spencer Schrock <sschrock@google.com> * fix typo in job context Signed-off-by: Spencer Schrock <sschrock@google.com> * typo part 2 Signed-off-by: Spencer Schrock <sschrock@google.com> * use github-script to get PR SHAs. Signed-off-by: Spencer Schrock <sschrock@google.com> * need to go through one more type to get to API response. Signed-off-by: Spencer Schrock <sschrock@google.com> * temporarily use monitor action to see the required permissions Signed-off-by: Spencer Schrock <sschrock@google.com> * spacing is hard Signed-off-by: Spencer Schrock <sschrock@google.com> * remove monitor and apply minimal permissions the read-all at the top might be too broad, but the monitor doesnt support graphql so best we can do for now. Signed-off-by: Spencer Schrock <sschrock@google.com> * try to set the checks Signed-off-by: Spencer Schrock <sschrock@google.com> * read the comment body Signed-off-by: Spencer Schrock <sschrock@google.com> * try to get around regex syntax error? Signed-off-by: Spencer Schrock <sschrock@google.com> * quote comment body Signed-off-by: Spencer Schrock <sschrock@google.com> * we want to pass an empty string to the args Signed-off-by: Spencer Schrock <sschrock@google.com> * fix the regex string Signed-off-by: Spencer Schrock <sschrock@google.com> * rest of repo has upgraded Signed-off-by: Spencer Schrock <sschrock@google.com> * seed 15 repos to analyze to start with Signed-off-by: Spencer Schrock <sschrock@google.com> * support gitlab repos in scdiff Signed-off-by: Spencer Schrock <sschrock@google.com> * rename pr step to config we also need the checks to run, so update the name to reflect that Signed-off-by: Spencer Schrock <sschrock@google.com> * switch from default token to a PAT By default, the GitHub Action token gets 1000 req/hour. If running all checks, the before/after each take about 1100 of core quota A PAT grants 5000/hr so the 2200 required should be fine if used infrequently. Ideally, the caller will always pass the check they care about into the command Signed-off-by: Spencer Schrock <sschrock@google.com> * escape comment body with bash Signed-off-by: Spencer Schrock <sschrock@google.com> * setup go manually Signed-off-by: Spencer Schrock <sschrock@google.com> * don't need to run on comment delete Signed-off-by: Spencer Schrock <sschrock@google.com> * limit scdiff to individuals with repo access Signed-off-by: Spencer Schrock <sschrock@google.com> --------- Signed-off-by: Spencer Schrock <sschrock@google.com>
2024-08-15 11:20:30 +03:00 · 2023-11-15 11:01:53 -08:00 · 2023-11-15 11:01:53 -08:00 · 288319ad12
commit 288319ad12
parent ea626de830
3 changed files with 133 additions and 5 deletions
--- a/.github/workflows/scdiff.yml
+++ b/.github/workflows/scdiff.yml
@ -0,0 +1,115 @@
+name: scdiff PR evaluation
+on:
+  issue_comment:
+    types: [created, edited]
+
+permissions: read-all
+
+env:
+  GO_VERSION: 1.21
+
+jobs:
+  share-link:
+    if: ${{ (github.event.issue.pull_request) && (contains(github.event.comment.body, '/scdiff generate')) }}
+    runs-on: [ubuntu-latest]
+    permissions:
+      pull-requests: write # to create the PR comment
+    steps:
+      - name: share link to workflow run
+        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1
+        with:
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: `[Here's a link to the scdiff run](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})`
+            })
+
+  golden-test:
+    if: ${{ (github.event.issue.pull_request) && (contains(github.event.comment.body, '/scdiff generate')) }}
+    runs-on: [ubuntu-latest]
+    steps:
+      - name: create file of repos to anlayze
+        run: |
+          cat <<EOF > $HOME/repos.txt
+          https://github.com/airbnb/lottie-web
+          https://github.com/apache/tomcat
+          https://github.com/Azure/azure-functions-dotnet-worker
+          https://github.com/cncf/xds
+          https://github.com/google/go-cmp
+          https://github.com/google/highwayhash
+          https://github.com/googleapis/google-api-php-client
+          https://github.com/jacoco/jacoco
+          https://github.com/ossf/scorecard
+          https://github.com/pallets/jinja
+          https://github.com/polymer/polymer
+          https://github.com/rust-random/getrandom
+          https://github.com/yaml/libyaml
+          https://gitlab.com/baserow/baserow
+          https://gitlab.com/cryptsetup/cryptsetup
+          EOF
+      # use shell syntax to escape, since the checks arg goes to CLI when calling scdiff
+      - name: escape comment body
+        id: comment
+        env:
+          BODY: ${{ github.event.comment.body }}
+        run: |
+          echo "body=$BODY" >> $GITHUB_OUTPUT
+      - name: configure scdiff
+        id: config
+        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1
+        with:
+          script: |
+            const allowedAssociations = ["COLLABORATOR", "MEMBER", "OWNER"];
+            authorAssociation = '${{ github.event.comment.author_association }}'
+            if (!allowedAssociations.includes(authorAssociation)) {
+              core.setFailed("You don't have access to run scdiff");
+            }
+
+            const response = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number,
+            })
+            core.setOutput('base', response.data.base.sha)
+            core.setOutput('head', response.data.head.sha)
+
+            checks = '""'
+            const commentBody = '${{ steps.comment.outputs.body }}'
+            const regex = /\/scdiff generate ([^ ]+)/;
+            const found = commentBody.match(regex);
+            if (found && found.length == 2) {
+              checks = found[1]
+            }
+            core.setOutput('checks', checks)
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ steps.config.outputs.base }}
+      - name: Setup Go
+        uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          check-latest: true
+      - name: generate before results
+        env:
+          GITHUB_AUTH_TOKEN: ${{ secrets.GH_AUTH_TOKEN }}
+          GITLAB_AUTH_TOKEN: ${{ secrets.GITLAB_TOKEN }}
+        run: |
+          go run cmd/internal/scdiff/main.go generate \
+            --repos $HOME/repos.txt \
+            --checks ${{ steps.config.outputs.checks }} > $HOME/before.json
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ steps.config.outputs.head }}
+      - name: generate after results
+        env:
+          GITHUB_AUTH_TOKEN: ${{ secrets.GH_AUTH_TOKEN }}
+          GITLAB_AUTH_TOKEN: ${{ secrets.GITLAB_TOKEN }}
+        run: |
+          go run cmd/internal/scdiff/main.go generate \
+            --repos $HOME/repos.txt \
+            --checks ${{ steps.config.outputs.checks }} > $HOME/after.json
+      - name: compare results
+        run: |
+          go run cmd/internal/scdiff/main.go compare $HOME/before.json $HOME/after.json
--- a/cmd/internal/scdiff/app/runner/runner.go
+++ b/cmd/internal/scdiff/app/runner/runner.go
@ -16,13 +16,16 @@ package runner

 import (
 	"context"
+	"errors"
 	"strings"

 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/checks"
 	"github.com/ossf/scorecard/v4/clients"
 	"github.com/ossf/scorecard/v4/clients/githubrepo"
+	"github.com/ossf/scorecard/v4/clients/gitlabrepo"
 	"github.com/ossf/scorecard/v4/clients/ossfuzz"
+	sce "github.com/ossf/scorecard/v4/errors"
 	"github.com/ossf/scorecard/v4/log"
 	"github.com/ossf/scorecard/v4/pkg"
 )
@ -37,7 +40,8 @@ type Runner struct {
 	ctx           context.Context
 	logger        *log.Logger
 	enabledChecks checker.CheckNameToFnMap
-	repoClient    clients.RepoClient
+	githubClient  clients.RepoClient
+	gitlabClient  clients.RepoClient
 	ossFuzz       clients.RepoClient
 	cii           clients.CIIBestPracticesClient
 	vuln          clients.VulnerabilitiesClient
@ -47,10 +51,15 @@ type Runner struct {
 func New(enabledChecks []string) Runner {
 	ctx := context.Background()
 	logger := log.NewLogger(log.DefaultLevel)
+	gitlabClient, err := gitlabrepo.CreateGitlabClient(ctx, "https://gitlab.com")
+	if err != nil {
+		logger.Error(err, "creating gitlab client")
+	}
 	return Runner{
 		ctx:           ctx,
 		logger:        logger,
-		repoClient:    githubrepo.CreateGithubRepoClient(ctx, logger),
+		githubClient:  githubrepo.CreateGithubRepoClient(ctx, logger),
+		gitlabClient:  gitlabClient,
 		ossFuzz:       ossfuzz.CreateOSSFuzzClient(ossfuzz.StatusURL),
 		cii:           clients.DefaultCIIBestPracticesClient(),
 		vuln:          clients.DefaultVulnerabilitiesClient(),
@ -61,12 +70,16 @@ func New(enabledChecks []string) Runner {
 //nolint:wrapcheck
 func (r *Runner) Run(repoURI string) (pkg.ScorecardResult, error) {
 	r.log("processing repo: " + repoURI)
-	// TODO (gitlab?)
+	repoClient := r.githubClient
 	repo, err := githubrepo.MakeGithubRepo(repoURI)
+	if errors.Is(err, sce.ErrorUnsupportedHost) {
+		repo, err = gitlabrepo.MakeGitlabRepo(repoURI)
+		repoClient = r.gitlabClient
+	}
 	if err != nil {
 		return pkg.ScorecardResult{}, err
 	}
-	return pkg.RunScorecard(r.ctx, repo, commit, commitDepth, r.enabledChecks, r.repoClient, r.ossFuzz, r.cii, r.vuln)
+	return pkg.RunScorecard(r.ctx, repo, commit, commitDepth, r.enabledChecks, repoClient, r.ossFuzz, r.cii, r.vuln)
 }

 // logs only if logger is set.
--- a/cmd/internal/scdiff/app/runner/runner_test.go
+++ b/cmd/internal/scdiff/app/runner/runner_test.go
@ -46,7 +46,7 @@ func TestRunner_Run(t *testing.T) {
 	mockRepo.EXPECT().Close().Return(nil)
 	r := Runner{
 		enabledChecks: checker.CheckNameToFnMap{},
-		repoClient:    mockRepo,
+		githubClient:  mockRepo,
 	}
 	const repo = "github.com/foo/bar"
 	result, err := r.Run(repo)