✨ Gitlab: CI-Tests check (#2833)

* gitlab: support ci-tests Signed-off-by: Raghav Kaul <raghavkaul@google.com> * update Signed-off-by: Raghav Kaul <raghavkaul@google.com> * update test Signed-off-by: Raghav Kaul <raghavkaul@google.com> * update gitlab workflows Signed-off-by: Raghav Kaul <raghavkaul@google.com> * fix test Signed-off-by: Raghav Kaul <raghavkaul@google.com> --------- Signed-off-by: Raghav Kaul <raghavkaul@google.com> Signed-off-by: raghavkaul <8695110+raghavkaul@users.noreply.github.com>
2024-08-16 11:50:37 +03:00 · 2023-04-24 13:58:27 -04:00 · 2023-04-24 13:58:27 -04:00 · 46c6fe700c
commit 46c6fe700c
parent a4e72a8696
8 changed files with 157 additions and 22 deletions
--- a/.github/workflows/gitlab.yml
+++ b/.github/workflows/gitlab.yml
@ -0,0 +1,63 @@
+# Copyright 2023 OpenSSF Scorecard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: gitlab-tests
+
+permissions: read-all
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  gitlab-integration-trusted:
+    runs-on: ubuntu-latest
+    environment: gitlab
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@03bee3930647ebbf994244c21ddbc0d4933aab4f # v1
+        with:
+          egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+      - name: Clone the code
+        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab
+        with:
+           fetch-depth: 0
+
+      - name: setup-go
+        uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v2.2.0
+        with:
+          go-version: '1.19'
+          check-latest: true
+
+      - name: Prepare test env
+        run: |
+            go mod download
+
+      - name: Run GitLab PAT E2E  #using retry because the GitHub token is being throttled.
+        uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd
+        env:
+          GITLAB_AUTH_TOKEN: ${{ secrets.GITLAB_TOKEN }}
+        with:
+          max_attempts: 3
+          retry_on: error
+          timeout_minutes: 30
+          command: make e2e-gitlab-token
+
+      - name: codecov
+        uses: codecov/codecov-action@40a12dcee2df644d47232dde008099a3e9e4f865 # 2.1.0
+        with:
+         files: ./e2e-coverage.out
+         verbose: true
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@ -56,6 +56,22 @@ jobs:
        run: |
            go mod download

+      - name: Run GitLab E2E  #using retry because the GitHub token is being throttled.
+        uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd
+        env:
+          GITLAB_AUTH_TOKEN: ${{ secrets.GITLAB_TOKEN }}
+        with:
+          max_attempts: 3
+          retry_on: error
+          timeout_minutes: 30
+          command: make e2e-gitlab
+
+      - name: codecov
+        uses: codecov/codecov-action@894ff025c7b54547a9a2a1e9f228beae737ad3c2 # 2.1.0
+        with:
+         files: ./e2e-coverage.out
+         verbose: true
+
      - name: Run GITHUB_TOKEN E2E  #using retry because the GitHub token is being throttled.
        uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd
        env:
@ -72,16 +88,3 @@ jobs:
         files: ./e2e-coverage.out
         verbose: true
      
-      - name: Run GitLab E2E  #using retry because the GitHub token is being throttled.
-        uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd
-        with:
-          max_attempts: 3
-          retry_on: error
-          timeout_minutes: 30
-          command: make e2e-gitlab
-
-      - name: codecov
-        uses: codecov/codecov-action@894ff025c7b54547a9a2a1e9f228beae737ad3c2 # 2.1.0
-        with:
-         files: ./e2e-coverage.out
-         verbose: true
--- a/13
+++ b/13
@ -324,6 +324,11 @@ ifndef GITHUB_AUTH_TOKEN
 	$(error GITHUB_AUTH_TOKEN is undefined)
 endif

+check-env-gitlab:
+ifndef GITLAB_AUTH_TOKEN
+	$(error GITLAB_AUTH_TOKEN is undefined)
+endif
+
 e2e-pat: ## Runs e2e tests. Requires GITHUB_AUTH_TOKEN env var to be set to GitHub personal access token
 e2e-pat: build-scorecard check-env | $(GINKGO)
 	# Run e2e tests. GITHUB_AUTH_TOKEN with personal access token must be exported to run this
@ -335,10 +340,12 @@ e2e-gh-token: build-scorecard check-env | $(GINKGO)
 	TOKEN_TYPE="GITHUB_TOKEN" $(GINKGO) --race -p -v -cover -coverprofile=e2e-coverage.out --keep-separate-coverprofiles ./...

 e2e-gitlab-token: ## Runs e2e tests that require a GITLAB_TOKEN
-	TEST_GITLAB_EXTERNAL=1 TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv --focus '.*GitLab Token' ./...
+e2e-gitlab-token: build-scorecard check-env-gitlab | $(GINKGO)
+	TEST_GITLAB_EXTERNAL=1 TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv --focus '.*GitLab' ./...

 e2e-gitlab: ## Runs e2e tests for GitLab only. TOKEN_TYPE is not used (since these are public APIs), but must be set to something
-	TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv --focus '.*GitLab' ./...
+e2e-gitlab: build-scorecard | $(GINKGO)
+	TEST_GITLAB_EXTERNAL=1 TOKEN_TYPE="PAT" $(GINKGO) --race -p -vv --focus ".*GitLab" ./...

 e2e-attestor: ## Runs e2e tests for scorecard-attestor
 	cd attestor/e2e; go test -covermode=atomic -coverprofile=e2e-coverage.out; cd ../..
@ -439,4 +446,4 @@ cron-github-server-ko: | $(KO) $(KOCACHE_PATH)
 			   --tags latest,$(GIT_VERSION),$(GIT_HASH) \
 			   github.com/ossf/scorecard/v4/clients/githubrepo/roundtripper/tokens/server

-###############################################################################
+###############################################################################
--- a/checks/evaluation/ci_tests.go
+++ b/checks/evaluation/ci_tests.go
@ -61,6 +61,7 @@ func CITests(name string, c *checker.CITestData, dl checker.DetailLogger) checke
 		if !foundCI {
 			// Log message says commit, but really we only care about PRs, and
 			// use only one commit (branch HEAD) to refer to all commits in a PR
+
 			dl.Debug(&checker.LogMessage{
 				Text: fmt.Sprintf("merged PR %d without CI test at HEAD: %s", r.PullRequestNumber, r.HeadSHA),
 			})
--- a/clients/gitlabrepo/checkruns.go
+++ b/clients/gitlabrepo/checkruns.go
@ -47,11 +47,11 @@ func checkRunsFrom(data []*gitlab.PipelineInfo, ref string) []clients.CheckRun {
 	var checkRuns []clients.CheckRun
 	for _, pipelineInfo := range data {
 		if strings.EqualFold(pipelineInfo.Ref, ref) {
+			// TODO: Can get more info from GitLab API here (e.g. pipeline name, URL)
+			// https://docs.gitlab.com/ee/api/pipelines.html#get-a-pipelines-test-report
 			checkRuns = append(checkRuns, clients.CheckRun{
-				Status:     pipelineInfo.Status,
-				Conclusion: "",
-				URL:        pipelineInfo.WebURL,
-				App:        clients.CheckRunApp{Slug: pipelineInfo.Source},
+				Status: pipelineInfo.Status,
+				URL:    pipelineInfo.WebURL,
 			})
 		}
 	}
--- a/clients/gitlabrepo/statuses.go
+++ b/clients/gitlabrepo/statuses.go
@ -46,7 +46,7 @@ func statusFromData(commitStatuses []*gitlab.CommitStatus) []clients.Status {
 	for _, commitStatus := range commitStatuses {
 		statuses = append(statuses, clients.Status{
 			State:     commitStatus.Status,
-			Context:   fmt.Sprint(commitStatus.ID),
+			Context:   commitStatus.Name,
 			URL:       commitStatus.TargetURL,
 			TargetURL: commitStatus.TargetURL,
 		})
--- a/docs/checks.md
+++ b/docs/checks.md
@ -130,7 +130,8 @@ Risk: `Low` (possible unknown vulnerabilities)

 This check tries to determine if the project runs tests before pull requests are
 merged. It is currently limited to repositories hosted on GitHub, and does not
-support other source hosting repositories (i.e., Forges).
+support other source hosting repositories (i.e., Forges). All commits that are
+part of a PR must be tested by a CI Test for the check to pass.

 Running tests helps developers catch mistakes early on, which can reduce the
 number of vulnerabilities that find their way into a project.
--- a/e2e/ci_tests_test.go
+++ b/e2e/ci_tests_test.go
@ -16,6 +16,7 @@ package e2e

 import (
 	"context"
+	"os"

 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@ -24,6 +25,7 @@ import (
 	"github.com/ossf/scorecard/v4/checks"
 	"github.com/ossf/scorecard/v4/clients"
 	"github.com/ossf/scorecard/v4/clients/githubrepo"
+	"github.com/ossf/scorecard/v4/clients/gitlabrepo"
 	scut "github.com/ossf/scorecard/v4/utests"
 )

@ -101,5 +103,63 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() {
 			Expect(scut.ValidateTestReturn(nil, "CI tests run", &expected, &result, &dl)).Should(BeTrue())
 			Expect(repoClient.Close()).Should(BeNil())
 		})
+		It("Should return use of CI tests at commit - GitLab", func() {
+			skipIfTokenIsNot(gitlabPATTokenType, "GitLab only")
+
+			dl := scut.TestDetailLogger{}
+			repo, err := gitlabrepo.MakeGitlabRepo("gitlab.com/gitlab-org/gitlab")
+			Expect(err).Should(BeNil())
+			repoClient, err := gitlabrepo.CreateGitlabClientWithToken(context.Background(), os.Getenv("GITLAB_AUTH_TOKEN"), repo)
+			Expect(err).Should(BeNil())
+			// url to commit is https://gitlab.com/gitlab-org/gitlab/-/commit/8ae23fa220d73fa07501aabd94214c9e83fe61a0
+			err = repoClient.InitRepo(repo, "8ae23fa220d73fa07501aabd94214c9e83fe61a0", 0)
+			Expect(err).Should(BeNil())
+			req := checker.CheckRequest{
+				Ctx:        context.Background(),
+				RepoClient: repoClient,
+				Repo:       repo,
+				Dlogger:    &dl,
+			}
+			expected := scut.TestReturn{
+				Error:         nil,
+				Score:         0,
+				NumberOfWarn:  0,
+				NumberOfInfo:  0,
+				NumberOfDebug: 13,
+			}
+			result := checks.CITests(&req)
+			Expect(result.Score).Should(BeNumerically("==", expected.Score))
+			Expect(result.Error).Should(BeNil())
+			Expect(repoClient.Close()).Should(BeNil())
+		})
+		It("Should return use of CI tests at commit - GitLab", func() {
+			skipIfTokenIsNot(gitlabPATTokenType, "GitLab only")
+
+			dl := scut.TestDetailLogger{}
+			repo, err := gitlabrepo.MakeGitlabRepo("gitlab.com/fdroid/fdroidclient")
+			Expect(err).Should(BeNil())
+			repoClient, err := gitlabrepo.CreateGitlabClientWithToken(context.Background(), os.Getenv("GITLAB_AUTH_TOKEN"), repo)
+			Expect(err).Should(BeNil())
+			// url to commit is https://gitlab.com/fdroid/fdroidclient/-/commit/a1d33881902cee33586a4fd4ee1538042a7bdedf
+			err = repoClient.InitRepo(repo, "a1d33881902cee33586a4fd4ee1538042a7bdedf", 0)
+			Expect(err).Should(BeNil())
+			req := checker.CheckRequest{
+				Ctx:        context.Background(),
+				RepoClient: repoClient,
+				Repo:       repo,
+				Dlogger:    &dl,
+			}
+			expected := scut.TestReturn{
+				Error:         nil,
+				Score:         2,
+				NumberOfWarn:  0,
+				NumberOfInfo:  0,
+				NumberOfDebug: 1,
+			}
+			result := checks.CITests(&req)
+			Expect(result.Score).Should(BeNumerically("==", expected.Score))
+			Expect(result.Error).Should(BeNil())
+			Expect(repoClient.Close()).Should(BeNil())
+		})
 	})
 })