diff --git a/.github/workflows/gitlab.yml b/.github/workflows/gitlab.yml new file mode 100644 index 00000000..d90d41ad --- /dev/null +++ b/.github/workflows/gitlab.yml @@ -0,0 +1,63 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: gitlab-tests + +permissions: read-all + +on: + push: + branches: + - main + +jobs: + gitlab-integration-trusted: + runs-on: ubuntu-latest + environment: gitlab + steps: + - name: Harden Runner + uses: step-security/harden-runner@03bee3930647ebbf994244c21ddbc0d4933aab4f # v1 + with: + egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + + - name: Clone the code + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab + with: + fetch-depth: 0 + + - name: setup-go + uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v2.2.0 + with: + go-version: '1.19' + check-latest: true + + - name: Prepare test env + run: | + go mod download + + - name: Run GitLab PAT E2E #using retry because the GitHub token is being throttled. + uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd + env: + GITLAB_AUTH_TOKEN: ${{ secrets.GITLAB_TOKEN }} + with: + max_attempts: 3 + retry_on: error + timeout_minutes: 30 + command: make e2e-gitlab-token + + - name: codecov + uses: codecov/codecov-action@40a12dcee2df644d47232dde008099a3e9e4f865 # 2.1.0 + with: + files: ./e2e-coverage.out + verbose: true \ No newline at end of file diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 62fc26e7..1585ef24 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -56,6 +56,22 @@ jobs: run: | go mod download + - name: Run GitLab E2E #using retry because the GitHub token is being throttled. + uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd + env: + GITLAB_AUTH_TOKEN: ${{ secrets.GITLAB_TOKEN }} + with: + max_attempts: 3 + retry_on: error + timeout_minutes: 30 + command: make e2e-gitlab + + - name: codecov + uses: codecov/codecov-action@894ff025c7b54547a9a2a1e9f228beae737ad3c2 # 2.1.0 + with: + files: ./e2e-coverage.out + verbose: true + - name: Run GITHUB_TOKEN E2E #using retry because the GitHub token is being throttled. uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd env: @@ -72,16 +88,3 @@ jobs: files: ./e2e-coverage.out verbose: true - - name: Run GitLab E2E #using retry because the GitHub token is being throttled. - uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - with: - max_attempts: 3 - retry_on: error - timeout_minutes: 30 - command: make e2e-gitlab - - - name: codecov - uses: codecov/codecov-action@894ff025c7b54547a9a2a1e9f228beae737ad3c2 # 2.1.0 - with: - files: ./e2e-coverage.out - verbose: true diff --git a/Makefile b/Makefile index 9f8eccc4..1e4880f0 100644 --- a/Makefile +++ b/Makefile @@ -324,6 +324,11 @@ ifndef GITHUB_AUTH_TOKEN $(error GITHUB_AUTH_TOKEN is undefined) endif +check-env-gitlab: +ifndef GITLAB_AUTH_TOKEN + $(error GITLAB_AUTH_TOKEN is undefined) +endif + e2e-pat: ## Runs e2e tests. Requires GITHUB_AUTH_TOKEN env var to be set to GitHub personal access token e2e-pat: build-scorecard check-env | $(GINKGO) # Run e2e tests. GITHUB_AUTH_TOKEN with personal access token must be exported to run this @@ -335,10 +340,12 @@ e2e-gh-token: build-scorecard check-env | $(GINKGO) TOKEN_TYPE="GITHUB_TOKEN" $(GINKGO) --race -p -v -cover -coverprofile=e2e-coverage.out --keep-separate-coverprofiles ./... e2e-gitlab-token: ## Runs e2e tests that require a GITLAB_TOKEN - TEST_GITLAB_EXTERNAL=1 TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv --focus '.*GitLab Token' ./... +e2e-gitlab-token: build-scorecard check-env-gitlab | $(GINKGO) + TEST_GITLAB_EXTERNAL=1 TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv --focus '.*GitLab' ./... e2e-gitlab: ## Runs e2e tests for GitLab only. TOKEN_TYPE is not used (since these are public APIs), but must be set to something - TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv --focus '.*GitLab' ./... +e2e-gitlab: build-scorecard | $(GINKGO) + TEST_GITLAB_EXTERNAL=1 TOKEN_TYPE="PAT" $(GINKGO) --race -p -vv --focus ".*GitLab" ./... e2e-attestor: ## Runs e2e tests for scorecard-attestor cd attestor/e2e; go test -covermode=atomic -coverprofile=e2e-coverage.out; cd ../.. @@ -439,4 +446,4 @@ cron-github-server-ko: | $(KO) $(KOCACHE_PATH) --tags latest,$(GIT_VERSION),$(GIT_HASH) \ github.com/ossf/scorecard/v4/clients/githubrepo/roundtripper/tokens/server -############################################################################### +############################################################################### \ No newline at end of file diff --git a/checks/evaluation/ci_tests.go b/checks/evaluation/ci_tests.go index b6686203..9033e13f 100644 --- a/checks/evaluation/ci_tests.go +++ b/checks/evaluation/ci_tests.go @@ -61,6 +61,7 @@ func CITests(name string, c *checker.CITestData, dl checker.DetailLogger) checke if !foundCI { // Log message says commit, but really we only care about PRs, and // use only one commit (branch HEAD) to refer to all commits in a PR + dl.Debug(&checker.LogMessage{ Text: fmt.Sprintf("merged PR %d without CI test at HEAD: %s", r.PullRequestNumber, r.HeadSHA), }) diff --git a/clients/gitlabrepo/checkruns.go b/clients/gitlabrepo/checkruns.go index 3508f4d3..3fe06f48 100644 --- a/clients/gitlabrepo/checkruns.go +++ b/clients/gitlabrepo/checkruns.go @@ -47,11 +47,11 @@ func checkRunsFrom(data []*gitlab.PipelineInfo, ref string) []clients.CheckRun { var checkRuns []clients.CheckRun for _, pipelineInfo := range data { if strings.EqualFold(pipelineInfo.Ref, ref) { + // TODO: Can get more info from GitLab API here (e.g. pipeline name, URL) + // https://docs.gitlab.com/ee/api/pipelines.html#get-a-pipelines-test-report checkRuns = append(checkRuns, clients.CheckRun{ - Status: pipelineInfo.Status, - Conclusion: "", - URL: pipelineInfo.WebURL, - App: clients.CheckRunApp{Slug: pipelineInfo.Source}, + Status: pipelineInfo.Status, + URL: pipelineInfo.WebURL, }) } } diff --git a/clients/gitlabrepo/statuses.go b/clients/gitlabrepo/statuses.go index 317df54d..1b1ede68 100644 --- a/clients/gitlabrepo/statuses.go +++ b/clients/gitlabrepo/statuses.go @@ -46,7 +46,7 @@ func statusFromData(commitStatuses []*gitlab.CommitStatus) []clients.Status { for _, commitStatus := range commitStatuses { statuses = append(statuses, clients.Status{ State: commitStatus.Status, - Context: fmt.Sprint(commitStatus.ID), + Context: commitStatus.Name, URL: commitStatus.TargetURL, TargetURL: commitStatus.TargetURL, }) diff --git a/docs/checks.md b/docs/checks.md index fe95ebd2..1349ba70 100644 --- a/docs/checks.md +++ b/docs/checks.md @@ -130,7 +130,8 @@ Risk: `Low` (possible unknown vulnerabilities) This check tries to determine if the project runs tests before pull requests are merged. It is currently limited to repositories hosted on GitHub, and does not -support other source hosting repositories (i.e., Forges). +support other source hosting repositories (i.e., Forges). All commits that are +part of a PR must be tested by a CI Test for the check to pass. Running tests helps developers catch mistakes early on, which can reduce the number of vulnerabilities that find their way into a project. diff --git a/e2e/ci_tests_test.go b/e2e/ci_tests_test.go index 2c691914..27550374 100644 --- a/e2e/ci_tests_test.go +++ b/e2e/ci_tests_test.go @@ -16,6 +16,7 @@ package e2e import ( "context" + "os" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -24,6 +25,7 @@ import ( "github.com/ossf/scorecard/v4/checks" "github.com/ossf/scorecard/v4/clients" "github.com/ossf/scorecard/v4/clients/githubrepo" + "github.com/ossf/scorecard/v4/clients/gitlabrepo" scut "github.com/ossf/scorecard/v4/utests" ) @@ -101,5 +103,63 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() { Expect(scut.ValidateTestReturn(nil, "CI tests run", &expected, &result, &dl)).Should(BeTrue()) Expect(repoClient.Close()).Should(BeNil()) }) + It("Should return use of CI tests at commit - GitLab", func() { + skipIfTokenIsNot(gitlabPATTokenType, "GitLab only") + + dl := scut.TestDetailLogger{} + repo, err := gitlabrepo.MakeGitlabRepo("gitlab.com/gitlab-org/gitlab") + Expect(err).Should(BeNil()) + repoClient, err := gitlabrepo.CreateGitlabClientWithToken(context.Background(), os.Getenv("GITLAB_AUTH_TOKEN"), repo) + Expect(err).Should(BeNil()) + // url to commit is https://gitlab.com/gitlab-org/gitlab/-/commit/8ae23fa220d73fa07501aabd94214c9e83fe61a0 + err = repoClient.InitRepo(repo, "8ae23fa220d73fa07501aabd94214c9e83fe61a0", 0) + Expect(err).Should(BeNil()) + req := checker.CheckRequest{ + Ctx: context.Background(), + RepoClient: repoClient, + Repo: repo, + Dlogger: &dl, + } + expected := scut.TestReturn{ + Error: nil, + Score: 0, + NumberOfWarn: 0, + NumberOfInfo: 0, + NumberOfDebug: 13, + } + result := checks.CITests(&req) + Expect(result.Score).Should(BeNumerically("==", expected.Score)) + Expect(result.Error).Should(BeNil()) + Expect(repoClient.Close()).Should(BeNil()) + }) + It("Should return use of CI tests at commit - GitLab", func() { + skipIfTokenIsNot(gitlabPATTokenType, "GitLab only") + + dl := scut.TestDetailLogger{} + repo, err := gitlabrepo.MakeGitlabRepo("gitlab.com/fdroid/fdroidclient") + Expect(err).Should(BeNil()) + repoClient, err := gitlabrepo.CreateGitlabClientWithToken(context.Background(), os.Getenv("GITLAB_AUTH_TOKEN"), repo) + Expect(err).Should(BeNil()) + // url to commit is https://gitlab.com/fdroid/fdroidclient/-/commit/a1d33881902cee33586a4fd4ee1538042a7bdedf + err = repoClient.InitRepo(repo, "a1d33881902cee33586a4fd4ee1538042a7bdedf", 0) + Expect(err).Should(BeNil()) + req := checker.CheckRequest{ + Ctx: context.Background(), + RepoClient: repoClient, + Repo: repo, + Dlogger: &dl, + } + expected := scut.TestReturn{ + Error: nil, + Score: 2, + NumberOfWarn: 0, + NumberOfInfo: 0, + NumberOfDebug: 1, + } + result := checks.CITests(&req) + Expect(result.Score).Should(BeNumerically("==", expected.Score)) + Expect(result.Error).Should(BeNil()) + Expect(repoClient.Close()).Should(BeNil()) + }) }) })