From 53c056081b7b06a38c604829b8967bf01f661e10 Mon Sep 17 00:00:00 2001 From: laurentsimon <64505099+laurentsimon@users.noreply.github.com> Date: Wed, 21 Jul 2021 13:40:16 -0700 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20[migration=20to=20score]=205:=20con?= =?UTF-8?q?tributors,=20vulnerabilities,=20packaging=20and=20sast=20(#729)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * contributors * packaging * vulnerabilities * fix errors * err * errors --- checker/check_result.go | 7 ++- checks/branch_protected.go | 6 +-- checks/code_review.go | 7 ++- checks/contributors.go | 32 ++++++-------- checks/packaging.go | 50 ++++++++++----------- checks/sast.go | 86 ++++++++++++++++++++++++------------- checks/vulnerabilities.go | 30 +++++++------ e2e/contributors_test.go | 37 +++++++++++++--- e2e/packaging_test.go | 21 +++++++-- e2e/sast_test.go | 20 +++++++-- e2e/vulnerabilities_test.go | 35 ++++++++++++--- 11 files changed, 216 insertions(+), 115 deletions(-) diff --git a/checker/check_result.go b/checker/check_result.go index 26b9a938..79f67995 100644 --- a/checker/check_result.go +++ b/checker/check_result.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "math" + "strings" scorecarderrors "github.com/ossf/scorecard/errors" ) @@ -213,10 +214,12 @@ func MakeOrResult(c *CheckRequest, checks ...CheckResult) CheckResult { //nolint for _, result := range checks[1:] { if result.Score >= bestResult.Score { - c.Dlogger.Info(bestResult.Reason) + i := strings.Index(bestResult.Reason, "-- score normalized") + c.Dlogger.Info(bestResult.Reason[:i]) bestResult = result } else { - c.Dlogger.Info(result.Reason) + i := strings.Index(result.Reason, "-- score normalized") + c.Dlogger.Info(result.Reason[:i]) } // Do not exit early so we can show all the details diff --git a/checks/branch_protected.go b/checks/branch_protected.go index 73f7d59f..142ff3cd 100644 --- a/checks/branch_protected.go +++ b/checks/branch_protected.go @@ -86,8 +86,7 @@ func checkReleaseAndDevBranchProtection(ctx context.Context, r repositories, dl name, err := resolveBranchName(branches, *release.TargetCommitish) if err != nil { // If the commitish branch is still not found, fail. - e := sce.Create(sce.ErrScorecardInternal, errInternalBranchNotFound.Error()) - r := checker.CreateRuntimeErrorResult(CheckBranchProtection, e) + r := checker.CreateRuntimeErrorResult(CheckBranchProtection, err) checks = append(checks, r) continue } @@ -107,8 +106,7 @@ func checkReleaseAndDevBranchProtection(ctx context.Context, r repositories, dl for b := range checkBranches { protected, err := isBranchProtected(branches, b) if err != nil { - e := sce.Create(sce.ErrScorecardInternal, errInternalBranchNotFound.Error()) - r := checker.CreateRuntimeErrorResult(CheckBranchProtection, e) + r := checker.CreateRuntimeErrorResult(CheckBranchProtection, err) checks = append(checks, r) } if !protected { diff --git a/checks/code_review.go b/checks/code_review.go index fde6a8a7..efc2ce8b 100644 --- a/checks/code_review.go +++ b/checks/code_review.go @@ -22,6 +22,7 @@ import ( "github.com/shurcooL/githubv4" "github.com/ossf/scorecard/checker" + sce "github.com/ossf/scorecard/errors" ) const ( @@ -82,7 +83,8 @@ func DoesCodeReview(c *checker.CheckRequest) checker.CheckResult { "labelsToAnalyze": githubv4.Int(labelsToAnalyze), } if err := c.GraphClient.Query(c.Ctx, &prHistory, vars); err != nil { - return checker.CreateRuntimeErrorResult(CheckCodeReview, err) + e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("c.GraphClient.Query: %v", err)) + return checker.CreateRuntimeErrorResult(CheckCodeReview, e) } return checker.MultiCheckOr2( isPrReviewRequired, @@ -161,7 +163,8 @@ func prowCodeReview(c *checker.CheckRequest) checker.CheckResult { func commitMessageHints(c *checker.CheckRequest) checker.CheckResult { commits, _, err := c.Client.Repositories.ListCommits(c.Ctx, c.Owner, c.Repo, &github.CommitsListOptions{}) if err != nil { - return checker.CreateRuntimeErrorResult(CheckCodeReview, err) + e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("Client.Repositories.ListCommits: %v", err)) + return checker.CreateRuntimeErrorResult(CheckCodeReview, e) } total := 0 diff --git a/checks/contributors.go b/checks/contributors.go index 7a4d1b32..f1030333 100644 --- a/checks/contributors.go +++ b/checks/contributors.go @@ -15,16 +15,18 @@ package checks import ( + "fmt" "strings" "github.com/google/go-github/v32/github" "github.com/ossf/scorecard/checker" + sce "github.com/ossf/scorecard/errors" ) const ( - minContributionsPerUser = 5 - minOrganizationCount = 2 + minContributionsPerUser = 5 + numberCompaniesForTopScore = 3 // CheckContributors is the registered name for Contributors. CheckContributors = "Contributors" ) @@ -37,7 +39,8 @@ func init() { func Contributors(c *checker.CheckRequest) checker.CheckResult { contribs, _, err := c.Client.Repositories.ListContributors(c.Ctx, c.Owner, c.Repo, &github.ListContributorsOptions{}) if err != nil { - return checker.MakeRetryResult(CheckContributors, err) + e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("Client.Repositories.ListContributors: %v", err)) + return checker.CreateRuntimeErrorResult(CheckContributors, e) } companies := map[string]struct{}{} @@ -47,11 +50,12 @@ func Contributors(c *checker.CheckRequest) checker.CheckResult { } u, _, err := c.Client.Users.Get(c.Ctx, contrib.GetLogin()) if err != nil { - return checker.MakeRetryResult(CheckContributors, err) + e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("Client.Users.Get: %v", err)) + return checker.CreateRuntimeErrorResult(CheckContributors, e) } orgs, _, err := c.Client.Organizations.List(c.Ctx, contrib.GetLogin(), nil) if err != nil { - c.Logf("unable to get org members for %s", contrib.GetLogin()) + c.Dlogger.Debug("unable to get org members for %s: %v", contrib.GetLogin(), err) } else if len(orgs) > 0 { companies[*orgs[0].Login] = struct{}{} continue @@ -72,17 +76,9 @@ func Contributors(c *checker.CheckRequest) checker.CheckResult { for c := range companies { names = append(names, c) } - c.Logf("companies found: %v", strings.Join(names, ",")) - if len(companies) >= minOrganizationCount { - return checker.CheckResult{ - Name: CheckContributors, - Pass: true, - Confidence: checker.MaxResultConfidence, - } - } - return checker.CheckResult{ - Name: CheckContributors, - Pass: false, - Confidence: checker.MaxResultConfidence, - } + + c.Dlogger.Info("contributors work for: %v", strings.Join(names, ",")) + + reason := fmt.Sprintf("%d different companies found", len(companies)) + return checker.CreateProportionalScoreResult(CheckContributors, reason, len(companies), numberCompaniesForTopScore) } diff --git a/checks/packaging.go b/checks/packaging.go index 4bbcad70..c49198a5 100644 --- a/checks/packaging.go +++ b/checks/packaging.go @@ -15,6 +15,7 @@ package checks import ( + "fmt" "path/filepath" "regexp" "strings" @@ -22,6 +23,7 @@ import ( "github.com/google/go-github/v32/github" "github.com/ossf/scorecard/checker" + sce "github.com/ossf/scorecard/errors" ) // CheckPackaging is the registered name for Packaging. @@ -36,14 +38,16 @@ func Packaging(c *checker.CheckRequest) checker.CheckResult { _, dc, _, err := c.Client.Repositories.GetContents(c.Ctx, c.Owner, c.Repo, ".github/workflows", &github.RepositoryContentGetOptions{}) if err != nil { - return checker.MakeRetryResult(CheckPackaging, err) + e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("Client.Repositories.GetContents: %v", err)) + return checker.CreateRuntimeErrorResult(CheckPackaging, e) } for _, f := range dc { fp := f.GetPath() fo, _, _, err := c.Client.Repositories.GetContents(c.Ctx, c.Owner, c.Repo, fp, &github.RepositoryContentGetOptions{}) if err != nil { - return checker.MakeRetryResult(CheckPackaging, err) + e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("Client.Repositories.GetContents: %v", err)) + return checker.CreateRuntimeErrorResult(CheckPackaging, e) } if fo == nil { // path is a directory, not a file. skip. @@ -51,7 +55,8 @@ func Packaging(c *checker.CheckRequest) checker.CheckResult { } fc, err := fo.GetContent() if err != nil { - return checker.MakeRetryResult(CheckPackaging, err) + e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("fo.GetContent: %v", err)) + return checker.CreateRuntimeErrorResult(CheckPackaging, e) } if !isPackagingWorkflow(fc, fp, c) { @@ -63,24 +68,19 @@ func Packaging(c *checker.CheckRequest) checker.CheckResult { Status: "success", }) if err != nil { - return checker.MakeRetryResult(CheckPackaging, err) + e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("Client.Actions.ListWorkflowRunsByFileName: %v", err)) + return checker.CreateRuntimeErrorResult(CheckPackaging, e) } if *runs.TotalCount > 0 { - c.Logf("found a completed run: %s", runs.WorkflowRuns[0].GetHTMLURL()) - return checker.CheckResult{ - Name: CheckPackaging, - Pass: true, - Confidence: checker.MaxResultConfidence, - } + c.Dlogger.Info("workflow %v used in run: %s", fp, runs.WorkflowRuns[0].GetHTMLURL()) + return checker.CreateMaxScoreResult(CheckPackaging, + "packaging workflow detected") } - c.Logf("!! no run completed") + c.Dlogger.Info("workflow %v not used in runs", fp) } - return checker.CheckResult{ - Name: CheckPackaging, - Pass: false, - Confidence: checker.MaxResultConfidence, - } + return checker.CreateMinScoreResult(CheckPackaging, + "no packaging workflow used") } func isPackagingWorkflow(s, fp string, c *checker.CheckRequest) bool { @@ -90,42 +90,42 @@ func isPackagingWorkflow(s, fp string, c *checker.CheckRequest) bool { r2 := regexp.MustCompile(`(?s)npm.*publish`) if r1.MatchString(s) && r2.MatchString(s) { - c.Logf("found node packaging workflow using npm: %s", fp) + c.Dlogger.Info("candidate node packaging workflow using npm: %s", fp) return true } } if strings.Contains(s, "uses: actions/setup-java@") { - // java packages with maven + // Java packages with maven. r1 := regexp.MustCompile(`(?s)mvn.*deploy`) if r1.MatchString(s) { - c.Logf("found java packaging workflow using maven: %s", fp) + c.Dlogger.Info("candidate java packaging workflow using maven: %s", fp) return true } - // java packages with gradle + // Java packages with gradle. r2 := regexp.MustCompile(`(?s)gradle.*publish`) if r2.MatchString(s) { - c.Logf("found java packaging workflow using gradle: %s", fp) + c.Dlogger.Info("candidate java packaging workflow using gradle: %s", fp) return true } } if strings.Contains(s, "actions/setup-python@") && strings.Contains(s, "pypa/gh-action-pypi-publish@master") { - c.Logf("found python packaging workflow using pypi: %s", fp) + c.Dlogger.Info("candidate python packaging workflow using pypi: %s", fp) return true } if strings.Contains(s, "uses: docker/build-push-action@") { - c.Logf("found docker publishing workflow: %s", fp) + c.Dlogger.Info("candidate docker publishing workflow: %s", fp) return true } if strings.Contains(s, "docker push") { - c.Logf("found docker publishing workflow: %s", fp) + c.Dlogger.Info("candidate docker publishing workflow: %s", fp) return true } - c.Logf("!! not a packaging workflow: %s", fp) + c.Dlogger.Debug("not a packaging workflow: %s", fp) return false } diff --git a/checks/sast.go b/checks/sast.go index 72b22185..d1fbe2f7 100644 --- a/checks/sast.go +++ b/checks/sast.go @@ -15,26 +15,19 @@ package checks import ( - "errors" + "fmt" + "strings" "github.com/google/go-github/v32/github" "github.com/ossf/scorecard/checker" + sce "github.com/ossf/scorecard/errors" ) -const ( - // CheckSAST is the registered name for SAST. - CheckSAST = "SAST" - sastPassThreshold = .75 -) +// CheckSAST is the registered name for SAST. +const CheckSAST = "SAST" -var ( - sastTools = map[string]bool{"github-code-scanning": true, "sonarcloud": true} - // ErrorNoChecks indicates no GitHub Check runs were found for this repo. - ErrorNoChecks = errors.New("no check runs found") - // ErrorNoMerges indicates no merges with SAST tool runs were found for this repo. - ErrorNoMerges = errors.New("no merges found") -) +var sastTools = map[string]bool{"github-code-scanning": true, "sonarcloud": true} //nolint:gochecknoinits func init() { @@ -42,10 +35,38 @@ func init() { } func SAST(c *checker.CheckRequest) checker.CheckResult { - return checker.MultiCheckOr( - CodeQLInCheckDefinitions, - SASTToolInCheckRuns, - )(c) + r1 := SASTToolInCheckRuns(c) + r2 := CodeQLInCheckDefinitions(c) + if r1.Error2 != nil { + return r1 + } + if r2.Error2 != nil { + return r2 + } + // Merge the results. + var result checker.CheckResult + if r1.Score == checker.MaxResultScore { + // All commits have a SAST tool check run, + // score is maximum. + result = r1 + result.Score = 10 + i := strings.Index(r2.Reason, "-- score normalized") + c.Dlogger.Info(r2.Reason[:i]) + } else { + // Not all commits have a check run, + // We compute the final score as follows: + // 5 points for CodeQL enabled, 5 points for + // SAST run on commits. + result = r2 + + //nolint + result.Score = 5 + r1.Score/10*5 + result.Reason = "not all commits are checked with a SAST tool" + i := strings.Index(r1.Reason, "-- score normalized") + c.Dlogger.Info(r1.Reason[:i]) + } + + return result } func SASTToolInCheckRuns(c *checker.CheckRequest) checker.CheckResult { @@ -53,7 +74,8 @@ func SASTToolInCheckRuns(c *checker.CheckRequest) checker.CheckResult { State: "closed", }) if err != nil { - return checker.MakeRetryResult(CheckSAST, err) + e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("Client.PullRequests.List: %v", err)) + return checker.CreateRuntimeErrorResult(CheckSAST, e) } totalMerged := 0 @@ -66,10 +88,11 @@ func SASTToolInCheckRuns(c *checker.CheckRequest) checker.CheckResult { crs, _, err := c.Client.Checks.ListCheckRunsForRef(c.Ctx, c.Owner, c.Repo, pr.GetHead().GetSHA(), &github.ListCheckRunsOptions{}) if err != nil { - return checker.MakeRetryResult(CheckSAST, err) + e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("Client.Checks.ListCheckRunsForRef: %v", err)) + return checker.CreateRuntimeErrorResult(CheckSAST, e) } if crs == nil { - return checker.MakeInconclusiveResult(CheckSAST, ErrorNoChecks) + return checker.CreateInconclusiveResult(CheckSAST, "no merges detected") } for _, cr := range crs.CheckRuns { if cr.GetStatus() != "completed" { @@ -79,32 +102,35 @@ func SASTToolInCheckRuns(c *checker.CheckRequest) checker.CheckResult { continue } if sastTools[cr.GetApp().GetSlug()] { - c.Logf("SAST Tool found: %s", cr.GetHTMLURL()) + c.Dlogger.Info("tool detected: %s", cr.GetHTMLURL()) totalTested++ break } } } - if totalTested == 0 { - return checker.MakeInconclusiveResult(CheckSAST, ErrorNoMerges) + if totalMerged == 0 { + return checker.CreateInconclusiveResult(CheckSAST, "no merges detected") } - return checker.MakeProportionalResult(CheckSAST, totalTested, totalMerged, sastPassThreshold) + reason := fmt.Sprintf("%v commits out of %v are checked with a SAST tool", totalTested, totalMerged) + return checker.CreateProportionalScoreResult(CheckSAST, reason, totalTested, totalMerged) } func CodeQLInCheckDefinitions(c *checker.CheckRequest) checker.CheckResult { searchQuery := ("github/codeql-action path:/.github/workflows repo:" + c.Owner + "/" + c.Repo) results, _, err := c.Client.Search.Code(c.Ctx, searchQuery, &github.SearchOptions{}) if err != nil { - return checker.MakeRetryResult(CheckSAST, err) + e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("Client.Search.Code: %v", err)) + return checker.CreateRuntimeErrorResult(CheckSAST, e) } for _, result := range results.CodeResults { - c.Logf("found CodeQL definition: %s", result.GetPath()) + c.Dlogger.Info("CodeQL definition detected: %s", result.GetPath()) } - return checker.CheckResult{ - Name: CheckSAST, - Pass: *results.Total > 0, - Confidence: checker.MaxResultConfidence, + // TODO: check if it's enabled as cron or presubmit. + // TODO: check which branches it is enabled on. We should find main. + if *results.Total > 0 { + return checker.CreateMaxScoreResult(CheckSAST, "tool detected: CodeQL") } + return checker.CreateMinScoreResult(CheckSAST, "CodeQL tool not detected") } diff --git a/checks/vulnerabilities.go b/checks/vulnerabilities.go index a9453992..c2d6f7ae 100644 --- a/checks/vulnerabilities.go +++ b/checks/vulnerabilities.go @@ -17,13 +17,14 @@ package checks import ( "bytes" "encoding/json" - "errors" + "fmt" "net/http" "strings" "github.com/google/go-github/v32/github" "github.com/ossf/scorecard/checker" + sce "github.com/ossf/scorecard/errors" ) const ( @@ -32,9 +33,6 @@ const ( osvQueryEndpoint = "https://api.osv.dev/v1/query" ) -// ErrNoCommits is the error for when there are no commits found. -var ErrNoCommits = errors.New("no commits found") - type osvQuery struct { Commit string `json:"commit"` } @@ -65,44 +63,50 @@ func HasUnfixedVulnerabilities(c *checker.CheckRequest) checker.CheckResult { }, }) if err != nil { - return checker.MakeRetryResult(CheckVulnerabilities, err) + e := sce.Create(sce.ErrScorecardInternal, "Client.Repositories.ListCommits") + return checker.CreateRuntimeErrorResult(CheckVulnerabilities, e) } if len(commits) != 1 || commits[0].SHA == nil { - return checker.MakeInconclusiveResult(CheckVulnerabilities, ErrNoCommits) + return checker.CreateInconclusiveResult(CheckVulnerabilities, "no commits found") } query, err := json.Marshal(&osvQuery{ Commit: *commits[0].SHA, }) if err != nil { - panic("!! failed to marshal OSV query.") + e := sce.Create(sce.ErrScorecardInternal, "json.Marshal") + return checker.CreateRuntimeErrorResult(CheckVulnerabilities, e) } req, err := http.NewRequestWithContext(c.Ctx, http.MethodPost, osvQueryEndpoint, bytes.NewReader(query)) if err != nil { - return checker.MakeRetryResult(CheckVulnerabilities, err) + e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("http.NewRequestWithContext: %v", err)) + return checker.CreateRuntimeErrorResult(CheckVulnerabilities, e) } // Use our own http client as the one from CheckRequest adds GitHub tokens to the headers. httpClient := &http.Client{} resp, err := httpClient.Do(req) if err != nil { - return checker.MakeRetryResult(CheckVulnerabilities, err) + e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("httpClient.Do: %v", err)) + return checker.CreateRuntimeErrorResult(CheckVulnerabilities, e) } defer resp.Body.Close() var osvResp osvResponse decoder := json.NewDecoder(resp.Body) if err := decoder.Decode(&osvResp); err != nil { - return checker.MakeRetryResult(CheckVulnerabilities, err) + e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("decoder.Decode: %v", err)) + return checker.CreateRuntimeErrorResult(CheckVulnerabilities, e) } + // TODO: take severity into account. vulnIDs := osvResp.getVulnerabilities() if len(vulnIDs) > 0 { - c.Logf("HEAD is vulnerable to %s", strings.Join(vulnIDs, ", ")) - return checker.MakeFailResult(CheckVulnerabilities, nil) + c.Dlogger.Warn("HEAD is vulnerable to %s", strings.Join(vulnIDs, ", ")) + return checker.CreateMinScoreResult(CheckVulnerabilities, "unfixed OSV detected") } - return checker.MakePassResult(CheckVulnerabilities) + return checker.CreateMaxScoreResult(CheckVulnerabilities, "no OSV detected") } diff --git a/e2e/contributors_test.go b/e2e/contributors_test.go index 03b9a0f3..5c01647d 100644 --- a/e2e/contributors_test.go +++ b/e2e/contributors_test.go @@ -23,13 +23,14 @@ import ( "github.com/ossf/scorecard/checker" "github.com/ossf/scorecard/checks" + scut "github.com/ossf/scorecard/utests" ) -var _ = Describe("E2E TEST:CodeReview", func() { +var _ = Describe("E2E TEST:Contributors", func() { Context("E2E TEST:Validating project contributors", func() { It("Should return valid project contributors", func() { - l := log{} - checkRequest := checker.CheckRequest{ + dl := scut.TestDetailLogger{} + req := checker.CheckRequest{ Ctx: context.Background(), Client: ghClient, HTTPClient: httpClient, @@ -37,14 +38,25 @@ var _ = Describe("E2E TEST:CodeReview", func() { Owner: "ossf", Repo: "scorecard", GraphClient: graphClient, - Logf: l.Logf, + Dlogger: &dl, } - result := checks.Contributors(&checkRequest) + expected := scut.TestReturn{ + Errors: nil, + Score: checker.MaxResultScore, + NumberOfWarn: 0, + NumberOfInfo: 1, + NumberOfDebug: 0, + } + result := checks.Contributors(&req) + // UPGRADEv2: to remove. + // Old version. Expect(result.Error).Should(BeNil()) Expect(result.Pass).Should(BeTrue()) + // New version. + Expect(scut.ValidateTestReturn(nil, "several contributors", &expected, &result, &dl)).Should(BeTrue()) }) It("Should return valid project contributors", func() { - l := log{} + dl := scut.TestDetailLogger{} checkRequest := checker.CheckRequest{ Ctx: context.Background(), Client: ghClient, @@ -53,11 +65,22 @@ var _ = Describe("E2E TEST:CodeReview", func() { Owner: "apache", Repo: "airflow", GraphClient: graphClient, - Logf: l.Logf, + Dlogger: &dl, + } + expected := scut.TestReturn{ + Errors: nil, + Score: checker.MaxResultScore, + NumberOfWarn: 0, + NumberOfInfo: 1, + NumberOfDebug: 0, } result := checks.Contributors(&checkRequest) + // UPGRADEv2: to remove. + // Old version. Expect(result.Error).Should(BeNil()) Expect(result.Pass).Should(BeTrue()) + // New version. + Expect(scut.ValidateTestReturn(nil, "several contributors", &expected, &result, &dl)).Should(BeTrue()) }) }) }) diff --git a/e2e/packaging_test.go b/e2e/packaging_test.go index e007c287..5f73ec30 100644 --- a/e2e/packaging_test.go +++ b/e2e/packaging_test.go @@ -22,13 +22,14 @@ import ( "github.com/ossf/scorecard/checker" "github.com/ossf/scorecard/checks" + scut "github.com/ossf/scorecard/utests" ) var _ = Describe("E2E TEST:Packaging", func() { Context("E2E TEST:Validating use of packaging in CI/CD", func() { It("Should return use of packaging in CI/CD", func() { - l := log{} - checkRequest := checker.CheckRequest{ + dl := scut.TestDetailLogger{} + req := checker.CheckRequest{ Ctx: context.Background(), Client: ghClient, HTTPClient: httpClient, @@ -36,11 +37,23 @@ var _ = Describe("E2E TEST:Packaging", func() { Owner: "apache", Repo: "orc", GraphClient: graphClient, - Logf: l.Logf, + Dlogger: &dl, } - result := checks.Packaging(&checkRequest) + expected := scut.TestReturn{ + Errors: nil, + Score: checker.MaxResultScore, + NumberOfWarn: 0, + NumberOfInfo: 2, + NumberOfDebug: 2, + } + result := checks.Packaging(&req) + + // UPGRADEv2: to remove. + // Old version. Expect(result.Error).Should(BeNil()) Expect(result.Pass).Should(BeTrue()) + // New version. + Expect(scut.ValidateTestReturn(nil, "use packaging", &expected, &result, &dl)).Should(BeTrue()) }) }) }) diff --git a/e2e/sast_test.go b/e2e/sast_test.go index f387a71d..bd0655d5 100644 --- a/e2e/sast_test.go +++ b/e2e/sast_test.go @@ -22,13 +22,14 @@ import ( "github.com/ossf/scorecard/checker" "github.com/ossf/scorecard/checks" + scut "github.com/ossf/scorecard/utests" ) var _ = Describe("E2E TEST:SAST", func() { Context("E2E TEST:Validating use of SAST tools", func() { It("Should return use of SAST tools", func() { - l := log{} - checkRequest := checker.CheckRequest{ + dl := scut.TestDetailLogger{} + req := checker.CheckRequest{ Ctx: context.Background(), Client: ghClient, HTTPClient: httpClient, @@ -36,11 +37,22 @@ var _ = Describe("E2E TEST:SAST", func() { Owner: "apache", Repo: "airflow", GraphClient: graphClient, - Logf: l.Logf, + Dlogger: &dl, } - result := checks.SAST(&checkRequest) + expected := scut.TestReturn{ + Errors: nil, + Score: checker.MaxResultScore, + NumberOfWarn: 0, + NumberOfInfo: 2, + NumberOfDebug: 2, + } + result := checks.SAST(&req) + // UPGRADEv2: to remove. + // Old version. Expect(result.Error).Should(BeNil()) Expect(result.Pass).Should(BeTrue()) + // New version. + Expect(scut.ValidateTestReturn(nil, "sast used", &expected, &result, &dl)).Should(BeTrue()) }) }) }) diff --git a/e2e/vulnerabilities_test.go b/e2e/vulnerabilities_test.go index 27c0c6a1..f9dba22d 100644 --- a/e2e/vulnerabilities_test.go +++ b/e2e/vulnerabilities_test.go @@ -23,13 +23,14 @@ import ( "github.com/ossf/scorecard/checker" "github.com/ossf/scorecard/checks" + scut "github.com/ossf/scorecard/utests" ) var _ = Describe("E2E TEST:Vulnerabilities", func() { Context("E2E TEST:Validating vulnerabilities status", func() { It("Should return that there are no vulnerabilities", func() { - l := log{} - checkRequest := checker.CheckRequest{ + dl := scut.TestDetailLogger{} + req := checker.CheckRequest{ Ctx: context.Background(), Client: ghClient, HTTPClient: httpClient, @@ -37,15 +38,26 @@ var _ = Describe("E2E TEST:Vulnerabilities", func() { Owner: "ossf", Repo: "scorecard", GraphClient: graphClient, - Logf: l.Logf, + Dlogger: &dl, } - result := checks.HasUnfixedVulnerabilities(&checkRequest) + expected := scut.TestReturn{ + Errors: nil, + Score: checker.MaxResultScore, + NumberOfWarn: 0, + NumberOfInfo: 0, + NumberOfDebug: 0, + } + result := checks.HasUnfixedVulnerabilities(&req) + // UPGRADEv2: to remove. + // Old version. Expect(result.Error).Should(BeNil()) Expect(result.Pass).Should(BeTrue()) + // New version. + Expect(scut.ValidateTestReturn(nil, "no osv vulnerabilities", &expected, &result, &dl)).Should(BeTrue()) }) It("Should return that there are vulnerabilities", func() { - l := log{} + dl := scut.TestDetailLogger{} checkRequest := checker.CheckRequest{ Ctx: context.Background(), Client: ghClient, @@ -54,11 +66,22 @@ var _ = Describe("E2E TEST:Vulnerabilities", func() { Owner: "oliverchang", Repo: "open62541", GraphClient: graphClient, - Logf: l.Logf, + Dlogger: &dl, + } + expected := scut.TestReturn{ + Errors: nil, + Score: checker.MinResultScore, + NumberOfWarn: 1, + NumberOfInfo: 0, + NumberOfDebug: 0, } result := checks.HasUnfixedVulnerabilities(&checkRequest) + // UPGRADEv2: to remove. + // Old version. Expect(result.Error).Should(BeNil()) Expect(result.Pass).Should(BeFalse()) + // New version. + Expect(scut.ValidateTestReturn(nil, "osv vulnerabilities", &expected, &result, &dl)).Should(BeTrue()) }) }) })