✨ [migration to score] 5: contributors, vulnerabilities, packaging and sast (#729)

* contributors

* packaging

* vulnerabilities

* fix errors

* err

* errors

checker/check_result.go

@@ -18,6 +18,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	"strings"
 
 	scorecarderrors "github.com/ossf/scorecard/errors"
 )
@@ -213,10 +214,12 @@ func MakeOrResult(c *CheckRequest, checks ...CheckResult) CheckResult {
 	//nolint
 	for _, result := range checks[1:] {
 		if result.Score >= bestResult.Score {
-			c.Dlogger.Info(bestResult.Reason)
+			i := strings.Index(bestResult.Reason, "-- score normalized")
+			c.Dlogger.Info(bestResult.Reason[:i])
 			bestResult = result
 		} else {
-			c.Dlogger.Info(result.Reason)
+			i := strings.Index(result.Reason, "-- score normalized")
+			c.Dlogger.Info(result.Reason[:i])
 		}
 
 		// Do not exit early so we can show all the details

checks/branch_protected.go

@@ -86,8 +86,7 @@ func checkReleaseAndDevBranchProtection(ctx context.Context, r repositories, dl
 		name, err := resolveBranchName(branches, *release.TargetCommitish)
 		if err != nil {
 			// If the commitish branch is still not found, fail.
-			e := sce.Create(sce.ErrScorecardInternal, errInternalBranchNotFound.Error())
+			r := checker.CreateRuntimeErrorResult(CheckBranchProtection, err)
-			r := checker.CreateRuntimeErrorResult(CheckBranchProtection, e)
 			checks = append(checks, r)
 			continue
 		}
@@ -107,8 +106,7 @@ func checkReleaseAndDevBranchProtection(ctx context.Context, r repositories, dl
 	for b := range checkBranches {
 		protected, err := isBranchProtected(branches, b)
 		if err != nil {
-			e := sce.Create(sce.ErrScorecardInternal, errInternalBranchNotFound.Error())
+			r := checker.CreateRuntimeErrorResult(CheckBranchProtection, err)
-			r := checker.CreateRuntimeErrorResult(CheckBranchProtection, e)
 			checks = append(checks, r)
 		}
 		if !protected {

checks/code_review.go

@@ -22,6 +22,7 @@ import (
 	"github.com/shurcooL/githubv4"
 
 	"github.com/ossf/scorecard/checker"
+	sce "github.com/ossf/scorecard/errors"
 )
 
 const (
@@ -82,7 +83,8 @@ func DoesCodeReview(c *checker.CheckRequest) checker.CheckResult {
 		"labelsToAnalyze":       githubv4.Int(labelsToAnalyze),
 	}
 	if err := c.GraphClient.Query(c.Ctx, &prHistory, vars); err != nil {
-		return checker.CreateRuntimeErrorResult(CheckCodeReview, err)
+		e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("c.GraphClient.Query: %v", err))
+		return checker.CreateRuntimeErrorResult(CheckCodeReview, e)
 	}
 	return checker.MultiCheckOr2(
 		isPrReviewRequired,
@@ -161,7 +163,8 @@ func prowCodeReview(c *checker.CheckRequest) checker.CheckResult {
 func commitMessageHints(c *checker.CheckRequest) checker.CheckResult {
 	commits, _, err := c.Client.Repositories.ListCommits(c.Ctx, c.Owner, c.Repo, &github.CommitsListOptions{})
 	if err != nil {
-		return checker.CreateRuntimeErrorResult(CheckCodeReview, err)
+		e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("Client.Repositories.ListCommits: %v", err))
+		return checker.CreateRuntimeErrorResult(CheckCodeReview, e)
 	}
 
 	total := 0

checks/contributors.go

@@ -15,16 +15,18 @@
 package checks
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/google/go-github/v32/github"
 
 	"github.com/ossf/scorecard/checker"
+	sce "github.com/ossf/scorecard/errors"
 )
 
 const (
-	minContributionsPerUser = 5
+	minContributionsPerUser    = 5
-	minOrganizationCount    = 2
+	numberCompaniesForTopScore = 3
 	// CheckContributors is the registered name for Contributors.
 	CheckContributors = "Contributors"
 )
@@ -37,7 +39,8 @@ func init() {
 func Contributors(c *checker.CheckRequest) checker.CheckResult {
 	contribs, _, err := c.Client.Repositories.ListContributors(c.Ctx, c.Owner, c.Repo, &github.ListContributorsOptions{})
 	if err != nil {
-		return checker.MakeRetryResult(CheckContributors, err)
+		e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("Client.Repositories.ListContributors: %v", err))
+		return checker.CreateRuntimeErrorResult(CheckContributors, e)
 	}
 
 	companies := map[string]struct{}{}
@@ -47,11 +50,12 @@ func Contributors(c *checker.CheckRequest) checker.CheckResult {
 		}
 		u, _, err := c.Client.Users.Get(c.Ctx, contrib.GetLogin())
 		if err != nil {
-			return checker.MakeRetryResult(CheckContributors, err)
+			e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("Client.Users.Get: %v", err))
+			return checker.CreateRuntimeErrorResult(CheckContributors, e)
 		}
 		orgs, _, err := c.Client.Organizations.List(c.Ctx, contrib.GetLogin(), nil)
 		if err != nil {
-			c.Logf("unable to get org members for %s", contrib.GetLogin())
+			c.Dlogger.Debug("unable to get org members for %s: %v", contrib.GetLogin(), err)
 		} else if len(orgs) > 0 {
 			companies[*orgs[0].Login] = struct{}{}
 			continue
@@ -72,17 +76,9 @@ func Contributors(c *checker.CheckRequest) checker.CheckResult {
 	for c := range companies {
 		names = append(names, c)
 	}
-	c.Logf("companies found: %v", strings.Join(names, ","))
+
-	if len(companies) >= minOrganizationCount {
+	c.Dlogger.Info("contributors work for: %v", strings.Join(names, ","))
-		return checker.CheckResult{
+
-			Name:       CheckContributors,
+	reason := fmt.Sprintf("%d different companies found", len(companies))
-			Pass:       true,
+	return checker.CreateProportionalScoreResult(CheckContributors, reason, len(companies), numberCompaniesForTopScore)
-			Confidence: checker.MaxResultConfidence,
-		}
-	}
-	return checker.CheckResult{
-		Name:       CheckContributors,
-		Pass:       false,
-		Confidence: checker.MaxResultConfidence,
-	}
 }

checks/packaging.go

@@ -15,6 +15,7 @@
 package checks
 
 import (
+	"fmt"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -22,6 +23,7 @@ import (
 	"github.com/google/go-github/v32/github"
 
 	"github.com/ossf/scorecard/checker"
+	sce "github.com/ossf/scorecard/errors"
 )
 
 // CheckPackaging is the registered name for Packaging.
@@ -36,14 +38,16 @@ func Packaging(c *checker.CheckRequest) checker.CheckResult {
 	_, dc, _, err := c.Client.Repositories.GetContents(c.Ctx, c.Owner, c.Repo, ".github/workflows",
 		&github.RepositoryContentGetOptions{})
 	if err != nil {
-		return checker.MakeRetryResult(CheckPackaging, err)
+		e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("Client.Repositories.GetContents: %v", err))
+		return checker.CreateRuntimeErrorResult(CheckPackaging, e)
 	}
 
 	for _, f := range dc {
 		fp := f.GetPath()
 		fo, _, _, err := c.Client.Repositories.GetContents(c.Ctx, c.Owner, c.Repo, fp, &github.RepositoryContentGetOptions{})
 		if err != nil {
-			return checker.MakeRetryResult(CheckPackaging, err)
+			e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("Client.Repositories.GetContents: %v", err))
+			return checker.CreateRuntimeErrorResult(CheckPackaging, e)
 		}
 		if fo == nil {
 			// path is a directory, not a file. skip.
@@ -51,7 +55,8 @@ func Packaging(c *checker.CheckRequest) checker.CheckResult {
 		}
 		fc, err := fo.GetContent()
 		if err != nil {
-			return checker.MakeRetryResult(CheckPackaging, err)
+			e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("fo.GetContent: %v", err))
+			return checker.CreateRuntimeErrorResult(CheckPackaging, e)
 		}
 
 		if !isPackagingWorkflow(fc, fp, c) {
@@ -63,24 +68,19 @@ func Packaging(c *checker.CheckRequest) checker.CheckResult {
 				Status: "success",
 			})
 		if err != nil {
-			return checker.MakeRetryResult(CheckPackaging, err)
+			e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("Client.Actions.ListWorkflowRunsByFileName: %v", err))
+			return checker.CreateRuntimeErrorResult(CheckPackaging, e)
 		}
 		if *runs.TotalCount > 0 {
-			c.Logf("found a completed run: %s", runs.WorkflowRuns[0].GetHTMLURL())
+			c.Dlogger.Info("workflow %v used in run: %s", fp, runs.WorkflowRuns[0].GetHTMLURL())
-			return checker.CheckResult{
+			return checker.CreateMaxScoreResult(CheckPackaging,
-				Name:       CheckPackaging,
+				"packaging workflow detected")
-				Pass:       true,
-				Confidence: checker.MaxResultConfidence,
-			}
 		}
-		c.Logf("!! no run completed")
+		c.Dlogger.Info("workflow %v not used in runs", fp)
 	}
 
-	return checker.CheckResult{
+	return checker.CreateMinScoreResult(CheckPackaging,
-		Name:       CheckPackaging,
+		"no packaging workflow used")
-		Pass:       false,
-		Confidence: checker.MaxResultConfidence,
-	}
 }
 
 func isPackagingWorkflow(s, fp string, c *checker.CheckRequest) bool {
@@ -90,42 +90,42 @@ func isPackagingWorkflow(s, fp string, c *checker.CheckRequest) bool {
 		r2 := regexp.MustCompile(`(?s)npm.*publish`)
 
 		if r1.MatchString(s) && r2.MatchString(s) {
-			c.Logf("found node packaging workflow using npm: %s", fp)
+			c.Dlogger.Info("candidate node packaging workflow using npm: %s", fp)
 			return true
 		}
 	}
 
 	if strings.Contains(s, "uses: actions/setup-java@") {
-		// java packages with maven
+		// Java packages with maven.
 		r1 := regexp.MustCompile(`(?s)mvn.*deploy`)
 		if r1.MatchString(s) {
-			c.Logf("found java packaging workflow using maven: %s", fp)
+			c.Dlogger.Info("candidate java packaging workflow using maven: %s", fp)
 			return true
 		}
 
-		// java packages with gradle
+		// Java packages with gradle.
 		r2 := regexp.MustCompile(`(?s)gradle.*publish`)
 		if r2.MatchString(s) {
-			c.Logf("found java packaging workflow using gradle: %s", fp)
+			c.Dlogger.Info("candidate java packaging workflow using gradle: %s", fp)
 			return true
 		}
 	}
 
 	if strings.Contains(s, "actions/setup-python@") && strings.Contains(s, "pypa/gh-action-pypi-publish@master") {
-		c.Logf("found python packaging workflow using pypi: %s", fp)
+		c.Dlogger.Info("candidate python packaging workflow using pypi: %s", fp)
 		return true
 	}
 
 	if strings.Contains(s, "uses: docker/build-push-action@") {
-		c.Logf("found docker publishing workflow: %s", fp)
+		c.Dlogger.Info("candidate docker publishing workflow: %s", fp)
 		return true
 	}
 
 	if strings.Contains(s, "docker push") {
-		c.Logf("found docker publishing workflow: %s", fp)
+		c.Dlogger.Info("candidate docker publishing workflow: %s", fp)
 		return true
 	}
 
-	c.Logf("!! not a packaging workflow: %s", fp)
+	c.Dlogger.Debug("not a packaging workflow: %s", fp)
 	return false
 }

checks/sast.go

@@ -15,26 +15,19 @@
 package checks
 
 import (
-	"errors"
+	"fmt"
+	"strings"
 
 	"github.com/google/go-github/v32/github"
 
 	"github.com/ossf/scorecard/checker"
+	sce "github.com/ossf/scorecard/errors"
 )
 
-const (
+// CheckSAST is the registered name for SAST.
-	// CheckSAST is the registered name for SAST.
+const CheckSAST = "SAST"
-	CheckSAST         = "SAST"
-	sastPassThreshold = .75
-)
 
-var (
+var sastTools = map[string]bool{"github-code-scanning": true, "sonarcloud": true}
-	sastTools = map[string]bool{"github-code-scanning": true, "sonarcloud": true}
-	// ErrorNoChecks indicates no GitHub Check runs were found for this repo.
-	ErrorNoChecks = errors.New("no check runs found")
-	// ErrorNoMerges indicates no merges with SAST tool runs were found for this repo.
-	ErrorNoMerges = errors.New("no merges found")
-)
 
 //nolint:gochecknoinits
 func init() {
@@ -42,10 +35,38 @@ func init() {
 }
 
 func SAST(c *checker.CheckRequest) checker.CheckResult {
-	return checker.MultiCheckOr(
+	r1 := SASTToolInCheckRuns(c)
-		CodeQLInCheckDefinitions,
+	r2 := CodeQLInCheckDefinitions(c)
-		SASTToolInCheckRuns,
+	if r1.Error2 != nil {
-	)(c)
+		return r1
+	}
+	if r2.Error2 != nil {
+		return r2
+	}
+	// Merge the results.
+	var result checker.CheckResult
+	if r1.Score == checker.MaxResultScore {
+		// All commits have a SAST tool check run,
+		// score is maximum.
+		result = r1
+		result.Score = 10
+		i := strings.Index(r2.Reason, "-- score normalized")
+		c.Dlogger.Info(r2.Reason[:i])
+	} else {
+		// Not all commits have a check run,
+		// We compute the final score as follows:
+		// 5 points for CodeQL enabled, 5 points for
+		// SAST run on commits.
+		result = r2
+
+		//nolint
+		result.Score = 5 + r1.Score/10*5
+		result.Reason = "not all commits are checked with a SAST tool"
+		i := strings.Index(r1.Reason, "-- score normalized")
+		c.Dlogger.Info(r1.Reason[:i])
+	}
+
+	return result
 }
 
 func SASTToolInCheckRuns(c *checker.CheckRequest) checker.CheckResult {
@@ -53,7 +74,8 @@ func SASTToolInCheckRuns(c *checker.CheckRequest) checker.CheckResult {
 		State: "closed",
 	})
 	if err != nil {
-		return checker.MakeRetryResult(CheckSAST, err)
+		e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("Client.PullRequests.List: %v", err))
+		return checker.CreateRuntimeErrorResult(CheckSAST, e)
 	}
 
 	totalMerged := 0
@@ -66,10 +88,11 @@ func SASTToolInCheckRuns(c *checker.CheckRequest) checker.CheckResult {
 		crs, _, err := c.Client.Checks.ListCheckRunsForRef(c.Ctx, c.Owner, c.Repo, pr.GetHead().GetSHA(),
 			&github.ListCheckRunsOptions{})
 		if err != nil {
-			return checker.MakeRetryResult(CheckSAST, err)
+			e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("Client.Checks.ListCheckRunsForRef: %v", err))
+			return checker.CreateRuntimeErrorResult(CheckSAST, e)
 		}
 		if crs == nil {
-			return checker.MakeInconclusiveResult(CheckSAST, ErrorNoChecks)
+			return checker.CreateInconclusiveResult(CheckSAST, "no merges detected")
 		}
 		for _, cr := range crs.CheckRuns {
 			if cr.GetStatus() != "completed" {
@@ -79,32 +102,35 @@ func SASTToolInCheckRuns(c *checker.CheckRequest) checker.CheckResult {
 				continue
 			}
 			if sastTools[cr.GetApp().GetSlug()] {
-				c.Logf("SAST Tool found: %s", cr.GetHTMLURL())
+				c.Dlogger.Info("tool detected: %s", cr.GetHTMLURL())
 				totalTested++
 				break
 			}
 		}
 	}
-	if totalTested == 0 {
+	if totalMerged == 0 {
-		return checker.MakeInconclusiveResult(CheckSAST, ErrorNoMerges)
+		return checker.CreateInconclusiveResult(CheckSAST, "no merges detected")
 	}
-	return checker.MakeProportionalResult(CheckSAST, totalTested, totalMerged, sastPassThreshold)
+	reason := fmt.Sprintf("%v commits out of %v are checked with a SAST tool", totalTested, totalMerged)
+	return checker.CreateProportionalScoreResult(CheckSAST, reason, totalTested, totalMerged)
 }
 
 func CodeQLInCheckDefinitions(c *checker.CheckRequest) checker.CheckResult {
 	searchQuery := ("github/codeql-action path:/.github/workflows repo:" + c.Owner + "/" + c.Repo)
 	results, _, err := c.Client.Search.Code(c.Ctx, searchQuery, &github.SearchOptions{})
 	if err != nil {
-		return checker.MakeRetryResult(CheckSAST, err)
+		e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("Client.Search.Code: %v", err))
+		return checker.CreateRuntimeErrorResult(CheckSAST, e)
 	}
 
 	for _, result := range results.CodeResults {
-		c.Logf("found CodeQL definition: %s", result.GetPath())
+		c.Dlogger.Info("CodeQL definition detected: %s", result.GetPath())
 	}
 
-	return checker.CheckResult{
+	// TODO: check if it's enabled as cron or presubmit.
-		Name:       CheckSAST,
+	// TODO: check which branches it is enabled on. We should find main.
-		Pass:       *results.Total > 0,
+	if *results.Total > 0 {
-		Confidence: checker.MaxResultConfidence,
+		return checker.CreateMaxScoreResult(CheckSAST, "tool detected: CodeQL")
 	}
+	return checker.CreateMinScoreResult(CheckSAST, "CodeQL tool not detected")
 }

checks/vulnerabilities.go

@@ -17,13 +17,14 @@ package checks
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
+	"fmt"
 	"net/http"
 	"strings"
 
 	"github.com/google/go-github/v32/github"
 
 	"github.com/ossf/scorecard/checker"
+	sce "github.com/ossf/scorecard/errors"
 )
 
 const (
@@ -32,9 +33,6 @@ const (
 	osvQueryEndpoint     = "https://api.osv.dev/v1/query"
 )
 
-// ErrNoCommits is the error for when there are no commits found.
-var ErrNoCommits = errors.New("no commits found")
-
 type osvQuery struct {
 	Commit string `json:"commit"`
 }
@@ -65,44 +63,50 @@ func HasUnfixedVulnerabilities(c *checker.CheckRequest) checker.CheckResult {
 		},
 	})
 	if err != nil {
-		return checker.MakeRetryResult(CheckVulnerabilities, err)
+		e := sce.Create(sce.ErrScorecardInternal, "Client.Repositories.ListCommits")
+		return checker.CreateRuntimeErrorResult(CheckVulnerabilities, e)
 	}
 
 	if len(commits) != 1 || commits[0].SHA == nil {
-		return checker.MakeInconclusiveResult(CheckVulnerabilities, ErrNoCommits)
+		return checker.CreateInconclusiveResult(CheckVulnerabilities, "no commits found")
 	}
 
 	query, err := json.Marshal(&osvQuery{
 		Commit: *commits[0].SHA,
 	})
 	if err != nil {
-		panic("!! failed to marshal OSV query.")
+		e := sce.Create(sce.ErrScorecardInternal, "json.Marshal")
+		return checker.CreateRuntimeErrorResult(CheckVulnerabilities, e)
 	}
 
 	req, err := http.NewRequestWithContext(c.Ctx, http.MethodPost, osvQueryEndpoint, bytes.NewReader(query))
 	if err != nil {
-		return checker.MakeRetryResult(CheckVulnerabilities, err)
+		e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("http.NewRequestWithContext: %v", err))
+		return checker.CreateRuntimeErrorResult(CheckVulnerabilities, e)
 	}
 
 	// Use our own http client as the one from CheckRequest adds GitHub tokens to the headers.
 	httpClient := &http.Client{}
 	resp, err := httpClient.Do(req)
 	if err != nil {
-		return checker.MakeRetryResult(CheckVulnerabilities, err)
+		e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("httpClient.Do: %v", err))
+		return checker.CreateRuntimeErrorResult(CheckVulnerabilities, e)
 	}
 	defer resp.Body.Close()
 
 	var osvResp osvResponse
 	decoder := json.NewDecoder(resp.Body)
 	if err := decoder.Decode(&osvResp); err != nil {
-		return checker.MakeRetryResult(CheckVulnerabilities, err)
+		e := sce.Create(sce.ErrScorecardInternal, fmt.Sprintf("decoder.Decode: %v", err))
+		return checker.CreateRuntimeErrorResult(CheckVulnerabilities, e)
 	}
 
+	// TODO: take severity into account.
 	vulnIDs := osvResp.getVulnerabilities()
 	if len(vulnIDs) > 0 {
-		c.Logf("HEAD is vulnerable to %s", strings.Join(vulnIDs, ", "))
+		c.Dlogger.Warn("HEAD is vulnerable to %s", strings.Join(vulnIDs, ", "))
-		return checker.MakeFailResult(CheckVulnerabilities, nil)
+		return checker.CreateMinScoreResult(CheckVulnerabilities, "unfixed OSV detected")
 	}
 
-	return checker.MakePassResult(CheckVulnerabilities)
+	return checker.CreateMaxScoreResult(CheckVulnerabilities, "no OSV detected")
 }

e2e/contributors_test.go

@@ -23,13 +23,14 @@ import (
 
 	"github.com/ossf/scorecard/checker"
 	"github.com/ossf/scorecard/checks"
+	scut "github.com/ossf/scorecard/utests"
 )
 
-var _ = Describe("E2E TEST:CodeReview", func() {
+var _ = Describe("E2E TEST:Contributors", func() {
 	Context("E2E TEST:Validating project contributors", func() {
 		It("Should return valid project contributors", func() {
-			l := log{}
+			dl := scut.TestDetailLogger{}
-			checkRequest := checker.CheckRequest{
+			req := checker.CheckRequest{
 				Ctx:         context.Background(),
 				Client:      ghClient,
 				HTTPClient:  httpClient,
@@ -37,14 +38,25 @@ var _ = Describe("E2E TEST:CodeReview", func() {
 				Owner:       "ossf",
 				Repo:        "scorecard",
 				GraphClient: graphClient,
-				Logf:        l.Logf,
+				Dlogger:     &dl,
 			}
-			result := checks.Contributors(&checkRequest)
+			expected := scut.TestReturn{
+				Errors:        nil,
+				Score:         checker.MaxResultScore,
+				NumberOfWarn:  0,
+				NumberOfInfo:  1,
+				NumberOfDebug: 0,
+			}
+			result := checks.Contributors(&req)
+			// UPGRADEv2: to remove.
+			// Old version.
 			Expect(result.Error).Should(BeNil())
 			Expect(result.Pass).Should(BeTrue())
+			// New version.
+			Expect(scut.ValidateTestReturn(nil, "several contributors", &expected, &result, &dl)).Should(BeTrue())
 		})
 		It("Should return valid project contributors", func() {
-			l := log{}
+			dl := scut.TestDetailLogger{}
 			checkRequest := checker.CheckRequest{
 				Ctx:         context.Background(),
 				Client:      ghClient,
@@ -53,11 +65,22 @@ var _ = Describe("E2E TEST:CodeReview", func() {
 				Owner:       "apache",
 				Repo:        "airflow",
 				GraphClient: graphClient,
-				Logf:        l.Logf,
+				Dlogger:     &dl,
+			}
+			expected := scut.TestReturn{
+				Errors:        nil,
+				Score:         checker.MaxResultScore,
+				NumberOfWarn:  0,
+				NumberOfInfo:  1,
+				NumberOfDebug: 0,
 			}
 			result := checks.Contributors(&checkRequest)
+			// UPGRADEv2: to remove.
+			// Old version.
 			Expect(result.Error).Should(BeNil())
 			Expect(result.Pass).Should(BeTrue())
+			// New version.
+			Expect(scut.ValidateTestReturn(nil, "several contributors", &expected, &result, &dl)).Should(BeTrue())
 		})
 	})
 })

e2e/packaging_test.go

@@ -22,13 +22,14 @@ import (
 
 	"github.com/ossf/scorecard/checker"
 	"github.com/ossf/scorecard/checks"
+	scut "github.com/ossf/scorecard/utests"
 )
 
 var _ = Describe("E2E TEST:Packaging", func() {
 	Context("E2E TEST:Validating use of packaging in CI/CD", func() {
 		It("Should return use of packaging in CI/CD", func() {
-			l := log{}
+			dl := scut.TestDetailLogger{}
-			checkRequest := checker.CheckRequest{
+			req := checker.CheckRequest{
 				Ctx:         context.Background(),
 				Client:      ghClient,
 				HTTPClient:  httpClient,
@@ -36,11 +37,23 @@ var _ = Describe("E2E TEST:Packaging", func() {
 				Owner:       "apache",
 				Repo:        "orc",
 				GraphClient: graphClient,
-				Logf:        l.Logf,
+				Dlogger:     &dl,
 			}
-			result := checks.Packaging(&checkRequest)
+			expected := scut.TestReturn{
+				Errors:        nil,
+				Score:         checker.MaxResultScore,
+				NumberOfWarn:  0,
+				NumberOfInfo:  2,
+				NumberOfDebug: 2,
+			}
+			result := checks.Packaging(&req)
+
+			// UPGRADEv2: to remove.
+			// Old version.
 			Expect(result.Error).Should(BeNil())
 			Expect(result.Pass).Should(BeTrue())
+			// New version.
+			Expect(scut.ValidateTestReturn(nil, "use packaging", &expected, &result, &dl)).Should(BeTrue())
 		})
 	})
 })

e2e/sast_test.go

@@ -22,13 +22,14 @@ import (
 
 	"github.com/ossf/scorecard/checker"
 	"github.com/ossf/scorecard/checks"
+	scut "github.com/ossf/scorecard/utests"
 )
 
 var _ = Describe("E2E TEST:SAST", func() {
 	Context("E2E TEST:Validating use of SAST tools", func() {
 		It("Should return use of SAST tools", func() {
-			l := log{}
+			dl := scut.TestDetailLogger{}
-			checkRequest := checker.CheckRequest{
+			req := checker.CheckRequest{
 				Ctx:         context.Background(),
 				Client:      ghClient,
 				HTTPClient:  httpClient,
@@ -36,11 +37,22 @@ var _ = Describe("E2E TEST:SAST", func() {
 				Owner:       "apache",
 				Repo:        "airflow",
 				GraphClient: graphClient,
-				Logf:        l.Logf,
+				Dlogger:     &dl,
 			}
-			result := checks.SAST(&checkRequest)
+			expected := scut.TestReturn{
+				Errors:        nil,
+				Score:         checker.MaxResultScore,
+				NumberOfWarn:  0,
+				NumberOfInfo:  2,
+				NumberOfDebug: 2,
+			}
+			result := checks.SAST(&req)
+			// UPGRADEv2: to remove.
+			// Old version.
 			Expect(result.Error).Should(BeNil())
 			Expect(result.Pass).Should(BeTrue())
+			// New version.
+			Expect(scut.ValidateTestReturn(nil, "sast used", &expected, &result, &dl)).Should(BeTrue())
 		})
 	})
 })

e2e/vulnerabilities_test.go

@@ -23,13 +23,14 @@ import (
 
 	"github.com/ossf/scorecard/checker"
 	"github.com/ossf/scorecard/checks"
+	scut "github.com/ossf/scorecard/utests"
 )
 
 var _ = Describe("E2E TEST:Vulnerabilities", func() {
 	Context("E2E TEST:Validating vulnerabilities status", func() {
 		It("Should return that there are no vulnerabilities", func() {
-			l := log{}
+			dl := scut.TestDetailLogger{}
-			checkRequest := checker.CheckRequest{
+			req := checker.CheckRequest{
 				Ctx:         context.Background(),
 				Client:      ghClient,
 				HTTPClient:  httpClient,
@@ -37,15 +38,26 @@ var _ = Describe("E2E TEST:Vulnerabilities", func() {
 				Owner:       "ossf",
 				Repo:        "scorecard",
 				GraphClient: graphClient,
-				Logf:        l.Logf,
+				Dlogger:     &dl,
 			}
-			result := checks.HasUnfixedVulnerabilities(&checkRequest)
+			expected := scut.TestReturn{
+				Errors:        nil,
+				Score:         checker.MaxResultScore,
+				NumberOfWarn:  0,
+				NumberOfInfo:  0,
+				NumberOfDebug: 0,
+			}
+			result := checks.HasUnfixedVulnerabilities(&req)
+			// UPGRADEv2: to remove.
+			// Old version.
 			Expect(result.Error).Should(BeNil())
 			Expect(result.Pass).Should(BeTrue())
+			// New version.
+			Expect(scut.ValidateTestReturn(nil, "no osv vulnerabilities", &expected, &result, &dl)).Should(BeTrue())
 		})
 
 		It("Should return that there are vulnerabilities", func() {
-			l := log{}
+			dl := scut.TestDetailLogger{}
 			checkRequest := checker.CheckRequest{
 				Ctx:         context.Background(),
 				Client:      ghClient,
@@ -54,11 +66,22 @@ var _ = Describe("E2E TEST:Vulnerabilities", func() {
 				Owner:       "oliverchang",
 				Repo:        "open62541",
 				GraphClient: graphClient,
-				Logf:        l.Logf,
+				Dlogger:     &dl,
+			}
+			expected := scut.TestReturn{
+				Errors:        nil,
+				Score:         checker.MinResultScore,
+				NumberOfWarn:  1,
+				NumberOfInfo:  0,
+				NumberOfDebug: 0,
 			}
 			result := checks.HasUnfixedVulnerabilities(&checkRequest)
+			// UPGRADEv2: to remove.
+			// Old version.
 			Expect(result.Error).Should(BeNil())
 			Expect(result.Pass).Should(BeFalse())
+			// New version.
+			Expect(scut.ValidateTestReturn(nil, "osv vulnerabilities", &expected, &result, &dl)).Should(BeTrue())
 		})
 	})
 })