✨ [RAW] End-to-end support for raw results for Binary-Artifacts (#1255)

* split binary artifact check

* fix

* missing file

* comments

* fix

* comments

* draft

* merge fix

* fix merge

* add indirection

* comments

* comments

* linter

* comments

* updates

* updates

* updates

* linter

* comments

checker/check_request.go

@@ -28,4 +28,6 @@ type CheckRequest struct {
 	OssFuzzRepo clients.RepoClient
 	Dlogger     DetailLogger
 	Repo        clients.Repo
+	// UPGRADEv6: return raw results instead of scores.
+	RawResults *RawResults
 }

checker/check_result.go

@@ -118,6 +118,31 @@ type CheckResult struct {
 	Details2 []CheckDetail `json:"-"` // Details of tests and sub-checks
 	Score    int           `json:"-"` // {[-1,0...10], -1 = Inconclusive}
 	Reason   string        `json:"-"` // A sentence describing the check result (score, etc)
+
+}
+
+// ====== Raw results for checks =========.
+
+// File represents a file.
+type File struct {
+	Path    string
+	Snippet string   // Snippet of code
+	Offset  int      // Offset in the file of Path (line for source/text files).
+	Type    FileType // Type of file.
+	// TODO: add hash.
+}
+
+// BinaryArtifactData contains the raw results
+// for the Binary-Artifact check.
+type BinaryArtifactData struct {
+	// Files contains a list of files.
+	Files []File
+}
+
+// RawResults contains results before a policy
+// is applied.
+type RawResults struct {
+	BinaryArtifactResults BinaryArtifactData
 }
 
 // CreateProportionalScore creates a proportional score.

checker/check_runner.go

@@ -128,10 +128,12 @@ func (r *Runner) Run(ctx context.Context, f CheckFn) CheckResult {
 		break
 	}
 
+	// Set details.
 	res.Details2 = l.messages2
 	for _, d := range l.messages2 {
 		res.Details = append(res.Details, d.Msg.Text)
 	}
+
 	if err := logStats(ctx, startTime, &res); err != nil {
 		panic(err)
 	}

checks/binary_artifact.go

@@ -31,11 +31,18 @@ func init() {
 
 // BinaryArtifacts  will check the repository contains binary artifacts.
 func BinaryArtifacts(c *checker.CheckRequest) checker.CheckResult {
-	rawData, err := raw.BinaryArtifacts(c)
+	rawResults, err := raw.BinaryArtifacts(c.RepoClient)
 	if err != nil {
 		e := sce.WithMessage(sce.ErrScorecardInternal, err.Error())
 		return checker.CreateRuntimeErrorResult(CheckBinaryArtifacts, e)
 	}
 
-	return evaluation.BinaryArtifacts(CheckBinaryArtifacts, c.Dlogger, &rawData)
+	// Return raw results.
+	if c.RawResults != nil {
+		c.RawResults.BinaryArtifactResults = rawResults
+		return checker.CheckResult{}
+	}
+
+	// Return the score evaluation.
+	return evaluation.BinaryArtifacts(CheckBinaryArtifacts, c.Dlogger, &rawResults)
 }

checks/evaluation/binary_artifacts.go

@@ -16,13 +16,12 @@ package evaluation
 
 import (
 	"github.com/ossf/scorecard/v3/checker"
-	"github.com/ossf/scorecard/v3/checks/raw"
 	sce "github.com/ossf/scorecard/v3/errors"
 )
 
 // BinaryArtifacts applies the score policy for the Binary-Artiacts check.
 func BinaryArtifacts(name string, dl checker.DetailLogger,
-	r *raw.BinaryArtifactData) checker.CheckResult {
+	r *checker.BinaryArtifactData) checker.CheckResult {
 	if r == nil {
 		e := sce.WithMessage(sce.ErrScorecardInternal, "empty raw data")
 		return checker.CreateRuntimeErrorResult(name, e)

checks/raw/binary_artifact.go

@@ -24,36 +24,25 @@ import (
 
 	"github.com/ossf/scorecard/v3/checker"
 	"github.com/ossf/scorecard/v3/checks/fileparser"
+	"github.com/ossf/scorecard/v3/clients"
 	sce "github.com/ossf/scorecard/v3/errors"
 )
 
-// File represents a file.
-type File struct {
-	Path string
-	// TODO: add hash if needed.
-}
-
-// BinaryArtifactData contains the raw results.
-type BinaryArtifactData struct {
-	// Files contains a list of files.
-	Files []File
-}
-
 // BinaryArtifacts retrieves the raw data for the Binary-Artifacts check.
-func BinaryArtifacts(c *checker.CheckRequest) (BinaryArtifactData, error) {
-	var files []File
-	err := fileparser.CheckFilesContentV6("*", false, c.RepoClient, checkBinaryFileContent, &files)
+func BinaryArtifacts(c clients.RepoClient) (checker.BinaryArtifactData, error) {
+	var files []checker.File
+	err := fileparser.CheckFilesContentV6("*", false, c, checkBinaryFileContent, &files)
 	if err != nil {
-		return BinaryArtifactData{}, err
+		return checker.BinaryArtifactData{}, fmt.Errorf("%w", err)
 	}
 
 	// No error, return the files.
-	return BinaryArtifactData{Files: files}, nil
+	return checker.BinaryArtifactData{Files: files}, nil
 }
 
 func checkBinaryFileContent(path string, content []byte,
 	data fileparser.FileCbData) (bool, error) {
-	pfiles, ok := data.(*[]File)
+	pfiles, ok := data.(*[]checker.File)
 	if !ok {
 		// This never happens.
 		panic("invalid type")
@@ -97,8 +86,10 @@ func checkBinaryFileContent(path string, content []byte,
 	exists1 := binaryFileTypes[t.Extension]
 	exists2 := binaryFileTypes[strings.ReplaceAll(filepath.Ext(path), ".", "")]
 	if exists1 || exists2 {
-		*pfiles = append(*pfiles, File{
-			Path: path,
+		*pfiles = append(*pfiles, checker.File{
+			Path:   path,
+			Type:   checker.FileTypeBinary,
+			Offset: 1,
 		})
 	}
 

cmd/root.go

@@ -43,6 +43,7 @@ import (
 
 var (
 	repo        string
+	raw         bool
 	local       string
 	checksToRun []string
 	metaData    []string
@@ -271,6 +272,12 @@ var rootCmd = &cobra.Command{
 			log.Fatal("--local option not supported yet")
 		}
 
+		var v6 bool
+		_, v6 = os.LookupEnv("SCORECARD_V6")
+		if raw && !v6 {
+			log.Fatal("--raw option not supported yet")
+		}
+
 		// Validate format.
 		if !validateFormat(format) {
 			log.Fatalf("unsupported format '%s'", format)
@@ -346,7 +353,7 @@ var rootCmd = &cobra.Command{
 
 		enabledChecks, err := getEnabledChecks(policy, checksToRun, supportedChecks, repoType)
 		if err != nil {
-			panic(err)
+			log.Fatal(err)
 		}
 
 		if format == formatDefault {
@@ -354,7 +361,12 @@ var rootCmd = &cobra.Command{
 				fmt.Fprintf(os.Stderr, "Starting [%s]\n", checkName)
 			}
 		}
-		repoResult, err := pkg.RunScorecards(ctx, repoURI, enabledChecks, repoClient, ossFuzzRepoClient, ciiClient)
+
+		if raw && format != "json" {
+			log.Fatalf("only json format is supported")
+		}
+
+		repoResult, err := pkg.RunScorecards(ctx, repoURI, raw, enabledChecks, repoClient, ossFuzzRepoClient, ciiClient)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -379,8 +391,12 @@ var rootCmd = &cobra.Command{
 			// TODO: support config files and update checker.MaxResultScore.
 			err = repoResult.AsSARIF(showDetails, *logLevel, os.Stdout, checkDocs, policy)
 		case formatJSON:
-			// UPGRADEv2: rename.
-			err = repoResult.AsJSON2(showDetails, *logLevel, checkDocs, os.Stdout)
+			if raw {
+				err = repoResult.AsRawJSON(os.Stdout)
+			} else {
+				err = repoResult.AsJSON2(showDetails, *logLevel, checkDocs, os.Stdout)
+			}
+
 		default:
 			err = sce.WithMessage(sce.ErrScorecardInternal,
 				fmt.Sprintf("invalid format flag: %v. Expected [default, json]", format))
@@ -538,4 +554,10 @@ func init() {
 	rootCmd.Flags().StringSliceVar(&checksToRun, "checks", []string{},
 		fmt.Sprintf("Checks to run. Possible values are: %s", strings.Join(checkNames, ",")))
 	rootCmd.Flags().StringVar(&policyFile, "policy", "", "policy to enforce")
+
+	var v6 bool
+	_, v6 = os.LookupEnv("SCORECARD_V6")
+	if v6 {
+		rootCmd.Flags().BoolVar(&raw, "raw", false, "generate raw results")
+	}
 }

cmd/serve.go

@@ -72,7 +72,7 @@ var serveCmd = &cobra.Command{
 			}
 			defer ossFuzzRepoClient.Close()
 			ciiClient := clients.DefaultCIIBestPracticesClient()
-			repoResult, err := pkg.RunScorecards(ctx, repo, checks.AllChecks, repoClient, ossFuzzRepoClient, ciiClient)
+			repoResult, err := pkg.RunScorecards(ctx, repo, false, checks.AllChecks, repoClient, ossFuzzRepoClient, ciiClient)
 			if err != nil {
 				sugar.Error(err)
 				rw.WriteHeader(http.StatusInternalServerError)

cron/worker/main.go

@@ -83,7 +83,7 @@ func processRequest(ctx context.Context,
 			continue
 		}
 		repo.AppendMetadata(repo.Metadata()...)
-		result, err := pkg.RunScorecards(ctx, repo, checksToRun, repoClient, ossFuzzRepoClient, ciiClient)
+		result, err := pkg.RunScorecards(ctx, repo, false, checksToRun, repoClient, ossFuzzRepoClient, ciiClient)
 		if errors.Is(err, sce.ErrRepoUnreachable) {
 			// Not accessible repo - continue.
 			continue

pkg/json_raw_results.go

@@ -0,0 +1,90 @@
+// Copyright 2021 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"github.com/ossf/scorecard/v3/checker"
+	sce "github.com/ossf/scorecard/v3/errors"
+)
+
+// Flat JSON structure to hold raw results.
+type jsonScorecardRawResult struct {
+	Date      string          `json:"date"`
+	Repo      jsonRepoV2      `json:"repo"`
+	Scorecard jsonScorecardV2 `json:"scorecard"`
+	Metadata  []string        `json:"metadata"`
+	Results   jsonRawResults  `json:"results"`
+}
+
+// TODO: separate each check extraction into its own file.
+type jsonBinaryFiles struct {
+	Path   string `json:"path"`
+	Offset int    `json:"offset,omitempty"`
+}
+
+type jsonRawResults struct {
+	// List of binaries found in the repo.
+	Binaries []jsonBinaryFiles `json:"binaries"`
+}
+
+//nolint:unparam
+func (r *jsonScorecardRawResult) addBinaryArtifactRawResults(ba *checker.BinaryArtifactData) error {
+	for _, v := range ba.Files {
+		r.Results.Binaries = append(r.Results.Binaries, jsonBinaryFiles{
+			Path: v.Path,
+		})
+	}
+	return nil
+}
+
+func (r *jsonScorecardRawResult) fillJSONRawResults(raw *checker.RawResults) error {
+	// Binary-Artifacts.
+	if err := r.addBinaryArtifactRawResults(&raw.BinaryArtifactResults); err != nil {
+		return sce.WithMessage(sce.ErrScorecardInternal, err.Error())
+	}
+	return nil
+}
+
+// AsRawJSON exports results as JSON for raw results.
+func (r *ScorecardResult) AsRawJSON(writer io.Writer) error {
+	encoder := json.NewEncoder(writer)
+	out := jsonScorecardRawResult{
+		Repo: jsonRepoV2{
+			Name:   r.Repo.Name,
+			Commit: r.Repo.CommitSHA,
+		},
+		Scorecard: jsonScorecardV2{
+			Version: r.Scorecard.Version,
+			Commit:  r.Scorecard.CommitSHA,
+		},
+		Date:     r.Date.Format("2006-01-02"),
+		Metadata: r.Metadata,
+	}
+
+	// if err := out.fillJSONRawResults(r.Checks[0].RawResults); err != nil {
+	if err := out.fillJSONRawResults(&r.RawResults); err != nil {
+		return err
+	}
+
+	if err := encoder.Encode(out); err != nil {
+		return sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("encoder.Encode: %v", err))
+	}
+
+	return nil
+}

pkg/scorecard.go

@@ -28,7 +28,7 @@ import (
 )
 
 func runEnabledChecks(ctx context.Context,
-	repo clients.Repo, checksToRun checker.CheckNameToFnMap,
+	repo clients.Repo, raw *checker.RawResults, checksToRun checker.CheckNameToFnMap,
 	repoClient clients.RepoClient, ossFuzzRepoClient clients.RepoClient, ciiClient clients.CIIBestPracticesClient,
 	resultsCh chan checker.CheckResult) {
 	request := checker.CheckRequest{
@@ -37,6 +37,7 @@ func runEnabledChecks(ctx context.Context,
 		OssFuzzRepo: ossFuzzRepoClient,
 		CIIClient:   ciiClient,
 		Repo:        repo,
+		RawResults:  raw,
 	}
 	wg := sync.WaitGroup{}
 	for checkName, checkFn := range checksToRun {
@@ -73,6 +74,7 @@ func getRepoCommitHash(r clients.RepoClient) (string, error) {
 // RunScorecards runs enabled Scorecard checks on a Repo.
 func RunScorecards(ctx context.Context,
 	repo clients.Repo,
+	raw bool,
 	checksToRun checker.CheckNameToFnMap,
 	repoClient clients.RepoClient,
 	ossFuzzRepoClient clients.RepoClient,
@@ -101,7 +103,12 @@ func RunScorecards(ctx context.Context,
 		Date: time.Now(),
 	}
 	resultsCh := make(chan checker.CheckResult)
-	go runEnabledChecks(ctx, repo, checksToRun, repoClient, ossFuzzRepoClient, ciiClient, resultsCh)
+	if raw {
+		go runEnabledChecks(ctx, repo, &ret.RawResults, checksToRun, repoClient, ossFuzzRepoClient, ciiClient, resultsCh)
+	} else {
+		go runEnabledChecks(ctx, repo, nil, checksToRun, repoClient, ossFuzzRepoClient, ciiClient, resultsCh)
+	}
+
 	for result := range resultsCh {
 		ret.Checks = append(ret.Checks, result)
 	}

pkg/scorecard_result.go

@@ -42,11 +42,12 @@ type RepoInfo struct {
 
 // ScorecardResult struct is returned on a successful Scorecard run.
 type ScorecardResult struct {
-	Repo      RepoInfo
-	Date      time.Time
-	Scorecard ScorecardInfo
-	Checks    []checker.CheckResult
-	Metadata  []string
+	Repo       RepoInfo
+	Date       time.Time
+	Scorecard  ScorecardInfo
+	Checks     []checker.CheckResult
+	RawResults checker.RawResults
+	Metadata   []string
 }
 
 func scoreToString(s float64) string {