🌱 Feature: Add scorecard attestation policy module (#2240)

* Add ability to parse policy.yaml

Temporary commit

Temporary commit

Temporary commit

Temporary commit

Temporary commit

Temporary commit

* Remove hidden options

* Fix cilint problems

* Add tests

* Add tests

* Address PR comments

* Refactor to standalone module
* Don't depend on evaluation package
* Remove everything but the Binary-Artifact

* Fix test failures

Signed-off-by: Raghav Kaul <raghavkaul@google.com>

* Address PR comments

* Use glob for binary artifact ignores
* Makefile

Signed-off-by: Raghav Kaul <raghavkaul@google.com>

Signed-off-by: Raghav Kaul <raghavkaul@google.com>

Makefile

@@ -55,8 +55,8 @@ all: update-dependencies all-targets-update-dependencies tree-status
 update-dependencies: ## Update go dependencies for all modules
 	# Update root go modules
 	go mod tidy && go mod verify
-	cd tools
-	go mod tidy && go mod verify
+	cd tools; go mod tidy && go mod verify; cd ../
+	cd attestor; go mod tidy && go mod verify; cd ../
 
 $(GOLANGCI_LINT): install
 check-linter: ## Install and run golang linter
@@ -70,9 +70,11 @@ check-osv: $(install)
 	go list -m -f '{{if not (or  .Main)}}{{.Path}}@{{.Version}}_{{.Replace}}{{end}}' all \
 			| stunning-tribble
 	# Checking the tools which also has go.mod
-	cd tools 
-	go list -m -f '{{if not (or  .Main)}}{{.Path}}@{{.Version}}_{{.Replace}}{{end}}' all \
-			| stunning-tribble 
+	cd tools; go list -m -f '{{if not (or  .Main)}}{{.Path}}@{{.Version}}_{{.Replace}}{{end}}' all \
+			| stunning-tribble ; cd ..
+	# Checking the attestor module for vulns
+	cd attestor; go list -m -f '{{if not (or  .Main)}}{{.Path}}@{{.Version}}_{{.Replace}}{{end}}' all \
+			| stunning-tribble ; cd ..
 
 add-projects: ## Adds new projects to ./cron/internal/data/projects.csv
 add-projects: ./cron/internal/data/projects.csv | build-add-script
@@ -276,7 +278,7 @@ cron-github-server-docker:
 
 ##@ Tests
 ################################# make test ###################################
-test-targets = unit-test e2e-pat e2e-gh-token ci-e2e
+test-targets = unit-test unit-test-attestor e2e-pat e2e-gh-token ci-e2e
 .PHONY: test $(test-targets)
 test: $(test-targets)
 
@@ -285,6 +287,9 @@ unit-test: ## Runs unit test without e2e
 	# run the go tests and gen the file coverage-all used to do the integration with codecov
 	SKIP_GINKGO=1 go test -race -covermode=atomic  -coverprofile=unit-coverage.out `go list ./...`
 
+unit-test-attestor: ## Runs unit tests on scorecard-attestor
+	cd attestor; SKIP_GINKGO=1 go test -covermode=atomic -coverprofile=unit-coverage-attestor.out `go list ./...`; cd ..;
+
 $(GINKGO): install
 
 check-env:

attestor/attestation_policy.go

@@ -0,0 +1,133 @@
+// Copyright 2021 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package policy
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/gobwas/glob"
+	"gopkg.in/yaml.v2"
+
+	"github.com/ossf/scorecard/v4/checker"
+	sce "github.com/ossf/scorecard/v4/errors"
+)
+
+//nolint:govet
+type AttestationPolicy struct {
+	// PreventBinaryArtifacts : set to true to require that this project's SCM repo is
+	// free of binary artifacts
+	PreventBinaryArtifacts bool `yaml:"preventBinaryArtifacts"`
+
+	// AllowedBinaryArtifacts : List of binary artifact paths to ignore
+	// when checking for binary artifacts in a repo
+	AllowedBinaryArtifacts []string `yaml:"allowedBinaryArtifacts"`
+}
+
+// Run attestation policy checks on raw data.
+func RunChecksForPolicy(policy *AttestationPolicy, raw *checker.RawResults,
+	dl checker.DetailLogger,
+) (PolicyResult, error) {
+	if policy.PreventBinaryArtifacts {
+		checkResult, err := CheckPreventBinaryArtifacts(policy.AllowedBinaryArtifacts, raw, dl)
+
+		if !checkResult || err != nil {
+			return checkResult, err
+		}
+	}
+
+	return Pass, nil
+}
+
+type PolicyResult = bool
+
+const (
+	Pass PolicyResult = true
+	Fail PolicyResult = false
+)
+
+func CheckPreventBinaryArtifacts(
+	allowedBinaryArtifacts []string,
+	results *checker.RawResults,
+	dl checker.DetailLogger,
+) (PolicyResult, error) {
+	for i := range results.BinaryArtifactResults.Files {
+		artifactFile := results.BinaryArtifactResults.Files[i]
+
+		ignoreArtifact := false
+
+		for j := range allowedBinaryArtifacts {
+			// Treat user input as paths and try to match prefixes
+			// This is a bit easier to use than forcing things to be file names
+			allowGlob := allowedBinaryArtifacts[j]
+
+			if g := glob.MustCompile(allowGlob); g.Match(artifactFile.Path) {
+				ignoreArtifact = true
+				dl.Info(&checker.LogMessage{Text: fmt.Sprintf(
+					"ignoring binary artifact at %s due to ignored glob path %s",
+					artifactFile.Path,
+					g,
+				)})
+			}
+		}
+
+		if !ignoreArtifact {
+			dl.Info(&checker.LogMessage{
+				Path: artifactFile.Path, Type: checker.FileTypeBinary,
+				Offset: artifactFile.Offset,
+				Text:   "binary detected",
+			})
+			return Fail, nil
+		}
+	}
+
+	return Pass, nil
+}
+
+// ParseFromFile takes a policy file and returns an AttestationPolicy.
+func ParseAttestationPolicyFromFile(policyFile string) (*AttestationPolicy, error) {
+	if policyFile != "" {
+		data, err := os.ReadFile(policyFile)
+		if err != nil {
+			return nil, sce.WithMessage(sce.ErrScorecardInternal,
+				fmt.Sprintf("os.ReadFile: %v", err))
+		}
+
+		sp, err := ParseAttestationPolicyFromYAML(data)
+		if err != nil {
+			return nil,
+				sce.WithMessage(
+					sce.ErrScorecardInternal,
+					fmt.Sprintf("spol.ParseFromYAML: %v", err),
+				)
+		}
+
+		return sp, nil
+	}
+
+	return nil, nil
+}
+
+// Parses a policy file and returns a AttestationPolicy.
+func ParseAttestationPolicyFromYAML(b []byte) (*AttestationPolicy, error) {
+	retPolicy := AttestationPolicy{}
+
+	err := yaml.Unmarshal(b, &retPolicy)
+	if err != nil {
+		return &retPolicy, sce.WithMessage(sce.ErrScorecardInternal, err.Error())
+	}
+
+	return &retPolicy, nil
+}

attestor/attestation_policy_test.go

@@ -0,0 +1,191 @@
+// Copyright 2022 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package policy
+
+import (
+	"encoding/json"
+	"errors"
+	"testing"
+
+	"github.com/ossf/scorecard/v4/checker"
+	sce "github.com/ossf/scorecard/v4/errors"
+	scut "github.com/ossf/scorecard/v4/utests"
+)
+
+func (a AttestationPolicy) ToJSON() string {
+	jsonbytes, err := json.Marshal(a)
+	if err != nil {
+		return ""
+	}
+
+	return string(jsonbytes)
+}
+
+func TestCheckPreventBinaryArtifacts(t *testing.T) {
+	t.Parallel()
+
+	dl := scut.TestDetailLogger{}
+
+	tests := []struct {
+		name                   string
+		raw                    *checker.RawResults
+		err                    error
+		allowedBinaryArtifacts []string
+		expected               PolicyResult
+	}{
+		{
+			name: "test with no artifacts",
+			raw: &checker.RawResults{
+				BinaryArtifactResults: checker.BinaryArtifactData{Files: []checker.File{}},
+			},
+			expected: Pass,
+			err:      nil,
+		},
+		{
+			name: "test with multiple artifacts",
+			raw: &checker.RawResults{
+				BinaryArtifactResults: checker.BinaryArtifactData{Files: []checker.File{
+					{Path: "a"},
+					{Path: "b"},
+				}},
+			},
+			expected: Fail,
+			err:      nil,
+		},
+		{
+			name:                   "test with multiple ignored artifacts",
+			allowedBinaryArtifacts: []string{"a", "b"},
+			raw: &checker.RawResults{
+				BinaryArtifactResults: checker.BinaryArtifactData{Files: []checker.File{
+					{Path: "a"},
+					{Path: "b"},
+				}},
+			},
+			expected: Pass,
+			err:      nil,
+		},
+		{
+			name:                   "test with some artifacts",
+			allowedBinaryArtifacts: []string{"a"},
+			raw: &checker.RawResults{
+				BinaryArtifactResults: checker.BinaryArtifactData{Files: []checker.File{
+					{Path: "a"},
+					{Path: "b/a"},
+				}},
+			},
+			expected: Fail,
+			err:      nil,
+		},
+
+		{
+			name:                   "test with glob ignored",
+			allowedBinaryArtifacts: []string{"a/*", "b/*"},
+			raw: &checker.RawResults{
+				BinaryArtifactResults: checker.BinaryArtifactData{Files: []checker.File{
+					{Path: "a/c/foo.txt"},
+					{Path: "b/c/foo.txt"},
+				}},
+			},
+			expected: Pass,
+			err:      nil,
+		},
+	}
+
+	for i := range tests {
+		tt := &tests[i]
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			actual, err := CheckPreventBinaryArtifacts(tt.allowedBinaryArtifacts, tt.raw, &dl)
+
+			if !errors.Is(err, tt.err) {
+				t.Fatalf("%s: expected %v, got %v", tt.name, tt.err, err)
+			}
+			if err != nil {
+				return
+			}
+
+			// Compare outputs only if the error is nil.
+			// TODO: compare objects.
+			if actual != tt.expected {
+				t.Fatalf("%s: invalid result", tt.name)
+			}
+		})
+	}
+}
+
+func TestAttestationPolicyRead(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		err      error
+		name     string
+		filename string
+		result   AttestationPolicy
+	}{
+		{
+			name:     "default attestation policy with everything on",
+			filename: "./testdata/policy-binauthz.yaml",
+			err:      nil,
+			result: AttestationPolicy{
+				PreventBinaryArtifacts: true,
+				AllowedBinaryArtifacts: []string{},
+			},
+		},
+		{
+			name:     "invalid attestation policy",
+			filename: "./testdata/policy-binauthz-invalid.yaml",
+			err:      sce.ErrScorecardInternal,
+		},
+		{
+			name:     "policy with allowlist of binary artifacts",
+			filename: "./testdata/policy-binauthz-allowlist.yaml",
+			err:      nil,
+			result: AttestationPolicy{
+				PreventBinaryArtifacts: true,
+				AllowedBinaryArtifacts: []string{"/a/b/c", "d"},
+			},
+		},
+		{
+			name:     "policy with allowlist of binary artifacts",
+			filename: "./testdata/policy-binauthz-missingparam.yaml",
+			err:      nil,
+			result: AttestationPolicy{
+				PreventBinaryArtifacts: true,
+				AllowedBinaryArtifacts: nil,
+			},
+		},
+	}
+
+	for i := range tests {
+		tt := &tests[i]
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			p, err := ParseAttestationPolicyFromFile(tt.filename)
+
+			if !errors.Is(err, tt.err) {
+				t.Fatalf("%s: expected %v, got %v", tt.name, tt.err, err)
+			}
+			if err != nil {
+				return
+			}
+
+			// Compare outputs only if the error is nil.
+			// TODO: compare objects.
+			if p.ToJSON() != tt.result.ToJSON() {
+				t.Fatalf("%s: invalid result", tt.name)
+			}
+		})
+	}
+}

attestor/go.mod

@@ -0,0 +1,47 @@
+module github.com/ossf/scorecard-attestor
+
+go 1.19
+
+require (
+	github.com/ossf/scorecard/v4 v4.6.0
+	gopkg.in/yaml.v2 v2.4.0
+)
+
+require (
+	cloud.google.com/go v0.102.1 // indirect
+	cloud.google.com/go/compute v1.7.0 // indirect
+	cloud.google.com/go/iam v0.3.0 // indirect
+	cloud.google.com/go/storage v1.23.0 // indirect
+	github.com/bombsimon/logrusr/v2 v2.0.1 // indirect
+	github.com/bradleyfalzon/ghinstallation/v2 v2.1.0 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/gobwas/glob v0.2.3
+	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.8 // indirect
+	github.com/google/go-github/v38 v38.1.0 // indirect
+	github.com/google/go-github/v45 v45.2.0 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/wire v0.5.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.5.1 // indirect
+	github.com/googleapis/go-type-adapters v1.0.0 // indirect
+	github.com/shurcooL/githubv4 v0.0.0-20201206200315-234843c633fa // indirect
+	github.com/shurcooL/graphql v0.0.0-20200928012149-18c5c3165e3a // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	gocloud.dev v0.26.0 // indirect
+	golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect
+	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
+	golang.org/x/oauth2 v0.0.0-20220718184931-c8730f7fcb92 // indirect
+	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
+	google.golang.org/api v0.92.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20220812140447-cec7f5303424 // indirect
+	google.golang.org/grpc v1.48.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+)

attestor/testdata/policy-binauthz-allowlist.yaml

@@ -0,0 +1,23 @@
+# Copyright 2021 Security Scorecard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this exe except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# PreventBinaryArtifacts : set to true to require that this project's SCM repo is
+# free of binary artifacts
+preventBinaryArtifacts: true
+
+# AllowedBinaryArtifacts : List of binary artifact paths to ignore
+# when checking for binary artifacts in a repo
+allowedBinaryArtifacts:
+# List of allowed binary artifact paths as strings
+    - /a/b/c
+    - d

attestor/testdata/policy-binauthz-invalid.yaml

@@ -0,0 +1,20 @@
+# Copyright 2021 Security Scorecard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this exe except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# PreventBinaryArtifacts : set to true to require that this project's SCM repo is
+# free of binary artifacts
+preventBinaryArtifacts: true
+
+# AllowedBinaryArtifacts : List of binary artifact paths to ignore
+# when checking for binary artifacts in a repo
+allowedBinaryArtifacts: true

attestor/testdata/policy-binauthz-missingparam.yaml

@@ -0,0 +1,16 @@
+# Copyright 2021 Security Scorecard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this exe except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# PreventBinaryArtifacts : set to true to require that this project's SCM repo is
+# free of binary artifacts
+preventBinaryArtifacts: true

attestor/testdata/policy-binauthz.yaml

@@ -0,0 +1,21 @@
+# Copyright 2021 Security Scorecard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this exe except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# PreventBinaryArtifacts : set to true to require that this project's SCM repo is
+# free of binary artifacts
+preventBinaryArtifacts: true
+
+# AllowedBinaryArtifacts : List of binary artifact paths to ignore
+# when checking for binary artifacts in a repo
+allowedBinaryArtifacts: []
+# List of allowed binary artifact paths as strings

checks/evaluation/vulnerabilities.go

@@ -38,8 +38,8 @@ func Vulnerabilities(name string, dl checker.DetailLogger,
 		score--
 	}
 
-	if score < 0 {
-		score = 0
+	if score < checker.MinResultScore {
+		score = checker.MinResultScore
 	}
 
 	if len(IDs) > 0 {