From cb721a8526fccd47bc234d9d2005eecd01e4b0ae Mon Sep 17 00:00:00 2001 From: AdamKorcz <44787359+AdamKorcz@users.noreply.github.com> Date: Tue, 5 Dec 2023 08:24:16 +0000 Subject: [PATCH] :seedling: convert binary artifact check to probe (#3508) * :seedling: convert binary artifact check to probe Signed-off-by: AdamKorcz * Reword motivation Signed-off-by: AdamKorcz * remove unused variable in test Signed-off-by: AdamKorcz * remove positiveOutcome() and length check Signed-off-by: AdamKorcz * fix wrong check name Signed-off-by: AdamKorcz * Split into two probes: One with and one without gradle-wrappers Signed-off-by: AdamKorcz * Add description about what Scorecard considers a verified binary Signed-off-by: Adam Korczynski * change 'trusted' to 'verified' Signed-off-by: Adam Korczynski * remove nil check Signed-off-by: Adam Korczynski * remove filtering Signed-off-by: Adam Korczynski * use const scores in tests Signed-off-by: Adam Korczynski * rename test Signed-off-by: Adam Korczynski * add sanity check in loop Signed-off-by: Adam Korczynski * rename binary file const Signed-off-by: Adam Korczynski --------- Signed-off-by: AdamKorcz Signed-off-by: Adam Korczynski --- checks/binary_artifact.go | 19 +- checks/evaluation/binary_artifacts.go | 37 +- checks/evaluation/binary_artifacts_test.go | 332 +++++------------- checks/raw/binary_artifact.go | 11 +- checks/raw/binary_artifact_test.go | 25 +- finding/finding.go | 2 + probes/entries.go | 4 + probes/freeOfAnyBinaryArtifacts/def.yml | 28 ++ probes/freeOfAnyBinaryArtifacts/impl.go | 66 ++++ probes/freeOfAnyBinaryArtifacts/impl_test.go | 158 +++++++++ .../freeOfUnverifiedBinaryArtifacts/def.yml | 28 ++ .../freeOfUnverifiedBinaryArtifacts/impl.go | 69 ++++ .../impl_test.go | 123 +++++++ 13 files changed, 638 insertions(+), 264 deletions(-) create mode 100644 probes/freeOfAnyBinaryArtifacts/def.yml create mode 100644 probes/freeOfAnyBinaryArtifacts/impl.go create mode 100644 probes/freeOfAnyBinaryArtifacts/impl_test.go create mode 100644 probes/freeOfUnverifiedBinaryArtifacts/def.yml create mode 100644 probes/freeOfUnverifiedBinaryArtifacts/impl.go create mode 100644 probes/freeOfUnverifiedBinaryArtifacts/impl_test.go diff --git a/checks/binary_artifact.go b/checks/binary_artifact.go index 63a7ec98..f57d1545 100644 --- a/checks/binary_artifact.go +++ b/checks/binary_artifact.go @@ -19,6 +19,8 @@ import ( "github.com/ossf/scorecard/v4/checks/evaluation" "github.com/ossf/scorecard/v4/checks/raw" sce "github.com/ossf/scorecard/v4/errors" + "github.com/ossf/scorecard/v4/probes" + "github.com/ossf/scorecard/v4/probes/zrunner" ) // CheckBinaryArtifacts is the exported name for Binary-Artifacts check. @@ -38,17 +40,22 @@ func init() { // BinaryArtifacts will check the repository contains binary artifacts. func BinaryArtifacts(c *checker.CheckRequest) checker.CheckResult { - rawData, err := raw.BinaryArtifacts(c.RepoClient) + rawData, err := raw.BinaryArtifacts(c) if err != nil { e := sce.WithMessage(sce.ErrScorecardInternal, err.Error()) return checker.CreateRuntimeErrorResult(CheckBinaryArtifacts, e) } - // Return raw results. - if c.RawResults != nil { - c.RawResults.BinaryArtifactResults = rawData + // Set the raw results. + pRawResults := getRawResults(c) + pRawResults.BinaryArtifactResults = rawData + + // Evaluate the probes. + findings, err := zrunner.Run(pRawResults, probes.BinaryArtifacts) + if err != nil { + e := sce.WithMessage(sce.ErrScorecardInternal, err.Error()) + return checker.CreateRuntimeErrorResult(CheckBinaryArtifacts, e) } - // Return the score evaluation. - return evaluation.BinaryArtifacts(CheckBinaryArtifacts, c.Dlogger, &rawData) + return evaluation.BinaryArtifacts(CheckBinaryArtifacts, findings, c.Dlogger) } diff --git a/checks/evaluation/binary_artifacts.go b/checks/evaluation/binary_artifacts.go index 59075912..fff943fb 100644 --- a/checks/evaluation/binary_artifacts.go +++ b/checks/evaluation/binary_artifacts.go @@ -18,33 +18,46 @@ import ( "github.com/ossf/scorecard/v4/checker" sce "github.com/ossf/scorecard/v4/errors" "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/freeOfUnverifiedBinaryArtifacts" ) // BinaryArtifacts applies the score policy for the Binary-Artifacts check. -func BinaryArtifacts(name string, dl checker.DetailLogger, - r *checker.BinaryArtifactData, +func BinaryArtifacts(name string, + findings []finding.Finding, + dl checker.DetailLogger, ) checker.CheckResult { - if r == nil { - e := sce.WithMessage(sce.ErrScorecardInternal, "empty raw data") + expectedProbes := []string{ + freeOfUnverifiedBinaryArtifacts.Probe, + } + + if !finding.UniqueProbesEqual(findings, expectedProbes) { + e := sce.WithMessage(sce.ErrScorecardInternal, "invalid probe results") return checker.CreateRuntimeErrorResult(name, e) } - // Apply the policy evaluation. - if r.Files == nil || len(r.Files) == 0 { + if findings[0].Outcome == finding.OutcomePositive { return checker.CreateMaxScoreResult(name, "no binaries found in the repo") } - score := checker.MaxResultScore - for _, f := range r.Files { + for i := range findings { + f := &findings[i] + if f.Outcome != finding.OutcomeNegative { + continue + } dl.Warn(&checker.LogMessage{ - Path: f.Path, Type: finding.FileTypeBinary, - Offset: f.Offset, + Path: f.Location.Path, + Type: f.Location.Type, + Offset: *f.Location.LineStart, Text: "binary detected", }) - // We remove one point for each binary. - score-- } + // There are only negative findings. + // Deduct the number of findings from max score + numberOfBinaryFilesFound := len(findings) + + score := checker.MaxResultScore - numberOfBinaryFilesFound + if score < checker.MinResultScore { score = checker.MinResultScore } diff --git a/checks/evaluation/binary_artifacts_test.go b/checks/evaluation/binary_artifacts_test.go index ed7a0fa9..d2f9c4f8 100644 --- a/checks/evaluation/binary_artifacts_test.go +++ b/checks/evaluation/binary_artifacts_test.go @@ -18,256 +18,121 @@ import ( "testing" "github.com/ossf/scorecard/v4/checker" + sce "github.com/ossf/scorecard/v4/errors" + "github.com/ossf/scorecard/v4/finding" scut "github.com/ossf/scorecard/v4/utests" ) // TestBinaryArtifacts tests the binary artifacts check. func TestBinaryArtifacts(t *testing.T) { t.Parallel() - //nolint:govet - type args struct { - name string - dl checker.DetailLogger - r *checker.BinaryArtifactData + lineStart := uint(123) + negativeFinding := finding.Finding{ + Probe: "freeOfUnverifiedBinaryArtifacts", + Outcome: finding.OutcomeNegative, + + Location: &finding.Location{ + Path: "path", + Type: finding.FileTypeBinary, + LineStart: &lineStart, + }, } + tests := []struct { - name string - args args - want checker.CheckResult - wantErr bool + name string + findings []finding.Finding + result scut.TestReturn }{ - { - name: "r nil", - args: args{ - name: "test_binary_artifacts_check_pass", - dl: &scut.TestDetailLogger{}, - }, - wantErr: true, - }, { name: "no binary artifacts", - args: args{ - name: "no binary artifacts", - dl: &scut.TestDetailLogger{}, - r: &checker.BinaryArtifactData{}, + findings: []finding.Finding{ + { + Probe: "freeOfUnverifiedBinaryArtifacts", + Outcome: finding.OutcomePositive, + }, }, - want: checker.CheckResult{ + result: scut.TestReturn{ Score: checker.MaxResultScore, }, }, { - name: "1 binary artifact", - args: args{ - name: "no binary artifacts", - dl: &scut.TestDetailLogger{}, - r: &checker.BinaryArtifactData{ - Files: []checker.File{ - { - Path: "test_binary_artifacts_check_pass", - Snippet: ` - package main - import "fmt" - func main() { - fmt.Println("Hello, world!") - }i`, - Offset: 0, - Type: 0, - }, - }, - }, + name: "one binary artifact", + findings: []finding.Finding{ + negativeFinding, }, - want: checker.CheckResult{ - Score: 9, + result: scut.TestReturn{ + Score: 9, + NumberOfWarn: 1, }, }, { - name: "many binary artifact", - args: args{ - name: "no binary artifacts", - dl: &scut.TestDetailLogger{}, - r: &checker.BinaryArtifactData{ - Files: []checker.File{ - { - Path: "test_binary_artifacts_check_pass", - Snippet: ` - package main - import "fmt" - func main() { - fmt.Println("Hello, world!") - }i`, - Offset: 0, - Type: 0, - }, - { - Path: "test_binary_artifacts_check_pass", - Snippet: ` - package main - import "fmt" - func main() { - fmt.Println("Hello, world!") - }i`, - Offset: 0, - Type: 0, - }, - { - Path: "test_binary_artifacts_check_pass", - Snippet: ` - package main - import "fmt" - func main() { - fmt.Println("Hello, world!") - }i`, - Offset: 0, - Type: 0, - }, - { - Path: "test_binary_artifacts_check_pass", - Snippet: ` - package main - import "fmt" - func main() { - fmt.Println("Hello, world!") - }i`, - Offset: 0, - Type: 0, - }, - { - Path: "test_binary_artifacts_check_pass", - Snippet: ` - package main - import "fmt" - func main() { - fmt.Println("Hello, world!") - }i`, - Offset: 0, - Type: 0, - }, - { - Path: "test_binary_artifacts_check_pass", - Snippet: ` - package main - import "fmt" - func main() { - fmt.Println("Hello, world!") - }i`, - Offset: 0, - Type: 0, - }, - { - Path: "test_binary_artifacts_check_pass", - Snippet: ` - package main - import "fmt" - func main() { - fmt.Println("Hello, world!") - }i`, - Offset: 0, - Type: 0, - }, - { - Path: "test_binary_artifacts_check_pass", - Snippet: ` - package main - import "fmt" - func main() { - fmt.Println("Hello, world!") - }i`, - Offset: 0, - Type: 0, - }, - { - Path: "test_binary_artifacts_check_pass", - Snippet: ` - package main - import "fmt" - func main() { - fmt.Println("Hello, world!") - }i`, - Offset: 0, - Type: 0, - }, - { - Path: "test_binary_artifacts_check_pass", - Snippet: ` - package main - import "fmt" - func main() { - fmt.Println("Hello, world!") - }i`, - Offset: 0, - Type: 0, - }, - { - Path: "test_binary_artifacts_check_pass", - Snippet: ` - package main - import "fmt" - func main() { - fmt.Println("Hello, world!") - }i`, - Offset: 0, - Type: 0, - }, - { - Path: "test_binary_artifacts_check_pass", - Snippet: ` - package main - import "fmt" - func main() { - fmt.Println("Hello, world!") - }i`, - Offset: 0, - Type: 0, - }, - { - Path: "test_binary_artifacts_check_pass", - Snippet: ` - package main - import "fmt" - func main() { - fmt.Println("Hello, world!") - }i`, - Offset: 0, - Type: 0, - }, - { - Path: "test_binary_artifacts_check_pass", - Snippet: ` - package main - import "fmt" - func main() { - fmt.Println("Hello, world!") - }i`, - Offset: 0, - Type: 0, - }, - { - Path: "test_binary_artifacts_check_pass", - Snippet: ` - package main - import "fmt" - func main() { - fmt.Println("Hello, world!") - }i`, - Offset: 0, - Type: 0, - }, - { - Path: "test_binary_artifacts_check_pass", - Snippet: ` - package main - import "fmt" - func main() { - fmt.Println("Hello, world!") - }i`, - Offset: 0, - Type: 0, - }, + name: "two binary artifact", + findings: []finding.Finding{ + { + Probe: "freeOfUnverifiedBinaryArtifacts", + Outcome: finding.OutcomeNegative, + Location: &finding.Location{ + Path: "path", + Type: finding.FileTypeBinary, + LineStart: &lineStart, + }, + }, + { + Probe: "freeOfUnverifiedBinaryArtifacts", + Outcome: finding.OutcomeNegative, + Location: &finding.Location{ + Path: "path", + Type: finding.FileTypeBinary, + LineStart: &lineStart, }, }, }, - want: checker.CheckResult{ - Score: 0, + result: scut.TestReturn{ + Score: 8, + NumberOfWarn: 2, + }, + }, + { + name: "five binary artifact", + findings: []finding.Finding{ + negativeFinding, + negativeFinding, + negativeFinding, + negativeFinding, + negativeFinding, + }, + result: scut.TestReturn{ + Score: 5, + NumberOfWarn: 5, + }, + }, + { + name: "twelve binary artifact - ensure score doesn't drop below min", + findings: []finding.Finding{ + negativeFinding, + negativeFinding, + negativeFinding, + negativeFinding, + negativeFinding, + negativeFinding, + negativeFinding, + negativeFinding, + negativeFinding, + negativeFinding, + negativeFinding, + negativeFinding, + }, + result: scut.TestReturn{ + Score: checker.MinResultScore, + NumberOfWarn: 12, + }, + }, + { + name: "invalid findings", + findings: []finding.Finding{}, + result: scut.TestReturn{ + Score: checker.InconclusiveResultScore, + Error: sce.ErrScorecardInternal, }, }, } @@ -275,15 +140,10 @@ func TestBinaryArtifacts(t *testing.T) { tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() - got := BinaryArtifacts(tt.args.name, tt.args.dl, tt.args.r) - if tt.wantErr { - if got.Error == nil { - t.Errorf("BinaryArtifacts() error = %v, wantErr %v", got.Error, tt.wantErr) - } - } else { - if got.Score != tt.want.Score { - t.Errorf("BinaryArtifacts() = %v, want %v", got.Score, tt.want.Score) - } + dl := scut.TestDetailLogger{} + got := BinaryArtifacts(tt.name, tt.findings, &dl) + if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) { + t.Errorf("got %v, expected %v", got, tt.result) } }) } diff --git a/checks/raw/binary_artifact.go b/checks/raw/binary_artifact.go index 03c129b5..c2fc925c 100644 --- a/checks/raw/binary_artifact.go +++ b/checks/raw/binary_artifact.go @@ -49,7 +49,8 @@ func mustParseConstraint(c string) *semver.Constraints { } // BinaryArtifacts retrieves the raw data for the Binary-Artifacts check. -func BinaryArtifacts(c clients.RepoClient) (checker.BinaryArtifactData, error) { +func BinaryArtifacts(req *checker.CheckRequest) (checker.BinaryArtifactData, error) { + c := req.RepoClient files := []checker.File{} err := fileparser.OnMatchingFileContentDo(c, fileparser.PathMatcher{ Pattern: "*", @@ -87,13 +88,11 @@ func excludeValidatedGradleWrappers(c clients.RepoClient, files []checker.File) } // It has been confirmed that latest commit has validated JARs! // Remove Gradle wrapper JARs from files. - filterFiles := []checker.File{} - for _, f := range files { - if filepath.Base(f.Path) != "gradle-wrapper.jar" { - filterFiles = append(filterFiles, f) + for i := range files { + if filepath.Base(files[i].Path) == "gradle-wrapper.jar" { + files[i].Type = finding.FileTypeBinaryVerified } } - files = filterFiles return files, nil } diff --git a/checks/raw/binary_artifact_test.go b/checks/raw/binary_artifact_test.go index ace73e34..22362c40 100644 --- a/checks/raw/binary_artifact_test.go +++ b/checks/raw/binary_artifact_test.go @@ -21,8 +21,10 @@ import ( "github.com/golang/mock/gomock" + "github.com/ossf/scorecard/v4/checker" "github.com/ossf/scorecard/v4/clients" mockrepo "github.com/ossf/scorecard/v4/clients/mockclients" + scut "github.com/ossf/scorecard/v4/utests" ) func strptr(s string) *string { @@ -126,7 +128,7 @@ func TestBinaryArtifacts(t *testing.T) { }, }, getFileContentCount: 3, - expect: 0, + expect: 1, }, { name: "gradle-wrapper.jar with non-verification action", @@ -210,7 +212,7 @@ func TestBinaryArtifacts(t *testing.T) { }, }, getFileContentCount: 3, - expect: 0, + expect: 1, }, } for _, tt := range tests { @@ -220,6 +222,7 @@ func TestBinaryArtifacts(t *testing.T) { ctrl := gomock.NewController(t) mockRepoClient := mockrepo.NewMockRepoClient(ctrl) + mockRepo := mockrepo.NewMockRepo(ctrl) for _, files := range tt.files { mockRepoClient.EXPECT().ListFiles(gomock.Any()).Return(files, nil) } @@ -240,7 +243,14 @@ func TestBinaryArtifacts(t *testing.T) { mockRepoClient.EXPECT().ListCommits().Return(tt.commits, nil) } - f, err := BinaryArtifacts(mockRepoClient) + dl := scut.TestDetailLogger{} + c := &checker.CheckRequest{ + RepoClient: mockRepoClient, + Repo: mockRepo, + Dlogger: &dl, + } + + f, err := BinaryArtifacts(c) if tt.err != nil { // If we expect an error, make sure it is the same @@ -261,6 +271,7 @@ func TestBinaryArtifacts_workflow_runs_unsupported(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) mockRepoClient := mockrepo.NewMockRepoClient(ctrl) + mockRepo := mockrepo.NewMockRepo(ctrl) const jarFile = "gradle-wrapper.jar" const verifyWorkflow = ".github/workflows/verify.yaml" files := []string{jarFile, verifyWorkflow} @@ -281,7 +292,13 @@ func TestBinaryArtifacts_workflow_runs_unsupported(t *testing.T) { }).AnyTimes() mockRepoClient.EXPECT().ListSuccessfulWorkflowRuns(gomock.Any()).Return(nil, clients.ErrUnsupportedFeature).AnyTimes() - got, err := BinaryArtifacts(mockRepoClient) + dl := scut.TestDetailLogger{} + c := &checker.CheckRequest{ + RepoClient: mockRepoClient, + Repo: mockRepo, + Dlogger: &dl, + } + got, err := BinaryArtifacts(c) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/finding/finding.go b/finding/finding.go index 39f06175..761fb876 100644 --- a/finding/finding.go +++ b/finding/finding.go @@ -40,6 +40,8 @@ const ( FileTypeText // FileTypeURL for URLs. FileTypeURL + // FileTypeBinaryVerified for verified binary files. + FileTypeBinaryVerified ) // Location represents the location of a finding. diff --git a/probes/entries.go b/probes/entries.go index b5600ebe..c9d125f8 100644 --- a/probes/entries.go +++ b/probes/entries.go @@ -18,6 +18,7 @@ import ( "github.com/ossf/scorecard/v4/checker" "github.com/ossf/scorecard/v4/finding" "github.com/ossf/scorecard/v4/probes/contributorsFromOrgOrCompany" + "github.com/ossf/scorecard/v4/probes/freeOfUnverifiedBinaryArtifacts" "github.com/ossf/scorecard/v4/probes/fuzzedWithCLibFuzzer" "github.com/ossf/scorecard/v4/probes/fuzzedWithClusterFuzzLite" "github.com/ossf/scorecard/v4/probes/fuzzedWithCppLibFuzzer" @@ -121,6 +122,9 @@ var ( CIIBestPractices = []ProbeImpl{ hasOpenSSFBadge.Run, } + BinaryArtifacts = []ProbeImpl{ + freeOfUnverifiedBinaryArtifacts.Run, + } ) //nolint:gochecknoinits diff --git a/probes/freeOfAnyBinaryArtifacts/def.yml b/probes/freeOfAnyBinaryArtifacts/def.yml new file mode 100644 index 00000000..858a882f --- /dev/null +++ b/probes/freeOfAnyBinaryArtifacts/def.yml @@ -0,0 +1,28 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: freeOfAnyBinaryArtifacts +short: Checks if the project has any binary files in its source tree. +motivation: > + Binary files are not readable so users can't see what they do. Many programming language systems can generate executables from source code (e.g., C/C++ generated machine code, Java .class files, Python .pyc files, and minified JavaScript). Users will often directly use executables if they are included in the source repository, leading to many dangerous behaviors. +implementation: > + The implementation looks for the presence of binary files. This is a more restrictive probe than "freeOfUnverifiededBinaryArtifacts" which excludes verified binary files. +outcome: + - If the probe finds binary files, it returns a number of negative outcomes equal to the number of binary files found. Each outcome includes a location of the file. + - If the probe finds no verified binary files, it returns a single positive outcome. +remediation: + effort: Medium + text: + - Remove the generated executable artifacts from the repository. + - Build from source. diff --git a/probes/freeOfAnyBinaryArtifacts/impl.go b/probes/freeOfAnyBinaryArtifacts/impl.go new file mode 100644 index 00000000..19df5a13 --- /dev/null +++ b/probes/freeOfAnyBinaryArtifacts/impl.go @@ -0,0 +1,66 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:stylecheck +package freeOfAnyBinaryArtifacts + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "freeOfAnyBinaryArtifacts" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + + r := raw.BinaryArtifactResults + var findings []finding.Finding + + // Apply the policy evaluation. + if len(r.Files) == 0 { + f, err := finding.NewWith(fs, Probe, + "Repository does not have any binary artifacts.", nil, + finding.OutcomePositive) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + } + for i := range r.Files { + file := &r.Files[i] + f, err := finding.NewWith(fs, Probe, "binary artifact detected", + nil, finding.OutcomeNegative) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + f = f.WithLocation(&finding.Location{ + Path: file.Path, + LineStart: &file.Offset, + Type: file.Type, + }) + findings = append(findings, *f) + } + + return findings, Probe, nil +} diff --git a/probes/freeOfAnyBinaryArtifacts/impl_test.go b/probes/freeOfAnyBinaryArtifacts/impl_test.go new file mode 100644 index 00000000..94046cb5 --- /dev/null +++ b/probes/freeOfAnyBinaryArtifacts/impl_test.go @@ -0,0 +1,158 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:stylecheck +package freeOfAnyBinaryArtifacts + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" +) + +func Test_Run(t *testing.T) { + t.Parallel() + //nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "1 binary artifact", + raw: &checker.RawResults{ + BinaryArtifactResults: checker.BinaryArtifactData{ + Files: []checker.File{ + { + Path: "test_binary_artifacts_check_pass", + Type: finding.FileTypeBinary, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "No binary artifacts", + raw: &checker.RawResults{ + BinaryArtifactResults: checker.BinaryArtifactData{}, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "many binary artifact", + raw: &checker.RawResults{ + BinaryArtifactResults: checker.BinaryArtifactData{ + Files: []checker.File{ + { + Path: "test_binary_artifacts_check_pass", + Type: finding.FileTypeBinary, + }, + { + Path: "test_binary_artifacts_check_pass", + Type: finding.FileTypeBinary, + }, + { + Path: "test_binary_artifacts_check_pass", + Type: finding.FileTypeBinary, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + finding.OutcomeNegative, + finding.OutcomeNegative, + }, + }, + { + name: "many binary artifact including gradle wrappers", + raw: &checker.RawResults{ + BinaryArtifactResults: checker.BinaryArtifactData{ + Files: []checker.File{ + { + Path: "test_binary_artifacts_check_pass", + Type: finding.FileTypeBinary, + }, + { + Path: "test_binary_artifacts_check_pass", + Type: finding.FileTypeBinary, + }, + { + Path: "test_binary_artifacts_check_pass", + Type: finding.FileTypeBinary, + }, + { + Path: "gradle-wrapper.jar", + Type: finding.FileTypeBinaryVerified, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + finding.OutcomeNegative, + finding.OutcomeNegative, + finding.OutcomeNegative, + }, + }, + { + name: "Is free of any binary artifacts", + raw: &checker.RawResults{ + BinaryArtifactResults: checker.BinaryArtifactData{ + Files: []checker.File{}, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} diff --git a/probes/freeOfUnverifiedBinaryArtifacts/def.yml b/probes/freeOfUnverifiedBinaryArtifacts/def.yml new file mode 100644 index 00000000..871475ea --- /dev/null +++ b/probes/freeOfUnverifiedBinaryArtifacts/def.yml @@ -0,0 +1,28 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: freeOfUnverifiedBinaryArtifacts +short: Checks if the project has binary files in its source tree. The probe skips verified binary files which currently are gradle-wrappers. +motivation: > + Binary files are not readable so users can't see what they do. Many programming language systems can generate executables from source code (e.g., C/C++ generated machine code, Java .class files, Python .pyc files, and minified JavaScript). Users will often directly use executables if they are included in the source repository, leading to many dangerous behaviors. +implementation: > + The implementation looks for the presence of binary files that are not "verified". A verified binary is one that Scorecard considers valid for building and/or releasing the project. This is a more permissive probe than "freeOfAnyBinaryArtifacts" which does not skip verified binary files. +outcome: + - If the probe finds unverified binary files, it returns a number of negative outcomes equal to the number of unverified binary files found. Each outcome includes a location of the file. + - If the probe finds no unverified binary files, it returns a single positive outcome. +remediation: + effort: Medium + text: + - Remove the generated executable artifacts from the repository. + - Build from source. diff --git a/probes/freeOfUnverifiedBinaryArtifacts/impl.go b/probes/freeOfUnverifiedBinaryArtifacts/impl.go new file mode 100644 index 00000000..ad934e50 --- /dev/null +++ b/probes/freeOfUnverifiedBinaryArtifacts/impl.go @@ -0,0 +1,69 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:stylecheck +package freeOfUnverifiedBinaryArtifacts + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "freeOfUnverifiedBinaryArtifacts" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + + r := raw.BinaryArtifactResults + + var findings []finding.Finding + + for i := range r.Files { + file := &r.Files[i] + if file.Type == finding.FileTypeBinaryVerified { + continue + } + f, err := finding.NewWith(fs, Probe, "binary artifact detected", + nil, finding.OutcomeNegative) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + f = f.WithLocation(&finding.Location{ + Path: file.Path, + LineStart: &file.Offset, + Type: file.Type, + }) + findings = append(findings, *f) + } + + if len(findings) == 0 { + f, err := finding.NewWith(fs, Probe, + "Repository does not have binary artifacts.", nil, + finding.OutcomePositive) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + } + return findings, Probe, nil +} diff --git a/probes/freeOfUnverifiedBinaryArtifacts/impl_test.go b/probes/freeOfUnverifiedBinaryArtifacts/impl_test.go new file mode 100644 index 00000000..ce03d740 --- /dev/null +++ b/probes/freeOfUnverifiedBinaryArtifacts/impl_test.go @@ -0,0 +1,123 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:stylecheck +package freeOfUnverifiedBinaryArtifacts + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" +) + +func Test_Run(t *testing.T) { + t.Parallel() + + //nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "1 binary artifact", + raw: &checker.RawResults{ + BinaryArtifactResults: checker.BinaryArtifactData{ + Files: []checker.File{ + { + Path: "test_binary_artifacts_check_pass", + Type: finding.FileTypeBinary, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "Two binary artifacts and one gradle wrapper (which is trusted)", + raw: &checker.RawResults{ + BinaryArtifactResults: checker.BinaryArtifactData{ + Files: []checker.File{ + { + Path: "test_binary_artifacts_check_pass", + Type: finding.FileTypeBinary, + }, + { + Path: "test_binary_artifacts_check_pass", + Type: finding.FileTypeBinary, + }, + { + Path: "test_binary_artifacts_check_pass", + Type: finding.FileTypeBinaryVerified, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + finding.OutcomeNegative, + }, + }, + { + name: "One gradle wrapper (which is trusted)", + raw: &checker.RawResults{ + BinaryArtifactResults: checker.BinaryArtifactData{ + Files: []checker.File{ + { + Path: "test_binary_artifacts_check_pass", + Type: finding.FileTypeBinaryVerified, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +}