diff --git a/checks/evaluation/finding.go b/checks/evaluation/finding.go index f11c8fad..11d1e6f9 100644 --- a/checks/evaluation/finding.go +++ b/checks/evaluation/finding.go @@ -29,3 +29,14 @@ func nonNegativeFindings(findings []finding.Finding) []finding.Finding { } return ff } + +func negativeFindings(findings []finding.Finding) []finding.Finding { + var ff []finding.Finding + for i := range findings { + f := &findings[i] + if f.Outcome == finding.OutcomeNegative { + ff = append(ff, *f) + } + } + return ff +} diff --git a/checks/evaluation/packaging.go b/checks/evaluation/packaging.go index a3c13fd3..da69d8c1 100644 --- a/checks/evaluation/packaging.go +++ b/checks/evaluation/packaging.go @@ -15,75 +15,46 @@ package evaluation import ( - "fmt" - "github.com/ossf/scorecard/v4/checker" sce "github.com/ossf/scorecard/v4/errors" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/packagedWithAutomatedWorkflow" ) // Packaging applies the score policy for the Packaging check. -func Packaging(name string, dl checker.DetailLogger, r *checker.PackagingData) checker.CheckResult { - if r == nil { - e := sce.WithMessage(sce.ErrScorecardInternal, "empty raw data") +func Packaging(name string, + findings []finding.Finding, + dl checker.DetailLogger, +) checker.CheckResult { + expectedProbes := []string{ + packagedWithAutomatedWorkflow.Probe, + } + + if !finding.UniqueProbesEqual(findings, expectedProbes) { + e := sce.WithMessage(sce.ErrScorecardInternal, "invalid probe results") return checker.CreateRuntimeErrorResult(name, e) } - pass := false - for _, p := range r.Packages { - if p.Msg != nil { - // This is a debug message. Let's just replay the message. - dl.Debug(&checker.LogMessage{ - Text: *p.Msg, + // Currently there is only a single packaging probe that returns + // a single positive or negative outcome. As such, in this evaluation, + // we return max score if the outcome is positive and lowest score if + // the outcome is negative. + maxScore := false + for _, f := range findings { + f := f + if f.Outcome == finding.OutcomePositive { + maxScore = true + // Log all findings except the negative ones. + dl.Info(&checker.LogMessage{ + Finding: &f, }) - continue } - - // Presence of a single non-debug message means the - // check passes. - pass = true - - msg, err := createLogMessage(p) - if err != nil { - return checker.CreateRuntimeErrorResult(name, err) - } - dl.Info(&msg) + } + if maxScore { + return checker.CreateMaxScoreResult(name, "packaging workflow detected") } - if pass { - return checker.CreateMaxScoreResult(name, - "publishing workflow detected") - } - - dl.Warn(&checker.LogMessage{ - Text: "no GitHub/GitLab publishing workflow detected", - }) - + checker.LogFindings(negativeFindings(findings), dl) return checker.CreateInconclusiveResult(name, - "no published package detected") -} - -func createLogMessage(p checker.Package) (checker.LogMessage, error) { - var msg checker.LogMessage - - if p.Msg != nil { - return msg, sce.WithMessage(sce.ErrScorecardInternal, "Msg should be nil") - } - - if p.File == nil { - return msg, sce.WithMessage(sce.ErrScorecardInternal, "File field is nil") - } - - if p.File != nil { - msg.Path = p.File.Path - msg.Type = p.File.Type - msg.Offset = p.File.Offset - } - - if len(p.Runs) == 0 { - return msg, sce.WithMessage(sce.ErrScorecardInternal, "no run data") - } - - msg.Text = fmt.Sprintf("GitHub/GitLab publishing workflow used in run %s", p.Runs[0].URL) - - return msg, nil + "packaging workflow not detected") } diff --git a/checks/evaluation/packaging_test.go b/checks/evaluation/packaging_test.go index eaf6a623..459552e3 100644 --- a/checks/evaluation/packaging_test.go +++ b/checks/evaluation/packaging_test.go @@ -16,153 +16,69 @@ package evaluation import ( "testing" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/ossf/scorecard/v4/checker" + sce "github.com/ossf/scorecard/v4/errors" + "github.com/ossf/scorecard/v4/finding" scut "github.com/ossf/scorecard/v4/utests" ) -func Test_createLogMessage(t *testing.T) { - msg := "msg" - t.Parallel() - tests := []struct { //nolint:govet - name string - args checker.Package - want checker.LogMessage - wantErr bool - }{ - { - name: "nil package", - args: checker.Package{}, - want: checker.LogMessage{}, - wantErr: true, - }, - { - name: "nil file", - args: checker.Package{ - File: nil, - }, - want: checker.LogMessage{}, - wantErr: true, - }, - { - name: "msg is not nil", - args: checker.Package{ - File: &checker.File{}, - Msg: &msg, - }, - want: checker.LogMessage{ - Text: "", - }, - wantErr: true, - }, - { - name: "file is not nil", - args: checker.Package{ - File: &checker.File{ - Path: "path", - }, - }, - want: checker.LogMessage{ - Path: "path", - }, - wantErr: true, - }, - { - name: "runs are not zero", - args: checker.Package{ - File: &checker.File{ - Path: "path", - }, - Runs: []checker.Run{ - {}, - }, - }, - want: checker.LogMessage{ - Text: "GitHub/GitLab publishing workflow used in run ", - Path: "path", - }, - }, - } - for _, tt := range tests { - tt := tt // Parallel testing - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - got, err := createLogMessage(tt.args) - if (err != nil) != tt.wantErr { - t.Errorf("createLogMessage() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !cmp.Equal(got, tt.want) { - t.Errorf("createLogMessage() got = %v, want %v", got, cmp.Diff(got, tt.want)) - } - }) - } -} - func TestPackaging(t *testing.T) { t.Parallel() - type args struct { //nolint:govet - name string - dl checker.DetailLogger - r *checker.PackagingData - } tests := []struct { - name string - args args - want checker.CheckResult + name string + findings []finding.Finding + result scut.TestReturn }{ { - name: "nil packaging data", - args: args{ - name: "name", - dl: nil, - r: nil, - }, - want: checker.CheckResult{ - Name: "name", - Version: 2, - Score: -1, - Reason: "internal error: empty raw data", - }, - }, - { - name: "empty packaging data", - args: args{ - name: "name", - dl: &scut.TestDetailLogger{}, - r: &checker.PackagingData{}, - }, - want: checker.CheckResult{ - Name: "name", - Version: 2, - Score: -1, - Reason: "no published package detected", - }, - }, - { - name: "runs are not zero", - args: args{ - dl: &scut.TestDetailLogger{}, - r: &checker.PackagingData{ - Packages: []checker.Package{ - { - File: &checker.File{ - Path: "path", - }, - Runs: []checker.Run{ - {}, - }, - }, - }, + name: "test positive outcome", + findings: []finding.Finding{ + { + Probe: "packagedWithAutomatedWorkflow", + Outcome: finding.OutcomePositive, }, }, - want: checker.CheckResult{ - Name: "", - Version: 2, - Score: 10, - Reason: "publishing workflow detected", + result: scut.TestReturn{ + Score: checker.MaxResultScore, + NumberOfInfo: 1, + }, + }, + { + name: "test positive outcome with wrong probes", + findings: []finding.Finding{ + { + Probe: "wrongProbe", + Outcome: finding.OutcomePositive, + }, + }, + result: scut.TestReturn{ + Score: -1, + Error: sce.ErrScorecardInternal, + }, + }, + { + name: "test inconclusive outcome", + findings: []finding.Finding{ + { + Probe: "packagedWithAutomatedWorkflow", + Outcome: finding.OutcomeNegative, + }, + }, + result: scut.TestReturn{ + Score: checker.InconclusiveResultScore, + NumberOfWarn: 1, + }, + }, + { + name: "test negative outcome with wrong probes", + findings: []finding.Finding{ + { + Probe: "wrongProbe", + Outcome: finding.OutcomeNegative, + }, + }, + result: scut.TestReturn{ + Score: -1, + Error: sce.ErrScorecardInternal, }, }, } @@ -170,8 +86,10 @@ func TestPackaging(t *testing.T) { tt := tt // Parallel testing t.Run(tt.name, func(t *testing.T) { t.Parallel() - if got := Packaging(tt.args.name, tt.args.dl, tt.args.r); !cmp.Equal(got, tt.want, cmpopts.IgnoreFields(checker.CheckResult{}, "Error")) { //nolint:lll - t.Errorf("Packaging() = %v, want %v", got, cmp.Diff(got, tt.want, cmpopts.IgnoreFields(checker.CheckResult{}, "Error"))) //nolint:lll + dl := scut.TestDetailLogger{} + got := Packaging(tt.name, tt.findings, &dl) + if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) { + t.Errorf("got %v, expected %v", got, tt.result) } }) } diff --git a/checks/packaging.go b/checks/packaging.go index 1ae19bda..cfa78782 100644 --- a/checks/packaging.go +++ b/checks/packaging.go @@ -22,6 +22,8 @@ import ( "github.com/ossf/scorecard/v4/clients/githubrepo" "github.com/ossf/scorecard/v4/clients/gitlabrepo" sce "github.com/ossf/scorecard/v4/errors" + "github.com/ossf/scorecard/v4/probes" + "github.com/ossf/scorecard/v4/probes/zrunner" ) // CheckPackaging is the registered name for Packaging. @@ -54,10 +56,14 @@ func Packaging(c *checker.CheckRequest) checker.CheckResult { return checker.CreateRuntimeErrorResult(CheckPackaging, e) } - // Set the raw results. - if c.RawResults != nil { - c.RawResults.PackagingResults = rawData + pRawResults := getRawResults(c) + pRawResults.PackagingResults = rawData + + findings, err := zrunner.Run(pRawResults, probes.Packaging) + if err != nil { + e := sce.WithMessage(sce.ErrScorecardInternal, err.Error()) + return checker.CreateRuntimeErrorResult(CheckPackaging, e) } - return evaluation.Packaging(CheckPackaging, c.Dlogger, &rawData) + return evaluation.Packaging(CheckPackaging, findings, c.Dlogger) } diff --git a/e2e/packaging_test.go b/e2e/packaging_test.go index 19750566..9c6eb42b 100644 --- a/e2e/packaging_test.go +++ b/e2e/packaging_test.go @@ -47,7 +47,7 @@ var _ = Describe("E2E TEST:"+checks.CheckPackaging, func() { Score: checker.InconclusiveResultScore, NumberOfWarn: 1, NumberOfInfo: 0, - NumberOfDebug: 4, + NumberOfDebug: 0, } result := checks.Packaging(&req) Expect(scut.ValidateTestReturn(nil, "use packaging", &expected, &result, &dl)).Should(BeTrue()) diff --git a/probes/entries.go b/probes/entries.go index e5b05282..17caef3d 100644 --- a/probes/entries.go +++ b/probes/entries.go @@ -33,6 +33,7 @@ import ( "github.com/ossf/scorecard/v4/probes/hasFSFOrOSIApprovedLicense" "github.com/ossf/scorecard/v4/probes/hasLicenseFile" "github.com/ossf/scorecard/v4/probes/hasLicenseFileAtTopDir" + "github.com/ossf/scorecard/v4/probes/packagedWithAutomatedWorkflow" "github.com/ossf/scorecard/v4/probes/securityPolicyContainsLinks" "github.com/ossf/scorecard/v4/probes/securityPolicyContainsText" "github.com/ossf/scorecard/v4/probes/securityPolicyContainsVulnerabilityDisclosure" @@ -80,6 +81,9 @@ var ( fuzzedWithPropertyBasedTypescript.Run, fuzzedWithPropertyBasedJavascript.Run, } + Packaging = []ProbeImpl{ + packagedWithAutomatedWorkflow.Run, + } License = []ProbeImpl{ hasLicenseFile.Run, hasFSFOrOSIApprovedLicense.Run, diff --git a/probes/packagedWithAutomatedWorkflow/def.yml b/probes/packagedWithAutomatedWorkflow/def.yml new file mode 100644 index 00000000..a7df0433 --- /dev/null +++ b/probes/packagedWithAutomatedWorkflow/def.yml @@ -0,0 +1,27 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: packagedWithAutomatedWorkflow +short: Checks whether the project uses automated packaging. +motivation: > + Packages give users of a project an easy way to download, install, update, and uninstall the software by a package manager. In particular, they make it easy for users to receive security patches as updates. +implementation: > + The implementation checks whether a project uses common patterns for packaging across multiple ecosystems. Scorecard gets this by checking the projects workflows for specific uses of actions and build commands such as `docker push` or `mvn deploy`. +outcome: + - If the project has a package without a debug message, the outcome is positive. + - If the project has a package with a debug message, the outcome is negative. +remediation: + effort: Low + text: + - Use a GitHub action to release your package to language-specific hubs. \ No newline at end of file diff --git a/probes/packagedWithAutomatedWorkflow/impl.go b/probes/packagedWithAutomatedWorkflow/impl.go new file mode 100644 index 00000000..bf261d94 --- /dev/null +++ b/probes/packagedWithAutomatedWorkflow/impl.go @@ -0,0 +1,73 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package packagedWithAutomatedWorkflow + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "packagedWithAutomatedWorkflow" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + + r := raw.PackagingResults + var findings []finding.Finding + for _, p := range r.Packages { + p := p + if p.Msg != nil { + continue + } + // Presence of a single non-debug message means the + // check passes. + f, err := finding.NewWith(fs, Probe, + "Project packages its releases by way of Github Actions.", nil, + finding.OutcomePositive) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + loc := &finding.Location{} + if p.File != nil { + loc.Path = p.File.Path + loc.Type = p.File.Type + loc.LineStart = &p.File.Offset + } + f = f.WithLocation(loc) + findings = append(findings, *f) + } + + if len(findings) > 0 { + return findings, Probe, nil + } + + f, err := finding.NewWith(fs, Probe, + "no GitHub/GitLab publishing workflow detected.", nil, + finding.OutcomeNegative) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + return []finding.Finding{*f}, Probe, nil +} diff --git a/probes/packagedWithAutomatedWorkflow/impl_test.go b/probes/packagedWithAutomatedWorkflow/impl_test.go new file mode 100644 index 00000000..0d9e6b38 --- /dev/null +++ b/probes/packagedWithAutomatedWorkflow/impl_test.go @@ -0,0 +1,94 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package packagedWithAutomatedWorkflow + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" +) + +func Test_Run(t *testing.T) { + msg := "msg" + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "debug msg is nil", + raw: &checker.RawResults{ + PackagingResults: checker.PackagingData{ + Packages: []checker.Package{ + {}, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "debug msg is not nil", + raw: &checker.RawResults{ + PackagingResults: checker.PackagingData{ + Packages: []checker.Package{ + { + Msg: &msg, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +}