mirror of
https://github.com/ossf/scorecard.git
synced 2024-09-11 00:45:36 +03:00
✨ checks/evaluation logs findings (#3409)
* checks/validation logs findings Signed-off-by: laurentsimon <laurentsimon@google.com> * gofmt file Signed-off-by: laurentsimon <laurentsimon@google.com> * linter Signed-off-by: laurentsimon <laurentsimon@google.com> * revert go.sum Signed-off-by: laurentsimon <laurentsimon@google.com> * typo Signed-off-by: laurentsimon <laurentsimon@google.com> * add unit tests and address comments Signed-off-by: laurentsimon <laurentsimon@google.com> * update comment Signed-off-by: laurentsimon <laurentsimon@google.com> * missing file Signed-off-by: laurentsimon <laurentsimon@google.com> * use option 1 Signed-off-by: laurentsimon <laurentsimon@google.com> * use got / want in test Signed-off-by: laurentsimon <laurentsimon@google.com> * missing tests updates Signed-off-by: laurentsimon <laurentsimon@google.com> --------- Signed-off-by: laurentsimon <laurentsimon@google.com>
This commit is contained in:
parent
52a4843bf1
commit
8b096ad4c0
@ -195,7 +195,7 @@ func CreateRuntimeErrorResult(name string, e error) CheckResult {
|
||||
}
|
||||
|
||||
// LogFindings logs the list of findings.
|
||||
func LogFindings(findings []finding.Finding, dl DetailLogger) error {
|
||||
func LogFindings(findings []finding.Finding, dl DetailLogger) {
|
||||
for i := range findings {
|
||||
f := &findings[i]
|
||||
switch f.Outcome {
|
||||
@ -213,6 +213,4 @@ func LogFindings(findings []finding.Finding, dl DetailLogger) error {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
@ -20,6 +20,7 @@ import (
|
||||
"github.com/ossf/scorecard/v4/checks/raw"
|
||||
sce "github.com/ossf/scorecard/v4/errors"
|
||||
"github.com/ossf/scorecard/v4/probes"
|
||||
"github.com/ossf/scorecard/v4/probes/zrunner"
|
||||
)
|
||||
|
||||
// CheckDependencyUpdateTool is the exported name for Automatic-Depdendency-Update.
|
||||
@ -49,12 +50,12 @@ func DependencyUpdateTool(c *checker.CheckRequest) checker.CheckResult {
|
||||
pRawResults.DependencyUpdateToolResults = rawData
|
||||
|
||||
// Evaluate the probes.
|
||||
findings, err := evaluateProbes(c, pRawResults, probes.DependencyToolUpdates)
|
||||
findings, err := zrunner.Run(pRawResults, probes.DependencyToolUpdates)
|
||||
if err != nil {
|
||||
e := sce.WithMessage(sce.ErrScorecardInternal, err.Error())
|
||||
return checker.CreateRuntimeErrorResult(CheckDependencyUpdateTool, e)
|
||||
}
|
||||
|
||||
// Return the score evaluation.
|
||||
return evaluation.DependencyUpdateTool(CheckDependencyUpdateTool, findings)
|
||||
return evaluation.DependencyUpdateTool(CheckDependencyUpdateTool, findings, c.Dlogger)
|
||||
}
|
||||
|
@ -51,7 +51,7 @@ func TestDependencyUpdateTool(t *testing.T) {
|
||||
CallSearchCommits: 0,
|
||||
expected: scut.TestReturn{
|
||||
NumberOfInfo: 1,
|
||||
NumberOfWarn: 3,
|
||||
NumberOfWarn: 0,
|
||||
Score: 10,
|
||||
},
|
||||
},
|
||||
@ -64,7 +64,7 @@ func TestDependencyUpdateTool(t *testing.T) {
|
||||
CallSearchCommits: 0,
|
||||
expected: scut.TestReturn{
|
||||
NumberOfInfo: 1,
|
||||
NumberOfWarn: 3,
|
||||
NumberOfWarn: 0,
|
||||
Score: 10,
|
||||
},
|
||||
},
|
||||
@ -103,7 +103,7 @@ func TestDependencyUpdateTool(t *testing.T) {
|
||||
CallSearchCommits: 1,
|
||||
expected: scut.TestReturn{
|
||||
NumberOfInfo: 1,
|
||||
NumberOfWarn: 3,
|
||||
NumberOfWarn: 0,
|
||||
Score: 10,
|
||||
},
|
||||
},
|
||||
@ -118,7 +118,7 @@ func TestDependencyUpdateTool(t *testing.T) {
|
||||
CallSearchCommits: 1,
|
||||
expected: scut.TestReturn{
|
||||
NumberOfInfo: 1,
|
||||
NumberOfWarn: 3,
|
||||
NumberOfWarn: 0,
|
||||
Score: 10,
|
||||
},
|
||||
},
|
||||
@ -136,7 +136,7 @@ func TestDependencyUpdateTool(t *testing.T) {
|
||||
CallSearchCommits: 1,
|
||||
expected: scut.TestReturn{
|
||||
NumberOfInfo: 1,
|
||||
NumberOfWarn: 3,
|
||||
NumberOfWarn: 0,
|
||||
Score: 10,
|
||||
},
|
||||
},
|
||||
|
@ -24,9 +24,10 @@ import (
|
||||
"github.com/ossf/scorecard/v4/probes/toolSonatypeLiftInstalled"
|
||||
)
|
||||
|
||||
// DependencyUpdateTool applies the score policy for the Dependency-Update-Tool check.
|
||||
// DependencyUpdateTool applies the score policy and logs the details
|
||||
// for the Dependency-Update-Tool check.
|
||||
func DependencyUpdateTool(name string,
|
||||
findings []finding.Finding,
|
||||
findings []finding.Finding, dl checker.DetailLogger,
|
||||
) checker.CheckResult {
|
||||
expectedProbes := []string{
|
||||
toolDependabotInstalled.Probe,
|
||||
@ -42,9 +43,13 @@ func DependencyUpdateTool(name string,
|
||||
for i := range findings {
|
||||
f := &findings[i]
|
||||
if f.Outcome == finding.OutcomePositive {
|
||||
// Log all findings except the negative ones.
|
||||
checker.LogFindings(nonNegativeFindings(findings), dl)
|
||||
return checker.CreateMaxScoreResult(name, "update tool detected")
|
||||
}
|
||||
}
|
||||
|
||||
// Log all findings.
|
||||
checker.LogFindings(findings, dl)
|
||||
return checker.CreateMinScoreResult(name, "no update tool detected")
|
||||
}
|
||||
|
@ -18,6 +18,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/ossf/scorecard/v4/checker"
|
||||
sce "github.com/ossf/scorecard/v4/errors"
|
||||
"github.com/ossf/scorecard/v4/finding"
|
||||
scut "github.com/ossf/scorecard/v4/utests"
|
||||
)
|
||||
@ -28,9 +29,7 @@ func TestDependencyUpdateTool(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
findings []finding.Finding
|
||||
err bool
|
||||
want checker.CheckResult
|
||||
expected scut.TestReturn
|
||||
result scut.TestReturn
|
||||
}{
|
||||
{
|
||||
name: "dependabot",
|
||||
@ -52,8 +51,9 @@ func TestDependencyUpdateTool(t *testing.T) {
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
},
|
||||
want: checker.CheckResult{
|
||||
Score: 10,
|
||||
result: scut.TestReturn{
|
||||
Score: checker.MaxResultScore,
|
||||
NumberOfInfo: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
@ -76,8 +76,9 @@ func TestDependencyUpdateTool(t *testing.T) {
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
},
|
||||
want: checker.CheckResult{
|
||||
Score: 10,
|
||||
result: scut.TestReturn{
|
||||
Score: checker.MaxResultScore,
|
||||
NumberOfInfo: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
@ -100,8 +101,9 @@ func TestDependencyUpdateTool(t *testing.T) {
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
},
|
||||
want: checker.CheckResult{
|
||||
Score: 10,
|
||||
result: scut.TestReturn{
|
||||
Score: checker.MaxResultScore,
|
||||
NumberOfInfo: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
@ -128,8 +130,9 @@ func TestDependencyUpdateTool(t *testing.T) {
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
},
|
||||
want: checker.CheckResult{
|
||||
Score: 10,
|
||||
result: scut.TestReturn{
|
||||
Score: checker.MaxResultScore,
|
||||
NumberOfInfo: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
@ -152,8 +155,9 @@ func TestDependencyUpdateTool(t *testing.T) {
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
},
|
||||
want: checker.CheckResult{
|
||||
Score: 0,
|
||||
result: scut.TestReturn{
|
||||
Score: checker.MinResultScore,
|
||||
NumberOfWarn: 4,
|
||||
},
|
||||
},
|
||||
{
|
||||
@ -172,9 +176,9 @@ func TestDependencyUpdateTool(t *testing.T) {
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
},
|
||||
err: true,
|
||||
want: checker.CheckResult{
|
||||
Score: -1,
|
||||
result: scut.TestReturn{
|
||||
Score: checker.InconclusiveResultScore,
|
||||
Error: sce.ErrScorecardInternal,
|
||||
},
|
||||
},
|
||||
{
|
||||
@ -201,8 +205,9 @@ func TestDependencyUpdateTool(t *testing.T) {
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
},
|
||||
want: checker.CheckResult{
|
||||
Score: -1,
|
||||
result: scut.TestReturn{
|
||||
Score: checker.InconclusiveResultScore,
|
||||
Error: sce.ErrScorecardInternal,
|
||||
},
|
||||
},
|
||||
}
|
||||
@ -211,13 +216,10 @@ func TestDependencyUpdateTool(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got := DependencyUpdateTool(tt.name, tt.findings)
|
||||
if tt.want.Score != got.Score {
|
||||
t.Errorf("DependencyUpdateTool() got Score = %v, want %v for %v", got.Score, tt.want.Score, tt.name)
|
||||
}
|
||||
if tt.err && got.Error == nil {
|
||||
t.Errorf("DependencyUpdateTool() error = %v, want %v for %v", got.Error, tt.want.Error, tt.name)
|
||||
return
|
||||
dl := scut.TestDetailLogger{}
|
||||
got := DependencyUpdateTool(tt.name, tt.findings, &dl)
|
||||
if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
|
||||
t.Errorf("got %v, expected %v", got, tt.result)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
31
checks/evaluation/finding.go
Normal file
31
checks/evaluation/finding.go
Normal file
@ -0,0 +1,31 @@
|
||||
// Copyright 2023 OpenSSF Scorecard Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package evaluation
|
||||
|
||||
import (
|
||||
"github.com/ossf/scorecard/v4/finding"
|
||||
)
|
||||
|
||||
func nonNegativeFindings(findings []finding.Finding) []finding.Finding {
|
||||
var ff []finding.Finding
|
||||
for i := range findings {
|
||||
f := &findings[i]
|
||||
if f.Outcome == finding.OutcomeNegative {
|
||||
continue
|
||||
}
|
||||
ff = append(ff, *f)
|
||||
}
|
||||
return ff
|
||||
}
|
@ -29,7 +29,7 @@ import (
|
||||
|
||||
// Fuzzing applies the score policy for the Fuzzing check.
|
||||
func Fuzzing(name string,
|
||||
findings []finding.Finding,
|
||||
findings []finding.Finding, dl checker.DetailLogger,
|
||||
) checker.CheckResult {
|
||||
// We have 7 unique probes, each should have a finding.
|
||||
expectedProbes := []string{
|
||||
@ -51,8 +51,12 @@ func Fuzzing(name string,
|
||||
for i := range findings {
|
||||
f := &findings[i]
|
||||
if f.Outcome == finding.OutcomePositive {
|
||||
// Log all findings except the negative ones.
|
||||
checker.LogFindings(nonNegativeFindings(findings), dl)
|
||||
return checker.CreateMaxScoreResult(name, "project is fuzzed")
|
||||
}
|
||||
}
|
||||
// Log all findings.
|
||||
checker.LogFindings(findings, dl)
|
||||
return checker.CreateMinScoreResult(name, "project is not fuzzed")
|
||||
}
|
||||
|
@ -16,190 +16,166 @@ package evaluation
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/ossf/scorecard/v4/checker"
|
||||
sce "github.com/ossf/scorecard/v4/errors"
|
||||
"github.com/ossf/scorecard/v4/finding"
|
||||
scut "github.com/ossf/scorecard/v4/utests"
|
||||
)
|
||||
|
||||
func TestFuzzing(t *testing.T) {
|
||||
t.Parallel()
|
||||
type args struct { //nolint
|
||||
tests := []struct {
|
||||
name string
|
||||
findings []finding.Finding
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want checker.CheckResult
|
||||
result scut.TestReturn
|
||||
}{
|
||||
{
|
||||
name: "Fuzzing - no fuzzing",
|
||||
args: args{
|
||||
name: "Fuzzing",
|
||||
findings: []finding.Finding{
|
||||
{
|
||||
Probe: "fuzzedWithClusterFuzzLite",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithGoNative",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithOneFuzz",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithOSSFuzz",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedHaskell",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedJavascript",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedTypescript",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
findings: []finding.Finding{
|
||||
{
|
||||
Probe: "fuzzedWithClusterFuzzLite",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithGoNative",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithOneFuzz",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithOSSFuzz",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedHaskell",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedJavascript",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedTypescript",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
},
|
||||
want: checker.CheckResult{
|
||||
Score: 0,
|
||||
Name: "Fuzzing",
|
||||
Version: 2,
|
||||
Reason: "project is not fuzzed",
|
||||
result: scut.TestReturn{
|
||||
Score: checker.MinResultScore,
|
||||
NumberOfWarn: 7,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Fuzzing - fuzzing GoNative",
|
||||
args: args{
|
||||
name: "Fuzzing",
|
||||
findings: []finding.Finding{
|
||||
{
|
||||
Probe: "fuzzedWithClusterFuzzLite",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithGoNative",
|
||||
Outcome: finding.OutcomePositive,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithOneFuzz",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithOSSFuzz",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedHaskell",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedJavascript",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedTypescript",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
findings: []finding.Finding{
|
||||
{
|
||||
Probe: "fuzzedWithClusterFuzzLite",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithGoNative",
|
||||
Outcome: finding.OutcomePositive,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithOneFuzz",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithOSSFuzz",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedHaskell",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedJavascript",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedTypescript",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
},
|
||||
want: checker.CheckResult{
|
||||
Score: 10,
|
||||
Name: "Fuzzing",
|
||||
Version: 2,
|
||||
Reason: "project is fuzzed",
|
||||
result: scut.TestReturn{
|
||||
Score: checker.MaxResultScore,
|
||||
NumberOfInfo: 1,
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
name: "Fuzzing - fuzzing missing GoNative finding",
|
||||
args: args{
|
||||
name: "Fuzzing",
|
||||
findings: []finding.Finding{
|
||||
{
|
||||
Probe: "fuzzedWithClusterFuzzLite",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithOneFuzz",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithOSSFuzz",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedHaskell",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedJavascript",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedTypescript",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
findings: []finding.Finding{
|
||||
{
|
||||
Probe: "fuzzedWithClusterFuzzLite",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithOneFuzz",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithOSSFuzz",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedHaskell",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedJavascript",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedTypescript",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
},
|
||||
want: checker.CheckResult{
|
||||
Score: -1,
|
||||
Name: "Fuzzing",
|
||||
Version: 2,
|
||||
Reason: "internal error: invalid probe results",
|
||||
result: scut.TestReturn{
|
||||
Score: checker.InconclusiveResultScore,
|
||||
Error: sce.ErrScorecardInternal,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Fuzzing - fuzzing invalid probe name",
|
||||
args: args{
|
||||
name: "Fuzzing",
|
||||
findings: []finding.Finding{
|
||||
{
|
||||
Probe: "fuzzedWithClusterFuzzLite",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithGoNative",
|
||||
Outcome: finding.OutcomePositive,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithOneFuzz",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithOSSFuzz",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedHaskell",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedJavascript",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedTypescript",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithInvalidProbeName",
|
||||
Outcome: finding.OutcomePositive,
|
||||
},
|
||||
findings: []finding.Finding{
|
||||
{
|
||||
Probe: "fuzzedWithClusterFuzzLite",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithGoNative",
|
||||
Outcome: finding.OutcomePositive,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithOneFuzz",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithOSSFuzz",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedHaskell",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedJavascript",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithPropertyBasedTypescript",
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
{
|
||||
Probe: "fuzzedWithInvalidProbeName",
|
||||
Outcome: finding.OutcomePositive,
|
||||
},
|
||||
},
|
||||
want: checker.CheckResult{
|
||||
Score: -1,
|
||||
Name: "Fuzzing",
|
||||
Version: 2,
|
||||
Reason: "internal error: invalid probe results",
|
||||
result: scut.TestReturn{
|
||||
Score: checker.InconclusiveResultScore,
|
||||
Error: sce.ErrScorecardInternal,
|
||||
},
|
||||
},
|
||||
}
|
||||
@ -207,8 +183,10 @@ func TestFuzzing(t *testing.T) {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := Fuzzing(tt.args.name, tt.args.findings); !cmp.Equal(got, tt.want, cmpopts.IgnoreFields(checker.CheckResult{}, "Error")) { //nolint:lll
|
||||
t.Errorf("Fuzzing() = %v, want %v", got, cmp.Diff(got, tt.want, cmpopts.IgnoreFields(checker.CheckResult{}, "Error"))) //nolint:lll
|
||||
dl := scut.TestDetailLogger{}
|
||||
got := Fuzzing(tt.name, tt.findings, &dl)
|
||||
if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
|
||||
t.Errorf("got %v, expected %v", got, tt.result)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
@ -25,7 +25,7 @@ import (
|
||||
)
|
||||
|
||||
// SecurityPolicy applies the score policy for the Security-Policy check.
|
||||
func SecurityPolicy(name string, findings []finding.Finding) checker.CheckResult {
|
||||
func SecurityPolicy(name string, findings []finding.Finding, dl checker.DetailLogger) checker.CheckResult {
|
||||
// We have 4 unique probes, each should have a finding.
|
||||
expectedProbes := []string{
|
||||
securityPolicyContainsVulnerabilityDisclosure.Probe,
|
||||
@ -64,9 +64,18 @@ func SecurityPolicy(name string, findings []finding.Finding) checker.CheckResult
|
||||
e := sce.WithMessage(sce.ErrScorecardInternal, "score calculation problem")
|
||||
return checker.CreateRuntimeErrorResult(name, e)
|
||||
}
|
||||
|
||||
// Log all findings.
|
||||
checker.LogFindings(findings, dl)
|
||||
return checker.CreateMinScoreResult(name, "security policy file not detected")
|
||||
}
|
||||
|
||||
// Log all findings.
|
||||
// NOTE: if the score is checker.MaxResultScore, then all findings are positive.
|
||||
// If the score is less than checker.MaxResultScore, some findings are negative,
|
||||
// so we log both positive and negative findings.
|
||||
checker.LogFindings(findings, dl)
|
||||
|
||||
return checker.CreateResultWithScore(name, "security policy file detected", score)
|
||||
}
|
||||
|
||||
|
@ -18,7 +18,9 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/ossf/scorecard/v4/checker"
|
||||
sce "github.com/ossf/scorecard/v4/errors"
|
||||
"github.com/ossf/scorecard/v4/finding"
|
||||
scut "github.com/ossf/scorecard/v4/utests"
|
||||
)
|
||||
|
||||
func TestSecurityPolicy(t *testing.T) {
|
||||
@ -27,8 +29,7 @@ func TestSecurityPolicy(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
findings []finding.Finding
|
||||
err bool
|
||||
want checker.CheckResult
|
||||
result scut.TestReturn
|
||||
}{
|
||||
{
|
||||
name: "missing findings links",
|
||||
@ -46,8 +47,9 @@ func TestSecurityPolicy(t *testing.T) {
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
},
|
||||
want: checker.CheckResult{
|
||||
Score: -1,
|
||||
result: scut.TestReturn{
|
||||
Score: checker.InconclusiveResultScore,
|
||||
Error: sce.ErrScorecardInternal,
|
||||
},
|
||||
},
|
||||
{
|
||||
@ -74,8 +76,9 @@ func TestSecurityPolicy(t *testing.T) {
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
},
|
||||
want: checker.CheckResult{
|
||||
Score: -1,
|
||||
result: scut.TestReturn{
|
||||
Score: checker.InconclusiveResultScore,
|
||||
Error: sce.ErrScorecardInternal,
|
||||
},
|
||||
},
|
||||
{
|
||||
@ -98,8 +101,10 @@ func TestSecurityPolicy(t *testing.T) {
|
||||
Outcome: finding.OutcomePositive,
|
||||
},
|
||||
},
|
||||
want: checker.CheckResult{
|
||||
Score: 0,
|
||||
result: scut.TestReturn{
|
||||
Score: checker.MinResultScore,
|
||||
NumberOfInfo: 1,
|
||||
NumberOfWarn: 3,
|
||||
},
|
||||
},
|
||||
{
|
||||
@ -122,8 +127,9 @@ func TestSecurityPolicy(t *testing.T) {
|
||||
Outcome: finding.OutcomeNegative,
|
||||
},
|
||||
},
|
||||
want: checker.CheckResult{
|
||||
Score: -1,
|
||||
result: scut.TestReturn{
|
||||
Score: checker.InconclusiveResultScore,
|
||||
Error: sce.ErrScorecardInternal,
|
||||
},
|
||||
},
|
||||
{
|
||||
@ -146,8 +152,10 @@ func TestSecurityPolicy(t *testing.T) {
|
||||
Outcome: finding.OutcomePositive,
|
||||
},
|
||||
},
|
||||
want: checker.CheckResult{
|
||||
Score: 6,
|
||||
result: scut.TestReturn{
|
||||
Score: 6,
|
||||
NumberOfInfo: 2,
|
||||
NumberOfWarn: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
@ -170,8 +178,9 @@ func TestSecurityPolicy(t *testing.T) {
|
||||
Outcome: finding.OutcomePositive,
|
||||
},
|
||||
},
|
||||
want: checker.CheckResult{
|
||||
Score: 10,
|
||||
result: scut.TestReturn{
|
||||
Score: checker.MaxResultScore,
|
||||
NumberOfInfo: 4,
|
||||
},
|
||||
},
|
||||
}
|
||||
@ -180,15 +189,10 @@ func TestSecurityPolicy(t *testing.T) {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got := SecurityPolicy("SecurityPolicy", tt.findings)
|
||||
if tt.err {
|
||||
if got.Score != -1 {
|
||||
t.Errorf("SecurityPolicy() = %v, want %v", got, tt.want)
|
||||
}
|
||||
}
|
||||
if got.Score != tt.want.Score {
|
||||
t.Errorf("SecurityPolicy() = %v, want %v for %v", got.Score, tt.want.Score, tt.name)
|
||||
dl := scut.TestDetailLogger{}
|
||||
got := SecurityPolicy(tt.name, tt.findings, &dl)
|
||||
if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
|
||||
t.Errorf("got %v, expected %v", got, tt.result)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
@ -20,6 +20,7 @@ import (
|
||||
"github.com/ossf/scorecard/v4/checks/raw"
|
||||
sce "github.com/ossf/scorecard/v4/errors"
|
||||
"github.com/ossf/scorecard/v4/probes"
|
||||
"github.com/ossf/scorecard/v4/probes/zrunner"
|
||||
)
|
||||
|
||||
// CheckFuzzing is the registered name for Fuzzing.
|
||||
@ -46,12 +47,12 @@ func Fuzzing(c *checker.CheckRequest) checker.CheckResult {
|
||||
pRawResults.FuzzingResults = rawData
|
||||
|
||||
// Evaluate the probes.
|
||||
findings, err := evaluateProbes(c, pRawResults, probes.Fuzzing)
|
||||
findings, err := zrunner.Run(pRawResults, probes.Fuzzing)
|
||||
if err != nil {
|
||||
e := sce.WithMessage(sce.ErrScorecardInternal, err.Error())
|
||||
return checker.CreateRuntimeErrorResult(CheckFuzzing, e)
|
||||
}
|
||||
|
||||
// Return the score evaluation.
|
||||
return evaluation.Fuzzing(CheckFuzzing, findings)
|
||||
return evaluation.Fuzzing(CheckFuzzing, findings, c.Dlogger)
|
||||
}
|
||||
|
@ -76,7 +76,7 @@ func TestFuzzing(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
expected: scut.TestReturn{
|
||||
NumberOfWarn: 6,
|
||||
NumberOfWarn: 0,
|
||||
NumberOfDebug: 0,
|
||||
NumberOfInfo: 1,
|
||||
Score: 10,
|
||||
|
@ -15,31 +15,9 @@
|
||||
package checks
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/ossf/scorecard/v4/checker"
|
||||
"github.com/ossf/scorecard/v4/finding"
|
||||
"github.com/ossf/scorecard/v4/probes"
|
||||
"github.com/ossf/scorecard/v4/probes/zrunner"
|
||||
)
|
||||
|
||||
// evaluateProbes runs the probes in probesToRun and logs its findings.
|
||||
func evaluateProbes(c *checker.CheckRequest, rawResults *checker.RawResults,
|
||||
probesToRun []probes.ProbeImpl,
|
||||
) ([]finding.Finding, error) {
|
||||
// Run the probes.
|
||||
findings, err := zrunner.Run(rawResults, probesToRun)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("zrunner.Run: %w", err)
|
||||
}
|
||||
|
||||
// Log the findings.
|
||||
if err := checker.LogFindings(findings, c.Dlogger); err != nil {
|
||||
return nil, fmt.Errorf("LogFindings: %w", err)
|
||||
}
|
||||
return findings, nil
|
||||
}
|
||||
|
||||
// getRawResults returns a pointer to the raw results in the CheckRequest
|
||||
// if the pointer is not nil. Else, it creates a new raw result.
|
||||
func getRawResults(c *checker.CheckRequest) *checker.RawResults {
|
@ -20,6 +20,7 @@ import (
|
||||
"github.com/ossf/scorecard/v4/checks/raw"
|
||||
sce "github.com/ossf/scorecard/v4/errors"
|
||||
"github.com/ossf/scorecard/v4/probes"
|
||||
"github.com/ossf/scorecard/v4/probes/zrunner"
|
||||
)
|
||||
|
||||
// CheckSecurityPolicy is the registred name for SecurityPolicy.
|
||||
@ -49,12 +50,12 @@ func SecurityPolicy(c *checker.CheckRequest) checker.CheckResult {
|
||||
pRawResults.SecurityPolicyResults = rawData
|
||||
|
||||
// Evaluate the probes.
|
||||
findings, err := evaluateProbes(c, pRawResults, probes.SecurityPolicy)
|
||||
findings, err := zrunner.Run(pRawResults, probes.SecurityPolicy)
|
||||
if err != nil {
|
||||
e := sce.WithMessage(sce.ErrScorecardInternal, err.Error())
|
||||
return checker.CreateRuntimeErrorResult(CheckSecurityPolicy, e)
|
||||
}
|
||||
|
||||
// Return the score evaluation.
|
||||
return evaluation.SecurityPolicy(CheckSecurityPolicy, findings)
|
||||
return evaluation.SecurityPolicy(CheckSecurityPolicy, findings, c.Dlogger)
|
||||
}
|
||||
|
@ -48,7 +48,7 @@ var _ = Describe("E2E TEST:"+checks.CheckDependencyUpdateTool, func() {
|
||||
expected := scut.TestReturn{
|
||||
Error: nil,
|
||||
Score: checker.MaxResultScore,
|
||||
NumberOfWarn: 3,
|
||||
NumberOfWarn: 0,
|
||||
NumberOfInfo: 1,
|
||||
NumberOfDebug: 0,
|
||||
}
|
||||
@ -75,7 +75,7 @@ var _ = Describe("E2E TEST:"+checks.CheckDependencyUpdateTool, func() {
|
||||
expected := scut.TestReturn{
|
||||
Error: nil,
|
||||
Score: checker.MaxResultScore,
|
||||
NumberOfWarn: 3,
|
||||
NumberOfWarn: 0,
|
||||
NumberOfInfo: 1,
|
||||
NumberOfDebug: 0,
|
||||
}
|
||||
|
@ -51,7 +51,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() {
|
||||
expected := scut.TestReturn{
|
||||
Error: nil,
|
||||
Score: checker.MaxResultScore,
|
||||
NumberOfWarn: 6,
|
||||
NumberOfWarn: 0,
|
||||
NumberOfInfo: 1,
|
||||
NumberOfDebug: 0,
|
||||
}
|
||||
@ -80,7 +80,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() {
|
||||
expected := scut.TestReturn{
|
||||
Error: nil,
|
||||
Score: checker.MaxResultScore,
|
||||
NumberOfWarn: 6,
|
||||
NumberOfWarn: 0,
|
||||
NumberOfInfo: 1,
|
||||
NumberOfDebug: 0,
|
||||
}
|
||||
@ -109,7 +109,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() {
|
||||
expected := scut.TestReturn{
|
||||
Error: nil,
|
||||
Score: checker.MaxResultScore,
|
||||
NumberOfWarn: 6,
|
||||
NumberOfWarn: 0,
|
||||
NumberOfInfo: 2,
|
||||
NumberOfDebug: 0,
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user