2024-02-15 22:32:35 +03:00
|
|
|
// Copyright 2024 OpenSSF Scorecard Authors
|
2024-01-02 20:23:47 +03:00
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
//nolint:stylecheck
|
2024-02-15 22:32:35 +03:00
|
|
|
package sastToolConfigured
|
2024-01-02 20:23:47 +03:00
|
|
|
|
|
|
|
import (
|
|
|
|
"testing"
|
|
|
|
|
|
|
|
"github.com/google/go-cmp/cmp"
|
|
|
|
"github.com/google/go-cmp/cmp/cmpopts"
|
|
|
|
|
2024-04-13 00:51:50 +03:00
|
|
|
"github.com/ossf/scorecard/v5/checker"
|
|
|
|
"github.com/ossf/scorecard/v5/finding"
|
|
|
|
"github.com/ossf/scorecard/v5/probes/internal/utils/test"
|
|
|
|
"github.com/ossf/scorecard/v5/probes/internal/utils/uerror"
|
2024-01-02 20:23:47 +03:00
|
|
|
)
|
|
|
|
|
|
|
|
func Test_Run(t *testing.T) {
|
|
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
2024-02-15 22:32:35 +03:00
|
|
|
err error
|
2024-01-02 20:23:47 +03:00
|
|
|
raw *checker.RawResults
|
2024-02-15 22:32:35 +03:00
|
|
|
name string
|
2024-01-02 20:23:47 +03:00
|
|
|
outcomes []finding.Outcome
|
|
|
|
}{
|
|
|
|
{
|
2024-02-15 22:32:35 +03:00
|
|
|
name: "no raw data",
|
|
|
|
raw: nil,
|
|
|
|
err: uerror.ErrNil,
|
|
|
|
outcomes: nil,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "no SAST tools detected",
|
|
|
|
raw: &checker.RawResults{
|
|
|
|
SASTResults: checker.SASTData{
|
|
|
|
Workflows: []checker.SASTWorkflow{},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
outcomes: []finding.Outcome{
|
2024-04-09 01:36:11 +03:00
|
|
|
finding.OutcomeFalse,
|
2024-02-15 22:32:35 +03:00
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "multiple tools detected",
|
2024-01-02 20:23:47 +03:00
|
|
|
err: nil,
|
|
|
|
raw: &checker.RawResults{
|
|
|
|
SASTResults: checker.SASTData{
|
|
|
|
Workflows: []checker.SASTWorkflow{
|
|
|
|
{
|
|
|
|
Type: checker.CodeQLWorkflow,
|
|
|
|
},
|
|
|
|
{
|
2024-02-15 22:32:35 +03:00
|
|
|
Type: checker.QodanaWorkflow,
|
2024-01-02 20:23:47 +03:00
|
|
|
},
|
|
|
|
{
|
|
|
|
Type: checker.PysaWorkflow,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
outcomes: []finding.Outcome{
|
2024-04-09 01:36:11 +03:00
|
|
|
finding.OutcomeTrue,
|
|
|
|
finding.OutcomeTrue,
|
|
|
|
finding.OutcomeTrue,
|
2024-01-02 20:23:47 +03:00
|
|
|
},
|
|
|
|
},
|
2024-02-15 22:32:35 +03:00
|
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
|
|
tt := tt // Re-initializing variable so it is not changed while executing the closure below
|
|
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
|
|
t.Parallel()
|
|
|
|
|
|
|
|
findings, s, err := Run(tt.raw)
|
|
|
|
if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) {
|
|
|
|
t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors()))
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if diff := cmp.Diff(Probe, s); diff != "" {
|
|
|
|
t.Errorf("mismatch (-want +got):\n%s", diff)
|
|
|
|
}
|
|
|
|
test.AssertOutcomes(t, findings, tt.outcomes)
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func Test_Run_tools(t *testing.T) {
|
|
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
|
|
name string
|
|
|
|
raw *checker.RawResults
|
|
|
|
tools []string
|
|
|
|
}{
|
2024-01-02 20:23:47 +03:00
|
|
|
{
|
2024-02-15 22:32:35 +03:00
|
|
|
name: "one tool",
|
2024-01-02 20:23:47 +03:00
|
|
|
raw: &checker.RawResults{
|
|
|
|
SASTResults: checker.SASTData{
|
|
|
|
Workflows: []checker.SASTWorkflow{
|
|
|
|
{
|
2024-02-15 22:32:35 +03:00
|
|
|
Type: checker.CodeQLWorkflow,
|
2024-01-02 20:23:47 +03:00
|
|
|
},
|
2024-02-15 22:32:35 +03:00
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
tools: []string{"CodeQL"},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "one tool, multiple times",
|
|
|
|
raw: &checker.RawResults{
|
|
|
|
SASTResults: checker.SASTData{
|
|
|
|
Workflows: []checker.SASTWorkflow{
|
2024-01-02 20:23:47 +03:00
|
|
|
{
|
|
|
|
Type: checker.CodeQLWorkflow,
|
|
|
|
},
|
|
|
|
{
|
2024-02-15 22:32:35 +03:00
|
|
|
Type: checker.CodeQLWorkflow,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
tools: []string{"CodeQL", "CodeQL"},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "multiple tools",
|
|
|
|
raw: &checker.RawResults{
|
|
|
|
SASTResults: checker.SASTData{
|
|
|
|
Workflows: []checker.SASTWorkflow{
|
|
|
|
{
|
|
|
|
Type: checker.SonarWorkflow,
|
2024-01-02 20:23:47 +03:00
|
|
|
},
|
|
|
|
{
|
|
|
|
Type: checker.PysaWorkflow,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
2024-02-15 22:32:35 +03:00
|
|
|
tools: []string{"Sonar", "Pysa"},
|
2024-01-02 20:23:47 +03:00
|
|
|
},
|
|
|
|
}
|
|
|
|
for _, tt := range tests {
|
2024-02-15 22:32:35 +03:00
|
|
|
tt := tt
|
2024-01-02 20:23:47 +03:00
|
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
|
|
t.Parallel()
|
|
|
|
findings, s, err := Run(tt.raw)
|
|
|
|
if err != nil {
|
2024-02-15 22:32:35 +03:00
|
|
|
t.Fatalf("expected no error: %v", err)
|
2024-01-02 20:23:47 +03:00
|
|
|
}
|
|
|
|
if diff := cmp.Diff(Probe, s); diff != "" {
|
|
|
|
t.Errorf("mismatch (-want +got):\n%s", diff)
|
|
|
|
}
|
2024-02-15 22:32:35 +03:00
|
|
|
assertTools(t, findings, tt.tools)
|
2024-01-02 20:23:47 +03:00
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
2024-02-15 22:32:35 +03:00
|
|
|
|
|
|
|
func assertTools(tb testing.TB, findings []finding.Finding, tools []string) {
|
|
|
|
tb.Helper()
|
|
|
|
if len(findings) != len(tools) {
|
|
|
|
tb.Fatalf("mismatch between number of finding (%d) and tools (%d)", len(findings), len(tools))
|
|
|
|
}
|
|
|
|
for i, f := range findings {
|
2024-04-09 01:36:11 +03:00
|
|
|
if f.Outcome != finding.OutcomeTrue {
|
2024-02-15 22:32:35 +03:00
|
|
|
tb.Errorf("outcome (%v) shouldn't have a tool field", f.Outcome)
|
|
|
|
}
|
|
|
|
tool, ok := f.Values[ToolKey]
|
|
|
|
if !ok {
|
|
|
|
tb.Errorf("no tool present")
|
|
|
|
}
|
|
|
|
if tool != tools[i] {
|
|
|
|
tb.Errorf("got: %s, wanted: %s", tool, tools[i])
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|