mirror of
https://github.com/ossf/scorecard.git
synced 2024-09-11 08:55:27 +03:00
342 lines
8.6 KiB
Go
342 lines
8.6 KiB
Go
// Copyright 2020 OpenSSF Scorecard Authors
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package checks
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/golang/mock/gomock"
|
|
|
|
"github.com/ossf/scorecard/v5/checker"
|
|
"github.com/ossf/scorecard/v5/clients"
|
|
mockrepo "github.com/ossf/scorecard/v5/clients/mockclients"
|
|
sce "github.com/ossf/scorecard/v5/errors"
|
|
scut "github.com/ossf/scorecard/v5/utests"
|
|
)
|
|
|
|
func Test_SAST(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
err error
|
|
searchRequest clients.SearchRequest
|
|
name string
|
|
path string
|
|
commits []clients.Commit
|
|
checkRuns []clients.CheckRun
|
|
searchresult clients.SearchResponse
|
|
expected scut.TestReturn
|
|
}{
|
|
{
|
|
name: "SAST checker should return min score when no PRs are found",
|
|
commits: []clients.Commit{},
|
|
searchresult: clients.SearchResponse{},
|
|
checkRuns: []clients.CheckRun{},
|
|
expected: scut.TestReturn{
|
|
Score: checker.MinResultScore,
|
|
NumberOfWarn: 1,
|
|
},
|
|
},
|
|
{
|
|
name: "SAST checker should return failed status when an error occurs",
|
|
err: errors.New("error"),
|
|
commits: []clients.Commit{},
|
|
searchresult: clients.SearchResponse{},
|
|
checkRuns: []clients.CheckRun{},
|
|
expected: scut.TestReturn{
|
|
Score: checker.InconclusiveResultScore,
|
|
Error: sce.ErrScorecardInternal,
|
|
},
|
|
},
|
|
{
|
|
name: "Successful SAST checker should return success status for github-advanced-security",
|
|
commits: []clients.Commit{
|
|
{
|
|
AssociatedMergeRequest: clients.PullRequest{
|
|
MergedAt: time.Now().Add(time.Hour - 1),
|
|
},
|
|
},
|
|
},
|
|
searchresult: clients.SearchResponse{},
|
|
checkRuns: []clients.CheckRun{
|
|
{
|
|
Status: "completed",
|
|
Conclusion: "success",
|
|
App: clients.CheckRunApp{
|
|
Slug: "github-advanced-security",
|
|
},
|
|
},
|
|
},
|
|
expected: scut.TestReturn{
|
|
Score: checker.MaxResultScore,
|
|
NumberOfInfo: 1,
|
|
NumberOfDebug: 1,
|
|
},
|
|
},
|
|
{
|
|
name: "Successful SAST checker should return success status for github-code-scanning",
|
|
commits: []clients.Commit{
|
|
{
|
|
AssociatedMergeRequest: clients.PullRequest{
|
|
MergedAt: time.Now().Add(time.Hour - 1),
|
|
},
|
|
},
|
|
},
|
|
searchresult: clients.SearchResponse{},
|
|
checkRuns: []clients.CheckRun{
|
|
{
|
|
Status: "completed",
|
|
Conclusion: "success",
|
|
App: clients.CheckRunApp{
|
|
Slug: "github-code-scanning",
|
|
},
|
|
},
|
|
},
|
|
expected: scut.TestReturn{
|
|
Score: checker.MaxResultScore,
|
|
NumberOfInfo: 1,
|
|
NumberOfDebug: 1,
|
|
},
|
|
},
|
|
{
|
|
name: "Successful SAST checker should return success status for lgtm",
|
|
commits: []clients.Commit{
|
|
{
|
|
AssociatedMergeRequest: clients.PullRequest{
|
|
MergedAt: time.Now().Add(time.Hour - 1),
|
|
},
|
|
},
|
|
},
|
|
searchresult: clients.SearchResponse{},
|
|
checkRuns: []clients.CheckRun{
|
|
{
|
|
Status: "completed",
|
|
Conclusion: "success",
|
|
App: clients.CheckRunApp{
|
|
Slug: "lgtm-com",
|
|
},
|
|
},
|
|
},
|
|
path: "",
|
|
expected: scut.TestReturn{
|
|
Score: checker.MaxResultScore,
|
|
NumberOfInfo: 1,
|
|
NumberOfDebug: 1,
|
|
},
|
|
},
|
|
{
|
|
name: "Successful SAST checker should return success status for sonarcloud",
|
|
commits: []clients.Commit{
|
|
{
|
|
AssociatedMergeRequest: clients.PullRequest{
|
|
MergedAt: time.Now().Add(time.Hour - 1),
|
|
},
|
|
},
|
|
},
|
|
searchresult: clients.SearchResponse{},
|
|
checkRuns: []clients.CheckRun{
|
|
{
|
|
Status: "completed",
|
|
Conclusion: "success",
|
|
App: clients.CheckRunApp{
|
|
Slug: "sonarcloud",
|
|
},
|
|
},
|
|
},
|
|
expected: scut.TestReturn{
|
|
Score: checker.MaxResultScore,
|
|
NumberOfInfo: 1,
|
|
NumberOfDebug: 1,
|
|
},
|
|
},
|
|
{
|
|
name: "Airflow Workflow has CodeQL but has no check runs.",
|
|
err: nil,
|
|
commits: []clients.Commit{
|
|
{
|
|
AssociatedMergeRequest: clients.PullRequest{
|
|
MergedAt: time.Now().Add(time.Hour - 1),
|
|
},
|
|
},
|
|
},
|
|
searchresult: clients.SearchResponse{},
|
|
path: ".github/workflows/airflow-codeql-workflow.yaml",
|
|
expected: scut.TestReturn{
|
|
Score: 7,
|
|
NumberOfWarn: 1,
|
|
NumberOfInfo: 1,
|
|
},
|
|
},
|
|
{
|
|
name: "Airflow Workflow has CodeQL and two check runs.",
|
|
err: nil,
|
|
commits: []clients.Commit{
|
|
{
|
|
AssociatedMergeRequest: clients.PullRequest{
|
|
MergedAt: time.Now().Add(time.Hour - 1),
|
|
},
|
|
},
|
|
},
|
|
searchresult: clients.SearchResponse{},
|
|
checkRuns: []clients.CheckRun{
|
|
{
|
|
Status: "completed",
|
|
Conclusion: "success",
|
|
App: clients.CheckRunApp{
|
|
Slug: "lgtm-com",
|
|
},
|
|
},
|
|
{
|
|
Status: "completed",
|
|
Conclusion: "success",
|
|
App: clients.CheckRunApp{
|
|
Slug: "lgtm-com",
|
|
},
|
|
},
|
|
},
|
|
path: ".github/workflows/airflow-codeql-workflow.yaml",
|
|
expected: scut.TestReturn{
|
|
Score: checker.MaxResultScore,
|
|
NumberOfInfo: 2,
|
|
NumberOfDebug: 1,
|
|
},
|
|
},
|
|
{
|
|
name: `Airflow Workflow has CodeQL and two check runs one of
|
|
which has wrong type of conclusion. The other is 'success'`,
|
|
err: nil,
|
|
commits: []clients.Commit{
|
|
{
|
|
AssociatedMergeRequest: clients.PullRequest{
|
|
MergedAt: time.Now().Add(time.Hour - 1),
|
|
},
|
|
},
|
|
},
|
|
searchresult: clients.SearchResponse{},
|
|
checkRuns: []clients.CheckRun{
|
|
{
|
|
Status: "completed",
|
|
Conclusion: "wrongConclusionValue",
|
|
App: clients.CheckRunApp{
|
|
Slug: "lgtm-com",
|
|
},
|
|
},
|
|
{
|
|
Status: "completed",
|
|
Conclusion: "success",
|
|
App: clients.CheckRunApp{
|
|
Slug: "lgtm-com",
|
|
},
|
|
},
|
|
},
|
|
path: ".github/workflows/airflow-codeql-workflow.yaml",
|
|
expected: scut.TestReturn{
|
|
Score: checker.MaxResultScore,
|
|
NumberOfInfo: 2,
|
|
NumberOfDebug: 1,
|
|
},
|
|
},
|
|
{
|
|
name: `Airflow Workflow has CodeQL and two commits none of which
|
|
ran the workflow.`,
|
|
err: nil,
|
|
commits: []clients.Commit{
|
|
{
|
|
AssociatedMergeRequest: clients.PullRequest{
|
|
MergedAt: time.Now().Add(time.Hour - 1),
|
|
},
|
|
},
|
|
{
|
|
AssociatedMergeRequest: clients.PullRequest{
|
|
MergedAt: time.Now().Add(time.Hour - 2),
|
|
},
|
|
},
|
|
},
|
|
searchresult: clients.SearchResponse{},
|
|
checkRuns: []clients.CheckRun{
|
|
{
|
|
Status: "notCompletedForTestingOnly",
|
|
Conclusion: "notSuccessForTestingOnly",
|
|
App: clients.CheckRunApp{
|
|
Slug: "lgtm-com",
|
|
},
|
|
},
|
|
{
|
|
Status: "notCompletedForTestingOnly",
|
|
Conclusion: "notSuccessForTestingOnly",
|
|
App: clients.CheckRunApp{
|
|
Slug: "lgtm-com",
|
|
},
|
|
},
|
|
},
|
|
path: ".github/workflows/airflow-codeql-workflow.yaml",
|
|
expected: scut.TestReturn{
|
|
Score: 7,
|
|
NumberOfWarn: 1,
|
|
NumberOfInfo: 1,
|
|
},
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
tt := tt
|
|
searchRequest := clients.SearchRequest{
|
|
Query: "github/codeql-action/analyze",
|
|
Path: "/.github/workflows",
|
|
}
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
ctrl := gomock.NewController(t)
|
|
mockRepoClient := mockrepo.NewMockRepoClient(ctrl)
|
|
mockRepoClient.EXPECT().ListCommits().DoAndReturn(func() ([]clients.Commit, error) {
|
|
if tt.err != nil {
|
|
return nil, tt.err
|
|
}
|
|
return tt.commits, tt.err
|
|
})
|
|
mockRepoClient.EXPECT().ListCheckRunsForRef("").Return(tt.checkRuns, nil).AnyTimes()
|
|
mockRepoClient.EXPECT().Search(searchRequest).Return(tt.searchresult, nil).AnyTimes()
|
|
mockRepoClient.EXPECT().ListFiles(gomock.Any()).DoAndReturn(
|
|
func(predicate func(string) (bool, error)) ([]string, error) {
|
|
if strings.Contains(tt.path, "pom") {
|
|
return []string{"pom.xml"}, nil
|
|
}
|
|
return []string{tt.path}, nil
|
|
}).AnyTimes()
|
|
mockRepoClient.EXPECT().GetFileReader(gomock.Any()).DoAndReturn(func(fn string) (io.ReadCloser, error) {
|
|
if tt.path == "" {
|
|
return io.NopCloser(strings.NewReader("")), nil
|
|
}
|
|
return os.Open("./testdata/" + tt.path)
|
|
}).AnyTimes()
|
|
|
|
dl := scut.TestDetailLogger{}
|
|
req := checker.CheckRequest{
|
|
RepoClient: mockRepoClient,
|
|
Ctx: context.TODO(),
|
|
Dlogger: &dl,
|
|
}
|
|
res := SAST(&req)
|
|
|
|
scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl)
|
|
})
|
|
}
|
|
}
|