Raw results for Fuzzing check (#1917)

* update

* update

* update

* update

* linter

* comments

* comments
This commit is contained in:
laurentsimon 2022-05-19 17:55:49 -07:00 committed by GitHub
parent fb45cd7e9d
commit 8d8bcf2f69
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 373 additions and 240 deletions

View File

@ -32,9 +32,34 @@ type RawResults struct {
ContributorsResults ContributorsData
MaintainedResults MaintainedData
SignedReleasesResults SignedReleasesData
FuzzingResults FuzzingData
LicenseResults LicenseData
}
// FuzzerName represents a fuzzing service.
type FuzzerName string
const (
// FuzzerNameCIFuzz is CIFuzz.
FuzzerNameCIFuzz FuzzerName = "CIFuzz"
// FuzzerNameOSSFuzz is OSSFuzz.
FuzzerNameOSSFuzz FuzzerName = "OSSFuzz"
// FuzzerNameGoBuiltin is the built-in Go fuzzer.
FuzzerNameGoBuiltin FuzzerName = "GoFuzzer"
)
// FuzzingData represents different fuzzing done.
type FuzzingData struct {
Fuzzers []Fuzzer
}
// Fuzzer represent the use of a fuzzer.
type Fuzzer struct {
Name FuzzerName
// TODO: CodeCoverage.
// TODO: (#1933)
}
// MaintainedData contains the raw results
// for the Maintained check.
type MaintainedData struct {
@ -233,21 +258,21 @@ type RepoAssociation string
const (
// RepoAssociationCollaborator has been invited to collaborate on the repository.
RepoAssociationCollaborator RepoAssociation = RepoAssociation("collaborator")
RepoAssociationCollaborator RepoAssociation = "collaborator"
// RepoAssociationContributor is an contributor to the repository.
RepoAssociationContributor RepoAssociation = RepoAssociation("contributor")
RepoAssociationContributor RepoAssociation = "contributor"
// RepoAssociationOwner is an owner of the repository.
RepoAssociationOwner RepoAssociation = RepoAssociation("owner")
RepoAssociationOwner RepoAssociation = "owner"
// RepoAssociationMember is a member of the organization that owns the repository.
RepoAssociationMember RepoAssociation = RepoAssociation("member")
RepoAssociationMember RepoAssociation = "member"
// RepoAssociationFirstTimer has previously committed to the repository.
RepoAssociationFirstTimer RepoAssociation = RepoAssociation("first-timer")
RepoAssociationFirstTimer RepoAssociation = "first-timer"
// RepoAssociationFirstTimeContributor has not previously committed to the repository.
RepoAssociationFirstTimeContributor RepoAssociation = RepoAssociation("first-timer-contributor")
RepoAssociationFirstTimeContributor RepoAssociation = "first-timer-contributor"
// RepoAssociationMannequin is a placeholder for an unclaimed user.
RepoAssociationMannequin RepoAssociation = RepoAssociation("unknown")
RepoAssociationMannequin RepoAssociation = "unknown"
// RepoAssociationNone has no association with the repository.
RepoAssociationNone RepoAssociation = RepoAssociation("none")
RepoAssociationNone RepoAssociation = "none"
)
// File represents a file.

View File

@ -0,0 +1,43 @@
// Copyright 2021 Security Scorecard Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package evaluation
import (
"github.com/ossf/scorecard/v4/checker"
sce "github.com/ossf/scorecard/v4/errors"
)
// Fuzzing applies the score policy for the Fuzzing check.
func Fuzzing(name string, dl checker.DetailLogger,
r *checker.FuzzingData,
) checker.CheckResult {
if r == nil {
e := sce.WithMessage(sce.ErrScorecardInternal, "empty raw data")
return checker.CreateRuntimeErrorResult(name, e)
}
for _, fuzzer := range r.Fuzzers {
switch fuzzer.Name {
case checker.FuzzerNameCIFuzz:
return checker.CreateMaxScoreResult(name, "project uses ClusterFuzzLite")
case checker.FuzzerNameOSSFuzz:
return checker.CreateMaxScoreResult(name, "project is fuzzed in OSS-Fuzz")
case checker.FuzzerNameGoBuiltin:
return checker.CreateMaxScoreResult(name, "project is fuzzed using Golang's fuzzing")
}
}
return checker.CreateMinScoreResult(name, "project is not fuzzed")
}

View File

@ -15,11 +15,9 @@
package checks
import (
"fmt"
"github.com/ossf/scorecard/v4/checker"
"github.com/ossf/scorecard/v4/checks/fileparser"
"github.com/ossf/scorecard/v4/clients"
"github.com/ossf/scorecard/v4/checks/evaluation"
"github.com/ossf/scorecard/v4/checks/raw"
sce "github.com/ossf/scorecard/v4/errors"
)
@ -34,57 +32,18 @@ func init() {
}
}
func checkCFLite(c *checker.CheckRequest) (bool, error) {
result := false
e := fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{
Pattern: ".clusterfuzzlite/Dockerfile",
CaseSensitive: true,
}, func(path string, content []byte, args ...interface{}) (bool, error) {
result = fileparser.CheckFileContainsCommands(content, "#")
return false, nil
}, nil)
if e != nil {
return result, fmt.Errorf("%w", e)
}
return result, nil
}
func checkOSSFuzz(c *checker.CheckRequest) (bool, error) {
if c.OssFuzzRepo == nil {
return false, nil
}
req := clients.SearchRequest{
Query: c.RepoClient.URI(),
Filename: "project.yaml",
}
result, err := c.OssFuzzRepo.Search(req)
if err != nil {
e := sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("Client.Search.Code: %v", err))
return false, e
}
return result.Hits > 0, nil
}
// Fuzzing runs Fuzzing check.
func Fuzzing(c *checker.CheckRequest) checker.CheckResult {
usingCFLite, e := checkCFLite(c)
if e != nil {
rawData, err := raw.Fuzzing(c)
if err != nil {
e := sce.WithMessage(sce.ErrScorecardInternal, err.Error())
return checker.CreateRuntimeErrorResult(CheckFuzzing, e)
}
if usingCFLite {
return checker.CreateMaxScoreResult(CheckFuzzing,
"project uses ClusterFuzzLite")
// Set the raw results.
if c.RawResults != nil {
c.RawResults.FuzzingResults = rawData
}
usingOSSFuzz, e := checkOSSFuzz(c)
if e != nil {
return checker.CreateRuntimeErrorResult(CheckFuzzing, e)
}
if usingOSSFuzz {
return checker.CreateMaxScoreResult(CheckFuzzing,
"project is fuzzed in OSS-Fuzz")
}
return checker.CreateMinScoreResult(CheckFuzzing, "project is not fuzzed")
return evaluation.Fuzzing(CheckFuzzing, c.Dlogger, &rawData)
}

View File

@ -27,187 +27,6 @@ import (
scut "github.com/ossf/scorecard/v4/utests"
)
// Test_checkOSSFuzz is a test function for checkOSSFuzz.
func Test_checkOSSFuzz(t *testing.T) {
t.Parallel()
//nolint
tests := []struct {
name string
want bool
response clients.SearchResponse
wantErr bool
wantFuzzErr bool
expected scut.TestReturn
}{
{
name: "Test_checkOSSFuzz failure",
want: false,
response: clients.SearchResponse{},
wantErr: false,
expected: scut.TestReturn{
NumberOfWarn: 0,
NumberOfDebug: 0,
NumberOfInfo: 0,
Score: 0,
},
},
{
name: "Test_checkOSSFuzz success",
want: true,
response: clients.SearchResponse{
Hits: 1,
},
wantErr: false,
expected: scut.TestReturn{
NumberOfWarn: 0,
NumberOfDebug: 0,
NumberOfInfo: 0,
Score: 0,
},
},
{
name: "error",
want: false,
wantErr: true,
expected: scut.TestReturn{
NumberOfWarn: 0,
NumberOfDebug: 0,
NumberOfInfo: 0,
Score: 0,
},
},
{
name: "Test_checkOSSFuzz fuzz error",
want: false,
wantFuzzErr: true,
expected: scut.TestReturn{
Error: nil,
NumberOfWarn: 0,
NumberOfDebug: 0,
NumberOfInfo: 0,
Score: 0,
},
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockFuzz := mockrepo.NewMockRepoClient(ctrl)
mockFuzz.EXPECT().URI().Return("github.com/ossf/scorecard").AnyTimes()
mockFuzz.EXPECT().Search(gomock.Any()).
DoAndReturn(func(q clients.SearchRequest) (clients.SearchResponse, error) {
if tt.wantErr {
//nolint
return clients.SearchResponse{}, errors.New("error")
}
return tt.response, nil
}).AnyTimes()
dl := scut.TestDetailLogger{}
req := checker.CheckRequest{
RepoClient: mockFuzz,
OssFuzzRepo: mockFuzz,
Dlogger: &dl,
}
if tt.wantFuzzErr {
req.OssFuzzRepo = nil
}
got, err := checkOSSFuzz(&req)
if (err != nil) != tt.wantErr {
t.Errorf("checkOSSFuzz() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("checkOSSFuzz() = %v, want %v for %v", got, tt.want, tt.name)
}
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &checker.CheckResult{}, &dl) {
t.Fatalf(tt.name)
}
})
}
}
// Test_checkCFLite is a test function for checkCFLite.
func Test_checkCFLite(t *testing.T) {
t.Parallel()
//nolint
tests := []struct {
name string
want bool
wantErr bool
fileName []string
fileContent string
expected scut.TestReturn
}{
{
name: "Test_checkCFLite success",
want: false,
wantErr: false,
fileName: []string{"docker-compose.yml"},
fileContent: `# .clusterfuzzlite/Dockerfile`,
expected: scut.TestReturn{
NumberOfWarn: 0,
NumberOfDebug: 0,
NumberOfInfo: 0,
Score: 0,
},
},
{
name: "Test_checkCFLite failure",
want: false,
wantErr: true,
fileName: []string{"docker-compose.yml"},
expected: scut.TestReturn{
NumberOfWarn: 0,
NumberOfDebug: 0,
NumberOfInfo: 0,
Score: 0,
},
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mockFuzz := mockrepo.NewMockRepoClient(ctrl)
mockFuzz.EXPECT().ListFiles(gomock.Any()).Return(tt.fileName, nil).AnyTimes()
mockFuzz.EXPECT().GetFileContent(gomock.Any()).DoAndReturn(func(f string) (string, error) {
if tt.wantErr {
//nolint
return "", errors.New("error")
}
return tt.fileContent, nil
}).AnyTimes()
dl := scut.TestDetailLogger{}
req := checker.CheckRequest{
RepoClient: mockFuzz,
Dlogger: &dl,
}
got, err := checkCFLite(&req)
if (err != nil) != tt.wantErr {
t.Errorf("checkCFLite() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("checkCFLite() = %v, want %v for test %v", got, tt.want, tt.name)
}
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &checker.CheckResult{}, &dl) {
t.Fatalf(tt.name)
}
})
}
}
// TestFuzzing is a test function for Fuzzing.
func TestFuzzing(t *testing.T) {
t.Parallel()

79
checks/raw/fuzzing.go Normal file
View File

@ -0,0 +1,79 @@
// Copyright 2021 Security Scorecard Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package raw
import (
"fmt"
"github.com/ossf/scorecard/v4/checker"
"github.com/ossf/scorecard/v4/checks/fileparser"
"github.com/ossf/scorecard/v4/clients"
sce "github.com/ossf/scorecard/v4/errors"
)
// Fuzzing runs Fuzzing check.
func Fuzzing(c *checker.CheckRequest) (checker.FuzzingData, error) {
var fuzzers []checker.Fuzzer
usingCFLite, e := checkCFLite(c)
if e != nil {
return checker.FuzzingData{}, fmt.Errorf("%w", e)
}
if usingCFLite {
fuzzers = append(fuzzers, checker.Fuzzer{Name: checker.FuzzerNameCIFuzz})
}
usingOSSFuzz, e := checkOSSFuzz(c)
if e != nil {
return checker.FuzzingData{}, fmt.Errorf("%w", e)
}
if usingOSSFuzz {
fuzzers = append(fuzzers, checker.Fuzzer{Name: checker.FuzzerNameOSSFuzz})
}
return checker.FuzzingData{Fuzzers: fuzzers}, nil
}
func checkCFLite(c *checker.CheckRequest) (bool, error) {
result := false
e := fileparser.OnMatchingFileContentDo(c.RepoClient, fileparser.PathMatcher{
Pattern: ".clusterfuzzlite/Dockerfile",
CaseSensitive: true,
}, func(path string, content []byte, args ...interface{}) (bool, error) {
result = fileparser.CheckFileContainsCommands(content, "#")
return false, nil
}, nil)
if e != nil {
return result, fmt.Errorf("%w", e)
}
return result, nil
}
func checkOSSFuzz(c *checker.CheckRequest) (bool, error) {
if c.OssFuzzRepo == nil {
return false, nil
}
req := clients.SearchRequest{
Query: c.RepoClient.URI(),
Filename: "project.yaml",
}
result, err := c.OssFuzzRepo.Search(req)
if err != nil {
e := sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("Client.Search.Code: %v", err))
return false, e
}
return result.Hits > 0, nil
}

157
checks/raw/fuzzing_test.go Normal file
View File

@ -0,0 +1,157 @@
// Copyright 2020 Security Scorecard Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package raw
import (
"errors"
"testing"
"github.com/golang/mock/gomock"
"github.com/ossf/scorecard/v4/checker"
"github.com/ossf/scorecard/v4/clients"
mockrepo "github.com/ossf/scorecard/v4/clients/mockclients"
)
// Test_checkOSSFuzz is a test function for checkOSSFuzz.
func Test_checkOSSFuzz(t *testing.T) {
t.Parallel()
//nolint
tests := []struct {
name string
want bool
response clients.SearchResponse
wantErr bool
wantFuzzErr bool
}{
{
name: "Test_checkOSSFuzz failure",
want: false,
response: clients.SearchResponse{},
wantErr: false,
},
{
name: "Test_checkOSSFuzz success",
want: true,
response: clients.SearchResponse{
Hits: 1,
},
wantErr: false,
},
{
name: "error",
want: false,
wantErr: true,
},
{
name: "Test_checkOSSFuzz fuzz error",
want: false,
wantFuzzErr: true,
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
mockFuzz := mockrepo.NewMockRepoClient(ctrl)
mockFuzz.EXPECT().URI().Return("github.com/ossf/scorecard").AnyTimes()
mockFuzz.EXPECT().Search(gomock.Any()).
DoAndReturn(func(q clients.SearchRequest) (clients.SearchResponse, error) {
if tt.wantErr {
//nolint
return clients.SearchResponse{}, errors.New("error")
}
return tt.response, nil
}).AnyTimes()
req := checker.CheckRequest{
RepoClient: mockFuzz,
OssFuzzRepo: mockFuzz,
}
if tt.wantFuzzErr {
req.OssFuzzRepo = nil
}
got, err := checkOSSFuzz(&req)
if (err != nil) != tt.wantErr {
t.Errorf("checkOSSFuzz() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("checkOSSFuzz() = %v, want %v for %v", got, tt.want, tt.name)
}
})
}
}
// Test_checkCFLite is a test function for checkCFLite.
func Test_checkCFLite(t *testing.T) {
t.Parallel()
//nolint
tests := []struct {
name string
want bool
wantErr bool
fileName []string
fileContent string
}{
{
name: "Test_checkCFLite success",
want: false,
wantErr: false,
fileName: []string{"docker-compose.yml"},
fileContent: `# .clusterfuzzlite/Dockerfile`,
},
{
name: "Test_checkCFLite failure",
want: false,
wantErr: true,
fileName: []string{"docker-compose.yml"},
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mockFuzz := mockrepo.NewMockRepoClient(ctrl)
mockFuzz.EXPECT().ListFiles(gomock.Any()).Return(tt.fileName, nil).AnyTimes()
mockFuzz.EXPECT().GetFileContent(gomock.Any()).DoAndReturn(func(f string) (string, error) {
if tt.wantErr {
//nolint
return "", errors.New("error")
}
return tt.fileContent, nil
}).AnyTimes()
req := checker.CheckRequest{
RepoClient: mockFuzz,
}
got, err := checkCFLite(&req)
if (err != nil) != tt.wantErr {
t.Errorf("checkCFLite() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("checkCFLite() = %v, want %v for test %v", got, tt.want, tt.name)
}
})
}
}

View File

@ -0,0 +1,24 @@
# Copyright 2022 Security Scorecard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
jobs:
publish:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Python Semantic Release
uses: relekang/python-semantic-release@v7.23.0
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
pypi_token: ${{ secrets.TEST_PYPI_API_TOKEN }}

View File

@ -186,6 +186,13 @@ type jsonWorkflowJob struct {
ID *string `json:"id"`
}
type jsonFuzzer struct {
Job *jsonWorkflowJob `json:"job,omitempty"`
File *jsonFile `json:"file,omitempty"`
Name string `json:"name"`
// TODO: (#1933)
}
//nolint
type jsonRawResults struct {
// Workflow results.
@ -216,10 +223,25 @@ type jsonRawResults struct {
DefaultBranchCommits []jsonDefaultBranchCommit `json:"default-branch-commits"`
// Archived status of the repo.
ArchivedStatus jsonArchivedStatus `json:"archived"`
// Fuzzers.
Fuzzers []jsonFuzzer `json:"fuzzers"`
// Releases.
Releases []jsonRelease `json:"releases"`
}
//nolint:unparam
func (r *jsonScorecardRawResult) addFuzzingRawResults(fd *checker.FuzzingData) error {
r.Results.Fuzzers = []jsonFuzzer{}
for _, f := range fd.Fuzzers {
fuzzer := jsonFuzzer{
// TODO: Job, File, Coverage.
Name: string(f.Name),
}
r.Results.Fuzzers = append(r.Results.Fuzzers, fuzzer)
}
return nil
}
func (r *jsonScorecardRawResult) addDangerousWorkflowRawResults(df *checker.DangerousWorkflowData) error {
r.Results.Workflows = []jsonWorkflow{}
for _, e := range df.Workflows {
@ -577,6 +599,11 @@ func (r *jsonScorecardRawResult) fillJSONRawResults(raw *checker.RawResults) err
return sce.WithMessage(sce.ErrScorecardInternal, err.Error())
}
// Fuzzers.
if err := r.addFuzzingRawResults(&raw.FuzzingResults); err != nil {
return sce.WithMessage(sce.ErrScorecardInternal, err.Error())
}
return nil
}