mirror of
https://github.com/ossf/scorecard.git
synced 2024-10-05 13:17:08 +03:00
✨ Detect fuzzing in Haskell by the presence of property tests. (#2843)
* Add Haskell as a language. Signed-off-by: Yoo Chung <chungyc@google.com> * Detect fuzzing in Haskell using presence of property-based testing. Signed-off-by: Yoo Chung <chungyc@google.com> * Mention fuzzing detection for Haskell in documentation. Signed-off-by: Yoo Chung <chungyc@google.com> * Fix pattern and test. Add test case. Signed-off-by: Yoo Chung <chungyc@google.com> --------- Signed-off-by: Yoo Chung <chungyc@google.com>
This commit is contained in:
parent
358de6bda0
commit
71eda75a0d
@ -28,10 +28,11 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
fuzzerOSSFuzz = "OSSFuzz"
|
||||
fuzzerClusterFuzzLite = "ClusterFuzzLite"
|
||||
oneFuzz = "OneFuzz"
|
||||
fuzzerBuiltInGo = "GoBuiltInFuzzer"
|
||||
fuzzerOSSFuzz = "OSSFuzz"
|
||||
fuzzerClusterFuzzLite = "ClusterFuzzLite"
|
||||
oneFuzz = "OneFuzz"
|
||||
fuzzerBuiltInGo = "GoBuiltInFuzzer"
|
||||
fuzzerPropertyBasedHaskell = "HaskellPropertyBasedTesting"
|
||||
// TODO: add more fuzzing check supports.
|
||||
)
|
||||
|
||||
@ -42,8 +43,12 @@ type filesWithPatternStr struct {
|
||||
|
||||
// Configurations for language-specified fuzzers.
|
||||
type languageFuzzConfig struct {
|
||||
URL, Desc *string
|
||||
filePattern, funcPattern, Name string
|
||||
URL, Desc *string
|
||||
|
||||
// Pattern is according to path.Match.
|
||||
filePattern string
|
||||
|
||||
funcPattern, Name string
|
||||
// TODO: add more language fuzzing-related fields.
|
||||
}
|
||||
|
||||
@ -59,6 +64,29 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{
|
||||
Desc: asPointer(
|
||||
"Go fuzzing intelligently walks through the source code to report failures and find vulnerabilities."),
|
||||
},
|
||||
// Fuzz patterns for Haskell based on property-based testing.
|
||||
//
|
||||
// Based on the import of one of these packages:
|
||||
// * https://hackage.haskell.org/package/QuickCheck
|
||||
// * https://hedgehog.qa/
|
||||
// * https://github.com/NorfairKing/validity
|
||||
// * https://hackage.haskell.org/package/smallcheck
|
||||
//
|
||||
// They can also be imported indirectly through these test frameworks:
|
||||
// * https://hspec.github.io/
|
||||
// * https://hackage.haskell.org/package/tasty
|
||||
//
|
||||
// This is not an exhaustive list.
|
||||
clients.Haskell: {
|
||||
filePattern: "*.hs",
|
||||
// Look for direct imports of QuickCheck, Hedgehog, validity, or SmallCheck,
|
||||
// or their indirect imports through the higher-level Hspec or Tasty testing frameworks.
|
||||
funcPattern: `import\s+(qualified\s+)?Test\.((Hspec|Tasty)\.)?(QuickCheck|Hedgehog|Validity|SmallCheck)`,
|
||||
Name: fuzzerPropertyBasedHaskell,
|
||||
Desc: asPointer(
|
||||
"Property-based testing in Haskell generates test instances randomly or exhaustively " +
|
||||
"and test that specific properties are satisfied."),
|
||||
},
|
||||
// TODO: add more language-specific fuzz patterns & configs.
|
||||
}
|
||||
|
||||
|
@ -316,6 +316,103 @@ func Test_checkFuzzFunc(t *testing.T) {
|
||||
},
|
||||
fileContent: "func TestFoo (t *testing.T)",
|
||||
},
|
||||
{
|
||||
name: "Haskell QuickCheck",
|
||||
want: true,
|
||||
fileName: []string{"ModuleSpec.hs"},
|
||||
langs: []clients.Language{
|
||||
{
|
||||
Name: clients.Haskell,
|
||||
NumLines: 50,
|
||||
},
|
||||
},
|
||||
fileContent: "import Test.QuickCheck",
|
||||
},
|
||||
{
|
||||
name: "Haskell Hedgehog",
|
||||
want: true,
|
||||
fileName: []string{"TestSpec.hs"},
|
||||
langs: []clients.Language{
|
||||
{
|
||||
Name: clients.Haskell,
|
||||
NumLines: 50,
|
||||
},
|
||||
},
|
||||
fileContent: "import Test.Hedgehog",
|
||||
},
|
||||
{
|
||||
name: "Haskell Validity",
|
||||
want: true,
|
||||
fileName: []string{"validity_test.hs"},
|
||||
langs: []clients.Language{
|
||||
{
|
||||
Name: clients.Haskell,
|
||||
NumLines: 50,
|
||||
},
|
||||
},
|
||||
fileContent: "import Test.Validity",
|
||||
},
|
||||
{
|
||||
name: "Haskell SmallCheck",
|
||||
want: true,
|
||||
fileName: []string{"SmallSpec.hs"},
|
||||
langs: []clients.Language{
|
||||
{
|
||||
Name: clients.Haskell,
|
||||
NumLines: 50,
|
||||
},
|
||||
},
|
||||
fileContent: "import Test.SmallCheck",
|
||||
},
|
||||
{
|
||||
name: "Haskell QuickCheck with qualified import",
|
||||
want: true,
|
||||
fileName: []string{"QualifiedSpec.hs"},
|
||||
langs: []clients.Language{
|
||||
{
|
||||
Name: clients.Haskell,
|
||||
NumLines: 50,
|
||||
},
|
||||
},
|
||||
fileContent: "import qualified Test.QuickCheck",
|
||||
},
|
||||
{
|
||||
name: "Haskell QuickCheck through Hspec",
|
||||
want: true,
|
||||
fileName: []string{"ArrowSpec.hs"},
|
||||
langs: []clients.Language{
|
||||
{
|
||||
Name: clients.Haskell,
|
||||
NumLines: 50,
|
||||
},
|
||||
},
|
||||
fileContent: "import Test.Hspec.QuickCheck",
|
||||
},
|
||||
{
|
||||
name: "Haskell QuickCheck through Tasty",
|
||||
want: true,
|
||||
fileName: []string{"test.hs"},
|
||||
langs: []clients.Language{
|
||||
{
|
||||
Name: clients.Haskell,
|
||||
NumLines: 50,
|
||||
},
|
||||
},
|
||||
fileContent: "import Test.Tasty.QuickCheck",
|
||||
},
|
||||
{
|
||||
name: "Haskell with no property-based testing",
|
||||
want: false,
|
||||
fileName: []string{"PropertySpec.hs"},
|
||||
wantErr: true,
|
||||
langs: []clients.Language{
|
||||
{
|
||||
Name: clients.Haskell,
|
||||
NumLines: 50,
|
||||
},
|
||||
},
|
||||
fileContent: "import Test.Hspec",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
@ -325,12 +422,12 @@ func Test_checkFuzzFunc(t *testing.T) {
|
||||
defer ctrl.Finish()
|
||||
mockClient := mockrepo.NewMockRepoClient(ctrl)
|
||||
mockClient.EXPECT().ListFiles(gomock.Any()).Return(tt.fileName, nil).AnyTimes()
|
||||
mockClient.EXPECT().GetFileContent(gomock.Any()).DoAndReturn(func(f string) (string, error) {
|
||||
mockClient.EXPECT().GetFileContent(gomock.Any()).DoAndReturn(func(f string) ([]byte, error) {
|
||||
if tt.wantErr {
|
||||
//nolint
|
||||
return "", errors.New("error")
|
||||
return nil, errors.New("error")
|
||||
}
|
||||
return tt.fileContent, nil
|
||||
return []byte(tt.fileContent), nil
|
||||
}).AnyTimes()
|
||||
req := checker.CheckRequest{
|
||||
RepoClient: mockClient,
|
||||
|
@ -75,7 +75,7 @@ The steps to writing a check are as follows:
|
||||
8. Create e2e tests in `e2e/mycheck_test.go`. Use a dedicated repo that will
|
||||
not change over time, so that it's reliable for the tests.
|
||||
|
||||
9. Update the `checks/checks.yaml` with a description of your check.
|
||||
9. Update the `docs/checks/internal/checks.yaml` with a description of your check.
|
||||
|
||||
10. Generate `docs/check.md` using `make generate-docs`. This will validate and
|
||||
generate `docs/check.md`.
|
||||
|
@ -71,6 +71,9 @@ const (
|
||||
// Dockerfile: https://docs.docker.com/engine/reference/builder/
|
||||
Dockerfile LanguageName = "dockerfile"
|
||||
|
||||
// Haskell: https://www.haskell.org/
|
||||
Haskell LanguageName = "haskell"
|
||||
|
||||
// Other indicates other languages not listed by the GitHub API.
|
||||
Other LanguageName = "other"
|
||||
|
||||
|
@ -336,7 +336,8 @@ This check tries to determine if the project uses
|
||||
[fuzzing](https://owasp.org/www-community/Fuzzing) by checking:
|
||||
1. if the repository name is included in the [OSS-Fuzz](https://github.com/google/oss-fuzz) project list;
|
||||
2. if [ClusterFuzzLite](https://google.github.io/clusterfuzzlite/) is deployed in the repository;
|
||||
3. if there are user-defined language-specified fuzzing functions (currently only supports [Go fuzzing](https://go.dev/doc/fuzz/)) in the repository.
|
||||
3. if there are user-defined language-specified fuzzing functions in the repository.
|
||||
- currently only supports [Go fuzzing](https://go.dev/doc/fuzz/) and a limited set of property-based testing libraries for Haskell.
|
||||
4. if it contains a [OneFuzz](https://github.com/microsoft/onefuzz) integration [detection file](https://github.com/microsoft/onefuzz/blob/main/docs/getting-started.md#detecting-the-use-of-onefuzz);
|
||||
|
||||
Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data
|
||||
|
@ -396,7 +396,8 @@ checks:
|
||||
[fuzzing](https://owasp.org/www-community/Fuzzing) by checking:
|
||||
1. if the repository name is included in the [OSS-Fuzz](https://github.com/google/oss-fuzz) project list;
|
||||
2. if [ClusterFuzzLite](https://google.github.io/clusterfuzzlite/) is deployed in the repository;
|
||||
3. if there are user-defined language-specified fuzzing functions (currently only supports [Go fuzzing](https://go.dev/doc/fuzz/)) in the repository.
|
||||
3. if there are user-defined language-specified fuzzing functions in the repository.
|
||||
- currently only supports [Go fuzzing](https://go.dev/doc/fuzz/) and a limited set of property-based testing libraries for Haskell.
|
||||
4. if it contains a [OneFuzz](https://github.com/microsoft/onefuzz) integration [detection file](https://github.com/microsoft/onefuzz/blob/main/docs/getting-started.md#detecting-the-use-of-onefuzz);
|
||||
|
||||
Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data
|
||||
|
Loading…
Reference in New Issue
Block a user