ossf/scorecard
1
1
Fork 0
mirror of https://github.com/ossf/scorecard.git synced 2024-09-11 00:45:36 +03:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

🌱 Enhance test output and management in ValidateTestReturn (#3810)

Browse Source 
* test failures should print the details they receive

this makes debugging failing tests easier.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* use GinkgoTB so the test helpers work instead of panicing

Signed-off-by: Spencer Schrock <sschrock@google.com>

* ValidateTestReturn will fail the test directly, no need for the bool return

Signed-off-by: Spencer Schrock <sschrock@google.com>

* clarify diff details

Signed-off-by: Spencer Schrock <sschrock@google.com>

---------

Signed-off-by: Spencer Schrock <sschrock@google.com>
This commit is contained in:
Spencer Schrock 2024-01-30 12:40:41 -08:00 committed by GitHub
parent 19047e8ab4
commit 83ff808f0d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
47 changed files with 109 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
checks/branch_protection_test.go
View File

@ -431,9 +431,7 @@ func TestReleaseAndDevBranchProtected(t *testing.T) {
RepoClient: mockRepoClient,
}
r := BranchProtection(&req)
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &r, &dl) {
t.Fail()
}
scut.ValidateTestReturn(t, tt.name, &tt.expected, &r, &dl)
ctrl.Finish()
})
}

4
checks/cii_best_practices_test.go
View File

@ -122,9 +122,7 @@ func TestCIIBestPractices(t *testing.T) {
}
res := CIIBestPractices(&req)
dl := scut.TestDetailLogger{}
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl) {
t.Fail()
}
scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl)
ctrl.Finish()
})
}

4
checks/dependency_update_tool_test.go
View File

@ -155,9 +155,7 @@ func TestDependencyUpdateTool(t *testing.T) {
}
res := DependencyUpdateTool(c)
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl) {
t.Fail()
}
scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl)
})
}
}

4
checks/evaluation/binary_artifacts_test.go
View File

@ -142,9 +142,7 @@ func TestBinaryArtifacts(t *testing.T) {
t.Parallel()
dl := scut.TestDetailLogger{}
got := BinaryArtifacts(tt.name, tt.findings, &dl)
if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
t.Errorf("got %v, expected %v", got, tt.result)
}
scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl)
})
}
}

4
checks/evaluation/branch_protection_test.go
View File

@ -573,9 +573,7 @@ func TestIsBranchProtected(t *testing.T) {
Score: score,
Error: err,
}
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, actual, &dl) {
t.Fail()
}
scut.ValidateTestReturn(t, tt.name, &tt.expected, actual, &dl)
})
}
}

4
checks/evaluation/ci_tests_test.go
View File

@ -111,9 +111,7 @@ func TestCITests(t *testing.T) {
t.Parallel()
dl := scut.TestDetailLogger{}
got := CITests(tt.name, tt.findings, &dl)
if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
t.Errorf("got %v, expected %v", got, tt.result)
}
scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl)
})
}
}

4
checks/evaluation/cii_best_practices_test.go
View File

@ -143,9 +143,7 @@ func TestCIIBestPractices(t *testing.T) {
t.Parallel()
dl := scut.TestDetailLogger{}
got := CIIBestPractices(tt.name, tt.findings, &dl)
if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
t.Errorf("got %v, expected %v", got, tt.result)
}
scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl)
})
}
}

4
checks/evaluation/code_review_test.go
View File

@ -216,9 +216,7 @@ func TestCodeReview(t *testing.T) {
dl := &scut.TestDetailLogger{}
res := CodeReview(tt.name, dl, tt.rawData)
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, dl) {
t.Error()
}
scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, dl)
})
}
}

4
checks/evaluation/contributors_test.go
View File

@ -83,9 +83,7 @@ func TestContributors(t *testing.T) {
t.Parallel()
dl := scut.TestDetailLogger{}
got := Contributors(tt.name, tt.findings, &dl)
if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
t.Error(tt.name)
}
scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl)
})
}
}

4
checks/evaluation/dangerous_workflow_test.go
View File

@ -252,9 +252,7 @@ func TestDangerousWorkflow(t *testing.T) {
t.Parallel()
dl := scut.TestDetailLogger{}
got := DangerousWorkflow(tt.name, tt.findings, &dl)
if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
t.Errorf("got %v, expected %v", got, tt.result)
}
scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl)
})
}
}

4
checks/evaluation/dependency_update_tool_test.go
View File

@ -164,9 +164,7 @@ func TestDependencyUpdateTool(t *testing.T) {
dl := scut.TestDetailLogger{}
got := DependencyUpdateTool(tt.name, tt.findings, &dl)
if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
t.Errorf("got %v, expected %v", got, tt.result)
}
scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl)
})
}
}

4
checks/evaluation/fuzzing_test.go
View File

@ -217,9 +217,7 @@ func TestFuzzing(t *testing.T) {
t.Parallel()
dl := scut.TestDetailLogger{}
got := Fuzzing(tt.name, tt.findings, &dl)
if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
t.Errorf("got %v, expected %v", got, tt.result)
}
scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl)
})
}
}

4
checks/evaluation/license_test.go
View File

@ -155,9 +155,7 @@ func TestLicense(t *testing.T) {
t.Parallel()
dl := scut.TestDetailLogger{}
got := License(tt.name, tt.findings, &dl)
if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
t.Errorf("got %v, expected %v", got, tt.result)
}
scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl)
})
}
}

4
checks/evaluation/maintained_test.go
View File

@ -127,9 +127,7 @@ func TestMaintained(t *testing.T) {
t.Parallel()
dl := scut.TestDetailLogger{}
got := Maintained(tt.name, tt.findings, &dl)
if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
t.Errorf("got %v, expected %v", got, tt.result)
}
scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl)
})
}
}

4
checks/evaluation/packaging_test.go
View File

@ -88,9 +88,7 @@ func TestPackaging(t *testing.T) {
t.Parallel()
dl := scut.TestDetailLogger{}
got := Packaging(tt.name, tt.findings, &dl)
if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
t.Errorf("got %v, expected %v", got, tt.result)
}
scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl)
})
}
}

4
checks/evaluation/pinned_dependencies_test.go
View File

@ -841,9 +841,7 @@ func Test_PinningDependencies(t *testing.T) {
ProcessingErrors: tt.processingErrors,
})
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &actual, &dl) {
t.Fail()
}
scut.ValidateTestReturn(t, tt.name, &tt.expected, &actual, &dl)
})
}
}

4
checks/evaluation/sast_test.go
View File

@ -301,9 +301,7 @@ func TestSAST(t *testing.T) {
t.Parallel()
dl := scut.TestDetailLogger{}
got := SAST(tt.name, tt.findings, &dl)
if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
t.Errorf("got %v, expected %v", got, tt.result)
}
scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl)
})
}
}

4
checks/evaluation/security_policy_test.go
View File

@ -190,9 +190,7 @@ func TestSecurityPolicy(t *testing.T) {
t.Parallel()
dl := scut.TestDetailLogger{}
got := SecurityPolicy(tt.name, tt.findings, &dl)
if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
t.Errorf("got %v, expected %v", got, tt.result)
}
scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl)
})
}
}

4
checks/evaluation/signed_releases_test.go
View File

@ -303,9 +303,7 @@ func TestSignedReleases(t *testing.T) {
t.Parallel()
dl := scut.TestDetailLogger{}
got := SignedReleases(tt.name, tt.findings, &dl)
if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
t.Errorf("got %v, expected %v", got, tt.result)
}
scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl)
})
}
}

4
checks/evaluation/vulnerabilities_test.go
View File

@ -139,9 +139,7 @@ func TestVulnerabilities(t *testing.T) {
t.Parallel()
dl := scut.TestDetailLogger{}
got := Vulnerabilities(tt.name, tt.findings, &dl)
if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
t.Errorf("got %v, expected %v", got, tt.result)
}
scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl)
})
}
}

4
checks/evaluation/webhooks_test.go
View File

@ -229,9 +229,7 @@ func TestWebhooks(t *testing.T) {
t.Parallel()
dl := scut.TestDetailLogger{}
got := Webhooks(tt.name, tt.findings, &dl)
if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
t.Errorf("got %v, expected %v", got, tt.result)
}
scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl)
})
}
}

4
checks/fuzzing_test.go
View File

@ -169,9 +169,7 @@ func TestFuzzing(t *testing.T) {
return
}
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &result, &dl) {
t.Fatalf(tt.name, tt.expected)
}
scut.ValidateTestReturn(t, tt.name, &tt.expected, &result, &dl)
})
}
}

4
checks/license_test.go
View File

@ -92,9 +92,7 @@ func TestLicenseFileSubdirectory(t *testing.T) {
res := License(&req)
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl) {
t.Fail()
}
scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl)
ctrl.Finish()
})

4
checks/permissions_test.go
View File

@ -458,9 +458,7 @@ func TestGithubTokenPermissions(t *testing.T) {
res := TokenPermissions(&c)
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl) {
t.Errorf("test failed: log message not present: %+v\n%+v", tt.expected, dl)
}
scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl)
})
}
}

4
checks/raw/security_policy_test.go
View File

@ -165,9 +165,7 @@ func TestSecurityPolicy(t *testing.T) {
res, err := SecurityPolicy(&c)
if !scut.ValidateTestReturn(t, tt.name, &tt.want, &checker.CheckResult{}, &dl) {
t.Errorf("test failed: log message not present: %+v , for test %v", tt.want, tt.name)
}
scut.ValidateTestReturn(t, tt.name, &tt.want, &checker.CheckResult{}, &dl)
if (err != nil) != tt.wantErr {
t.Errorf("SecurityPolicy() error = %v, wantErr %v", err, tt.wantErr)

4
checks/raw/vulnerabilities_test.go
View File

@ -114,9 +114,7 @@ func TestVulnerabilities(t *testing.T) {
}
}
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &checker.CheckResult{}, &dl) {
t.Fatalf("Test %s failed", tt.name)
}
scut.ValidateTestReturn(t, tt.name, &tt.expected, &checker.CheckResult{}, &dl)
})
}
}

4
checks/raw/webhooks_test.go
View File

@ -130,9 +130,7 @@ func TestWebhooks(t *testing.T) {
}
}
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &checker.CheckResult{}, &dl) {
t.Fatalf("Test %s failed", tt.name)
}
scut.ValidateTestReturn(t, tt.name, &tt.expected, &checker.CheckResult{}, &dl)
})
}
}

4
checks/security_policy_test.go
View File

@ -197,9 +197,7 @@ func TestSecurityPolicy(t *testing.T) {
res := SecurityPolicy(c)
if !scut.ValidateTestReturn(t, tt.name, &tt.want, &res, &dl) {
t.Errorf("test failed: log message not present: %+v on %+v", tt.want, res)
}
scut.ValidateTestReturn(t, tt.name, &tt.want, &res, &dl)
})
}
}

12
e2e/binary_artifacts_test.go
View File

@ -57,7 +57,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() {
}
result := checks.BinaryArtifacts(&req)
Expect(scut.ValidateTestReturn(nil, "no binary artifacts", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "no binary artifacts", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return binary artifacts present in source code", func() {
@ -84,7 +84,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() {
NumberOfDebug: 0,
}
result := checks.BinaryArtifacts(&req)
Expect(scut.ValidateTestReturn(nil, "binary artifacts", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "binary artifacts", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return binary artifacts present at commit in source code", func() {
@ -111,7 +111,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() {
NumberOfDebug: 0,
}
result := checks.BinaryArtifacts(&req)
Expect(scut.ValidateTestReturn(nil, "binary artifacts", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "binary artifacts", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return no binary artifacts present in source code", func() {
@ -139,7 +139,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() {
}
result := checks.BinaryArtifacts(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "binary artifacts", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "binary artifacts", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return binary artifacts present at commit in source code", func() {
@ -167,7 +167,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() {
}
result := checks.BinaryArtifacts(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "binary artifacts", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "binary artifacts", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return binary artifacts present at commit in source code when using local repoClient", func() {
@ -206,7 +206,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() {
}
result := checks.BinaryArtifacts(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "binary artifacts", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "binary artifacts", &expected, &result, &dl)
Expect(x.Close()).Should(BeNil())
})
})

8
e2e/branch_protection_test.go
View File

@ -56,7 +56,7 @@ var _ = Describe("E2E TEST PAT:"+checks.CheckBranchProtection, func() {
// Old version.
// New version.
Expect(scut.ValidateTestReturn(nil, "branch protection accessible", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "branch protection accessible", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should fail to return branch protection on other repositories", func() {
@ -84,7 +84,7 @@ var _ = Describe("E2E TEST PAT:"+checks.CheckBranchProtection, func() {
result := checks.BranchProtection(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "branch protection accessible none", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "branch protection accessible none", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should fail to return branch protection on other repositories patch", func() {
@ -112,7 +112,7 @@ var _ = Describe("E2E TEST PAT:"+checks.CheckBranchProtection, func() {
result := checks.BranchProtection(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "branch protection accessible patch", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "branch protection accessible patch", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
})
@ -169,7 +169,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBranchProtection+" (repo rules)", func(
}
result := checks.BranchProtection(&req)
Expect(result.Error).Should(BeNil())
Expect(scut.ValidateTestReturn(nil, "repo rules accessible", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "repo rules accessible", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
})

8
e2e/ci_tests_test.go
View File

@ -51,7 +51,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() {
NumberOfDebug: 0,
}
result := checks.CITests(&req)
Expect(scut.ValidateTestReturn(nil, "CI tests run", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "CI tests run", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return use of CI tests at commit", func() {
@ -75,7 +75,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() {
NumberOfDebug: 0,
}
result := checks.CITests(&req)
Expect(scut.ValidateTestReturn(nil, "CI tests run", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "CI tests run", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return absence of CI tests in a repo with unsquashed merges", func() {
@ -99,7 +99,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() {
NumberOfDebug: 12,
}
result := checks.CITests(&req)
Expect(scut.ValidateTestReturn(nil, "CI tests run", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "CI tests run", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return use of CI tests at commit - GitLab", func() {
@ -127,7 +127,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() {
NumberOfDebug: 22,
}
result := checks.CITests(&req)
Expect(scut.ValidateTestReturn(nil, "CI tests at commit - GitLab", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "CI tests at commit - GitLab", &expected, &result, &dl)
Expect(result.Error).Should(BeNil())
Expect(repoClient.Close()).Should(BeNil())
})

2
e2e/cii_best_practices_test.go
View File

@ -51,7 +51,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCIIBestPractices, func() {
}
result := checks.CIIBestPractices(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "passing badge", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "passing badge", &expected, &result, &dl)
})
})
})

10
e2e/code_review_test.go
View File

@ -54,7 +54,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCodeReview, func() {
NumberOfDebug: 0,
}
result := checks.CodeReview(&req)
Expect(scut.ValidateTestReturn(nil, "use code reviews", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "use code reviews", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return use of implicit code reviews at commit", func() {
@ -97,7 +97,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCodeReview, func() {
NumberOfDebug: 18,
}
result := checks.CodeReview(&req)
Expect(scut.ValidateTestReturn(nil, "use code reviews", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "use code reviews", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return partial score for a single-maintainer project with some unreviewed human changesets", func() {
@ -119,7 +119,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCodeReview, func() {
NumberOfDebug: 10,
}
result := checks.CodeReview(&req)
Expect(scut.ValidateTestReturn(nil, "use code reviews", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "use code reviews", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
})
@ -149,7 +149,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCodeReview, func() {
NumberOfDebug: 1,
}
result := checks.CodeReview(&req)
Expect(scut.ValidateTestReturn(nil, "use code reviews", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "use code reviews", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
// GitLab doesn't seem to preserve merge requests (pull requests in github) and some users had data lost in
@ -179,7 +179,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCodeReview, func() {
NumberOfDebug: 1,
}
result := checks.CodeReview(&req)
Expect(scut.ValidateTestReturn(nil, "use code reviews", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "use code reviews", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
})

2
e2e/contributors_test.go
View File

@ -54,7 +54,7 @@ var _ = Describe("E2E TEST:"+checks.CheckContributors, func() {
}
result := checks.Contributors(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "several contributors", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "several contributors", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})

6
e2e/dangerous_workflow_test.go
View File

@ -53,7 +53,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() {
}
result := checks.DangerousWorkflow(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "dangerous workflow", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "dangerous workflow", &expected, &result, &dl)
})
It("Should return dangerous workflow at commit", func() {
dl := scut.TestDetailLogger{}
@ -77,7 +77,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() {
}
result := checks.DangerousWorkflow(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "dangerous workflow", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "dangerous workflow", &expected, &result, &dl)
})
It("Should return dangerous workflow for local repoClient", func() {
dl := scut.TestDetailLogger{}
@ -113,7 +113,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() {
}
result := checks.DangerousWorkflow(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "dangerous workflow", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "dangerous workflow", &expected, &result, &dl)
})
})
})

4
e2e/dependency_update_tool_test.go
View File

@ -55,7 +55,7 @@ var _ = Describe("E2E TEST:"+checks.CheckDependencyUpdateTool, func() {
result := checks.DependencyUpdateTool(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "dependabot", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "dependabot", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return repo uses renovatebot", func() {
@ -81,7 +81,7 @@ var _ = Describe("E2E TEST:"+checks.CheckDependencyUpdateTool, func() {
}
result := checks.DependencyUpdateTool(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "renovatebot", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "renovatebot", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
})

10
e2e/fuzzing_test.go
View File

@ -57,7 +57,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() {
NumberOfDebug: 0,
}
result := checks.Fuzzing(&req)
Expect(scut.ValidateTestReturn(nil, "use fuzzing", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "use fuzzing", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
Expect(ossFuzzRepoClient.Close()).Should(BeNil())
})
@ -89,7 +89,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() {
NumberOfDebug: 0,
}
result := checks.Fuzzing(&req)
Expect(scut.ValidateTestReturn(nil, "use fuzzing", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "use fuzzing", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
Expect(ossFuzzRepoClient.Close()).Should(BeNil())
})
@ -118,7 +118,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() {
NumberOfDebug: 0,
}
result := checks.Fuzzing(&req)
Expect(scut.ValidateTestReturn(nil, "use fuzzing", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "use fuzzing", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
Expect(ossFuzzRepoClient.Close()).Should(BeNil())
})
@ -147,7 +147,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() {
NumberOfDebug: 0,
}
result := checks.Fuzzing(&req)
Expect(scut.ValidateTestReturn(nil, "use fuzzing", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "use fuzzing", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
Expect(ossFuzzRepoClient.Close()).Should(BeNil())
})
@ -197,7 +197,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() {
NumberOfDebug: 0,
}
result := checks.Fuzzing(&req)
Expect(scut.ValidateTestReturn(nil, "no fuzzing", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "no fuzzing", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
Expect(ossFuzzRepoClient.Close()).Should(BeNil())
})

18
e2e/license_test.go
View File

@ -54,8 +54,7 @@ var _ = Describe("E2E TEST:"+checks.CheckLicense, func() {
}
result := checks.License(&req)
Expect(scut.ValidateTestReturn(nil, "license found", &expected, &result,
&dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "license found", &expected, &result, &dl)
})
It("Should return license check works at commitSHA", func() {
dl := scut.TestDetailLogger{}
@ -79,8 +78,7 @@ var _ = Describe("E2E TEST:"+checks.CheckLicense, func() {
}
result := checks.License(&req)
Expect(scut.ValidateTestReturn(nil, "license found", &expected, &result,
&dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "license found", &expected, &result, &dl)
})
It("Should return license check works for the local repoclient", func() {
dl := scut.TestDetailLogger{}
@ -116,8 +114,7 @@ var _ = Describe("E2E TEST:"+checks.CheckLicense, func() {
}
result := checks.License(&req)
Expect(scut.ValidateTestReturn(nil, "license found", &expected, &result,
&dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "license found", &expected, &result, &dl)
})
It("Should return license check works - GitLab", func() {
skipIfTokenIsNot(gitlabPATTokenType, "GitLab only")
@ -142,8 +139,7 @@ var _ = Describe("E2E TEST:"+checks.CheckLicense, func() {
}
result := checks.License(&req)
Expect(scut.ValidateTestReturn(nil, "license found", &expected, &result,
&dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "license found", &expected, &result, &dl)
})
It("Should return license check works for unrecognized license type - GitLab", func() {
skipIfTokenIsNot(gitlabPATTokenType, "GitLab only")
@ -170,8 +166,7 @@ var _ = Describe("E2E TEST:"+checks.CheckLicense, func() {
}
result := checks.License(&req)
Expect(scut.ValidateTestReturn(nil, "license found", &expected, &result,
&dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "license found", &expected, &result, &dl)
})
It("Should return license check works at commitSHA - GitLab", func() {
skipIfTokenIsNot(gitlabPATTokenType, "GitLab only")
@ -196,8 +191,7 @@ var _ = Describe("E2E TEST:"+checks.CheckLicense, func() {
}
result := checks.License(&req)
Expect(scut.ValidateTestReturn(nil, "license found", &expected, &result,
&dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "license found", &expected, &result, &dl)
})
})
})

4
e2e/maintained_test.go
View File

@ -53,7 +53,7 @@ var _ = Describe("E2E TEST:"+checks.CheckMaintained, func() {
}
result := checks.Maintained(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "active repo", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "active repo", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return valid maintained status - GitLab", func() {
@ -82,7 +82,7 @@ var _ = Describe("E2E TEST:"+checks.CheckMaintained, func() {
}
result := checks.Maintained(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "active repo", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "active repo", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
})

2
e2e/packaging_test.go
View File

@ -50,7 +50,7 @@ var _ = Describe("E2E TEST:"+checks.CheckPackaging, func() {
NumberOfDebug: 0,
}
result := checks.Packaging(&req)
Expect(scut.ValidateTestReturn(nil, "use packaging", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "use packaging", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
})

6
e2e/permissions_test.go
View File

@ -53,7 +53,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() {
}
result := checks.TokenPermissions(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "token permissions", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "token permissions", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return token permission at commit", func() {
@ -78,7 +78,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() {
}
result := checks.TokenPermissions(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "token permissions", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "token permissions", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return token permission for a local repo client", func() {
@ -115,7 +115,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() {
}
result := checks.TokenPermissions(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "token permissions", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "token permissions", &expected, &result, &dl)
Expect(x.Close()).Should(BeNil())
})
})

6
e2e/pinned_dependencies_test.go
View File

@ -55,7 +55,7 @@ var _ = Describe("E2E TEST:"+checks.CheckPinnedDependencies, func() {
NumberOfDebug: 0,
}
result := checks.PinningDependencies(&req)
Expect(scut.ValidateTestReturn(nil, "dependencies check", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "dependencies check", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return dependencies check at commit", func() {
@ -80,7 +80,7 @@ var _ = Describe("E2E TEST:"+checks.CheckPinnedDependencies, func() {
NumberOfDebug: 0,
}
result := checks.PinningDependencies(&req)
Expect(scut.ValidateTestReturn(nil, "dependencies check", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "dependencies check", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return dependencies check for a local repoClient", func() {
@ -116,7 +116,7 @@ var _ = Describe("E2E TEST:"+checks.CheckPinnedDependencies, func() {
NumberOfDebug: 0,
}
result := checks.PinningDependencies(&req)
Expect(scut.ValidateTestReturn(nil, "dependencies check", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "dependencies check", &expected, &result, &dl)
Expect(x.Close()).Should(BeNil())
})
})

2
e2e/sast_test.go
View File

@ -51,7 +51,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSAST, func() {
}
result := checks.SAST(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "sast used", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "sast used", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
})

10
e2e/security_policy_test.go
View File

@ -55,7 +55,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSecurityPolicy, func() {
}
result := checks.SecurityPolicy(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "policy found", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "policy found", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return valid security policy at commitSHA", func() {
@ -81,7 +81,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSecurityPolicy, func() {
}
result := checks.SecurityPolicy(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "policy found", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "policy found", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return valid security policy for local repoClient at head", func() {
@ -118,7 +118,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSecurityPolicy, func() {
}
result := checks.SecurityPolicy(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "policy found", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "policy found", &expected, &result, &dl)
Expect(x.Close()).Should(BeNil())
})
It("Should return valid security policy - GitLab", func() {
@ -149,7 +149,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSecurityPolicy, func() {
}
result := checks.SecurityPolicy(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "policy found", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "policy found", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return valid security policy at commitSHA - GitLab", func() {
@ -181,7 +181,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSecurityPolicy, func() {
}
result := checks.SecurityPolicy(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "policy found", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "policy found", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
})

2
e2e/signedreleases_test.go
View File

@ -51,7 +51,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSignedReleases, func() {
}
result := checks.SignedReleases(&req)
// New version.
Expect(scut.ValidateTestReturn(nil, "verified release", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "verified release", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
})

6
e2e/vulnerabilities_test.go
View File

@ -53,7 +53,7 @@ var _ = Describe("E2E TEST:"+checks.CheckVulnerabilities, func() {
}
result := checks.Vulnerabilities(&checkRequest)
// New version.
Expect(scut.ValidateTestReturn(nil, "osv vulnerabilities", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "osv vulnerabilities", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return that there are vulnerabilities at commit", func() {
@ -80,7 +80,7 @@ var _ = Describe("E2E TEST:"+checks.CheckVulnerabilities, func() {
}
result := checks.Vulnerabilities(&checkRequest)
// New version.
Expect(scut.ValidateTestReturn(nil, "osv vulnerabilities", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "osv vulnerabilities", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return that there are vulnerable packages", func() {
@ -107,7 +107,7 @@ var _ = Describe("E2E TEST:"+checks.CheckVulnerabilities, func() {
}
result := checks.Vulnerabilities(&checkRequest)
// New version.
Expect(scut.ValidateTestReturn(nil, "osv vulnerabilities", &expected, &result, &dl)).Should(BeTrue())
scut.ValidateTestReturn(GinkgoTB(), "osv vulnerabilities", &expected, &result, &dl)
Expect(repoClient.Close()).Should(BeNil())
})
})

42
utests/utlib.go
View File

@ -16,12 +16,12 @@
package utests
import (
"errors"
"fmt"
"log"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/ossf/scorecard/v4/checker"
)
@ -74,7 +74,18 @@ func (l *TestDetailLogger) Flush() []checker.CheckDetail {
return ret
}
func getTestReturn(cr *checker.CheckResult, logger *TestDetailLogger) (*TestReturn, error) {
//nolint:gocritic // not worried about test helper param size
func logDetail(tb testing.TB, level string, detail checker.CheckDetail) {
tb.Helper()
if detail.Msg.Finding != nil {
tb.Logf("%s: %s", level, detail.Msg.Finding.Message)
} else {
tb.Logf("%s: %s", level, detail.Msg.Text)
}
}
func getTestReturn(tb testing.TB, cr *checker.CheckResult, logger *TestDetailLogger) (*TestReturn, error) {
tb.Helper()
ret := new(TestReturn)
for _, v := range logger.messages {
switch v.Type {
@ -83,10 +94,13 @@ func getTestReturn(cr *checker.CheckResult, logger *TestDetailLogger) (*TestRetu
return nil, fmt.Errorf("invalid type %v", v.Type)
case checker.DetailInfo:
ret.NumberOfInfo++
logDetail(tb, "INFO", v)
case checker.DetailDebug:
ret.NumberOfDebug++
logDetail(tb, "DEBUG", v)
case checker.DetailWarn:
ret.NumberOfWarn++
logDetail(tb, "WARN", v)
}
}
ret.Score = cr.Score
@ -94,29 +108,23 @@ func getTestReturn(cr *checker.CheckResult, logger *TestDetailLogger) (*TestRetu
return ret, nil
}
func errCmp(e1, e2 error) bool {
return errors.Is(e1, e2) || errors.Is(e2, e1)
}
// ValidateTestReturn validates expected TestReturn with actual checker.CheckResult values.
//
//nolint:thelper
// ValidateTestReturn validates expected [TestReturn] with actual [checker.CheckResult] values.
// All test management is handled by this function.
func ValidateTestReturn(
t *testing.T,
tb testing.TB,
name string,
expected *TestReturn,
actual *checker.CheckResult,
logger *TestDetailLogger,
) bool {
actualTestReturn, err := getTestReturn(actual, logger)
) {
tb.Helper()
actualTestReturn, err := getTestReturn(tb, actual, logger)
if err != nil {
panic(err)
tb.Fatal(err)
}
if !cmp.Equal(*expected, *actualTestReturn, cmp.Comparer(errCmp)) {
log.Println(name+":", cmp.Diff(*expected, *actualTestReturn))
return false
if diff := cmp.Diff(*expected, *actualTestReturn, cmpopts.EquateErrors()); diff != "" {
tb.Error(name + ": (-expected +actual)" + diff)
}
return true
}
// ValidatePinningDependencies tests that at least one entry returns true for isExpectedMessage.