ossf/scorecard
1
1
Fork 0
mirror of https://github.com/ossf/scorecard.git synced 2024-09-11 00:45:36 +03:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

Update test utils (#933)

Browse Source 
Co-authored-by: Azeem Shaikh <azeems@google.com>
This commit is contained in:
Azeem Shaikh 2021-08-30 14:12:57 -07:00 committed by GitHub
parent dbb23450e5
commit d9f5209803
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 191 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
checks/branch_protection_test.go
View File

@ -90,7 +90,7 @@ func TestReleaseAndDevBranchProtected(t *testing.T) {
{
name: "Only development branch",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: 1,
NumberOfWarn: 6,
NumberOfInfo: 2,
@ -138,7 +138,7 @@ func TestReleaseAndDevBranchProtected(t *testing.T) {
{
name: "Take worst of release and development",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: 5,
NumberOfWarn: 8,
NumberOfInfo: 9,
@ -219,7 +219,7 @@ func TestReleaseAndDevBranchProtected(t *testing.T) {
{
name: "Both release and development are OK",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: 9,
NumberOfWarn: 4,
NumberOfInfo: 14,
@ -300,7 +300,7 @@ func TestReleaseAndDevBranchProtected(t *testing.T) {
{
name: "Ignore a non-branch targetcommitish",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: 1,
NumberOfWarn: 6,
NumberOfInfo: 2,
@ -348,7 +348,7 @@ func TestReleaseAndDevBranchProtected(t *testing.T) {
{
name: "TargetCommittish nil",
expected: scut.TestReturn{
Errors: []error{sce.ErrScorecardInternal},
Error: sce.ErrScorecardInternal,
Score: checker.InconclusiveResultScore,
NumberOfWarn: 0,
NumberOfInfo: 0,
@ -396,7 +396,7 @@ func TestReleaseAndDevBranchProtected(t *testing.T) {
{
name: "Non-admin check with protected release and development",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: 1,
NumberOfWarn: 2,
NumberOfInfo: 0,
@ -442,7 +442,9 @@ func TestReleaseAndDevBranchProtected(t *testing.T) {
dl := scut.TestDetailLogger{}
r := checkReleaseAndDevBranchProtection(context.Background(), mockRepoClient, m,
&dl, "testowner", "testrepo")
scut.ValidateTestReturn(t, tt.name, &tt.expected, &r, &dl)
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &r, &dl) {
t.Fail()
}
ctrl.Finish()
})
}
@ -459,7 +461,7 @@ func TestIsBranchProtected(t *testing.T) {
{
name: "Nothing is enabled",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: 1,
NumberOfWarn: 6,
NumberOfInfo: 2,
@ -502,7 +504,7 @@ func TestIsBranchProtected(t *testing.T) {
{
name: "Nothing is enabled and values in github.Protection are nil",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: 1,
NumberOfWarn: 4,
NumberOfInfo: 2,
@ -513,7 +515,7 @@ func TestIsBranchProtected(t *testing.T) {
{
name: "Required status check enabled",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: 2,
NumberOfWarn: 6,
NumberOfInfo: 3,
@ -556,7 +558,7 @@ func TestIsBranchProtected(t *testing.T) {
{
name: "Required status check enabled without checking for status string",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: 2,
NumberOfWarn: 6,
NumberOfInfo: 3,
@ -599,7 +601,7 @@ func TestIsBranchProtected(t *testing.T) {
{
name: "Required pull request enabled",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: 2,
NumberOfWarn: 5,
NumberOfInfo: 3,
@ -642,7 +644,7 @@ func TestIsBranchProtected(t *testing.T) {
{
name: "Required admin enforcement enabled",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: 3,
NumberOfWarn: 5,
NumberOfInfo: 3,
@ -685,7 +687,7 @@ func TestIsBranchProtected(t *testing.T) {
{
name: "Required linear history enabled",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: 2,
NumberOfWarn: 5,
NumberOfInfo: 3,
@ -728,7 +730,7 @@ func TestIsBranchProtected(t *testing.T) {
{
name: "Allow force push enabled",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: 0,
NumberOfWarn: 7,
NumberOfInfo: 1,
@ -771,7 +773,7 @@ func TestIsBranchProtected(t *testing.T) {
{
name: "Allow deletions enabled",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: 0,
NumberOfWarn: 7,
NumberOfInfo: 1,
@ -814,7 +816,7 @@ func TestIsBranchProtected(t *testing.T) {
{
name: "Branches are protected",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: 9,
NumberOfWarn: 2,
NumberOfInfo: 7,
@ -860,8 +862,12 @@ func TestIsBranchProtected(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
dl := scut.TestDetailLogger{}
score := IsBranchProtected(tt.protection, "test", &dl)
scut.ValidateTestValues(t, tt.name, &tt.expected, score, nil, &dl)
actual := &checker.CheckResult{
Score: IsBranchProtected(tt.protection, "test", &dl),
}
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, actual, &dl) {
t.Fail()
}
})
}
}

36
checks/permissions_test.go
View File

@ -36,7 +36,7 @@ func TestGithubTokenPermissions(t *testing.T) {
name: "run workflow codeql write test",
filename: "./testdata/github-workflow-permissions-run-codeql-write.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -47,7 +47,7 @@ func TestGithubTokenPermissions(t *testing.T) {
name: "run workflow no codeql write test",
filename: "./testdata/github-workflow-permissions-run-no-codeql-write.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore - 1,
NumberOfWarn: 1,
NumberOfInfo: 1,
@ -58,7 +58,7 @@ func TestGithubTokenPermissions(t *testing.T) {
name: "run workflow write test",
filename: "./testdata/github-workflow-permissions-run-writes-2.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 3,
NumberOfInfo: 2,
@ -69,7 +69,7 @@ func TestGithubTokenPermissions(t *testing.T) {
name: "run package workflow write test",
filename: "./testdata/github-workflow-permissions-run-package-workflow-write.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 2,
@ -80,7 +80,7 @@ func TestGithubTokenPermissions(t *testing.T) {
name: "run package write test",
filename: "./testdata/github-workflow-permissions-run-package-write.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 1,
NumberOfInfo: 1,
@ -91,7 +91,7 @@ func TestGithubTokenPermissions(t *testing.T) {
name: "run writes test",
filename: "./testdata/github-workflow-permissions-run-writes.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -102,7 +102,7 @@ func TestGithubTokenPermissions(t *testing.T) {
name: "write all test",
filename: "./testdata/github-workflow-permissions-writeall.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 1,
NumberOfInfo: 0,
@ -113,7 +113,7 @@ func TestGithubTokenPermissions(t *testing.T) {
name: "read all test",
filename: "./testdata/github-workflow-permissions-readall.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -124,7 +124,7 @@ func TestGithubTokenPermissions(t *testing.T) {
name: "no permission test",
filename: "./testdata/github-workflow-permissions-absent.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 1,
NumberOfInfo: 0,
@ -135,7 +135,7 @@ func TestGithubTokenPermissions(t *testing.T) {
name: "writes test",
filename: "./testdata/github-workflow-permissions-writes.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -146,7 +146,7 @@ func TestGithubTokenPermissions(t *testing.T) {
name: "reads test",
filename: "./testdata/github-workflow-permissions-reads.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 10,
@ -157,7 +157,7 @@ func TestGithubTokenPermissions(t *testing.T) {
name: "nones test",
filename: "./testdata/github-workflow-permissions-nones.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 10,
@ -168,7 +168,7 @@ func TestGithubTokenPermissions(t *testing.T) {
name: "none test",
filename: "./testdata/github-workflow-permissions-none.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -179,7 +179,7 @@ func TestGithubTokenPermissions(t *testing.T) {
name: "status/checks write",
filename: "./testdata/github-workflow-permissions-status-checks.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore - 1,
NumberOfWarn: 2,
NumberOfInfo: 2,
@ -190,7 +190,7 @@ func TestGithubTokenPermissions(t *testing.T) {
name: "sec-events/deployments write",
filename: "./testdata/github-workflow-permissions-secevent-deployments.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore - 2,
NumberOfWarn: 2,
NumberOfInfo: 3,
@ -201,7 +201,7 @@ func TestGithubTokenPermissions(t *testing.T) {
name: "contents write",
filename: "./testdata/github-workflow-permissions-contents.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 1,
NumberOfInfo: 2,
@ -212,7 +212,7 @@ func TestGithubTokenPermissions(t *testing.T) {
name: "actions write",
filename: "./testdata/github-workflow-permissions-actions.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 1,
NumberOfInfo: 2,
@ -223,7 +223,7 @@ func TestGithubTokenPermissions(t *testing.T) {
name: "packages write",
filename: "./testdata/github-workflow-permissions-packages.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 1,
NumberOfInfo: 1,

108
checks/pinned_dependencies_test.go
View File

@ -36,7 +36,7 @@ func TestGithubWorkflowPinning(t *testing.T) {
name: "empty file",
filename: "./testdata/github-workflow-empty",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -47,7 +47,7 @@ func TestGithubWorkflowPinning(t *testing.T) {
name: "comments only",
filename: "./testdata/github-workflow-comments",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -58,7 +58,7 @@ func TestGithubWorkflowPinning(t *testing.T) {
name: "Pinned workflow",
filename: "./testdata/workflow-pinned.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -69,7 +69,7 @@ func TestGithubWorkflowPinning(t *testing.T) {
name: "Non-pinned workflow",
filename: "./testdata/workflow-not-pinned.yaml",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 1,
NumberOfInfo: 0,
@ -93,7 +93,13 @@ func TestGithubWorkflowPinning(t *testing.T) {
}
dl := scut.TestDetailLogger{}
s, e := testIsGitHubActionsWorkflowPinned(tt.filename, content, &dl)
scut.ValidateTestValues(t, tt.name, &tt.expected, s, e, &dl)
actual := checker.CheckResult{
Score: s,
Error2: e,
}
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &actual, &dl) {
t.Fail()
}
})
}
}
@ -109,7 +115,7 @@ func TestDockerfilePinning(t *testing.T) {
name: "invalid dockerfile",
filename: "./testdata/Dockerfile-invalid",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -120,7 +126,7 @@ func TestDockerfilePinning(t *testing.T) {
name: "invalid dockerfile sh",
filename: "./testdata/script-sh",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -131,7 +137,7 @@ func TestDockerfilePinning(t *testing.T) {
name: "empty file",
filename: "./testdata/Dockerfile-empty",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -142,7 +148,7 @@ func TestDockerfilePinning(t *testing.T) {
name: "comments only",
filename: "./testdata/Dockerfile-comments",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -153,7 +159,7 @@ func TestDockerfilePinning(t *testing.T) {
name: "Pinned dockerfile",
filename: "./testdata/Dockerfile-pinned",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -164,7 +170,7 @@ func TestDockerfilePinning(t *testing.T) {
name: "Pinned dockerfile as",
filename: "./testdata/Dockerfile-pinned-as",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -175,7 +181,7 @@ func TestDockerfilePinning(t *testing.T) {
name: "Non-pinned dockerfile as",
filename: "./testdata/Dockerfile-not-pinned-as",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 3, // TODO: should be 2, https://github.com/ossf/scorecard/issues/701.
NumberOfInfo: 0,
@ -186,7 +192,7 @@ func TestDockerfilePinning(t *testing.T) {
name: "Non-pinned dockerfile",
filename: "./testdata/Dockerfile-not-pinned",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 1,
NumberOfInfo: 0,
@ -210,7 +216,13 @@ func TestDockerfilePinning(t *testing.T) {
}
dl := scut.TestDetailLogger{}
s, e := testValidateDockerfileIsPinned(tt.filename, content, &dl)
scut.ValidateTestValues(t, tt.name, &tt.expected, s, e, &dl)
actual := checker.CheckResult{
Score: s,
Error2: e,
}
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &actual, &dl) {
t.Fail()
}
})
}
}
@ -226,7 +238,7 @@ func TestDockerfileScriptDownload(t *testing.T) {
name: "curl | sh",
filename: "testdata/Dockerfile-curl-sh",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 4,
NumberOfInfo: 0,
@ -237,7 +249,7 @@ func TestDockerfileScriptDownload(t *testing.T) {
name: "empty file",
filename: "./testdata/Dockerfile-empty",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -248,7 +260,7 @@ func TestDockerfileScriptDownload(t *testing.T) {
name: "invalid file sh",
filename: "./testdata/script.sh",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -259,7 +271,7 @@ func TestDockerfileScriptDownload(t *testing.T) {
name: "comments only",
filename: "./testdata/Dockerfile-comments",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -270,7 +282,7 @@ func TestDockerfileScriptDownload(t *testing.T) {
name: "wget | /bin/sh",
filename: "testdata/Dockerfile-wget-bin-sh",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 3,
NumberOfInfo: 0,
@ -281,7 +293,7 @@ func TestDockerfileScriptDownload(t *testing.T) {
name: "wget no exec",
filename: "testdata/Dockerfile-script-ok",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -292,7 +304,7 @@ func TestDockerfileScriptDownload(t *testing.T) {
name: "curl file sh",
filename: "testdata/Dockerfile-curl-file-sh",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 12,
NumberOfInfo: 0,
@ -303,7 +315,7 @@ func TestDockerfileScriptDownload(t *testing.T) {
name: "proc substitution",
filename: "testdata/Dockerfile-proc-subs",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 6,
NumberOfInfo: 0,
@ -314,7 +326,7 @@ func TestDockerfileScriptDownload(t *testing.T) {
name: "wget file",
filename: "testdata/Dockerfile-wget-file",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 10,
NumberOfInfo: 0,
@ -325,7 +337,7 @@ func TestDockerfileScriptDownload(t *testing.T) {
name: "gsutil file",
filename: "testdata/Dockerfile-gsutil-file",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 17,
NumberOfInfo: 0,
@ -336,7 +348,7 @@ func TestDockerfileScriptDownload(t *testing.T) {
name: "aws file",
filename: "testdata/Dockerfile-aws-file",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 15,
NumberOfInfo: 0,
@ -347,7 +359,7 @@ func TestDockerfileScriptDownload(t *testing.T) {
name: "pkg managers",
filename: "testdata/Dockerfile-pkg-managers",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 27,
NumberOfInfo: 0,
@ -358,7 +370,7 @@ func TestDockerfileScriptDownload(t *testing.T) {
name: "download with some python",
filename: "testdata/Dockerfile-some-python",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 1,
NumberOfInfo: 0,
@ -382,7 +394,13 @@ func TestDockerfileScriptDownload(t *testing.T) {
}
dl := scut.TestDetailLogger{}
s, e := testValidateDockerfileIsFreeOfInsecureDownloads(tt.filename, content, &dl)
scut.ValidateTestValues(t, tt.name, &tt.expected, s, e, &dl)
actual := checker.CheckResult{
Score: s,
Error2: e,
}
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &actual, &dl) {
t.Fail()
}
})
}
}
@ -398,7 +416,7 @@ func TestShellScriptDownload(t *testing.T) {
name: "sh script",
filename: "testdata/script-sh",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 7,
NumberOfInfo: 0,
@ -409,7 +427,7 @@ func TestShellScriptDownload(t *testing.T) {
name: "empty file",
filename: "./testdata/script-empty.sh",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -420,7 +438,7 @@ func TestShellScriptDownload(t *testing.T) {
name: "comments",
filename: "./testdata/script-comments.sh",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -431,7 +449,7 @@ func TestShellScriptDownload(t *testing.T) {
name: "bash script",
filename: "testdata/script-bash",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 7,
NumberOfInfo: 0,
@ -442,7 +460,7 @@ func TestShellScriptDownload(t *testing.T) {
name: "sh script 2",
filename: "testdata/script.sh",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 7,
NumberOfInfo: 0,
@ -453,7 +471,7 @@ func TestShellScriptDownload(t *testing.T) {
name: "pkg managers",
filename: "testdata/script-pkg-managers",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 24,
NumberOfInfo: 0,
@ -477,7 +495,13 @@ func TestShellScriptDownload(t *testing.T) {
}
dl := scut.TestDetailLogger{}
s, e := testValidateShellScriptIsFreeOfInsecureDownloads(tt.filename, content, &dl)
scut.ValidateTestValues(t, tt.name, &tt.expected, s, e, &dl)
actual := checker.CheckResult{
Score: s,
Error2: e,
}
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &actual, &dl) {
t.Fail()
}
})
}
}
@ -493,7 +517,7 @@ func TestGitHubWorflowRunDownload(t *testing.T) {
name: "workflow curl default",
filename: "testdata/github-workflow-curl-default",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 1,
NumberOfInfo: 0,
@ -504,7 +528,7 @@ func TestGitHubWorflowRunDownload(t *testing.T) {
name: "workflow curl no default",
filename: "testdata/github-workflow-curl-no-default",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 1,
NumberOfInfo: 0,
@ -515,7 +539,7 @@ func TestGitHubWorflowRunDownload(t *testing.T) {
name: "wget across steps",
filename: "testdata/github-workflow-wget-across-steps",
expected: scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 2,
NumberOfInfo: 0,
@ -539,7 +563,13 @@ func TestGitHubWorflowRunDownload(t *testing.T) {
}
dl := scut.TestDetailLogger{}
s, e := testValidateGitHubWorkflowScriptFreeOfInsecureDownloads(tt.filename, content, &dl)
scut.ValidateTestValues(t, tt.name, &tt.expected, s, e, &dl)
actual := checker.CheckResult{
Score: s,
Error2: e,
}
if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &actual, &dl) {
t.Fail()
}
})
}
}

4
e2e/binary_artifacts_test.go
View File

@ -47,7 +47,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 0,
@ -79,7 +79,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() {
}
// TODO: upload real binaries to the repo as well.
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 35,
NumberOfInfo: 0,

16
e2e/branch_protection_test.go
View File

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// nolint: dupl
package e2e
import (
@ -22,6 +23,7 @@ import (
"github.com/ossf/scorecard/v2/checker"
"github.com/ossf/scorecard/v2/checks"
"github.com/ossf/scorecard/v2/clients/githubrepo"
scut "github.com/ossf/scorecard/v2/utests"
)
@ -29,18 +31,21 @@ var _ = Describe("E2E TEST:"+checks.CheckBranchProtection, func() {
Context("E2E TEST:Validating branch protection", func() {
It("Should fail to return branch protection on other repositories", func() {
dl := scut.TestDetailLogger{}
repoClient := githubrepo.CreateGithubRepoClient(context.Background(), ghClient, graphClient)
err := repoClient.InitRepo("apache", "airflow")
Expect(err).Should(BeNil())
req := checker.CheckRequest{
Ctx: context.Background(),
Client: ghClient,
HTTPClient: httpClient,
RepoClient: nil,
RepoClient: repoClient,
Owner: "apache",
Repo: "airflow",
GraphClient: graphClient,
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: 1,
NumberOfWarn: 3,
NumberOfInfo: 0,
@ -57,18 +62,21 @@ var _ = Describe("E2E TEST:"+checks.CheckBranchProtection, func() {
Context("E2E TEST:Validating branch protection", func() {
It("Should fail to return branch protection on other repositories", func() {
dl := scut.TestDetailLogger{}
repoClient := githubrepo.CreateGithubRepoClient(context.Background(), ghClient, graphClient)
err := repoClient.InitRepo("ossf-tests", "scorecard-check-branch-protection-e2e")
Expect(err).Should(BeNil())
req := checker.CheckRequest{
Ctx: context.Background(),
Client: ghClient,
HTTPClient: httpClient,
RepoClient: nil,
RepoClient: repoClient,
Owner: "ossf-tests",
Repo: "scorecard-check-branch-protection-e2e",
GraphClient: graphClient,
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: 9,
NumberOfWarn: 1,
NumberOfInfo: 8,

4
e2e/ci_tests_test.go
View File

@ -45,11 +45,11 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 0,
NumberOfDebug: 24,
NumberOfDebug: 30,
}
result := checks.CITests(&req)
// UPGRADEv2: to remove.

2
e2e/cii_best_practices_test.go
View File

@ -40,7 +40,7 @@ var _ = Describe("E2E TEST:CIIBestPractices", func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: 5,
NumberOfWarn: 0,
NumberOfInfo: 0,

2
e2e/code_review_test.go
View File

@ -48,7 +48,7 @@ var _ = Describe("E2E TEST:CodeReview", func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 2,

4
e2e/contributors_test.go
View File

@ -45,7 +45,7 @@ var _ = Describe("E2E TEST:"+checks.CheckContributors, func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -75,7 +75,7 @@ var _ = Describe("E2E TEST:"+checks.CheckContributors, func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,

4
e2e/dependency_update_tool_test.go
View File

@ -47,7 +47,7 @@ var _ = Describe("E2E TEST:"+checks.CheckDependencyUpdateTool, func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -78,7 +78,7 @@ var _ = Describe("E2E TEST:"+checks.CheckDependencyUpdateTool, func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,

2
e2e/fuzzing_test.go
View File

@ -45,7 +45,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 0,

2
e2e/maintained_test.go
View File

@ -45,7 +45,7 @@ var _ = Describe("E2E TEST:"+checks.CheckMaintained, func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 0,

2
e2e/packaging_test.go
View File

@ -45,7 +45,7 @@ var _ = Describe("E2E TEST:"+checks.CheckPackaging, func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 2,

2
e2e/permissions_test.go
View File

@ -44,7 +44,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 1,
NumberOfInfo: 2,

2
e2e/pinned_dependencies_test.go
View File

@ -47,7 +47,7 @@ var _ = Describe("E2E TEST:"+checks.CheckPinnedDependencies, func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 154,
NumberOfInfo: 0,

3
e2e/sast_test.go
View File

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// nolint: dupl
package e2e
import (
@ -44,7 +45,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSAST, func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: 7,
NumberOfWarn: 1,
NumberOfInfo: 1,

4
e2e/security_policy_test.go
View File

@ -45,7 +45,7 @@ var _ = Describe("E2E TEST:SecurityPolicy", func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,
@ -76,7 +76,7 @@ var _ = Describe("E2E TEST:SecurityPolicy", func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 1,

2
e2e/signedreleases_test.go
View File

@ -45,7 +45,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSignedReleases, func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 5,

4
e2e/vulnerabilities_test.go
View File

@ -46,7 +46,7 @@ var _ = Describe("E2E TEST:Vulnerabilities", func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 0,
@ -78,7 +78,7 @@ var _ = Describe("E2E TEST:Vulnerabilities", func() {
Dlogger: &dl,
}
expected := scut.TestReturn{
Errors: nil,
Error: nil,
Score: checker.MinResultScore,
NumberOfWarn: 1,
NumberOfInfo: 0,

102
utests/utlib.go
View File

@ -18,30 +18,21 @@ package utests
import (
"errors"
"fmt"
"log"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/ossf/scorecard/v2/checker"
)
func validateDetailTypes(messages []checker.CheckDetail, nw, ni, nd int) bool {
enw := 0
eni := 0
end := 0
for _, v := range messages {
switch v.Type {
default:
panic(fmt.Sprintf("invalid type %v", v.Type))
case checker.DetailInfo:
eni++
case checker.DetailDebug:
end++
case checker.DetailWarn:
enw++
}
}
return enw == nw &&
eni == ni &&
end == nd
// TestReturn encapsulates expected CheckResult return values.
type TestReturn struct {
Error error
Score int
NumberOfWarn int
NumberOfInfo int
NumberOfDebug int
}
// TestDetailLogger implements `checker.DetailLogger`.
@ -49,15 +40,6 @@ type TestDetailLogger struct {
messages []checker.CheckDetail
}
// TestReturn encapsulates expected CheckResult return values.
type TestReturn struct {
Errors []error
Score int
NumberOfWarn int
NumberOfInfo int
NumberOfDebug int
}
// Info implements DetailLogger.Info.
func (l *TestDetailLogger) Info(desc string, args ...interface{}) {
cd := checker.CheckDetail{Type: checker.DetailInfo, Msg: checker.LogMessage{Text: fmt.Sprintf(desc, args...)}}
@ -77,7 +59,8 @@ func (l *TestDetailLogger) Debug(desc string, args ...interface{}) {
}
// UPGRADEv3: to rename.
//nolint:revive
// Info3 implements DetailLogger.Info3.
func (l *TestDetailLogger) Info3(msg *checker.LogMessage) {
cd := checker.CheckDetail{
Type: checker.DetailInfo,
@ -87,7 +70,7 @@ func (l *TestDetailLogger) Info3(msg *checker.LogMessage) {
l.messages = append(l.messages, cd)
}
//nolint:revive
// Warn3 implements DetailLogger.Warn3.
func (l *TestDetailLogger) Warn3(msg *checker.LogMessage) {
cd := checker.CheckDetail{
Type: checker.DetailWarn,
@ -97,7 +80,7 @@ func (l *TestDetailLogger) Warn3(msg *checker.LogMessage) {
l.messages = append(l.messages, cd)
}
//nolint:revive
// Debug3 implements DetailLogger.Debug3.
func (l *TestDetailLogger) Debug3(msg *checker.LogMessage) {
cd := checker.CheckDetail{
Type: checker.DetailDebug,
@ -107,38 +90,43 @@ func (l *TestDetailLogger) Debug3(msg *checker.LogMessage) {
l.messages = append(l.messages, cd)
}
// ValidateTestValues validates returned score and log values.
// nolint: thelper
func ValidateTestValues(t *testing.T, name string, te *TestReturn,
score int, err error, dl *TestDetailLogger) bool {
for _, we := range te.Errors {
if !errors.Is(err, we) {
if t != nil {
t.Errorf("%v: invalid error returned: %v is not of type %v",
name, err, we)
}
fmt.Printf("%v: invalid error returned: %v is not of type %v",
name, err, we)
return false
func getTestReturn(cr *checker.CheckResult, logger *TestDetailLogger) (*TestReturn, error) {
ret := new(TestReturn)
for _, v := range logger.messages {
switch v.Type {
default:
// nolint: goerr113
return nil, fmt.Errorf("invalid type %v", v.Type)
case checker.DetailInfo:
ret.NumberOfInfo++
case checker.DetailDebug:
ret.NumberOfDebug++
case checker.DetailWarn:
ret.NumberOfWarn++
}
}
if score != te.Score ||
!validateDetailTypes(dl.messages, te.NumberOfWarn,
te.NumberOfInfo, te.NumberOfDebug) {
if t != nil {
t.Errorf("%v: Got (score=%v) expected (%v)\n%v",
name, score, te.Score, dl.messages)
}
return false
}
return true
ret.Score = cr.Score
ret.Error = cr.Error
return ret, nil
}
func errCmp(e1, e2 error) bool {
return errors.Is(e1, e2) || errors.Is(e2, e1)
}
// ValidateTestReturn validates expected TestReturn with actual checker.CheckResult values.
// nolint: thelper
func ValidateTestReturn(t *testing.T, name string, te *TestReturn,
tr *checker.CheckResult, dl *TestDetailLogger) bool {
return ValidateTestValues(t, name, te, tr.Score, tr.Error2, dl)
func ValidateTestReturn(t *testing.T, name string, expected *TestReturn,
actual *checker.CheckResult, logger *TestDetailLogger) bool {
actualTestReturn, err := getTestReturn(actual, logger)
if err != nil {
panic(err)
}
if !cmp.Equal(*actualTestReturn, *expected, cmp.Comparer(errCmp)) {
log.Println(cmp.Diff(*actualTestReturn, *expected))
return false
}
return true
}
// ValidateLogMessage tests that at least one log message returns true for isExpectedMessage.