2022-12-12 23:27:14 +03:00
|
|
|
// Copyright 2021 Security Scorecard Authors
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package e2e
|
|
|
|
|
|
|
|
import (
|
|
|
|
"os"
|
|
|
|
"strings"
|
|
|
|
|
|
|
|
. "github.com/onsi/ginkgo/v2"
|
|
|
|
. "github.com/onsi/gomega"
|
|
|
|
"gopkg.in/yaml.v2"
|
|
|
|
|
|
|
|
"github.com/ossf/scorecard/v4/attestor/command"
|
|
|
|
"github.com/ossf/scorecard/v4/attestor/policy"
|
|
|
|
)
|
|
|
|
|
|
|
|
var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
|
|
|
|
Context("E2E TEST:Validating scorecard attestation policy", func() {
|
|
|
|
It("Should attest to repos based on policy", func() {
|
|
|
|
tt := []struct {
|
|
|
|
name string
|
|
|
|
repoURL string
|
|
|
|
commit string
|
|
|
|
policy policy.AttestationPolicy
|
|
|
|
expected policy.PolicyResult
|
|
|
|
}{
|
|
|
|
{
|
|
|
|
name: "test good repo",
|
|
|
|
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-good",
|
|
|
|
policy: policy.AttestationPolicy{
|
|
|
|
PreventBinaryArtifacts: true,
|
|
|
|
PreventKnownVulnerabilities: true,
|
|
|
|
PreventUnpinnedDependencies: true,
|
|
|
|
},
|
|
|
|
expected: policy.Pass,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "test bad repo with vulnerabilities prevented but no known vulnerabilities",
|
|
|
|
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
|
|
|
|
policy: policy.AttestationPolicy{
|
|
|
|
PreventKnownVulnerabilities: true,
|
|
|
|
},
|
|
|
|
expected: policy.Pass,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "test bad repo with ignored binary artifact",
|
|
|
|
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
|
|
|
|
policy: policy.AttestationPolicy{
|
|
|
|
PreventBinaryArtifacts: true,
|
|
|
|
AllowedBinaryArtifacts: []string{"test-binary-artifact-*"},
|
|
|
|
PreventKnownVulnerabilities: true,
|
|
|
|
},
|
|
|
|
expected: policy.Pass,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "test bad repo with ignored binary artifact",
|
|
|
|
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
|
|
|
|
policy: policy.AttestationPolicy{
|
|
|
|
PreventBinaryArtifacts: true,
|
|
|
|
PreventKnownVulnerabilities: true,
|
|
|
|
},
|
|
|
|
expected: policy.Fail,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "test bad repo with ignored dep by path",
|
|
|
|
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
|
|
|
|
policy: policy.AttestationPolicy{
|
|
|
|
PreventUnpinnedDependencies: true,
|
|
|
|
AllowedUnpinnedDependencies: []policy.Dependency{{Filepath: "Dockerfile"}},
|
|
|
|
},
|
|
|
|
expected: policy.Pass,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "test bad repo without ignored dep",
|
|
|
|
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
|
|
|
|
policy: policy.AttestationPolicy{
|
|
|
|
PreventUnpinnedDependencies: true,
|
|
|
|
},
|
|
|
|
expected: policy.Fail,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "test bad repo with ignored dep by name",
|
|
|
|
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
|
|
|
|
policy: policy.AttestationPolicy{
|
|
|
|
PreventUnpinnedDependencies: true,
|
|
|
|
AllowedUnpinnedDependencies: []policy.Dependency{{PackageName: "static-debian11"}, {PackageName: "golang"}},
|
|
|
|
},
|
|
|
|
expected: policy.Pass,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "test bad repo with everything ignored",
|
|
|
|
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
|
|
|
|
policy: policy.AttestationPolicy{
|
|
|
|
PreventBinaryArtifacts: true,
|
|
|
|
AllowedBinaryArtifacts: []string{"test-binary-artifact-*"},
|
|
|
|
PreventKnownVulnerabilities: true,
|
|
|
|
PreventUnpinnedDependencies: true,
|
|
|
|
AllowedUnpinnedDependencies: []policy.Dependency{{Filepath: "Dockerfile"}},
|
|
|
|
},
|
|
|
|
expected: policy.Pass,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "test repo with simple code review requirements",
|
|
|
|
repoURL: "https://github.com/ossf/scorecard",
|
|
|
|
commit: "fa0592fab28aa92560f04e1ae8649dfff566ae2b",
|
|
|
|
policy: policy.AttestationPolicy{
|
|
|
|
EnsureCodeReviewed: true,
|
|
|
|
CodeReviewRequirements: policy.CodeReviewRequirements{
|
|
|
|
MinReviewers: 1,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
expected: policy.Pass,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "test code reviews required but repo doesn't have code reviews",
|
|
|
|
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
|
|
|
|
policy: policy.AttestationPolicy{
|
|
|
|
PreventBinaryArtifacts: true,
|
|
|
|
PreventKnownVulnerabilities: true,
|
|
|
|
PreventUnpinnedDependencies: true,
|
|
|
|
EnsureCodeReviewed: true,
|
|
|
|
},
|
|
|
|
expected: policy.Fail,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "test code reviews required with min reviewers",
|
|
|
|
repoURL: "https://github.com/ossf/scorecard",
|
|
|
|
commit: "fa0592fab28aa92560f04e1ae8649dfff566ae2b",
|
|
|
|
policy: policy.AttestationPolicy{
|
|
|
|
PreventBinaryArtifacts: true,
|
2022-12-13 03:46:43 +03:00
|
|
|
PreventKnownVulnerabilities: false,
|
2022-12-12 23:27:14 +03:00
|
|
|
PreventUnpinnedDependencies: true,
|
|
|
|
EnsureCodeReviewed: true,
|
|
|
|
CodeReviewRequirements: policy.CodeReviewRequirements{
|
|
|
|
MinReviewers: 1,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
expected: policy.Pass,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "test code reviews required with min reviewers and required reviewers",
|
|
|
|
repoURL: "https://github.com/ossf/scorecard",
|
|
|
|
commit: "fa0592fab28aa92560f04e1ae8649dfff566ae2b",
|
|
|
|
policy: policy.AttestationPolicy{
|
|
|
|
PreventBinaryArtifacts: true,
|
2022-12-13 03:46:43 +03:00
|
|
|
PreventKnownVulnerabilities: false,
|
2022-12-12 23:27:14 +03:00
|
|
|
PreventUnpinnedDependencies: true,
|
|
|
|
EnsureCodeReviewed: true,
|
|
|
|
CodeReviewRequirements: policy.CodeReviewRequirements{
|
|
|
|
MinReviewers: 1,
|
|
|
|
RequiredApprovers: []string{"spencerschrock", "laurentsimon", "naveensrinivasan", "azeemshaikh38"},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
expected: policy.Pass,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "test code reviews required with too many min reviewers but matching required reviewers",
|
|
|
|
repoURL: "https://github.com/ossf/scorecard",
|
|
|
|
commit: "fa0592fab28aa92560f04e1ae8649dfff566ae2b",
|
|
|
|
policy: policy.AttestationPolicy{
|
|
|
|
PreventBinaryArtifacts: true,
|
2022-12-13 03:46:43 +03:00
|
|
|
PreventKnownVulnerabilities: false,
|
2022-12-12 23:27:14 +03:00
|
|
|
PreventUnpinnedDependencies: true,
|
|
|
|
EnsureCodeReviewed: true,
|
|
|
|
CodeReviewRequirements: policy.CodeReviewRequirements{
|
|
|
|
MinReviewers: 2,
|
|
|
|
RequiredApprovers: []string{"spencerschrock", "laurentsimon", "naveensrinivasan", "azeemshaikh38"},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
expected: policy.Fail,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, tc := range tt {
|
|
|
|
f, err := os.CreateTemp("/tmp", strings.ReplaceAll(tc.name, " ", "-"))
|
|
|
|
Expect(err).Should(BeNil())
|
|
|
|
defer os.Remove(f.Name())
|
|
|
|
|
|
|
|
buf, err := yaml.Marshal(tc.policy)
|
|
|
|
Expect(err).Should(BeNil())
|
|
|
|
|
|
|
|
nbytes, err := f.Write(buf)
|
|
|
|
Expect(err).Should(BeNil())
|
|
|
|
Expect(nbytes).Should(BeNumerically(">", 0))
|
|
|
|
|
|
|
|
result, err := command.RunCheckWithParams(tc.repoURL, tc.commit, f.Name())
|
|
|
|
Expect(err).Should(BeNil())
|
|
|
|
Expect(result).Should(BeEquivalentTo(tc.expected))
|
|
|
|
}
|
|
|
|
})
|
|
|
|
})
|
|
|
|
})
|