mirror of
https://github.com/ossf/scorecard.git
synced 2024-07-14 19:40:37 +03:00
📖 Generate probe markdown documentation (#4184)
* generate probe markdown documentation Walks the various probes def.yaml files and puts them in a single markdown document. This doesn't currently include the remediation, but neither does the existing checks.md document either. In order to avoid duplicating yaml definitions, this existing ones were moved to an internal directory so they can be reused. Signed-off-by: Spencer Schrock <sschrock@google.com> * fix linter Signed-off-by: Spencer Schrock <sschrock@google.com> * add probe doc generation to Makefile Note: There is no validate-docs step for the probes code, as the def.yml fields are validated elsewhere currently in the unit tests. Signed-off-by: Spencer Schrock <sschrock@google.com> * fix license for new yaml package Signed-off-by: Spencer Schrock <sschrock@google.com> --------- Signed-off-by: Spencer Schrock <sschrock@google.com>
This commit is contained in:
Spencer Schrock
GitHub
parent
5d08c1cc11
commit
0d57c0224a
2
.github/workflows/main.yml
vendored
2
.github/workflows/main.yml
vendored
@ -163,7 +163,7 @@ jobs:
|
||||
command: |
|
||||
go env -w GOFLAGS=-mod=mod
|
||||
make generate-docs
|
||||
- name: ensure checks.yaml and checks.md match
|
||||
- name: ensure generated check and probe documentation match the source
|
||||
run: git diff --exit-code
|
||||
build-proto:
|
||||
name: build-proto
|
||||
|
||||
6
Makefile
6
Makefile
@ -165,10 +165,14 @@ cmd/internal/nuget/nuget_mockclient.go: cmd/internal/nuget/client.go | $(MOCKGEN
|
||||
# Generating MockNugetClient
|
||||
$(MOCKGEN) -source=cmd/internal/nuget/client.go -destination=cmd/internal/nuget/nuget_mockclient.go -package=nuget -copyright_file=clients/mockclients/license.txt
|
||||
|
||||
PROBE_DEFINITION_FILES = $(shell find ./probes/ -name "def.yml")
|
||||
generate-docs: ## Generates docs
|
||||
generate-docs: validate-docs docs/checks.md docs/checks/internal/checks.yaml docs/checks/internal/*.go docs/checks/internal/generate/*.go
|
||||
generate-docs: validate-docs docs/checks.md docs/checks/internal/checks.yaml docs/checks/internal/*.go docs/checks/internal/generate/*.go \
|
||||
docs/probes.md $(PROBE_DEFINITION_FILES) docs/probes/internal/generate/*.go
|
||||
# Generating checks.md
|
||||
go run ./docs/checks/internal/generate/main.go docs/checks.md
|
||||
# Generating probes.md
|
||||
go run ./docs/probes/internal/generate/main.go probes/ > docs/probes.md
|
||||
|
||||
validate-docs: docs/checks/internal/generate/main.go
|
||||
# Validating checks.yaml
|
||||
|
||||
571
docs/probes.md
Normal file
571
docs/probes.md
Normal file
@ -0,0 +1,571 @@
|
||||
<!-- Do not edit this file manually! Edit the individual probe's def.yml instead. -->
|
||||
# Probe Documentation
|
||||
|
||||
This page describes each Scorecard probe in detail, including description, motivation,
|
||||
and outcomes. If you have ideas for additions or new detection techniques,
|
||||
please [contribute](../CONTRIBUTING.md)!
|
||||
|
||||
## archived
|
||||
|
||||
**Description**: Check that the project is archived
|
||||
|
||||
**Motivation**: An archived project will not received security patches, and is not actively tested or used.
|
||||
|
||||
**Implementation**: The probe checks the Archived Status of a project.
|
||||
|
||||
**Outcomes**: If the project is archived, the outcome is OutcomeTrue.
|
||||
If the project is not archived, the outcome is OutcomeFalse.
|
||||
|
||||
|
||||
## blocksDeleteOnBranches
|
||||
|
||||
**Description**: Check that the project blocks non-admins from deleting branches.
|
||||
|
||||
**Motivation**: Allowing non-admins to delete project branches has a similar effect to performing force pushes.
|
||||
|
||||
**Implementation**: Checks the protection rules of default and release branches.
|
||||
|
||||
**Outcomes**: The probe returns one OutcomeTrue for each branch that is disallowed from users deleting it, and one OutcomeFalse for branches where users are able to delete it. Scorecard only considers default and releases branches.
|
||||
|
||||
|
||||
## blocksForcePushOnBranches
|
||||
|
||||
**Description**: Check that the project blocks force push on its branches.
|
||||
|
||||
**Motivation**: Allowing force pushes to branches could allow those with write access to make insecure changes to the behavior of the project.
|
||||
|
||||
**Implementation**: Checks the protection rules of default and release branches.
|
||||
|
||||
**Outcomes**: The probe returns one OutcomeTrue for each branch that is blocked from force pushes, and one OutcomeFalse for branches that allows force push.
|
||||
Returns OutcomeNotAvailable if Scorecard cannot fetch the data from the repository.
|
||||
|
||||
|
||||
## branchProtectionAppliesToAdmins
|
||||
|
||||
**Description**: Check that the project's branch protection rules apply to project admins.
|
||||
|
||||
**Motivation**: Admins may be able to bypass branch protection settings which could defeat the purpose of having them.
|
||||
|
||||
**Implementation**: Checks the protection rules of default and release branches.
|
||||
|
||||
**Outcomes**: The probe returns one OutcomeTrue for each branch that enforces branch protection rules on admins, and one OutcomeFalse for branches that don't.
|
||||
|
||||
|
||||
## branchesAreProtected
|
||||
|
||||
**Description**: Check that the project uses protected branches.
|
||||
|
||||
**Motivation**: Unprotected branches may allow actions that could compromise the project's security.
|
||||
|
||||
**Implementation**: Checks the protection rules of default and release branches.
|
||||
|
||||
**Outcomes**: The probe returns one OutcomeTrue for each branch that is protected, and one OutcomeFalse for branches that are not protected. Scorecard only considers default and releases branches.
|
||||
|
||||
|
||||
## codeApproved
|
||||
|
||||
**Description**: Check that all recent changesets have been approved by someone who is not the author of the changeset.
|
||||
|
||||
**Motivation**: To ensure that the review process works, the proposed changes should have a minimum number of approvals.
|
||||
|
||||
**Implementation**: This probe looks for whether all changes over the last `--commit-depth` commits have been approved before merge. Commits are grouped by the changeset they were introduced in, and each changesets must have at least one approval. Reviewed, bot authored changesets (e.g. dependabot) are not counted.
|
||||
|
||||
**Outcomes**: If all commits were approved, the probe returns OutcomeTrue
|
||||
If any commits were not approved, the probe returns OutcomeFalse
|
||||
If there are no changes, the probe returns OutcomeNotApplicable
|
||||
|
||||
|
||||
## codeReviewOneReviewers
|
||||
|
||||
**Description**: Check that at least one reviewers review a change before merging.
|
||||
|
||||
**Motivation**: To ensure that the review process works, the proposed changes should have a minimum number of approvals.
|
||||
|
||||
**Implementation**: This probe looks for whether all changes over the last `--commit-depth` commits have been approved by a minimum number of reviewers. Commits are grouped by the Pull Request they were introduced in. Only unique reviewer logins that aren't the same as the changeset author are counted.
|
||||
|
||||
**Outcomes**: If all the changes had at least one reviewers, the probe returns OutcomeTrue (1)
|
||||
If the changes had fewer than one reviewers, the prove returns OutcomeFalse (0)
|
||||
|
||||
|
||||
## contributorsFromOrgOrCompany
|
||||
|
||||
**Description**: Checks whether a project has a contributions from users associated with a company or organization.
|
||||
|
||||
**Motivation**: This probe tries to determine if the project has recent contributors from multiple organizations. For some projects, having a diverse group of contributors is an indicator of project health.
|
||||
|
||||
**Implementation**: The probe looks at the Company field on the user profile for authors of recent commits. To receive the highest score, the project must have had contributors from at least 3 different companies in the last 30 commits.
|
||||
|
||||
**Outcomes**: If the project has no contributing organizations, the probe returns 1 OutcomeFalse
|
||||
If the project has contributing organizations, the probe returns 1 OutcomeTrue per organization.
|
||||
|
||||
|
||||
## createdRecently
|
||||
|
||||
**Description**: Checks if the project was created in the last 90 days.
|
||||
|
||||
**Motivation**: Recently created repositories have been used for malicious forks / typosquatting attacks in the past. A newly created repo is not a strong signal on its own, but can be a useful piece of information.
|
||||
|
||||
**Implementation**: The implementation checks the creation date is within the last 90 days.
|
||||
|
||||
**Outcomes**: If the project was created within the last 90 days, the outcome is OutcomeTrue.
|
||||
If the project is older than 90 days, the outcome is OutcomeFalse. The finding will include a "lookBackDays" value which is the time period that the probe looks back in.
|
||||
|
||||
|
||||
## dependencyUpdateToolConfigured
|
||||
|
||||
**Description**: Check that a dependency update tool config is present.
|
||||
|
||||
**Motivation**: Out-of-date dependencies make a project vulnerable to known flaws and prone to attacks. Tools can help the process of updating dependencies by scanning for outdated or insecure requirements, and opening a pull request to update them if found.
|
||||
|
||||
**Implementation**: The implementation looks for the presence of various config files for different dependency update tools.
|
||||
|
||||
**Outcomes**: If a dependency update tool is configured, the probe returns OutcomeTrue for each configuration.
|
||||
If no tool is detected, the probe returns OutcomeFalse.
|
||||
|
||||
|
||||
## dismissesStaleReviews
|
||||
|
||||
**Description**: Check that the project dismisses stale reviews when new commits are pushed.
|
||||
|
||||
**Motivation**: When a project does not dismiss stale reviews, contributors can bring their pull requests to an approved state and then make unreviewed commits.
|
||||
|
||||
**Implementation**: Checks the protection rules of default and release branches.
|
||||
|
||||
**Outcomes**: The probe returns one OutcomeTrue for each branch that dismisses the stale status of PRs, and one OutcomeFalse for branches that don't.
|
||||
|
||||
|
||||
## fuzzed
|
||||
|
||||
**Description**: Check that the project is fuzzed
|
||||
|
||||
**Motivation**: Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data into a program to expose bugs. Regular fuzzing is important to detect vulnerabilities that may be exploited by others, especially since attackers can also use fuzzing to find the same flaws.
|
||||
|
||||
**Implementation**: The implementation looks for various fuzzing function signatures, imports, configuration files, and external integration data.
|
||||
|
||||
**Outcomes**: If a fuzzing tool is found, one finding per tool with OutcomeTrue is returned.
|
||||
If no fuzzing tool is found, or the project uses a tool we don't detect, one finding with OutcomeFalse is returned.
|
||||
|
||||
|
||||
## hasBinaryArtifacts
|
||||
|
||||
**Description**: Checks if the project has any binary files in its source tree.
|
||||
|
||||
**Motivation**: Binary files are not human readable so users and reviewers can't easily see what they do.
|
||||
|
||||
**Implementation**: The implementation looks for the presence of binary files. This is a more restrictive probe than "hasUnverifiedBinaryArtifacts" which excludes verified binary files.
|
||||
|
||||
**Outcomes**: If the probe finds binary files, it returns one OutcomeTrue for each binary file found.
|
||||
If the probe finds no binary files, it returns a single OutcomeFalse.
|
||||
|
||||
|
||||
## hasDangerousWorkflowScriptInjection
|
||||
|
||||
**Description**: Check whether the project has GitHub Actions workflows that enable script injection.
|
||||
|
||||
**Motivation**: Script injections allow attackers to use untrusted input to access privileged resources (code execution, secret exfiltration, etc.)
|
||||
|
||||
**Implementation**: The probe analyzes the repository's workflows for known dangerous patterns.
|
||||
|
||||
**Outcomes**: The probe returns one finding with OutcomeTrue for each dangerous script injection pattern detected.
|
||||
If no dangerous patterns are found, the probe returns one finding with OutcomeFalse.
|
||||
|
||||
|
||||
## hasDangerousWorkflowUntrustedCheckout
|
||||
|
||||
**Description**: Check whether the project has GitHub Actions workflows that does untrusted checkouts.
|
||||
|
||||
**Motivation**: GitHub workflows triggered with pull_request_target or workflow_run have write permission to the target repository and access to target repository secrets. Combined with a dangerous checkout of PR contents, attackers may be able to compromise the repository, for example, by using build scripts controlled by the PR author.
|
||||
|
||||
**Implementation**: The probe iterates through the workflows looking for pull_request_target and workflow_run triggers which checkout references from a PR. This check does not detect whether untrusted code checkouts are used safely, for example, only on pull request that have been assigned a label.
|
||||
|
||||
**Outcomes**: The probe returns one finding with OutcomeTrue per untrusted checkout.
|
||||
The probe returns one finding with OutcomeFalse if no untrusted checkouts are detected.
|
||||
|
||||
|
||||
## hasFSFOrOSIApprovedLicense
|
||||
|
||||
**Description**: Check that the project has an FSF or OSI approved license.
|
||||
|
||||
**Motivation**: A license can give users information about how the source code may or may not be used. Using a recognized license facilitates security or legal reviews for potential users.
|
||||
|
||||
**Implementation**: The implementation checks whether a license file is present and is of an approved format. The list of FSF or OSI approved license is taken from https://spdx.org/licenses/ .
|
||||
|
||||
**Outcomes**: If a license file is found and is of an approved format, the probe returns a single OutcomeTrue.
|
||||
If a license file is missing the probe returns a single OutcomeNotApplicable.
|
||||
If the license is not of an approved format, the probe returns a single OutcomeFalse.
|
||||
|
||||
|
||||
## hasLicenseFile
|
||||
|
||||
**Description**: Check that the project has a license file
|
||||
|
||||
**Motivation**: A license can give users information about how the source code may or may not be used. The lack of a license will impede any kind of security review or audit and creates a legal risk for potential users.
|
||||
|
||||
**Implementation**: The implementation checks whether a license file is present.
|
||||
|
||||
**Outcomes**: If license files are found, the probe returns OutcomeTrue for each license file.
|
||||
If a license file is not found, the probe returns a single OutcomeFalse.
|
||||
|
||||
|
||||
## hasNoGitHubWorkflowPermissionUnknown
|
||||
|
||||
**Description**: Checks that GitHub workflows have workflows with unknown permissions
|
||||
|
||||
**Motivation**: Unknown permissions may be a result of a bug or another error from fetching the permission levels.
|
||||
|
||||
**Implementation**: The probe checks the permission levels of a projects workflows and collects the workflows that have unknown permissions.
|
||||
|
||||
**Outcomes**: The probe returns 1 false outcome per workflow without unknown permission level(s).
|
||||
The probe returns 1 true outcome if the project has no workflows with unknown permission levels.
|
||||
|
||||
|
||||
## hasOSVVulnerabilities
|
||||
|
||||
**Description**: Check whether the project has known vulnerabilities
|
||||
|
||||
**Motivation**: This check determines whether the project has open, unfixed vulnerabilities in its own codebase or its dependencies using the OSV (Open Source Vulnerabilities) service. An open vulnerability may be exploited by attackers and should be fixed as soon as possible.
|
||||
|
||||
**Implementation**: The implementation fetches data from OSV.dev about the project which shows whether a given project has known, unfixed vulnerabilities. The implementation uses the number of known, unfixed vulnerabilities to score.
|
||||
|
||||
**Outcomes**: The probe returns one true outcome for each vulnerability found in OSV.
|
||||
If there are no known vulnerabilities detected, the probe returns one false outcome.
|
||||
|
||||
|
||||
## hasOpenSSFBadge
|
||||
|
||||
**Description**: This check determines whether the project has an OpenSSF (formerly CII) Best Practices Badge.
|
||||
|
||||
**Motivation**: The OpenSSF Best Practices badge indicates whether or not the project uses a set of security-focused best development practices for open source software.
|
||||
|
||||
**Implementation**: The probe checks the badge level using the OpenSSF Best Practices Badge API.
|
||||
|
||||
**Outcomes**: If the project has a badge, the probe returns one OutcomeTrue finding. The finding includes the badge level as an entry in the `Values` map with the "badgeLevel" key.
|
||||
If the project does not have a badge, the probe returns one OutcomeFalse.
|
||||
|
||||
|
||||
## hasPermissiveLicense
|
||||
|
||||
**Description**: Check that the project has an permissive license.
|
||||
|
||||
**Motivation**: A permissive license allows users to use the analyzed component to be used in derivative works. Non-permissive licenses (as copyleft licenses) might be a legal risk for potential users.
|
||||
|
||||
**Implementation**: The implementation checks whether a permissive license is present
|
||||
|
||||
**Outcomes**: If a license file is found and is permissive, the probe returns a single OutcomeTrue.
|
||||
If a license file is missing the probe returns a single OutcomeFalse.
|
||||
If the license is not permissive, the probe returns a single OutcomeFalse.
|
||||
|
||||
|
||||
## hasRecentCommits
|
||||
|
||||
**Description**: Check whether the project has at least one commit per week over the last 90 days.
|
||||
|
||||
**Motivation**: A project which is not active might not be patched, have its dependencies patched, or be actively tested and used. A lack of active maintenance should signal that potential users should investigate further to judge the situation. A project may not need further features or maintenance; In this case, the probe results can be disregarded.
|
||||
|
||||
**Implementation**: The implementation checks the number of commits made in the last 90 days by any user type.
|
||||
|
||||
**Outcomes**: If the project has commits from the last 90 days, the probe returns one OutcomeTrue with a "commitsWithinThreshold" value which contains the number of commits that the probe found within the threshold. The probe will also return a "lookBackDays" value which is the number of days that the probe includes in its threshold - which is 90.
|
||||
If the project does not have commits in the last 90 days, the probe returns a single OutcomeFalse.
|
||||
|
||||
|
||||
## hasReleaseSBOM
|
||||
|
||||
**Description**: Check that the project publishes an SBOM as part of its release artifacts.
|
||||
|
||||
**Motivation**: An SBOM can give users information about how the source code components and dependencies. They help facilitate sotware supplychain security and aid in identifying upstream vulnerabilities in a codebase.
|
||||
|
||||
**Implementation**: The implementation checks whether a SBOM artifact is included in release artifacts.
|
||||
|
||||
**Outcomes**: If SBOM artifacts are found, the probe returns OutcomeTrue for each SBOM artifact up to 5.
|
||||
If an SBOM artifact is not found, the probe returns a single OutcomeFalse.
|
||||
|
||||
|
||||
## hasSBOM
|
||||
|
||||
**Description**: Check that the project has an SBOM file
|
||||
|
||||
**Motivation**: An SBOM can give users information about how the source code components and dependencies. They help facilitate sotware supplychain security and aid in identifying upstream vulnerabilities in a codebase.
|
||||
|
||||
**Implementation**: The implementation checks whether an SBOM file is present in the source code.
|
||||
|
||||
**Outcomes**: If an SBOM file(s) is found, the probe returns OutcomeTrue for each SBOM artifact up to 5.
|
||||
If an SBOM file is not found, the probe returns a single OutcomeFalse.
|
||||
|
||||
|
||||
## hasUnverifiedBinaryArtifacts
|
||||
|
||||
**Description**: Checks if the project has binary files in its source tree. The probe skips verified binary files which currently are gradle-wrappers.
|
||||
|
||||
**Motivation**: Binary files are not human readable so users and reviewers can't easily see what they do.
|
||||
|
||||
**Implementation**: The implementation looks for the presence of binary files that are not "verified". A verified binary is one that Scorecard considers valid for building and/or releasing the project. This is a more permissive probe than "hasBinaryArtifacts" which does not skip verified binary files.
|
||||
|
||||
**Outcomes**: If the probe finds unverified binary files, it returns OutcomeTrue for each unverified binary file found.
|
||||
If the probe finds no unverified binary files, it returns OutcomeFalse.
|
||||
|
||||
|
||||
## issueActivityByProjectMember
|
||||
|
||||
**Description**: Checks that a collaborator, member or owner has participated in issues in the last 90 days.
|
||||
|
||||
**Motivation**: A project which does not respond to issues may not be actively maintained. A lack of active maintenance should signal that potential users should investigate further to judge the situation. However a project may simply not have any recent issues; In this case, the probe results can be disregarded.
|
||||
|
||||
**Implementation**: The probe checks whether collaborators, members or owners of a project have participated in issues in the last 90 days.
|
||||
|
||||
**Outcomes**: If collaborators, members or owners have participated in issues in the last 90 days, the probe returns one OutcomeTrue. The probe also returns a "numberOfIssuesUpdatedWithinThreshold" value with represents the number of issues on the repository which project collaborators, members or owners have shown activity in.
|
||||
If collaborators, members or owners have NOT participated in issues in the last 90 days, the probe returns a single OutcomeFalse.
|
||||
|
||||
|
||||
## jobLevelPermissions
|
||||
|
||||
**Description**: Checks that GitHub workflows do not have "write" permissions at the "job" level.
|
||||
|
||||
**Motivation**: In some circumstances, having "write" permissions at the "job" level may enable attackers to escalate privileges.
|
||||
|
||||
**Implementation**: The probe checks the permission level, the workflow type and the permission type of each workflow in the project.
|
||||
|
||||
**Outcomes**: The probe returns 1 false outcome per workflow with "write" permissions at the "job" level.
|
||||
The probe returns 1 true outcome if the project has no workflows "write" permissions a the "job" level.
|
||||
|
||||
|
||||
## packagedWithAutomatedWorkflow
|
||||
|
||||
**Description**: Checks whether the project uses automated packaging.
|
||||
|
||||
**Motivation**: Packages give users of a project an easy way to download, install, update, and uninstall the software by a package manager. In particular, they make it easy for users to receive security patches as updates.
|
||||
|
||||
**Implementation**: The implementation checks whether a project uses common patterns for packaging across multiple ecosystems. Scorecard gets this by checking the projects workflows for specific uses of actions and build commands such as `docker push` or `mvn deploy`.
|
||||
|
||||
**Outcomes**: If the project uses a packaging mechanism we detect, the outcome is positive.
|
||||
If the project doesn't use automated packaing we can detect, the outcome is negative.
|
||||
|
||||
|
||||
## pinsDependencies
|
||||
|
||||
**Description**: Check that the project pins dependencies to a specific digest.
|
||||
|
||||
**Motivation**: Pinned dependencies ensure that checking and deployment are all done with the same software, reducing deployment risks, simplifying debugging, and enabling reproducibility. They can help mitigate compromised dependencies from undermining the security of the project (in the case where you've evaluated the pinned dependency, you are confident it's not compromised, and a later version is released that is compromised).
|
||||
|
||||
**Implementation**: The probe works by looking for unpinned dependencies in Dockerfiles, shell scripts, and GitHub workflows which are used during the build and release process of a project. Special considerations for Go modules treat full semantic versions as pinned due to how the Go tool verifies downloaded content against the hashes when anyone first downloaded the module.
|
||||
|
||||
**Outcomes**: For supported ecosystem, the probe returns OutcomeTrue per pinned dependency.
|
||||
For supported ecosystem, the probe returns OutcomeFalse per unpinned dependency.
|
||||
If the project has no supported dependencies, the probe returns OutcomeNotApplicable.
|
||||
|
||||
|
||||
## releasesAreSigned
|
||||
|
||||
**Description**: Check that the projects GitHub and GitLab releases are signed.
|
||||
|
||||
**Motivation**: Signed releases allow consumers to verify their artifacts before consuming them.
|
||||
|
||||
**Implementation**: The implementation checks whether a signature file is present in release assets. The probe checks the last 5 releases on GitHub and GitLab.
|
||||
|
||||
**Outcomes**: For each of the last 5 releases, the probe returns OutcomeTrue, if the release has a signature file in the release assets.
|
||||
For each of the last 5 releases, the probe returns OutcomeFalse, if the release does not have a signature file in the release assets.
|
||||
If the project has no releases, the probe returns OutcomeNotApplicable.
|
||||
|
||||
|
||||
## releasesHaveProvenance
|
||||
|
||||
**Description**: Check that the projects releases on GitHub and GitLab have provenance.
|
||||
|
||||
**Motivation**: Provenance give users security-critical, verifiable information so that consumers can verify their artifacts before consuming them.
|
||||
|
||||
**Implementation**: The probe checks whether any of the assets in any of the last five releases on GitHub or GitLab have a provenance file.
|
||||
|
||||
**Outcomes**: For each of the last 5 releases, the probe returns OutcomeTrue, if the release has a provenance file in the release assets.
|
||||
For each of the last 5 releases, the probe returns OutcomeFalse, if the release does not have a provenance file in the release assets.
|
||||
If the project has no releases, the probe returns OutcomeNotApplicable.
|
||||
|
||||
|
||||
## releasesHaveVerifiedProvenance
|
||||
|
||||
**Description**: Checks if the project releases with provenance attestations that have been verified
|
||||
|
||||
**Motivation**: Package provenance attestations provide a greater guarantee of authenticity and integrity than package signatures alone, since the attestation can be performed over a hash of both the package contents and metadata. Developers can attest to particular qualities of the build, such as the build environment, build steps or builder identity.
|
||||
|
||||
**Implementation**: This probe checks how many packages published by the repository are associated with verified SLSA provenance attestations. It uses data from a ProjectPackageClient, which associates a GitHub/GitLab project with a package in a package manager. Using the data from the package manager (whom we rely on to verify the provenance attestation), this probe returns a finding for each release. For now, only NPM is supported.
|
||||
|
||||
**Outcomes**: For each release, the probe returns OutcomeTrue or OutcomeFalse, depending on if the package has a verified provenance attestation.
|
||||
If we didn't find a package or didn't find releases, return OutcomeNotAvailable.
|
||||
|
||||
|
||||
## requiresApproversForPullRequests
|
||||
|
||||
**Description**: Check that the project requires approvers for pull requests.
|
||||
|
||||
**Motivation**: Requiring approvers for pull requests makes it harder to introduce vulnerable code to the project.
|
||||
|
||||
**Implementation**: The probe checks the number of required approvers in default and release branches of the project.
|
||||
|
||||
**Outcomes**: The probe returns one OutcomeTrue for each branch that requires approval for PRs, and one OutcomeFalse for branches that don't.
|
||||
|
||||
|
||||
## requiresCodeOwnersReview
|
||||
|
||||
**Description**: Check that the project requires dedicated code owners to review PRs.
|
||||
|
||||
**Motivation**: Code owners are expected to have deep knowledge about a code; Having experienced reviewers for PRs is expected to prevent security issues.
|
||||
|
||||
**Implementation**: The probe checks which branches require code owner reviews. The probe only considers default and release branches.
|
||||
|
||||
**Outcomes**: The probe returns one OutcomeTrue for each branch that requires code owner review for PRs, and one OutcomeFalse for branches that don't.
|
||||
|
||||
|
||||
## requiresLastPushApproval
|
||||
|
||||
**Description**: Check that the project requires approval of the most recent push.
|
||||
|
||||
**Motivation**: Requiring approval of the most recent push prevents contributors from sneaking malicious commits into a PR after it has been approved.
|
||||
|
||||
**Implementation**: The probe checks the protection rules of default and release branches branches.
|
||||
|
||||
**Outcomes**: The probe returns one OutcomeTrue for each branch that requires approval of the most recent push, and one OutcomeFalse for branches that don't.
|
||||
|
||||
|
||||
## requiresPRsToChangeCode
|
||||
|
||||
**Description**: Check that the project requires pull requests to change code.
|
||||
|
||||
**Motivation**: Changing code through pull requests promotes testing and reviews of the suggested change.
|
||||
|
||||
**Implementation**: The probe checks which branches require pull requests to change the branches' code. The probe only considers default and release branches.
|
||||
|
||||
**Outcomes**: The probe returns one OutcomeTrue for each branch that requires pull requests to change code, and one OutcomeFalse for branches that don't.
|
||||
|
||||
|
||||
## requiresUpToDateBranches
|
||||
|
||||
**Description**: Check that the project requires PRs to be in sync with the base branch.
|
||||
|
||||
**Motivation**: Requiring PRs to be in sync with the base branch is good practice.
|
||||
|
||||
**Implementation**: The probe checks the branch protection rules of default and release branches in the repository.
|
||||
|
||||
**Outcomes**: The probe returns one OutcomeTrue for each branch that requires PRs to be in sync with the base branch, and one OutcomeFalse for branches that don't.
|
||||
|
||||
|
||||
## runsStatusChecksBeforeMerging
|
||||
|
||||
**Description**: Check that the project runs required status checks
|
||||
|
||||
**Motivation**: Required status checks can check for common errors and resolve issues in PRs.
|
||||
|
||||
**Implementation**: The probe checks the rules for default and release branches in the projects repository.
|
||||
|
||||
**Outcomes**: The probe returns one OutcomeTrue for each branch that runs required status checks, and one OutcomeFalse for branches that don't.
|
||||
|
||||
|
||||
## sastToolConfigured
|
||||
|
||||
**Description**: Check that the project uses a SAST tool
|
||||
|
||||
**Motivation**: SAST is testing run on source code before the application is run. Using SAST tools can prevent known classes of bugs from being inadvertently introduced in the codebase.
|
||||
|
||||
**Implementation**: The implementation checks for evidence of various SAST tools. This includes configuration files, GitHub Action workflows, and GitHub PR check annotations.
|
||||
|
||||
**Outcomes**: If the project uses a SAST tool we can detect, the probe returns one finding per tool with OutcomeTrue.
|
||||
If the project does not use a SAST tool, or uses a tool we dont currently detect, the probe returns one finding with OutcomeFalse.
|
||||
|
||||
|
||||
## sastToolRunsOnAllCommits
|
||||
|
||||
**Description**: Checks that a SAST tool runs on all commits in the projects CI.
|
||||
|
||||
**Motivation**: SAST is testing run on source code before the application is run. Using SAST tools can prevent known classes of bugs from being inadvertently introduced in the codebase.
|
||||
|
||||
**Implementation**: The implementation iterates through the projects commits and checks whether any of the check runs for the commits associated merge request was any of the SAST tools that Scorecard supports.
|
||||
|
||||
**Outcomes**: If the project had no commits merged, the probe returns a finding with OutcomeNotApplicable.
|
||||
If the project runs SAST tools successfully on every pull request before merging, the probe returns one finding with OutcomeTrue (1). In addition, the finding will include two values. 1) How many commits were tested by a SAST tool, and 2) How many commits in total were merged.
|
||||
If the project does not run any SAST tools successfully on every pull request before merging, the probe returns one finding with OutcomeFalse (0). In addition, the finding will include two values. 1) How many commits were tested by a SAST tool, and 2) How many commits in total were merged.
|
||||
|
||||
|
||||
## securityPolicyContainsLinks
|
||||
|
||||
**Description**: Check that the security policy contains web or email links.
|
||||
|
||||
**Motivation**: URLs point users to additional information as well as online disclosure forms. Emails provide a point of contact for vulnerability disclosure.
|
||||
|
||||
**Implementation**: The implementation looks for strings "http(s)://" to find URLs; and for strings "...@..." for email addresses.
|
||||
|
||||
**Outcomes**: If links are found, one finding with OutcomeTrue is returned for each security policy file.
|
||||
If no links are found, one finding with OutcomeFalse is returned for each security policy file.
|
||||
If no security policy files are found, one finding with OutcomeFalse is returned.
|
||||
|
||||
|
||||
## securityPolicyContainsText
|
||||
|
||||
**Description**: Check that the security policy contains enough text and not just links.
|
||||
|
||||
**Motivation**: Telling security researchers how to privately disclose problems with your project is important. The more details available, the better.
|
||||
|
||||
**Implementation**: The implementation checks that the content of the SECURITY.md contains more than just a link or an email address. It does this by comparing the length of the content to the lengths of the links and email addresses.
|
||||
|
||||
**Outcomes**: If explanatory text is found in a security policy, one finding with OutcomeTrue is returned for each file.
|
||||
If no additional text is found in a security policy, one finding with OutcomeFalse is returned for each file.
|
||||
If no security policy is found, one finding with OutcomeFalse is returned.
|
||||
|
||||
|
||||
## securityPolicyContainsVulnerabilityDisclosure
|
||||
|
||||
**Description**: Check that the security policy indicates a vulnerability disclosure process.
|
||||
|
||||
**Motivation**: If someone finds a vulnerability in the project, it is important for them to be able to communicate it to the maintainers.
|
||||
|
||||
**Implementation**: The implementation looks for strings "Disclos" and "Vuln".
|
||||
|
||||
**Outcomes**: If information about the disclosure process is found in a security policy file, the probe returns one finding with OutcomeTrue for each file.
|
||||
If no information about the disclosure process is found, the probe returns one finding with OutcomeFalse for each file.
|
||||
If no security policy is found, the probe returns one finding with OutcomeFalse.
|
||||
|
||||
|
||||
## securityPolicyPresent
|
||||
|
||||
**Description**: Check if a security policy is defined in the repository or in the org's .github repository.
|
||||
|
||||
**Motivation**: A security policy (typically a SECURITY.md file) can give users information about what constitutes a vulnerability and how to report one securely so that information about a bug is not publicly visible. If you have a large organization, having a unified security policy across all your repositories may simplify the vulnerability disclosure response.
|
||||
|
||||
**Implementation**: The implementation looks for the presence of security policy files in the repository or in '<org>/.github' repository. See https://github.com/ossf/scorecard/blob/main/checks/raw/security_policy.go#L139 for a detailed list of filenames.
|
||||
|
||||
**Outcomes**: If a security policy file is found, one finding with OutcomeTrue is returned.
|
||||
If no security file is found, one finding with OutcomeFalse is returned.
|
||||
|
||||
|
||||
## testsRunInCI
|
||||
|
||||
**Description**: Checks that the project runs tests in the CI for example with GitHub Actions or Prow.
|
||||
|
||||
**Motivation**: Running tests helps developers catch mistakes early on, which can reduce the number of vulnerabilities that find their way into a project.
|
||||
|
||||
**Implementation**: The probe checks for tests in the projects CI jobs in the recent commits (~30).
|
||||
|
||||
**Outcomes**: The probe returns one OutcomeTrue for each PR that ran CI tests and one OutcomeFalse for each PR that did not run CI tests.
|
||||
The probe returns a single OutcomeNotApplicable if the projects has had no pull requests.
|
||||
|
||||
|
||||
## topLevelPermissions
|
||||
|
||||
**Description**: Checks that the project does not have any top-level write permissions in its workflows.
|
||||
|
||||
**Motivation**: In some circumstances, having "write" permissions at the "top" level may enable attackers to escalate privileges.
|
||||
|
||||
**Implementation**: The probe checks the permission level, the workflow type and the permission type of each workflow in the project.
|
||||
|
||||
**Outcomes**: The probe returns 1 false outcome per workflow with "write" permissions at the "top" level.
|
||||
The probe returns 1 true outcome if the project has no workflows "write" permissions a the "top" level.
|
||||
|
||||
|
||||
## webhooksUseSecrets
|
||||
|
||||
**Description**: This check determines whether the webhooks defined in the repository have secrets configured to authenticate the origins of requests.
|
||||
|
||||
**Motivation**: Webhooks without secret authorization have the potential to make projects accessible to third-parties.
|
||||
|
||||
**Implementation**: The probe checks all webhooks of a project and checks whether each uses secret authentication.
|
||||
|
||||
**Outcomes**: The probe returns one OutcomeTrue per webhook with secret authorization.
|
||||
The probe returns one OutcomeFalse per webhook without secret authorization.
|
||||
Projects without webhooks receive an OutcomeNotApplicable.
|
||||
|
||||
89
docs/probes/internal/generate/main.go
Normal file
89
docs/probes/internal/generate/main.go
Normal file
@ -0,0 +1,89 @@
|
||||
// Copyright 2024 OpenSSF Scorecard Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
pyaml "github.com/ossf/scorecard/v5/internal/probes/yaml"
|
||||
)
|
||||
|
||||
func printField(w io.Writer, name string, value any) {
|
||||
// some fields have extra newlines we can get rid of
|
||||
if v, ok := value.(string); ok {
|
||||
value = strings.TrimSpace(v)
|
||||
}
|
||||
|
||||
fmt.Fprint(w, "**", name, "**: ", value, "\n\n")
|
||||
}
|
||||
|
||||
func printProbe(w io.Writer, p *pyaml.Probe) {
|
||||
// short, motivation, implementation, outcome, remediation, ecosystem
|
||||
fmt.Fprint(w, "\n"+"## "+p.ID+"\n\n")
|
||||
printField(w, "Description", p.Short)
|
||||
printField(w, "Motivation", p.Motivation)
|
||||
printField(w, "Implementation", p.Implementation)
|
||||
printField(w, "Outcomes", "\n\n"+strings.Join(p.Outcomes, "\n"))
|
||||
// TODO remediation
|
||||
// TODO ecosystem
|
||||
}
|
||||
|
||||
func walk(path string, d fs.DirEntry, err error) error {
|
||||
// no special handling of errors, we can stop now
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !strings.EqualFold(filepath.Base(path), "def.yml") {
|
||||
return nil
|
||||
}
|
||||
var probe pyaml.Probe
|
||||
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read probe definition: %w", err)
|
||||
}
|
||||
err = yaml.Unmarshal(content, &probe)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse yaml: %w", err)
|
||||
}
|
||||
printProbe(os.Stdout, &probe)
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 2 {
|
||||
log.Fatalf("usage: %s <probe directory>", os.Args[0])
|
||||
}
|
||||
probeDir := os.Args[1]
|
||||
|
||||
fmt.Fprint(os.Stdout, `<!-- Do not edit this file manually! Edit the individual probe's def.yml instead. -->
|
||||
# Probe Documentation
|
||||
|
||||
This page describes each Scorecard probe in detail, including description, motivation,
|
||||
and outcomes. If you have ideas for additions or new detection techniques,
|
||||
please [contribute](../CONTRIBUTING.md)!
|
||||
`)
|
||||
if err := filepath.WalkDir(probeDir, walk); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
@ -22,6 +22,7 @@ import (
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/ossf/scorecard/v5/clients"
|
||||
pyaml "github.com/ossf/scorecard/v5/internal/probes/yaml"
|
||||
)
|
||||
|
||||
// RemediationEffort indicates the estimated effort necessary to remediate a finding.
|
||||
@ -50,33 +51,12 @@ type Remediation struct {
|
||||
Effort RemediationEffort `json:"effort"`
|
||||
}
|
||||
|
||||
type yamlRemediation struct {
|
||||
OnOutcome Outcome `yaml:"onOutcome"`
|
||||
Text []string `yaml:"text"`
|
||||
Markdown []string `yaml:"markdown"`
|
||||
Effort RemediationEffort `yaml:"effort"`
|
||||
}
|
||||
|
||||
type yamlEcosystem struct {
|
||||
Languages []string `yaml:"languages"`
|
||||
Clients []string `yaml:"clients"`
|
||||
}
|
||||
|
||||
var supportedClients = map[string]bool{
|
||||
"github": true,
|
||||
"gitlab": true,
|
||||
"localdir": true,
|
||||
}
|
||||
|
||||
type yamlProbe struct {
|
||||
ID string `yaml:"id"`
|
||||
Short string `yaml:"short"`
|
||||
Motivation string `yaml:"motivation"`
|
||||
Implementation string `yaml:"implementation"`
|
||||
Ecosystem yamlEcosystem `yaml:"ecosystem"`
|
||||
Remediation yamlRemediation `yaml:"remediation"`
|
||||
}
|
||||
|
||||
type probe struct {
|
||||
ID string
|
||||
Short string
|
||||
@ -104,9 +84,9 @@ func probeFromBytes(content []byte, probeID string) (*probe, error) {
|
||||
Remediation: &Remediation{
|
||||
Text: strings.Join(r.Remediation.Text, "\n"),
|
||||
Markdown: strings.Join(r.Remediation.Markdown, "\n"),
|
||||
Effort: r.Remediation.Effort,
|
||||
Effort: toRemediationEffort(r.Remediation.Effort),
|
||||
},
|
||||
RemediateOnOutcome: r.Remediation.OnOutcome,
|
||||
RemediateOnOutcome: Outcome(r.Remediation.OnOutcome),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@ -119,11 +99,11 @@ func newProbe(loc embed.FS, probeID string) (*probe, error) {
|
||||
return probeFromBytes(content, probeID)
|
||||
}
|
||||
|
||||
func validate(r *yamlProbe, probeID string) error {
|
||||
func validate(r *pyaml.Probe, probeID string) error {
|
||||
if err := validateID(r.ID, probeID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateRemediation(r.Remediation); err != nil {
|
||||
if err := validateRemediation(&r.Remediation); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateEcosystem(r.Ecosystem); err != nil {
|
||||
@ -140,11 +120,11 @@ func validateID(actual, expected string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateRemediation(r yamlRemediation) error {
|
||||
if err := validateRemediationOutcomeTrigger(r.OnOutcome); err != nil {
|
||||
func validateRemediation(r *pyaml.Remediation) error {
|
||||
if err := validateRemediationOutcomeTrigger(Outcome(r.OnOutcome)); err != nil {
|
||||
return fmt.Errorf("remediation: %w", err)
|
||||
}
|
||||
switch r.Effort {
|
||||
switch toRemediationEffort(r.Effort) {
|
||||
case RemediationEffortHigh, RemediationEffortMedium, RemediationEffortLow:
|
||||
return nil
|
||||
default:
|
||||
@ -152,7 +132,7 @@ func validateRemediation(r yamlRemediation) error {
|
||||
}
|
||||
}
|
||||
|
||||
func validateEcosystem(r yamlEcosystem) error {
|
||||
func validateEcosystem(r pyaml.Ecosystem) error {
|
||||
if err := validateSupportedLanguages(r); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -171,7 +151,7 @@ func validateRemediationOutcomeTrigger(o Outcome) error {
|
||||
}
|
||||
}
|
||||
|
||||
func validateSupportedLanguages(r yamlEcosystem) error {
|
||||
func validateSupportedLanguages(r pyaml.Ecosystem) error {
|
||||
for _, lang := range r.Languages {
|
||||
switch clients.LanguageName(lang) {
|
||||
case clients.Go, clients.Python, clients.JavaScript,
|
||||
@ -189,7 +169,7 @@ func validateSupportedLanguages(r yamlEcosystem) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateSupportedClients(r yamlEcosystem) error {
|
||||
func validateSupportedClients(r pyaml.Ecosystem) error {
|
||||
for _, lang := range r.Clients {
|
||||
if _, ok := supportedClients[lang]; !ok {
|
||||
return fmt.Errorf("%w: %v", errInvalid, fmt.Sprintf("client '%v'", r))
|
||||
@ -198,8 +178,8 @@ func validateSupportedClients(r yamlEcosystem) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseFromYAML(content []byte) (*yamlProbe, error) {
|
||||
r := yamlProbe{}
|
||||
func parseFromYAML(content []byte) (*pyaml.Probe, error) {
|
||||
r := pyaml.Probe{}
|
||||
|
||||
err := yaml.Unmarshal(content, &r)
|
||||
if err != nil {
|
||||
@ -216,16 +196,11 @@ func (r *RemediationEffort) UnmarshalYAML(n *yaml.Node) error {
|
||||
return fmt.Errorf("%w: %w", errInvalid, err)
|
||||
}
|
||||
|
||||
switch n.Value {
|
||||
case "Low":
|
||||
*r = RemediationEffortLow
|
||||
case "Medium":
|
||||
*r = RemediationEffortMedium
|
||||
case "High":
|
||||
*r = RemediationEffortHigh
|
||||
default:
|
||||
*r = toRemediationEffort(n.Value)
|
||||
if *r == RemediationEffortNone {
|
||||
return fmt.Errorf("%w: effort:%q", errInvalid, str)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -242,3 +217,16 @@ func (r *RemediationEffort) String() string {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func toRemediationEffort(s string) RemediationEffort {
|
||||
switch s {
|
||||
case "Low":
|
||||
return RemediationEffortLow
|
||||
case "Medium":
|
||||
return RemediationEffortMedium
|
||||
case "High":
|
||||
return RemediationEffortHigh
|
||||
default:
|
||||
return RemediationEffortNone
|
||||
}
|
||||
}
|
||||
|
||||
37
internal/probes/yaml/yaml.go
Normal file
37
internal/probes/yaml/yaml.go
Normal file
@ -0,0 +1,37 @@
|
||||
// Copyright 2024 OpenSSF Scorecard Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package yaml
|
||||
|
||||
type Remediation struct {
|
||||
OnOutcome string `yaml:"onOutcome"`
|
||||
Effort string `yaml:"effort"`
|
||||
Text []string `yaml:"text"`
|
||||
Markdown []string `yaml:"markdown"`
|
||||
}
|
||||
|
||||
type Ecosystem struct {
|
||||
Languages []string `yaml:"languages"`
|
||||
Clients []string `yaml:"clients"`
|
||||
}
|
||||
|
||||
type Probe struct {
|
||||
Remediation Remediation `yaml:"remediation"`
|
||||
ID string `yaml:"id"`
|
||||
Short string `yaml:"short"`
|
||||
Motivation string `yaml:"motivation"`
|
||||
Implementation string `yaml:"implementation"`
|
||||
Ecosystem Ecosystem `yaml:"ecosystem"`
|
||||
Outcomes []string `yaml:"outcome"`
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user