From f9f910d43742e929349dcbc7629dfc2437b71f94 Mon Sep 17 00:00:00 2001 From: Latortuga <42878263+latortuga71@users.noreply.github.com> Date: Tue, 22 Nov 2022 11:11:36 -0500 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Commit=20depth=20feature=20(#2407)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * :seedling: Bump actions/dependency-review-action from 2.4.1 to 2.5.1 Bumps [actions/dependency-review-action](https://github.com/actions/dependency-review-action) from 2.4.1 to 2.5.1. - [Release notes](https://github.com/actions/dependency-review-action/releases) - [Commits](https://github.com/actions/dependency-review-action/compare/9c96258789e5d9e85fe4ca86115ba4cc62b780cf...0efb1d1d84fc9633afcdaad14c485cbbc90ef46c) --- updated-dependencies: - dependency-name: actions/dependency-review-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: latortuga71 * commit_depth feature Signed-off-by: latortuga71 * added more descriptive comments, changed numberofcommits variable name, moved paging for commits into seperate function. small changes Signed-off-by: latortuga71 linter Signed-off-by: latortuga71 * added unit tests Signed-off-by: latortuga71 added test in e2e Signed-off-by: latortuga71 * :seedling: Bump github.com/spf13/cobra from 1.6.0 to 1.6.1 (#2397) Bumps [github.com/spf13/cobra](https://github.com/spf13/cobra) from 1.6.0 to 1.6.1. - [Release notes](https://github.com/spf13/cobra/releases) - [Commits](https://github.com/spf13/cobra/compare/v1.6.0...v1.6.1) --- updated-dependencies: - dependency-name: github.com/spf13/cobra dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: latortuga71 * :seedling: Bump github.com/onsi/ginkgo/v2 from 2.1.6 to 2.4.0 Bumps [github.com/onsi/ginkgo/v2](https://github.com/onsi/ginkgo) from 2.1.6 to 2.4.0. - [Release notes](https://github.com/onsi/ginkgo/releases) - [Changelog](https://github.com/onsi/ginkgo/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/ginkgo/compare/v2.1.6...v2.4.0) --- updated-dependencies: - dependency-name: github.com/onsi/ginkgo/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: latortuga71 * :seedling: Bump cloud.google.com/go/pubsub from 1.25.1 to 1.26.0 Bumps [cloud.google.com/go/pubsub](https://github.com/googleapis/google-cloud-go) from 1.25.1 to 1.26.0. - [Release notes](https://github.com/googleapis/google-cloud-go/releases) - [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-cloud-go/compare/pubsub/v1.25.1...pubsub/v1.26.0) --- updated-dependencies: - dependency-name: cloud.google.com/go/pubsub dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: latortuga71 * :seedling: Bump github.com/xanzy/go-gitlab from 0.73.1 to 0.74.0 Bumps [github.com/xanzy/go-gitlab](https://github.com/xanzy/go-gitlab) from 0.73.1 to 0.74.0. - [Release notes](https://github.com/xanzy/go-gitlab/releases) - [Changelog](https://github.com/xanzy/go-gitlab/blob/master/releases_test.go) - [Commits](https://github.com/xanzy/go-gitlab/compare/v0.73.1...v0.74.0) --- updated-dependencies: - dependency-name: github.com/xanzy/go-gitlab dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: latortuga71 * :seedling: Bump github.com/onsi/gomega from 1.20.2 to 1.23.0 (#2409) Bumps [github.com/onsi/gomega](https://github.com/onsi/gomega) from 1.20.2 to 1.23.0. - [Release notes](https://github.com/onsi/gomega/releases) - [Changelog](https://github.com/onsi/gomega/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/gomega/compare/v1.20.2...v1.23.0) --- updated-dependencies: - dependency-name: github.com/onsi/gomega dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: latortuga71 * :seedling: Bump github.com/onsi/ginkgo/v2 from 2.1.6 to 2.4.0 in /tools Bumps [github.com/onsi/ginkgo/v2](https://github.com/onsi/ginkgo) from 2.1.6 to 2.4.0. - [Release notes](https://github.com/onsi/ginkgo/releases) - [Changelog](https://github.com/onsi/ginkgo/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/ginkgo/compare/v2.1.6...v2.4.0) --- updated-dependencies: - dependency-name: github.com/onsi/ginkgo/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: latortuga71 * :seedling: Bump github.com/golangci/golangci-lint in /tools Bumps [github.com/golangci/golangci-lint](https://github.com/golangci/golangci-lint) from 1.50.0 to 1.50.1. - [Release notes](https://github.com/golangci/golangci-lint/releases) - [Changelog](https://github.com/golangci/golangci-lint/blob/master/CHANGELOG.md) - [Commits](https://github.com/golangci/golangci-lint/compare/v1.50.0...v1.50.1) --- updated-dependencies: - dependency-name: github.com/golangci/golangci-lint dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: latortuga71 * :seedling: Bump goreleaser/goreleaser-action from 2.9.1 to 3.2.0 (#2363) Bumps [goreleaser/goreleaser-action](https://github.com/goreleaser/goreleaser-action) from 2.9.1 to 3.2.0. - [Release notes](https://github.com/goreleaser/goreleaser-action/releases) - [Commits](https://github.com/goreleaser/goreleaser-action/compare/b953231f81b8dfd023c58e0854a721e35037f28b...b508e2e3ef3b19d4e4146d4f8fb3ba9db644a757) --- updated-dependencies: - dependency-name: goreleaser/goreleaser-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: latortuga71 * :seedling: Bump github.com/goreleaser/goreleaser in /tools (#2373) Bumps [github.com/goreleaser/goreleaser](https://github.com/goreleaser/goreleaser) from 1.11.5 to 1.12.3. - [Release notes](https://github.com/goreleaser/goreleaser/releases) - [Changelog](https://github.com/goreleaser/goreleaser/blob/main/.goreleaser.yaml) - [Commits](https://github.com/goreleaser/goreleaser/compare/v1.11.5...v1.12.3) --- updated-dependencies: - dependency-name: github.com/goreleaser/goreleaser dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: latortuga71 * ✨ CLI for scorecard-attestor (#2309) * Reorganize Signed-off-by: Raghav Kaul * Working commit Signed-off-by: Raghav Kaul * Compile with local scorecard; go mod tidy Signed-off-by: Raghav Kaul * Add signing code Heavily borrowed from https://github.com/grafeas/kritis/blob/master/cmd/kritis/signer/main.go Signed-off-by: Raghav Kaul * Update deps * Naming * Makefile Signed-off-by: Raghav Kaul * Edit license, add lint.yml Signed-off-by: Raghav Kaul * checks: go mod tidy, license Signed-off-by: Raghav Kaul * Address PR comments * Split into checker/signer files * Naming convention Signed-off-by: Raghav Kaul * License, remove golangci.yml Signed-off-by: Raghav Kaul * Address PR comments * Use cobra Signed-off-by: Raghav Kaul * Add tests for root command Signed-off-by: Raghav Kaul * Filter out checks that aren't needed for policy evaluation Signed-off-by: Raghav Kaul * Add `make` targets for attestor; submit coverage stats Signed-off-by: Raghav Kaul * Improvements * Use sclog instead of glog * Remove unneeded subcommands * Formatting Signed-off-by: Raghav Kaul * Flags: Make note-name constant and fix messaging Signed-off-by: Raghav Kaul * Remove SupportedRequestTypes Signed-off-by: Raghav Kaul * go mod tidy Signed-off-by: Raghav Kaul * go mod tidy, makefile Signed-off-by: Raghav Kaul * Fix GH actions run Signed-off-by: Raghav Kaul Signed-off-by: Raghav Kaul Signed-off-by: latortuga71 * fix workflow (#2417) Signed-off-by: Spencer Schrock Signed-off-by: Spencer Schrock Signed-off-by: latortuga71 * Bump scorecard-action (#2416) Signed-off-by: Spencer Schrock Signed-off-by: Spencer Schrock Signed-off-by: latortuga71 * Fail unit-test job if codecov upload fails (#2415) Signed-off-by: Spencer Schrock Signed-off-by: Spencer Schrock Signed-off-by: latortuga71 * 🌱 Enable comparison for alternative isText implementation (#2414) * use more performant IsText Signed-off-by: Spencer Schrock * AB test isText implementations Signed-off-by: Spencer Schrock * Add comparison env var to release test. Signed-off-by: Spencer Schrock * go mod tidy for attestor Signed-off-by: Spencer Schrock Signed-off-by: Spencer Schrock Signed-off-by: latortuga71 * 🐛 modify alternative isText to accept carriage returns (#2421) * modify IsText from golang.org/x/tools/godoc/util to accept carriage returns. Signed-off-by: Spencer Schrock * add TODO reminder to cleanup after release tests Signed-off-by: Spencer Schrock Signed-off-by: Spencer Schrock Signed-off-by: latortuga71 * :seedling: Bump github.com/onsi/gomega from 1.23.0 to 1.24.0 Bumps [github.com/onsi/gomega](https://github.com/onsi/gomega) from 1.23.0 to 1.24.0. - [Release notes](https://github.com/onsi/gomega/releases) - [Changelog](https://github.com/onsi/gomega/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/gomega/compare/v1.23.0...v1.24.0) --- updated-dependencies: - dependency-name: github.com/onsi/gomega dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: latortuga71 * :seedling: Bump github/codeql-action from 2.1.29 to 2.1.30 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.1.29 to 2.1.30. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6...18fe527fa8b29f134bb91f32f1a5dc5abb15ed7f) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: latortuga71 * revert failing unit-test on ci error (#2422) Signed-off-by: Spencer Schrock Signed-off-by: Spencer Schrock Signed-off-by: latortuga71 * :sparkles: Improved Security Policy Check (#2195) * :sparkles: Improved Security Policy Check (#2137) * Examines and awards points for linked content (URLs / Emails) * Examines and awards points for hints of disclosure and vulnerability practices * Examines and awards points for hints of elaboration of timelines Signed-off-by: Scott Hissam * Repaired Security Policy to correctly use linked content length for evaluation Signed-off-by: Scott Hissam * gofmt'ed changes Signed-off-by: Scott Hissam * Repaired the case in the evaluation which was too sensitive to content length over the length of the linked content for urls and emails Signed-off-by: Scott Hissam * added unit test cases for the new content-based Security Policy checks Signed-off-by: Scott Hissam * reverted the direct (mistaken) change to checks.md and updated the checks.yaml for generate-docs Signed-off-by: Scott Hissam * :sparkles: Improved Security Policy Check (#2137) (revisted based on comments) * replaced reason strings with log.Info & log.Warn (as seen in --show-details) * internal assertion check for nil (*pinfo) and empty pfile * internal switched to FileTypeText over FileTypeSource * internal implement type SecurityPolicyInformationType/SecurityPolicyInformation revised SecurityPolicyData to support only one file * revised expected unit-test results and revised unit-test to reflect the new SecurityPolicyData type Signed-off-by: Scott Hissam * revised the score value based on observation of one *or more* url(s) or one email(s) found; unit tests update accordingly Signed-off-by: Scott Hissam * revised the score value based on observation of one *or more* url(s) or one email(s) found; unit tests update accordingly Signed-off-by: Scott Hissam * revised the score value based on observation of one *or more* url(s) or one email(s) found; e2e tests update accordingly Signed-off-by: Scott Hissam * Addressed PR comments; added telemetry for policy hits in security policy file to track hits by line number Signed-off-by: Scott Hissam * Resolved merge conflict with checks.yaml Signed-off-by: Scott Hissam * updated raw results to emit all the raw information for the new security policy check Signed-off-by: Scott Hissam * Resolved merge conflicts and lint errors with json_raw_results.go Signed-off-by: Scott Hissam * Addressed review comments to reorganize security policy data struct to support the potential for multiple security policy files. Signed-off-by: Scott Hissam * Added logic to the security policy to process multiple security policy files only after future improvements to aggregating scoring across such files are designed. For now the security policy behaves as originally designed to stop once one of the expected policy files are found in the repo Signed-off-by: Scott Hissam * added comments regarding the capacity to support multiple policy files and removed unneeded break statements in the code Signed-off-by: Scott Hissam * Addressed review comments to remove the dependency on the path in the filename from the code and introduced FileSize to checker.File type and removed the SecurityContentLength which was used to hold that information for the new security policy assessment Signed-off-by: Scott Hissam * restored reporting full security policy path and filename for policies found in the org level repos Signed-off-by: Scott Hissam * Resolved conflicts in checks.yaml for documentation Signed-off-by: Scott Hissam * ✨ CLI for scorecard-attestor (#2309) * Reorganize Signed-off-by: Raghav Kaul * Working commit Signed-off-by: Raghav Kaul * Compile with local scorecard; go mod tidy Signed-off-by: Raghav Kaul * Add signing code Heavily borrowed from https://github.com/grafeas/kritis/blob/master/cmd/kritis/signer/main.go Signed-off-by: Raghav Kaul * Update deps * Naming * Makefile Signed-off-by: Raghav Kaul * Edit license, add lint.yml Signed-off-by: Raghav Kaul * checks: go mod tidy, license Signed-off-by: Raghav Kaul * Address PR comments * Split into checker/signer files * Naming convention Signed-off-by: Raghav Kaul * License, remove golangci.yml Signed-off-by: Raghav Kaul * Address PR comments * Use cobra Signed-off-by: Raghav Kaul * Add tests for root command Signed-off-by: Raghav Kaul * Filter out checks that aren't needed for policy evaluation Signed-off-by: Raghav Kaul * Add `make` targets for attestor; submit coverage stats Signed-off-by: Raghav Kaul * Improvements * Use sclog instead of glog * Remove unneeded subcommands * Formatting Signed-off-by: Raghav Kaul * Flags: Make note-name constant and fix messaging Signed-off-by: Raghav Kaul * Remove SupportedRequestTypes Signed-off-by: Raghav Kaul * go mod tidy Signed-off-by: Raghav Kaul * go mod tidy, makefile Signed-off-by: Raghav Kaul * Fix GH actions run Signed-off-by: Raghav Kaul Signed-off-by: Raghav Kaul Signed-off-by: Scott Hissam * removed whitespace before stanza for Run attestor e2e Signed-off-by: Scott Hissam * resolved code review and doc review comments Signed-off-by: Scott Hissam * repaired the link for the maintainer's guide for supporting the coordinated vulnerability disclosure guidelines Signed-off-by: Scott Hissam Signed-off-by: Scott Hissam Signed-off-by: latortuga71 * :seedling: Bump github/codeql-action from 2.1.30 to 2.1.31 (#2431) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.1.30 to 2.1.31. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/18fe527fa8b29f134bb91f32f1a5dc5abb15ed7f...c3b6fce4ee2ca25bc1066aa3bf73962fda0e8898) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: latortuga71 * enable more performant isText (#2433) Signed-off-by: Spencer Schrock Signed-off-by: Spencer Schrock Signed-off-by: latortuga71 * modified tests,InitRepo Function, Added GetCommitDepth Function to Client Interface Signed-off-by: latortuga71 * removed getcommitdepth function Signed-off-by: latortuga71 * added TODO Signed-off-by: latortuga71 * :seedling: Bump github.com/onsi/ginkgo/v2 from 2.4.0 to 2.5.0 in /tools (#2436) Bumps [github.com/onsi/ginkgo/v2](https://github.com/onsi/ginkgo) from 2.4.0 to 2.5.0. - [Release notes](https://github.com/onsi/ginkgo/releases) - [Changelog](https://github.com/onsi/ginkgo/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/ginkgo/compare/v2.4.0...v2.5.0) --- updated-dependencies: - dependency-name: github.com/onsi/ginkgo/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: latortuga71 * :seedling: Bump github.com/onsi/ginkgo/v2 from 2.4.0 to 2.5.0 Bumps [github.com/onsi/ginkgo/v2](https://github.com/onsi/ginkgo) from 2.4.0 to 2.5.0. - [Release notes](https://github.com/onsi/ginkgo/releases) - [Changelog](https://github.com/onsi/ginkgo/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/ginkgo/compare/v2.4.0...v2.5.0) --- updated-dependencies: - dependency-name: github.com/onsi/ginkgo/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: latortuga71 * 🌱 Code Review: treat merging a PR as code review (#2413) * Merges on Github count as a code review by the maintainer Signed-off-by: Raghav Kaul * Update Raw Results * More detailed information for Changesets * If there's no Revision ID, use the Commit SHA instead Signed-off-by: Raghav Kaul * Check that pull request had atleast one reviewer that wasn't its author * Add field for Pull Request Merged-By to Github and Gitlab * Note, this check can be bypassed if an author opens a PR with other people's commits Signed-off-by: Raghav Kaul Signed-off-by: Raghav Kaul Signed-off-by: latortuga71 * Trivial: Fix typo (exepted -> expected) (#2440) Signed-off-by: Michael Scovetta Signed-off-by: Michael Scovetta Signed-off-by: latortuga71 * :seedling: Bump step-security/harden-runner from 1.5.0 to 2.0.0 (#2443) Bumps [step-security/harden-runner](https://github.com/step-security/harden-runner) from 1.5.0 to 2.0.0. - [Release notes](https://github.com/step-security/harden-runner/releases) - [Commits](https://github.com/step-security/harden-runner/compare/2e205a28d0e1da00c5f53b161f4067b052c61f34...ebacdc22ef6c2cfb85ee5ded8f2e640f4c776dd5) --- updated-dependencies: - dependency-name: step-security/harden-runner dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: latortuga71 * 🌱 cron: support reading prefix from file for controller input files (7/n) (#2445) * add prefix marker file to config Signed-off-by: Spencer Schrock * Read the new config values, if they exist. Signed-off-by: Spencer Schrock * Add function to fetch prefix file config value. Signed-off-by: Spencer Schrock * Read prefix file if prefix not set. Signed-off-by: Spencer Schrock * Add tests to verify how List works with various prefixes Signed-off-by: Spencer Schrock * Add tests for getPrefix Signed-off-by: Spencer Schrock * Remove panics from iterator helper functions Signed-off-by: Spencer Schrock Signed-off-by: Spencer Schrock Signed-off-by: latortuga71 * Detect SECURITY.markdown in addition to SECURITY.md (#2447) GitHub probably supports many more file extensions for Markdown files, but at the very least, `.md` and `.markdown` have been standardized in RFC 7763. Signed-off-by: favonia Signed-off-by: favonia Signed-off-by: latortuga71 * Add Pinned-Dependency, Vulnerability, and Code-Review checks to attestor (#2430) Signed-off-by: Raghav Kaul Signed-off-by: Raghav Kaul Signed-off-by: latortuga71 * :seedling: cron: expose the stackdriver prefix as a config variable so it can be changed. (#2446) * Expose the stackdriver prefix as a config variable so it can be changed. Signed-off-by: Caleb Brown * fix linter warning Signed-off-by: Caleb Brown Signed-off-by: Caleb Brown Co-authored-by: Spencer Schrock Signed-off-by: latortuga71 * Only write to the rawBucket if the value exists. (#2451) Signed-off-by: Caleb Brown Signed-off-by: Caleb Brown Signed-off-by: latortuga71 * :seedling: Bump golang.org/x/tools from 0.2.0 to 0.3.0 (#2448) * :seedling: Bump golang.org/x/tools from 0.2.0 to 0.3.0 Bumps [golang.org/x/tools](https://github.com/golang/tools) from 0.2.0 to 0.3.0. - [Release notes](https://github.com/golang/tools/releases) - [Commits](https://github.com/golang/tools/compare/v0.2.0...v0.3.0) --- updated-dependencies: - dependency-name: golang.org/x/tools dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * bump attestor modules Signed-off-by: Spencer Schrock Signed-off-by: dependabot[bot] Signed-off-by: Spencer Schrock Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Spencer Schrock Signed-off-by: latortuga71 * Move cron monitoring to a non-internal location. (#2453) This allows external workers (e.g. criticality_score) to use the same monitoring code. Signed-off-by: Caleb Brown Signed-off-by: Caleb Brown Signed-off-by: latortuga71 * :seedling: Bump actions/dependency-review-action from 2.5.1 to 3.0.0 (#2455) Bumps [actions/dependency-review-action](https://github.com/actions/dependency-review-action) from 2.5.1 to 3.0.0. - [Release notes](https://github.com/actions/dependency-review-action/releases) - [Commits](https://github.com/actions/dependency-review-action/compare/0efb1d1d84fc9633afcdaad14c485cbbc90ef46c...30d582111533d59ab793fd9f971817241654f3ec) --- updated-dependencies: - dependency-name: actions/dependency-review-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: latortuga71 * 🌱 [cron] generalize some of the transfer logic so it is easy to build new transfer agents (#2454) * Generalize the transfer logic so it is easy to build new transfer agents This change moves code that reads shards and produces summaries into the data package so that it can be reused to create new transfer agents, similar to the BigQuery transfer agent in cron/internal/bq. Signed-off-by: Caleb Brown * Lint fix and commentary. Signed-off-by: Caleb Brown Signed-off-by: Caleb Brown Signed-off-by: latortuga71 * :seedling: Bump github.com/google/addlicense in /tools (#2459) Bumps [github.com/google/addlicense](https://github.com/google/addlicense) from 1.0.0 to 1.1.0. - [Release notes](https://github.com/google/addlicense/releases) - [Changelog](https://github.com/google/addlicense/blob/master/.goreleaser.yaml) - [Commits](https://github.com/google/addlicense/compare/v1.0.0...v1.1.0) --- updated-dependencies: - dependency-name: github.com/google/addlicense dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: latortuga71 * :seedling: Bump github.com/google/go-containerregistry Bumps [github.com/google/go-containerregistry](https://github.com/google/go-containerregistry) from 0.12.0 to 0.12.1. - [Release notes](https://github.com/google/go-containerregistry/releases) - [Changelog](https://github.com/google/go-containerregistry/blob/main/.goreleaser.yml) - [Commits](https://github.com/google/go-containerregistry/compare/v0.12.0...v0.12.1) --- updated-dependencies: - dependency-name: github.com/google/go-containerregistry dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: latortuga71 * go mod tidy Signed-off-by: Spencer Schrock Signed-off-by: latortuga71 * Added <= instead of == incase negative int is passed Signed-off-by: latortuga71 * missed test fix Signed-off-by: latortuga71 Signed-off-by: dependabot[bot] Signed-off-by: latortuga71 Signed-off-by: Raghav Kaul Signed-off-by: Spencer Schrock Signed-off-by: Scott Hissam Signed-off-by: Michael Scovetta Signed-off-by: favonia Signed-off-by: Caleb Brown Signed-off-by: Latortuga <42878263+latortuga71@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: raghavkaul <8695110+raghavkaul@users.noreply.github.com> Co-authored-by: Spencer Schrock Co-authored-by: scott hissam Co-authored-by: Michael Scovetta Co-authored-by: favonia Co-authored-by: Caleb Brown --- .github/workflows/codeql-analysis.yml | 1 + Makefile | 1 + attestor/command/check.go | 1 + attestor/e2e/command_test.go | 1 + checks/binary_artifact_test.go | 2 +- checks/license_test.go | 2 +- checks/raw/security_policy.go | 3 +- clients/githubrepo/client.go | 14 +++- clients/githubrepo/graphql.go | 62 +++++++++++--- clients/githubrepo/graphql_e2e_test.go | 112 ++++++++++++++++++++++++- clients/gitlabrepo/client.go | 10 ++- clients/localdir/client.go | 21 +++-- clients/localdir/client_test.go | 2 +- clients/mockclients/repo_client.go | 8 +- clients/repo_client.go | 2 +- cmd/root.go | 4 +- cmd/serve.go | 2 +- cron/internal/worker/main.go | 2 +- dependencydiff/dependencydiff.go | 4 +- e2e/binary_artifacts_test.go | 12 +-- e2e/branch_protection_test.go | 8 +- e2e/ci_tests_test.go | 6 +- e2e/code_review_test.go | 6 +- e2e/contributors_test.go | 2 +- e2e/dangerous_workflow_test.go | 6 +- e2e/dependency_update_tool_test.go | 4 +- e2e/fuzzing_test.go | 10 +-- e2e/license_test.go | 6 +- e2e/maintained_test.go | 2 +- e2e/packaging_test.go | 2 +- e2e/permissions_test.go | 6 +- e2e/pinned_dependencies_test.go | 6 +- e2e/sast_test.go | 2 +- e2e/security_policy_test.go | 10 +-- e2e/signedreleases_test.go | 2 +- e2e/vulnerabilities_test.go | 6 +- options/flags.go | 9 ++ options/options.go | 6 +- pkg/scorecard.go | 3 +- pkg/scorecard_test.go | 7 +- 40 files changed, 275 insertions(+), 100 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 15a2f73f..729dedd0 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -61,6 +61,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL + uses: github/codeql-action/init@678fc3afe258fb2e0cdc165ccf77b85719de7b3c # v1 with: languages: ${{ matrix.language }} diff --git a/Makefile b/Makefile index d1dcddeb..04675651 100644 --- a/Makefile +++ b/Makefile @@ -243,6 +243,7 @@ build-attestor: ## Runs go build on scorecard attestor # Run go build on scorecard attestor cd attestor/; CGO_ENABLED=0 go build -trimpath -a -tags netgo -ldflags '$(LDFLAGS)' -o scorecard-attestor + build-attestor-docker: ## Build scorecard-attestor Docker image build-attestor-docker: DOCKER_BUILDKIT=1 docker build . --file attestor/Dockerfile \ diff --git a/attestor/command/check.go b/attestor/command/check.go index 7185c459..c8c234f0 100644 --- a/attestor/command/check.go +++ b/attestor/command/check.go @@ -97,6 +97,7 @@ func runCheck() (policy.PolicyResult, error) { ctx, repo, commitSHA, + 0, enabledChecks, repoClient, ossFuzzRepoClient, diff --git a/attestor/e2e/command_test.go b/attestor/e2e/command_test.go index 1be27a2a..e90fc311 100644 --- a/attestor/e2e/command_test.go +++ b/attestor/e2e/command_test.go @@ -18,6 +18,7 @@ import ( "strings" "testing" + "github.com/spf13/cobra" "github.com/ossf/scorecard-attestor/command" diff --git a/checks/binary_artifact_test.go b/checks/binary_artifact_test.go index d3697945..94b4c8ce 100644 --- a/checks/binary_artifact_test.go +++ b/checks/binary_artifact_test.go @@ -71,7 +71,7 @@ func TestBinaryArtifacts(t *testing.T) { ctx := context.Background() client := localdir.CreateLocalDirClient(ctx, logger) - if err := client.InitRepo(repo, clients.HeadSHA); err != nil { + if err := client.InitRepo(repo, clients.HeadSHA, 0); err != nil { t.Errorf("InitRepo: %v", err) } diff --git a/checks/license_test.go b/checks/license_test.go index 450b8626..bb2b6319 100644 --- a/checks/license_test.go +++ b/checks/license_test.go @@ -75,7 +75,7 @@ func TestLicenseFileSubdirectory(t *testing.T) { ctx := context.Background() client := localdir.CreateLocalDirClient(ctx, logger) - if err := client.InitRepo(repo, clients.HeadSHA); err != nil { + if err := client.InitRepo(repo, clients.HeadSHA, 0); err != nil { t.Errorf("InitRepo: %v", err) } diff --git a/checks/raw/security_policy.go b/checks/raw/security_policy.go index 9469d625..997d78d6 100644 --- a/checks/raw/security_policy.go +++ b/checks/raw/security_policy.go @@ -63,8 +63,9 @@ func SecurityPolicy(c *checker.CheckRequest) (checker.SecurityPolicyData, error) // https#://docs.github.com/en/github/building-a-strong-community/creating-a-default-community-health-file. // TODO(1491): Make this non-GitHub specific. logger := log.NewLogger(log.InfoLevel) + // HAD TO HARD CODE TO 30 dotGitHubClient := githubrepo.CreateGithubRepoClient(c.Ctx, logger) - err = dotGitHubClient.InitRepo(c.Repo.Org(), clients.HeadSHA) + err = dotGitHubClient.InitRepo(c.Repo.Org(), clients.HeadSHA, 0) switch { case err == nil: defer dotGitHubClient.Close() diff --git a/clients/githubrepo/client.go b/clients/githubrepo/client.go index 18288f9b..bd2c1e74 100644 --- a/clients/githubrepo/client.go +++ b/clients/githubrepo/client.go @@ -55,10 +55,11 @@ type Client struct { languages *languagesHandler ctx context.Context tarball tarballHandler + commitDepth int } // InitRepo sets up the GitHub repo in local storage for improving performance and GitHub token usage efficiency. -func (client *Client) InitRepo(inputRepo clients.Repo, commitSHA string) error { +func (client *Client) InitRepo(inputRepo clients.Repo, commitSHA string, commitDepth int) error { ghRepo, ok := inputRepo.(*repoURL) if !ok { return fmt.Errorf("%w: %v", errInputRepoType, inputRepo) @@ -69,7 +70,11 @@ func (client *Client) InitRepo(inputRepo clients.Repo, commitSHA string) error { if err != nil { return sce.WithMessage(sce.ErrRepoUnreachable, err.Error()) } - + if commitDepth <= 0 { + client.commitDepth = 30 // default + } else { + client.commitDepth = commitDepth + } client.repo = repo client.repourl = &repoURL{ owner: repo.Owner.GetLogin(), @@ -82,7 +87,7 @@ func (client *Client) InitRepo(inputRepo clients.Repo, commitSHA string) error { client.tarball.init(client.ctx, client.repo, commitSHA) // Setup GraphQL. - client.graphClient.init(client.ctx, client.repourl) + client.graphClient.init(client.ctx, client.repourl, client.commitDepth) // Setup contributorsHandler. client.contributors.init(client.ctx, client.repourl) @@ -138,6 +143,7 @@ func (client *Client) ListCommits() ([]clients.Commit, error) { // ListIssues implements RepoClient.ListIssues. func (client *Client) ListIssues() ([]clients.Issue, error) { + // here you would need to pass commitDepth or something return client.graphClient.getIssues() } @@ -295,7 +301,7 @@ func CreateOssFuzzRepoClient(ctx context.Context, logger *log.Logger) (clients.R } ossFuzzRepoClient := CreateGithubRepoClient(ctx, logger) - if err := ossFuzzRepoClient.InitRepo(ossFuzzRepo, clients.HeadSHA); err != nil { + if err := ossFuzzRepoClient.InitRepo(ossFuzzRepo, clients.HeadSHA, 0); err != nil { return nil, fmt.Errorf("error during InitRepo: %w", err) } return ossFuzzRepoClient, nil diff --git a/clients/githubrepo/graphql.go b/clients/githubrepo/graphql.go index 18254a12..7b0477dd 100644 --- a/clients/githubrepo/graphql.go +++ b/clients/githubrepo/graphql.go @@ -36,7 +36,6 @@ const ( issueCommentsToAnalyze = 30 reviewsToAnalyze = 30 labelsToAnalyze = 30 - commitsToAnalyze = 30 ) var errNotCached = errors.New("result not cached") @@ -100,7 +99,12 @@ type graphqlData struct { } } `graphql:"associatedPullRequests(first: $pullRequestsToAnalyze)"` } - } `graphql:"history(first: $commitsToAnalyze)"` + PageInfo struct { + StartCursor githubv4.String + EndCursor githubv4.String + HasNextPage bool + } + } `graphql:"history(first: $commitsToAnalyze, after: $historyCursor)"` } `graphql:"... on Commit"` } `graphql:"object(expression: $commitExpression)"` Issues struct { @@ -183,9 +187,10 @@ type graphqlHandler struct { commits []clients.Commit issues []clients.Issue archived bool + commitDepth int } -func (handler *graphqlHandler) init(ctx context.Context, repourl *repoURL) { +func (handler *graphqlHandler) init(ctx context.Context, repourl *repoURL, commitDepth int) { handler.ctx = ctx handler.repourl = repourl handler.data = new(graphqlData) @@ -195,6 +200,32 @@ func (handler *graphqlHandler) init(ctx context.Context, repourl *repoURL) { handler.setupCheckRunsOnce = new(sync.Once) handler.checkRuns = checkRunCache{} handler.logger = log.NewLogger(log.DefaultLevel) + handler.commitDepth = commitDepth +} + +func populateCommits(handler *graphqlHandler, vars map[string]interface{}) ([]clients.Commit, error) { + var allCommits []clients.Commit + var commitsLeft githubv4.Int + commitsLeft, ok := vars["commitsToAnalyze"].(githubv4.Int) + if !ok { + return nil, nil + } + for vars["commitsToAnalyze"] = githubv4.Int(100); commitsLeft > 0; commitsLeft = commitsLeft - 100 { + if commitsLeft < 100 { + vars["commitsToAnalyze"] = commitsLeft + } + err := handler.client.Query(handler.ctx, handler.data, vars) + if err != nil { + return nil, fmt.Errorf("failed to populate commits: %w", err) + } + vars["historyCursor"] = handler.data.Repository.Object.Commit.History.PageInfo.EndCursor + tmp, err := commitsFrom(handler.data, handler.repourl.owner, handler.repourl.repo) + if err != nil { + return nil, fmt.Errorf("failed to populate commits: %w", err) + } + allCommits = append(allCommits, tmp...) + } + return allCommits, nil } func (handler *graphqlHandler) setup() error { @@ -208,19 +239,24 @@ func (handler *graphqlHandler) setup() error { "issueCommentsToAnalyze": githubv4.Int(issueCommentsToAnalyze), "reviewsToAnalyze": githubv4.Int(reviewsToAnalyze), "labelsToAnalyze": githubv4.Int(labelsToAnalyze), - "commitsToAnalyze": githubv4.Int(commitsToAnalyze), + "commitsToAnalyze": githubv4.Int(handler.commitDepth), "commitExpression": githubv4.String(commitExpression), + "historyCursor": (*githubv4.String)(nil), + } + // if NumberOfCommits set to < 99 we are required by the graphql to page by 100 commits. + if handler.commitDepth > 99 { + handler.commits, handler.errSetup = populateCommits(handler, vars) + handler.issues = issuesFrom(handler.data) + handler.archived = bool(handler.data.Repository.IsArchived) + return } if err := handler.client.Query(handler.ctx, handler.data, vars); err != nil { handler.errSetup = sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("githubv4.Query: %v", err)) return } - handler.archived = bool(handler.data.Repository.IsArchived) handler.commits, handler.errSetup = commitsFrom(handler.data, handler.repourl.owner, handler.repourl.repo) - if handler.errSetup != nil { - return - } handler.issues = issuesFrom(handler.data) + handler.archived = bool(handler.data.Repository.IsArchived) }) return handler.errSetup } @@ -232,10 +268,16 @@ func (handler *graphqlHandler) setupCheckRuns() error { "owner": githubv4.String(handler.repourl.owner), "name": githubv4.String(handler.repourl.repo), "pullRequestsToAnalyze": githubv4.Int(pullRequestsToAnalyze), - "commitsToAnalyze": githubv4.Int(commitsToAnalyze), + "commitsToAnalyze": githubv4.Int(handler.commitDepth), "commitExpression": githubv4.String(commitExpression), "checksToAnalyze": githubv4.Int(checksToAnalyze), } + // TODO(#2224): + // sast and ci checks causes cache miss if commits dont match number of check runs. + // paging for this needs to be implemented if using higher than 100 --number-of-commits + if handler.commitDepth > 99 { + vars["commitsToAnalyze"] = githubv4.Int(99) + } if err := handler.client.Query(handler.ctx, handler.checkData, vars); err != nil { // quit early without setting crsErrSetup for "Resource not accessible by integration" error // for whatever reason, this check doesn't work with a GITHUB_TOKEN, only a PAT @@ -325,7 +367,7 @@ func parseCheckRuns(data *checkRunsGraphqlData) checkRunCache { return checkCache } -//nolint +// nolint func commitsFrom(data *graphqlData, repoOwner, repoName string) ([]clients.Commit, error) { ret := make([]clients.Commit, 0) for _, commit := range data.Repository.Object.Commit.History.Nodes { diff --git a/clients/githubrepo/graphql_e2e_test.go b/clients/githubrepo/graphql_e2e_test.go index ea0aa9b4..a47d6531 100644 --- a/clients/githubrepo/graphql_e2e_test.go +++ b/clients/githubrepo/graphql_e2e_test.go @@ -16,11 +16,15 @@ package githubrepo import ( "context" + "net/http" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/shurcooL/githubv4" "github.com/ossf/scorecard/v4/clients" + "github.com/ossf/scorecard/v4/clients/githubrepo/roundtripper" + "github.com/ossf/scorecard/v4/log" ) var _ = Describe("E2E TEST: githubrepo.graphqlHandler", func() { @@ -32,6 +36,108 @@ var _ = Describe("E2E TEST: githubrepo.graphqlHandler", func() { } }) + Context("E2E TEST: Confirm Paging Commits Works", func() { + It("Should only have 1 commit", func() { + _repourl := &repoURL{ + owner: "ossf", + repo: "scorecard", + commitSHA: clients.HeadSHA, + } + _vars := map[string]interface{}{ + "owner": githubv4.String("ossf"), + "name": githubv4.String("scorecard"), + "pullRequestsToAnalyze": githubv4.Int(1), + "issuesToAnalyze": githubv4.Int(30), + "issueCommentsToAnalyze": githubv4.Int(30), + "reviewsToAnalyze": githubv4.Int(30), + "labelsToAnalyze": githubv4.Int(30), + "commitsToAnalyze": githubv4.Int(1), + "commitExpression": githubv4.String("heads/main"), + "historyCursor": (*githubv4.String)(nil), + } + _ctx := context.Background() + _logger := log.NewLogger(log.DebugLevel) + _rt := roundtripper.NewTransport(_ctx, _logger) + _httpClient := &http.Client{ + Transport: _rt, + } + _graphClient := githubv4.NewClient(_httpClient) + _handler := &graphqlHandler{ + client: _graphClient, + } + _handler.init(context.Background(), _repourl, 1) + commits, err := populateCommits(_handler, _vars) + Expect(err).To(BeNil()) + Expect(len(commits)).Should(BeEquivalentTo(1)) + }) + It("Should have 30 commits", func() { + _repourl := &repoURL{ + owner: "ossf", + repo: "scorecard", + commitSHA: clients.HeadSHA, + } + _vars := map[string]interface{}{ + "owner": githubv4.String("ossf"), + "name": githubv4.String("scorecard"), + "pullRequestsToAnalyze": githubv4.Int(1), + "issuesToAnalyze": githubv4.Int(30), + "issueCommentsToAnalyze": githubv4.Int(30), + "reviewsToAnalyze": githubv4.Int(30), + "labelsToAnalyze": githubv4.Int(30), + "commitsToAnalyze": githubv4.Int(30), + "commitExpression": githubv4.String("heads/main"), + "historyCursor": (*githubv4.String)(nil), + } + _ctx := context.Background() + _logger := log.NewLogger(log.DebugLevel) + _rt := roundtripper.NewTransport(_ctx, _logger) + _httpClient := &http.Client{ + Transport: _rt, + } + _graphClient := githubv4.NewClient(_httpClient) + _handler := &graphqlHandler{ + client: _graphClient, + } + _handler.init(context.Background(), _repourl, 30) + commits, err := populateCommits(_handler, _vars) + Expect(err).To(BeNil()) + Expect(len(commits)).Should(BeEquivalentTo(30)) + }) + It("Should have 101 commits", func() { + _repourl := &repoURL{ + owner: "ossf", + repo: "scorecard", + commitSHA: clients.HeadSHA, + } + _vars := map[string]interface{}{ + "owner": githubv4.String("ossf"), + "name": githubv4.String("scorecard"), + "pullRequestsToAnalyze": githubv4.Int(1), + "issuesToAnalyze": githubv4.Int(30), + "issueCommentsToAnalyze": githubv4.Int(30), + "reviewsToAnalyze": githubv4.Int(30), + "labelsToAnalyze": githubv4.Int(30), + "commitsToAnalyze": githubv4.Int(101), + "commitExpression": githubv4.String("heads/main"), + "historyCursor": (*githubv4.String)(nil), + } + _ctx := context.Background() + _logger := log.NewLogger(log.DebugLevel) + _rt := roundtripper.NewTransport(_ctx, _logger) + _httpClient := &http.Client{ + Transport: _rt, + } + _graphClient := githubv4.NewClient(_httpClient) + _handler := &graphqlHandler{ + client: _graphClient, + } + _handler.init(context.Background(), _repourl, 101) + commits, err := populateCommits(_handler, _vars) + Expect(err).To(BeNil()) + Expect(len(commits)).Should(BeEquivalentTo(101)) + }) + }) + Context("E2E TEST: Validate query cost", func() { It("Should not have increased for HEAD query", func() { repourl := &repoURL{ @@ -39,7 +145,7 @@ var _ = Describe("E2E TEST: githubrepo.graphqlHandler", func() { repo: "scorecard", commitSHA: clients.HeadSHA, } - graphqlhandler.init(context.Background(), repourl) + graphqlhandler.init(context.Background(), repourl, 30) Expect(graphqlhandler.setup()).Should(BeNil()) Expect(graphqlhandler.data).ShouldNot(BeNil()) Expect(graphqlhandler.data.RateLimit.Cost).ShouldNot(BeNil()) @@ -51,7 +157,7 @@ var _ = Describe("E2E TEST: githubrepo.graphqlHandler", func() { repo: "scorecard", commitSHA: "de5224bbc56eceb7a25aece55d2d53bbc561ed2d", } - graphqlhandler.init(context.Background(), repourl) + graphqlhandler.init(context.Background(), repourl, 30) Expect(graphqlhandler.setup()).Should(BeNil()) Expect(graphqlhandler.data).ShouldNot(BeNil()) Expect(graphqlhandler.data.RateLimit.Cost).ShouldNot(BeNil()) @@ -63,7 +169,7 @@ var _ = Describe("E2E TEST: githubrepo.graphqlHandler", func() { repo: "scorecard", commitSHA: clients.HeadSHA, } - graphqlhandler.init(context.Background(), repourl) + graphqlhandler.init(context.Background(), repourl, 30) Expect(graphqlhandler.setupCheckRuns()).Should(BeNil()) Expect(graphqlhandler.checkData).ShouldNot(BeNil()) Expect(graphqlhandler.checkData.RateLimit.Cost).ShouldNot(BeNil()) diff --git a/clients/gitlabrepo/client.go b/clients/gitlabrepo/client.go index fca89383..d981dc11 100644 --- a/clients/gitlabrepo/client.go +++ b/clients/gitlabrepo/client.go @@ -52,10 +52,11 @@ type Client struct { languages *languagesHandler ctx context.Context // tarball tarballHandler + commitDepth int } // InitRepo sets up the GitLab project in local storage for improving performance and GitLab token usage efficiency. -func (client *Client) InitRepo(inputRepo clients.Repo, commitSHA string) error { +func (client *Client) InitRepo(inputRepo clients.Repo, commitSHA string, commitDepth int) error { glRepo, ok := inputRepo.(*repoURL) if !ok { return fmt.Errorf("%w: %v", errInputRepoType, inputRepo) @@ -66,9 +67,12 @@ func (client *Client) InitRepo(inputRepo clients.Repo, commitSHA string) error { if err != nil { return sce.WithMessage(sce.ErrRepoUnreachable, err.Error()) } - + if commitDepth <= 0 { + client.commitDepth = 30 // default + } else { + client.commitDepth = commitDepth + } client.repo = repo - client.repourl = &repoURL{ hostname: inputRepo.URI(), projectID: fmt.Sprint(repo.ID), diff --git a/clients/localdir/client.go b/clients/localdir/client.go index c3615295..dea9455f 100644 --- a/clients/localdir/client.go +++ b/clients/localdir/client.go @@ -39,21 +39,26 @@ var ( //nolint:govet type localDirClient struct { - logger *log.Logger - ctx context.Context - path string - once sync.Once - errFiles error - files []string + logger *log.Logger + ctx context.Context + path string + once sync.Once + errFiles error + files []string + commitDepth int } // InitRepo sets up the local repo. -func (client *localDirClient) InitRepo(inputRepo clients.Repo, commitSHA string) error { +func (client *localDirClient) InitRepo(inputRepo clients.Repo, commitSHA string, commitDepth int) error { localRepo, ok := inputRepo.(*repoLocal) if !ok { return fmt.Errorf("%w: %v", errInputRepoType, inputRepo) } - + if commitDepth <= 0 { + client.commitDepth = 30 // default + } else { + client.commitDepth = commitDepth + } client.path = strings.TrimPrefix(localRepo.URI(), "file://") return nil diff --git a/clients/localdir/client_test.go b/clients/localdir/client_test.go index 233bef5a..53a28220 100644 --- a/clients/localdir/client_test.go +++ b/clients/localdir/client_test.go @@ -76,7 +76,7 @@ func TestClient_CreationAndCaching(t *testing.T) { } client := CreateLocalDirClient(ctx, logger) - if err := client.InitRepo(repo, clients.HeadSHA); err != nil { + if err := client.InitRepo(repo, clients.HeadSHA, 30); err != nil { t.Errorf("InitRepo: %v", err) } diff --git a/clients/mockclients/repo_client.go b/clients/mockclients/repo_client.go index d518c9a0..d29882bf 100644 --- a/clients/mockclients/repo_client.go +++ b/clients/mockclients/repo_client.go @@ -140,17 +140,17 @@ func (mr *MockRepoClientMockRecorder) GetFileContent(filename interface{}) *gomo } // InitRepo mocks base method. -func (m *MockRepoClient) InitRepo(repo clients.Repo, commitSHA string) error { +func (m *MockRepoClient) InitRepo(repo clients.Repo, commitSHA string, commitDepth int) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "InitRepo", repo, commitSHA) + ret := m.ctrl.Call(m, "InitRepo", repo, commitSHA, commitDepth) ret0, _ := ret[0].(error) return ret0 } // InitRepo indicates an expected call of InitRepo. -func (mr *MockRepoClientMockRecorder) InitRepo(repo, commitSHA interface{}) *gomock.Call { +func (mr *MockRepoClientMockRecorder) InitRepo(repo, commitSHA, commitDepth interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitRepo", reflect.TypeOf((*MockRepoClient)(nil).InitRepo), repo, commitSHA) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitRepo", reflect.TypeOf((*MockRepoClient)(nil).InitRepo), repo, commitSHA, commitDepth) } // IsArchived mocks base method. diff --git a/clients/repo_client.go b/clients/repo_client.go index 20784ae4..607ebb64 100644 --- a/clients/repo_client.go +++ b/clients/repo_client.go @@ -28,7 +28,7 @@ const HeadSHA = "HEAD" // RepoClient interface is used by Scorecard checks to access a repo. type RepoClient interface { - InitRepo(repo Repo, commitSHA string) error + InitRepo(repo Repo, commitSHA string, commitDepth int) error URI() string IsArchived() (bool, error) ListFiles(predicate func(string) (bool, error)) ([]string, error) diff --git a/cmd/root.go b/cmd/root.go index 86711d26..1979880e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -90,10 +90,11 @@ func rootCmd(o *options.Options) error { ctx := context.Background() logger := sclog.NewLogger(sclog.ParseLevel(o.LogLevel)) repoURI, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient, err := checker.GetClients( - ctx, o.Repo, o.Local, logger) + ctx, o.Repo, o.Local, logger) // MODIFIED if err != nil { return fmt.Errorf("GetClients: %w", err) } + defer repoClient.Close() if ossFuzzRepoClient != nil { defer ossFuzzRepoClient.Close() @@ -127,6 +128,7 @@ func rootCmd(o *options.Options) error { ctx, repoURI, o.Commit, + o.CommitDepth, enabledChecks, repoClient, ossFuzzRepoClient, diff --git a/cmd/serve.go b/cmd/serve.go index 078451e5..e0a51661 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -70,7 +70,7 @@ func serveCmd(o *options.Options) *cobra.Command { ciiClient := clients.DefaultCIIBestPracticesClient() checksToRun := checks.GetAll() repoResult, err := pkg.RunScorecards( - ctx, repo, clients.HeadSHA /*commitSHA*/, checksToRun, repoClient, + ctx, repo, clients.HeadSHA /*commitSHA*/, o.CommitDepth, checksToRun, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient) if err != nil { logger.Error(err, "running enabled scorecard checks on repo") diff --git a/cron/internal/worker/main.go b/cron/internal/worker/main.go index 1bc8a3be..593019ec 100644 --- a/cron/internal/worker/main.go +++ b/cron/internal/worker/main.go @@ -164,7 +164,7 @@ func processRequest(ctx context.Context, delete(checksToRun, check) } - result, err := pkg.RunScorecards(ctx, repo, commitSHA, checksToRun, + result, err := pkg.RunScorecards(ctx, repo, commitSHA, 0, checksToRun, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient) if errors.Is(err, sce.ErrRepoUnreachable) { // Not accessible repo - continue. diff --git a/dependencydiff/dependencydiff.go b/dependencydiff/dependencydiff.go index 637073a1..20723ad9 100644 --- a/dependencydiff/dependencydiff.go +++ b/dependencydiff/dependencydiff.go @@ -92,8 +92,7 @@ func GetDependencyDiffResults( func initRepoAndClientByChecks(dCtx *dependencydiffContext, dSrcRepo string) error { repo, repoClient, ossFuzzClient, ciiClient, vulnsClient, err := checker.GetClients( - dCtx.ctx, dSrcRepo, "", dCtx.logger, - ) + dCtx.ctx, dSrcRepo, "", dCtx.logger) if err != nil { return fmt.Errorf("error getting the github repo and clients: %w", err) } @@ -162,6 +161,7 @@ func getScorecardCheckResults(dCtx *dependencydiffContext) error { // TODO (#2065): In future versions, ideally, this should be // the commitSHA corresponding to d.Version instead of HEAD. clients.HeadSHA, + 0, checksToRun, dCtx.ghRepoClient, dCtx.ossFuzzClient, diff --git a/e2e/binary_artifacts_test.go b/e2e/binary_artifacts_test.go index 236f7eae..0a3ef3b9 100644 --- a/e2e/binary_artifacts_test.go +++ b/e2e/binary_artifacts_test.go @@ -39,7 +39,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() { repo, err := githubrepo.MakeGithubRepo("ossf/scorecard") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ @@ -65,7 +65,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-binary-artifacts-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ @@ -92,7 +92,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-binary-artifacts-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, "5b48dea88825662d67ed94b609b45cf7705333b6") + err = repoClient.InitRepo(repo, "5b48dea88825662d67ed94b609b45cf7705333b6", 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ @@ -119,7 +119,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-binary-artifacts-e2e-4-binaries") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ @@ -147,7 +147,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-binary-artifacts-e2e-4-binaries") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, "d994b3e1a8912283f9958a7c1e0aa480ca24a7ce") + err = repoClient.InitRepo(repo, "d994b3e1a8912283f9958a7c1e0aa480ca24a7ce", 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ @@ -186,7 +186,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() { Expect(err).Should(BeNil()) x := localdir.CreateLocalDirClient(context.Background(), logger) - err = x.InitRepo(repo, clients.HeadSHA) + err = x.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ diff --git a/e2e/branch_protection_test.go b/e2e/branch_protection_test.go index a1279dc5..c1c45b1a 100644 --- a/e2e/branch_protection_test.go +++ b/e2e/branch_protection_test.go @@ -36,7 +36,7 @@ var _ = Describe("E2E TEST PAT:"+checks.CheckBranchProtection, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-branch-protection-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), @@ -66,7 +66,7 @@ var _ = Describe("E2E TEST PAT:"+checks.CheckBranchProtection, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-branch-protection-e2e-none") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), @@ -94,7 +94,7 @@ var _ = Describe("E2E TEST PAT:"+checks.CheckBranchProtection, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-branch-protection-e2e-patch-1") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), @@ -126,7 +126,7 @@ var _ = Describe("E2E TEST GITHUB_TOKEN:"+checks.CheckBranchProtection, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-branch-protection-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), diff --git a/e2e/ci_tests_test.go b/e2e/ci_tests_test.go index 4fdd82ba..83e5923b 100644 --- a/e2e/ci_tests_test.go +++ b/e2e/ci_tests_test.go @@ -34,7 +34,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), @@ -58,7 +58,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, "0a6850647e531b08f68118ff8ca20577a5b4062c") + err = repoClient.InitRepo(repo, "0a6850647e531b08f68118ff8ca20577a5b4062c", 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), @@ -82,7 +82,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() { repo, err := githubrepo.MakeGithubRepo("duo-labs/parliament") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, "1ead655ec85bdbe0739e4a4125ce36eb48a329bc") + err = repoClient.InitRepo(repo, "1ead655ec85bdbe0739e4a4125ce36eb48a329bc", 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), diff --git a/e2e/code_review_test.go b/e2e/code_review_test.go index f18b51a4..fb923d2a 100644 --- a/e2e/code_review_test.go +++ b/e2e/code_review_test.go @@ -37,7 +37,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCodeReview, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ @@ -62,7 +62,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCodeReview, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, "0a6850647e531b08f68118ff8ca20577a5b4062c") + err = repoClient.InitRepo(repo, "0a6850647e531b08f68118ff8ca20577a5b4062c", 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ @@ -86,7 +86,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCodeReview, func() { repo, err := githubrepo.MakeGithubRepo("spring-projects/spring-framework") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, "ca5e453f87f7e84033bb90a2fb54ee9f7fc94d61") + err = repoClient.InitRepo(repo, "ca5e453f87f7e84033bb90a2fb54ee9f7fc94d61", 0) Expect(err).Should(BeNil()) reviewData, err := raw.CodeReview(repoClient) diff --git a/e2e/contributors_test.go b/e2e/contributors_test.go index 273d51cd..2d02f6c8 100644 --- a/e2e/contributors_test.go +++ b/e2e/contributors_test.go @@ -34,7 +34,7 @@ var _ = Describe("E2E TEST:"+checks.CheckContributors, func() { repo, err := githubrepo.MakeGithubRepo("ossf/scorecard") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), diff --git a/e2e/dangerous_workflow_test.go b/e2e/dangerous_workflow_test.go index 4b0fdcc0..e3b826e5 100644 --- a/e2e/dangerous_workflow_test.go +++ b/e2e/dangerous_workflow_test.go @@ -36,7 +36,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-dangerous-workflow-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), @@ -60,7 +60,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-dangerous-workflow-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, "8db326e9ba20517feeefd157524a89184ed41f7f") + err = repoClient.InitRepo(repo, "8db326e9ba20517feeefd157524a89184ed41f7f", 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), @@ -95,7 +95,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() { Expect(err).Should(BeNil()) x := localdir.CreateLocalDirClient(context.Background(), logger) - err = x.InitRepo(repo, clients.HeadSHA) + err = x.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ diff --git a/e2e/dependency_update_tool_test.go b/e2e/dependency_update_tool_test.go index 9b6ab593..baf8686e 100644 --- a/e2e/dependency_update_tool_test.go +++ b/e2e/dependency_update_tool_test.go @@ -36,7 +36,7 @@ var _ = Describe("E2E TEST:"+checks.CheckDependencyUpdateTool, func() { repo, err := githubrepo.MakeGithubRepo("ossf/scorecard") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ @@ -63,7 +63,7 @@ var _ = Describe("E2E TEST:"+checks.CheckDependencyUpdateTool, func() { repo, err := githubrepo.MakeGithubRepo("netlify/netlify-cms") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ diff --git a/e2e/fuzzing_test.go b/e2e/fuzzing_test.go index eadab88a..7f83d684 100644 --- a/e2e/fuzzing_test.go +++ b/e2e/fuzzing_test.go @@ -35,7 +35,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { repo, err := githubrepo.MakeGithubRepo("tensorflow/tensorflow") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) ossFuzzRepoClient, err := githubrepo.CreateOssFuzzRepoClient(context.Background(), logger) Expect(err).Should(BeNil()) @@ -63,7 +63,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-fuzzing-cflite") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) ossFuzzRepoClient, err := githubrepo.CreateOssFuzzRepoClient(context.Background(), logger) Expect(err).Should(BeNil()) @@ -91,7 +91,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-fuzzing-golang") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) ossFuzzRepoClient, err := githubrepo.CreateOssFuzzRepoClient(context.Background(), logger) Expect(err).Should(BeNil()) @@ -119,7 +119,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-fuzzing-golang") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) ossFuzzRepoClient, err := githubrepo.CreateOssFuzzRepoClient(context.Background(), logger) Expect(err).Should(BeNil()) @@ -139,7 +139,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-packaging-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) ossFuzzRepoClient, err := githubrepo.CreateOssFuzzRepoClient(context.Background(), logger) Expect(err).Should(BeNil()) diff --git a/e2e/license_test.go b/e2e/license_test.go index 31ed4924..3a0bf3bb 100644 --- a/e2e/license_test.go +++ b/e2e/license_test.go @@ -36,7 +36,7 @@ var _ = Describe("E2E TEST:"+checks.CheckLicense, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-license-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), @@ -61,7 +61,7 @@ var _ = Describe("E2E TEST:"+checks.CheckLicense, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-license-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, "c3a8778e73ea95f937c228a34ee57d5e006f7304") + err = repoClient.InitRepo(repo, "c3a8778e73ea95f937c228a34ee57d5e006f7304", 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), @@ -97,7 +97,7 @@ var _ = Describe("E2E TEST:"+checks.CheckLicense, func() { Expect(err).Should(BeNil()) x := localdir.CreateLocalDirClient(context.Background(), logger) - err = x.InitRepo(repo, clients.HeadSHA) + err = x.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ diff --git a/e2e/maintained_test.go b/e2e/maintained_test.go index 23e9dd29..e4694d3c 100644 --- a/e2e/maintained_test.go +++ b/e2e/maintained_test.go @@ -34,7 +34,7 @@ var _ = Describe("E2E TEST:"+checks.CheckMaintained, func() { repo, err := githubrepo.MakeGithubRepo("apache/airflow") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), diff --git a/e2e/packaging_test.go b/e2e/packaging_test.go index a0abfa33..810d61e4 100644 --- a/e2e/packaging_test.go +++ b/e2e/packaging_test.go @@ -34,7 +34,7 @@ var _ = Describe("E2E TEST:"+checks.CheckPackaging, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-packaging-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), diff --git a/e2e/permissions_test.go b/e2e/permissions_test.go index 4bed2107..66f04eb8 100644 --- a/e2e/permissions_test.go +++ b/e2e/permissions_test.go @@ -36,7 +36,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-token-permissions-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), @@ -61,7 +61,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-token-permissions-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, "35a3425d1e682c32946b7d36adcfd772cf772e63") + err = repoClient.InitRepo(repo, "35a3425d1e682c32946b7d36adcfd772cf772e63", 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), @@ -97,7 +97,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() { Expect(err).Should(BeNil()) x := localdir.CreateLocalDirClient(context.Background(), logger) - err = x.InitRepo(repo, clients.HeadSHA) + err = x.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ diff --git a/e2e/pinned_dependencies_test.go b/e2e/pinned_dependencies_test.go index aa6c08a2..6b2e72bc 100644 --- a/e2e/pinned_dependencies_test.go +++ b/e2e/pinned_dependencies_test.go @@ -38,7 +38,7 @@ var _ = Describe("E2E TEST:"+checks.CheckPinnedDependencies, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-pinned-dependencies-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ @@ -63,7 +63,7 @@ var _ = Describe("E2E TEST:"+checks.CheckPinnedDependencies, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-pinned-dependencies-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, "c8bfd7cf04ea7af741e1d07af98fabfcc1b6ffb1") + err = repoClient.InitRepo(repo, "c8bfd7cf04ea7af741e1d07af98fabfcc1b6ffb1", 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ @@ -99,7 +99,7 @@ var _ = Describe("E2E TEST:"+checks.CheckPinnedDependencies, func() { Expect(err).Should(BeNil()) x := localdir.CreateLocalDirClient(context.Background(), logger) - err = x.InitRepo(repo, clients.HeadSHA) + err = x.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ diff --git a/e2e/sast_test.go b/e2e/sast_test.go index 657fd97b..40d06262 100644 --- a/e2e/sast_test.go +++ b/e2e/sast_test.go @@ -34,7 +34,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSAST, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), diff --git a/e2e/security_policy_test.go b/e2e/security_policy_test.go index 1c5c9a5d..ec5bd215 100644 --- a/e2e/security_policy_test.go +++ b/e2e/security_policy_test.go @@ -36,7 +36,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSecurityPolicy, func() { repo, err := githubrepo.MakeGithubRepo("tensorflow/tensorflow") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ @@ -62,7 +62,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSecurityPolicy, func() { repo, err := githubrepo.MakeGithubRepo("tensorflow/tensorflow") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, "e0cb70344e46276b37d65824f95eca478080de4a") + err = repoClient.InitRepo(repo, "e0cb70344e46276b37d65824f95eca478080de4a", 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ @@ -88,7 +88,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSecurityPolicy, func() { repo, err := githubrepo.MakeGithubRepo("randombit/botan") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ @@ -114,7 +114,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSecurityPolicy, func() { repo, err := githubrepo.MakeGithubRepo("randombit/botan") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, "bab40cdd29d19e0638cf1301dfd355c52b94d1c0") + err = repoClient.InitRepo(repo, "bab40cdd29d19e0638cf1301dfd355c52b94d1c0", 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ @@ -151,7 +151,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSecurityPolicy, func() { Expect(err).Should(BeNil()) x := localdir.CreateLocalDirClient(context.Background(), logger) - err = x.InitRepo(repo, clients.HeadSHA) + err = x.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ diff --git a/e2e/signedreleases_test.go b/e2e/signedreleases_test.go index 65d46252..0f2ea85c 100644 --- a/e2e/signedreleases_test.go +++ b/e2e/signedreleases_test.go @@ -34,7 +34,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSignedReleases, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-signed-releases-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), diff --git a/e2e/vulnerabilities_test.go b/e2e/vulnerabilities_test.go index 11abb347..1983a18a 100644 --- a/e2e/vulnerabilities_test.go +++ b/e2e/vulnerabilities_test.go @@ -33,7 +33,7 @@ var _ = Describe("E2E TEST:"+checks.CheckVulnerabilities, func() { repo, err := githubrepo.MakeGithubRepo("ossf/scorecard") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) dl := scut.TestDetailLogger{} @@ -62,7 +62,7 @@ var _ = Describe("E2E TEST:"+checks.CheckVulnerabilities, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-vulnerabilities-open62541") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, clients.HeadSHA) + err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) dl := scut.TestDetailLogger{} @@ -89,7 +89,7 @@ var _ = Describe("E2E TEST:"+checks.CheckVulnerabilities, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-vulnerabilities-open62541") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo, "de6367caa31b59e2156f83b04c2f30611b7ac393") + err = repoClient.InitRepo(repo, "de6367caa31b59e2156f83b04c2f30611b7ac393", 0) Expect(err).Should(BeNil()) dl := scut.TestDetailLogger{} diff --git a/options/flags.go b/options/flags.go index f9b3b281..a9b103d9 100644 --- a/options/flags.go +++ b/options/flags.go @@ -59,6 +59,8 @@ const ( // FlagFormat is the flag name for specifying output format. FlagFormat = "format" + + FlagCommitDepth = "commit-depth" ) // Command is an interface for handling options for command-line utilities. @@ -132,6 +134,13 @@ func (o *Options) AddFlags(cmd *cobra.Command) { "show extra details about each check", ) + cmd.Flags().IntVar( + &o.CommitDepth, + FlagCommitDepth, + o.CommitDepth, + "number of commits to check, commits begin backwards from the HEAD", + ) + checkNames := []string{} for checkName := range checks.GetAll() { checkNames = append(checkNames, checkName) diff --git a/options/options.go b/options/options.go index 4424f2e7..25fafc0b 100644 --- a/options/options.go +++ b/options/options.go @@ -41,8 +41,8 @@ type Options struct { ResultsFile string ChecksToRun []string Metadata []string + CommitDepth int ShowDetails bool - // Feature flags. EnableSarif bool `env:"ENABLE_SARIF"` EnableScorecardV6 bool `env:"SCORECARD_V6"` @@ -54,7 +54,6 @@ func New() *Options { if err := env.Parse(opts); err != nil { fmt.Printf("could not parse env vars, using default options: %v", err) } - // Defaulting. // TODO(options): Consider moving this to a separate function/method. if opts.Commit == "" { @@ -66,7 +65,6 @@ func New() *Options { if opts.LogLevel == "" { opts.LogLevel = DefaultLogLevel } - return opts } @@ -75,7 +73,6 @@ const ( DefaultCommit = clients.HeadSHA // Formats. - // FormatJSON specifies that results should be output in JSON format. FormatJSON = "json" // FormatSarif specifies that results should be output in SARIF format. @@ -86,7 +83,6 @@ const ( FormatRaw = "raw" // Environment variables. - // EnvVarEnableSarif is the environment variable which controls enabling // SARIF logging. EnvVarEnableSarif = "ENABLE_SARIF" diff --git a/pkg/scorecard.go b/pkg/scorecard.go index ae1eacc0..5563e5d1 100644 --- a/pkg/scorecard.go +++ b/pkg/scorecard.go @@ -84,13 +84,14 @@ func getRepoCommitHash(r clients.RepoClient) (string, error) { func RunScorecards(ctx context.Context, repo clients.Repo, commitSHA string, + commitDepth int, checksToRun checker.CheckNameToFnMap, repoClient clients.RepoClient, ossFuzzRepoClient clients.RepoClient, ciiClient clients.CIIBestPracticesClient, vulnsClient clients.VulnerabilitiesClient, ) (ScorecardResult, error) { - if err := repoClient.InitRepo(repo, commitSHA); err != nil { + if err := repoClient.InitRepo(repo, commitSHA, commitDepth); err != nil { // No need to call sce.WithMessage() since InitRepo will do that for us. //nolint:wrapcheck return ScorecardResult{}, err diff --git a/pkg/scorecard_test.go b/pkg/scorecard_test.go index 0c4f283b..74b9ddd0 100644 --- a/pkg/scorecard_test.go +++ b/pkg/scorecard_test.go @@ -101,7 +101,7 @@ func Test_getRepoCommitHashLocal(t *testing.T) { t.Errorf("MakeLocalDirRepo: %v", err) return } - if err := localDirClient.InitRepo(localRepo, clients.HeadSHA); err != nil { + if err := localDirClient.InitRepo(localRepo, clients.HeadSHA, 0); err != nil { t.Errorf("InitRepo: %v", err) return } @@ -146,7 +146,7 @@ func TestRunScorecards(t *testing.T) { mockRepoClient := mockrepo.NewMockRepoClient(ctrl) repo := mockrepo.NewMockRepo(ctrl) - mockRepoClient.EXPECT().InitRepo(repo, tt.args.commitSHA).Return(nil) + mockRepoClient.EXPECT().InitRepo(repo, tt.args.commitSHA, 0).Return(nil) mockRepoClient.EXPECT().Close().DoAndReturn(func() error { return nil @@ -163,8 +163,7 @@ func TestRunScorecards(t *testing.T) { }, nil }) defer ctrl.Finish() - got, err := RunScorecards(context.Background(), repo, tt.args.commitSHA, nil, - mockRepoClient, nil, nil, nil) + got, err := RunScorecards(context.Background(), repo, tt.args.commitSHA, 0, nil, mockRepoClient, nil, nil, nil) if (err != nil) != tt.wantErr { t.Errorf("RunScorecards() error = %v, wantErr %v", err, tt.wantErr) return