mirror of
https://github.com/ossf/scorecard.git
synced 2024-08-15 11:20:30 +03:00
Enable revive linters which are used in google3 (#793)
Co-authored-by: Azeem Shaikh <azeems@google.com>
This commit is contained in:
parent
a66b53ebe4
commit
83e9f52501
@ -4,6 +4,12 @@ run:
|
||||
deadline: 5m
|
||||
issues:
|
||||
new-from-rev: ""
|
||||
include:
|
||||
# revive `package-comments` and `exported` rules.
|
||||
- EXC0012
|
||||
- EXC0013
|
||||
- EXC0014
|
||||
- EXC0015
|
||||
skip-files:
|
||||
- cron/data/request.pb.go # autogenerated
|
||||
linters:
|
||||
|
@ -24,6 +24,7 @@ import (
|
||||
"github.com/ossf/scorecard/v2/clients"
|
||||
)
|
||||
|
||||
// CheckRequest struct encapsulates all data to be passed into a CheckFn.
|
||||
type CheckRequest struct {
|
||||
Ctx context.Context
|
||||
Client *github.Client
|
||||
|
@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package checker includes structs and functions used for running a check.
|
||||
package checker
|
||||
|
||||
import (
|
||||
@ -34,12 +35,15 @@ const migrationThresholdPassValue = 8
|
||||
// UPGRADEv2: to remove.
|
||||
var ErrorDemoninatorZero = errors.New("internal error: denominator is 0")
|
||||
|
||||
// Types of details.
|
||||
// DetailType defines types of details.
|
||||
type DetailType int
|
||||
|
||||
const (
|
||||
// DetailInfo is info-level log.
|
||||
DetailInfo DetailType = iota
|
||||
// DetailWarn is warn log.
|
||||
DetailWarn
|
||||
// DetailDebug is debug log.
|
||||
DetailDebug
|
||||
)
|
||||
|
||||
@ -50,6 +54,7 @@ type CheckDetail struct {
|
||||
Msg string // A short string explaining why the details was recorded/logged..
|
||||
}
|
||||
|
||||
// DetailLogger logs map to CheckDetail struct.
|
||||
type DetailLogger interface {
|
||||
Info(desc string, args ...interface{})
|
||||
Warn(desc string, args ...interface{})
|
||||
@ -63,7 +68,8 @@ const (
|
||||
InconclusiveResultScore = -1
|
||||
)
|
||||
|
||||
//nolint
|
||||
// nolint
|
||||
// CheckResult captures result from a check run.
|
||||
type CheckResult struct {
|
||||
// Old structure
|
||||
Error error `json:"-"`
|
||||
@ -82,7 +88,7 @@ type CheckResult struct {
|
||||
Reason string `json:"-"` // A sentence describing the check result (score, etc)
|
||||
}
|
||||
|
||||
// CreateProportionalScore() creates a proportional score.
|
||||
// CreateProportionalScore creates a proportional score.
|
||||
func CreateProportionalScore(success, total int) int {
|
||||
if total == 0 {
|
||||
return 0
|
||||
@ -115,6 +121,7 @@ func AggregateScoresWithWeight(scores map[int]int) int {
|
||||
return int(math.Floor(float64(r) / float64(ws)))
|
||||
}
|
||||
|
||||
// NormalizeReason - placeholder function if we want to update range of scores.
|
||||
func NormalizeReason(reason string, score int) string {
|
||||
return fmt.Sprintf("%v -- score normalized to %d", reason, score)
|
||||
}
|
||||
|
@ -29,14 +29,17 @@ import (
|
||||
|
||||
const checkRetries = 3
|
||||
|
||||
// Runner runs a check with retries.
|
||||
type Runner struct {
|
||||
CheckName string
|
||||
Repo string
|
||||
CheckRequest CheckRequest
|
||||
}
|
||||
|
||||
// CheckFn defined for convenience.
|
||||
type CheckFn func(*CheckRequest) CheckResult
|
||||
|
||||
// CheckNameToFnMap defined here for convenience.
|
||||
type CheckNameToFnMap map[string]CheckFn
|
||||
|
||||
// UPGRADEv2: messages2 will ultimately
|
||||
@ -80,6 +83,7 @@ func logStats(ctx context.Context, startTime time.Time, result *CheckResult) err
|
||||
return nil
|
||||
}
|
||||
|
||||
// Run runs a given check.
|
||||
func (r *Runner) Run(ctx context.Context, f CheckFn) CheckResult {
|
||||
ctx, err := tag.New(ctx, tag.Upsert(stats.CheckName, r.CheckName))
|
||||
if err != nil {
|
||||
|
@ -23,6 +23,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
// CheckActive is the exported check name for Active.
|
||||
CheckActive = "Active"
|
||||
lookBackDays = 90
|
||||
commitsPerWeek = 1
|
||||
@ -34,6 +35,7 @@ func init() {
|
||||
registerCheck(CheckActive, IsActive)
|
||||
}
|
||||
|
||||
// IsActive runs Active check.
|
||||
func IsActive(c *checker.CheckRequest) checker.CheckResult {
|
||||
archived, err := c.RepoClient.IsArchived()
|
||||
if err != nil {
|
||||
|
@ -20,6 +20,7 @@ import (
|
||||
"github.com/ossf/scorecard/v2/checker"
|
||||
)
|
||||
|
||||
// CheckAutomaticDependencyUpdate is the exported name for Automatic-Depdendency-Update.
|
||||
const CheckAutomaticDependencyUpdate = "Automatic-Dependency-Update"
|
||||
|
||||
//nolint
|
||||
|
@ -26,6 +26,7 @@ import (
|
||||
sce "github.com/ossf/scorecard/v2/errors"
|
||||
)
|
||||
|
||||
// CheckBinaryArtifacts is the exported name for Binary-Artifacts check.
|
||||
const CheckBinaryArtifacts string = "Binary-Artifacts"
|
||||
|
||||
//nolint
|
||||
|
@ -25,6 +25,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
// CheckBranchProtection is the exported name for Branch-Protected check.
|
||||
CheckBranchProtection = "Branch-Protection"
|
||||
minReviews = 2
|
||||
)
|
||||
@ -45,6 +46,7 @@ type repositories interface {
|
||||
*github.Protection, *github.Response, error)
|
||||
}
|
||||
|
||||
// BranchProtection runs Branch-Protection check.
|
||||
func BranchProtection(c *checker.CheckRequest) checker.CheckResult {
|
||||
// Checks branch protection on both release and development branch.
|
||||
return checkReleaseAndDevBranchProtection(c.Ctx, c.Client.Repositories, c.Dlogger, c.Owner, c.Repo)
|
||||
@ -178,6 +180,7 @@ func getProtectionAndCheck(ctx context.Context, r repositories, dl checker.Detai
|
||||
return IsBranchProtected(protection, branch, dl), nil
|
||||
}
|
||||
|
||||
// IsBranchProtected checks branch protection rules on a Git branch.
|
||||
func IsBranchProtected(protection *github.Protection, branch string, dl checker.DetailLogger) int {
|
||||
totalScore := 15
|
||||
score := 0
|
||||
|
@ -41,6 +41,7 @@ func init() {
|
||||
registerCheck(CheckCITests, CITests)
|
||||
}
|
||||
|
||||
// CITests runs CI-Tests check.
|
||||
func CITests(c *checker.CheckRequest) checker.CheckResult {
|
||||
prs, _, err := c.Client.PullRequests.List(c.Ctx, c.Owner, c.Repo, &github.PullRequestListOptions{
|
||||
State: "closed",
|
||||
|
@ -37,6 +37,7 @@ type response struct {
|
||||
BadgeLevel string `json:"badge_level"`
|
||||
}
|
||||
|
||||
// CIIBestPractices runs CII-Best-Practices check.
|
||||
func CIIBestPractices(c *checker.CheckRequest) checker.CheckResult {
|
||||
repoURL := fmt.Sprintf("https://github.com/%s/%s", c.Owner, c.Repo)
|
||||
url := fmt.Sprintf("https://bestpractices.coreinfrastructure.org/projects.json?url=%s", repoURL)
|
||||
|
@ -36,6 +36,7 @@ func init() {
|
||||
registerCheck(CheckContributors, Contributors)
|
||||
}
|
||||
|
||||
// Contributors run Contributors check.
|
||||
func Contributors(c *checker.CheckRequest) checker.CheckResult {
|
||||
contribs, _, err := c.Client.Repositories.ListContributors(c.Ctx, c.Owner, c.Repo, &github.ListContributorsOptions{})
|
||||
if err != nil {
|
||||
|
@ -117,6 +117,7 @@ func CheckFilesContent(shellPathFnPattern string,
|
||||
return nil
|
||||
}
|
||||
|
||||
// FileCb represents a callback fn.
|
||||
type FileCb func(path string,
|
||||
dl checker.DetailLogger, data FileCbData) (bool, error)
|
||||
|
||||
@ -142,6 +143,7 @@ func CheckIfFileExists(checkName string, c *checker.CheckRequest, onFile FileCb,
|
||||
return nil
|
||||
}
|
||||
|
||||
// FileGetCbDataAsBoolPointer returns callback data as bool.
|
||||
func FileGetCbDataAsBoolPointer(data FileCbData) *bool {
|
||||
pdata, ok := data.(*bool)
|
||||
if !ok {
|
||||
|
@ -31,6 +31,7 @@ func init() {
|
||||
registerCheck(CheckFuzzing, Fuzzing)
|
||||
}
|
||||
|
||||
// Fuzzing runs Fuzzing check.
|
||||
func Fuzzing(c *checker.CheckRequest) checker.CheckResult {
|
||||
url := fmt.Sprintf("github.com/%s/%s", c.Owner, c.Repo)
|
||||
searchString := url + " repo:google/oss-fuzz in:file filename:project.yaml"
|
||||
|
@ -34,6 +34,7 @@ func init() {
|
||||
registerCheck(CheckPackaging, Packaging)
|
||||
}
|
||||
|
||||
// Packaging runs Packaging check.
|
||||
func Packaging(c *checker.CheckRequest) checker.CheckResult {
|
||||
_, dc, _, err := c.Client.Repositories.GetContents(c.Ctx, c.Owner, c.Repo, ".github/workflows",
|
||||
&github.RepositoryContentGetOptions{})
|
||||
|
@ -24,6 +24,7 @@ import (
|
||||
sce "github.com/ossf/scorecard/v2/errors"
|
||||
)
|
||||
|
||||
// CheckTokenPermissions is the exported name for Token-Permissions check.
|
||||
const CheckTokenPermissions = "Token-Permissions"
|
||||
|
||||
//nolint:gochecknoinits
|
||||
@ -38,6 +39,7 @@ type permissionCbData struct {
|
||||
writePermissions map[string]bool
|
||||
}
|
||||
|
||||
// TokenPermissions runs Token-Permissions check.
|
||||
func TokenPermissions(c *checker.CheckRequest) checker.CheckResult {
|
||||
data := permissionCbData{writePermissions: make(map[string]bool)}
|
||||
err := CheckFilesContent(".github/workflows/*", false,
|
||||
|
@ -33,6 +33,7 @@ func init() {
|
||||
registerCheck(CheckSAST, SAST)
|
||||
}
|
||||
|
||||
// SAST runs SAST check.
|
||||
func SAST(c *checker.CheckRequest) checker.CheckResult {
|
||||
sastScore, sastErr := SASTToolInCheckRuns(c)
|
||||
if sastErr != nil {
|
||||
|
@ -33,6 +33,7 @@ func init() {
|
||||
registerCheck(CheckSecurityPolicy, SecurityPolicy)
|
||||
}
|
||||
|
||||
// SecurityPolicy runs Security-Policy check.
|
||||
func SecurityPolicy(c *checker.CheckRequest) checker.CheckResult {
|
||||
var r bool
|
||||
// Check repository for repository-specific policy.
|
||||
|
@ -35,6 +35,7 @@ func init() {
|
||||
registerCheck(CheckSignedReleases, SignedReleases)
|
||||
}
|
||||
|
||||
// SignedReleases runs Signed-Releases check.
|
||||
func SignedReleases(c *checker.CheckRequest) checker.CheckResult {
|
||||
releases, _, err := c.Client.Repositories.ListReleases(c.Ctx, c.Owner, c.Repo, &github.ListOptions{})
|
||||
if err != nil {
|
||||
|
@ -34,6 +34,7 @@ func init() {
|
||||
registerCheck(CheckSignedTags, SignedTags)
|
||||
}
|
||||
|
||||
// SignedTags runs Signed-Tags check.
|
||||
func SignedTags(c *checker.CheckRequest) checker.CheckResult {
|
||||
type ref struct {
|
||||
Name githubv4.String
|
||||
|
@ -54,6 +54,7 @@ func (resp *osvResponse) getVulnerabilities() []string {
|
||||
return ids
|
||||
}
|
||||
|
||||
// HasUnfixedVulnerabilities runs Vulnerabilities check.
|
||||
func HasUnfixedVulnerabilities(c *checker.CheckRequest) checker.CheckResult {
|
||||
commits, err := c.RepoClient.ListCommits()
|
||||
if err != nil {
|
||||
|
@ -14,11 +14,13 @@
|
||||
|
||||
package clients
|
||||
|
||||
// BranchRef holds data about a Git branch.
|
||||
type BranchRef struct {
|
||||
Name string
|
||||
BranchProtectionRule BranchProtectionRule
|
||||
}
|
||||
|
||||
// BranchProtectionRule specifies rules for a Git branch.
|
||||
type BranchProtectionRule struct {
|
||||
RequiredApprovingReviewCount int
|
||||
}
|
||||
|
@ -16,13 +16,16 @@ package clients
|
||||
|
||||
import "time"
|
||||
|
||||
// Commit represents a Git commit.
|
||||
type Commit struct {
|
||||
CommittedDate time.Time
|
||||
Message string
|
||||
SHA string
|
||||
Committer User
|
||||
CommittedDate time.Time
|
||||
Message string
|
||||
SHA string
|
||||
Committer User
|
||||
AuthoredByCommitter bool
|
||||
}
|
||||
|
||||
// User represents a Git user.
|
||||
type User struct {
|
||||
Login string
|
||||
}
|
||||
|
@ -24,6 +24,7 @@ import (
|
||||
"github.com/ossf/scorecard/v2/clients"
|
||||
)
|
||||
|
||||
// Client is GitHub-specific implementation of RepoClient.
|
||||
type Client struct {
|
||||
repo *github.Repository
|
||||
repoClient *github.Client
|
||||
@ -32,6 +33,7 @@ type Client struct {
|
||||
tarball tarballHandler
|
||||
}
|
||||
|
||||
// InitRepo sets up the GitHub repo in local storage for improving performance and GitHub token usage efficiency.
|
||||
func (client *Client) InitRepo(owner, repoName string) error {
|
||||
// Sanity check
|
||||
repo, _, err := client.repoClient.Repositories.Get(client.ctx, owner, repoName)
|
||||
@ -54,34 +56,42 @@ func (client *Client) InitRepo(owner, repoName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListFiles implements RepoClient.ListFiles.
|
||||
func (client *Client) ListFiles(predicate func(string) (bool, error)) ([]string, error) {
|
||||
return client.tarball.listFiles(predicate)
|
||||
}
|
||||
|
||||
// GetFileContent implements RepoClient.GetFileContent.
|
||||
func (client *Client) GetFileContent(filename string) ([]byte, error) {
|
||||
return client.tarball.getFileContent(filename)
|
||||
}
|
||||
|
||||
// ListMergedPRs implements RepoClient.ListMergedPRs.
|
||||
func (client *Client) ListMergedPRs() ([]clients.PullRequest, error) {
|
||||
return client.graphClient.getMergedPRs()
|
||||
}
|
||||
|
||||
// ListCommits implements RepoClient.ListCommits.
|
||||
func (client *Client) ListCommits() ([]clients.Commit, error) {
|
||||
return client.graphClient.getCommits()
|
||||
}
|
||||
|
||||
// IsArchived implements RepoClient.IsArchived.
|
||||
func (client *Client) IsArchived() (bool, error) {
|
||||
return client.graphClient.isArchived()
|
||||
}
|
||||
|
||||
// GetDefaultBranch implements RepoClient.GetDefaultBranch.
|
||||
func (client *Client) GetDefaultBranch() (clients.BranchRef, error) {
|
||||
return client.graphClient.getDefaultBranch()
|
||||
}
|
||||
|
||||
// Close implements RepoClient.Close.
|
||||
func (client *Client) Close() error {
|
||||
return client.tarball.cleanup()
|
||||
}
|
||||
|
||||
// CreateGithubRepoClient returns a Client which implements RepoClient interface.
|
||||
func CreateGithubRepoClient(ctx context.Context,
|
||||
client *github.Client, graphClient *githubv4.Client) clients.RepoClient {
|
||||
return &Client{
|
||||
|
@ -131,7 +131,7 @@ func pullRequestFrom(data *graphqlData) []clients.PullRequest {
|
||||
toAppend := clients.PullRequest{
|
||||
Number: int(pr.Number),
|
||||
MergedAt: pr.MergedAt.Time,
|
||||
MergeCommit: clients.MergeCommit{
|
||||
MergeCommit: clients.Commit{
|
||||
AuthoredByCommitter: bool(pr.MergeCommit.AuthoredByCommitter),
|
||||
},
|
||||
}
|
||||
|
@ -24,6 +24,7 @@ import (
|
||||
"go.opencensus.io/tag"
|
||||
)
|
||||
|
||||
// MakeGitHubTransport wraps input RoundTripper with GitHub authorization logic.
|
||||
func MakeGitHubTransport(innerTransport http.RoundTripper, accessTokens []string) http.RoundTripper {
|
||||
return &githubTransport{
|
||||
innerTransport: innerTransport,
|
||||
|
@ -18,23 +18,22 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// PullRequest struct represents a PR as returned by RepoClient.
|
||||
// nolint: govet
|
||||
type PullRequest struct {
|
||||
MergedAt time.Time
|
||||
MergeCommit MergeCommit
|
||||
MergeCommit Commit
|
||||
Number int
|
||||
Labels []Label
|
||||
Reviews []Review
|
||||
}
|
||||
|
||||
type MergeCommit struct {
|
||||
AuthoredByCommitter bool
|
||||
}
|
||||
|
||||
// Label represents a PR label.
|
||||
type Label struct {
|
||||
Name string
|
||||
}
|
||||
|
||||
// Review represents a PR review.
|
||||
type Review struct {
|
||||
State string
|
||||
}
|
||||
|
@ -12,31 +12,37 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package clients defines the interface for RepoClient and related structs.
|
||||
package clients
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// ErrRepoUnavailable is returned when RepoClient is unable to reach the repo.
|
||||
// UPGRADEv2: use ErrRepoUnreachable instead.
|
||||
type ErrRepoUnavailable struct {
|
||||
innerError error
|
||||
}
|
||||
|
||||
// Error returns the error string.
|
||||
func (e *ErrRepoUnavailable) Error() string {
|
||||
return fmt.Sprintf("repo cannot be accessed: %v", e.innerError)
|
||||
}
|
||||
|
||||
// Unwrap returns the wrapped error.
|
||||
func (e *ErrRepoUnavailable) Unwrap() error {
|
||||
return e.innerError
|
||||
}
|
||||
|
||||
// NewRepoUnavailableError returns a wrapped error of type ErrRepoUnavailable.
|
||||
func NewRepoUnavailableError(err error) error {
|
||||
return &ErrRepoUnavailable{
|
||||
innerError: err,
|
||||
}
|
||||
}
|
||||
|
||||
// RepoClient interface is used by Scorecard checks to access a repo.
|
||||
type RepoClient interface {
|
||||
InitRepo(owner, repo string) error
|
||||
IsArchived() (bool, error)
|
||||
|
@ -194,6 +194,7 @@ type rubyGemsSearchResults struct {
|
||||
SourceCodeURI string `json:"source_code_uri"`
|
||||
}
|
||||
|
||||
// Execute runs the Scorecard commandline.
|
||||
func Execute() {
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
|
@ -87,7 +87,7 @@ func transferDataToBq(ctx context.Context, bucketURL string, summary *bucketSumm
|
||||
}
|
||||
|
||||
shardFileURI := data.GetBlobFilename("shard-*", creationTime)
|
||||
if err := StartDataTransferJob(ctx, bucketURL, shardFileURI, creationTime); err != nil {
|
||||
if err := startDataTransferJob(ctx, bucketURL, shardFileURI, creationTime); err != nil {
|
||||
return fmt.Errorf("error during StartDataTransferJob: %w", err)
|
||||
}
|
||||
|
||||
|
@ -62,7 +62,7 @@ func createBQLoader(ctx context.Context, projectID, datasetName, tableName strin
|
||||
return bqClient, loader, nil
|
||||
}
|
||||
|
||||
func StartDataTransferJob(ctx context.Context, bucketURL, fileURI string, partitionDate time.Time) error {
|
||||
func startDataTransferJob(ctx context.Context, bucketURL, fileURI string, partitionDate time.Time) error {
|
||||
projectID, datasetName, tableName, err := getBQConfig()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting BQ config: %w", err)
|
||||
|
@ -112,34 +112,42 @@ func getIntConfigValue(envVar string, byteValue []byte, fieldName, configName st
|
||||
}
|
||||
}
|
||||
|
||||
// GetProjectID returns the cloud projectID for the cron job.
|
||||
func GetProjectID() (string, error) {
|
||||
return getStringConfigValue(projectID, configYAML, "ProjectID", "project-id")
|
||||
}
|
||||
|
||||
// GetResultDataBucketURL returns the bucketURL for storing cron job results.
|
||||
func GetResultDataBucketURL() (string, error) {
|
||||
return getStringConfigValue(resultDataBucketURL, configYAML, "ResultDataBucketURL", "result-data-bucket-url")
|
||||
}
|
||||
|
||||
// GetRequestTopicURL returns the topic name for sending cron job PubSub requests.
|
||||
func GetRequestTopicURL() (string, error) {
|
||||
return getStringConfigValue(requestTopicURL, configYAML, "RequestTopicURL", "request-topic-url")
|
||||
}
|
||||
|
||||
// GetRequestSubscriptionURL returns the subscription name of the PubSub topic for cron job reuests.
|
||||
func GetRequestSubscriptionURL() (string, error) {
|
||||
return getStringConfigValue(requestSubscriptionURL, configYAML, "RequestSubscriptionURL", "request-subscription-url")
|
||||
}
|
||||
|
||||
// GetBigQueryDataset returns the BQ dataset name to transfer cron job results.
|
||||
func GetBigQueryDataset() (string, error) {
|
||||
return getStringConfigValue(bigqueryDataset, configYAML, "BigQueryDataset", "bigquery-dataset")
|
||||
}
|
||||
|
||||
// GetBigQueryTable returns the table name to transfer cron job results.
|
||||
func GetBigQueryTable() (string, error) {
|
||||
return getStringConfigValue(bigqueryTable, configYAML, "BigQueryTable", "bigquery-table")
|
||||
}
|
||||
|
||||
// GetShardSize returns the shard_size for the cron job.
|
||||
func GetShardSize() (int, error) {
|
||||
return getIntConfigValue(shardSize, configYAML, "ShardSize", "shard-size")
|
||||
}
|
||||
|
||||
// GetMetricExporter returns the opencensus exporter type.
|
||||
func GetMetricExporter() (string, error) {
|
||||
return getStringConfigValue(metricExporter, configYAML, "MetricExporter", "metric-exporter")
|
||||
}
|
||||
|
@ -28,7 +28,7 @@ import (
|
||||
"github.com/ossf/scorecard/v2/cron/pubsub"
|
||||
)
|
||||
|
||||
func PublishToRepoRequestTopic(ctx context.Context, iter data.Iterator, datetime time.Time) (int32, error) {
|
||||
func publishToRepoRequestTopic(ctx context.Context, iter data.Iterator, datetime time.Time) (int32, error) {
|
||||
var shardNum int32
|
||||
request := data.ScorecardBatchRequest{
|
||||
JobTime: timestamppb.New(datetime),
|
||||
@ -98,7 +98,7 @@ func main() {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
shardNum, err := PublishToRepoRequestTopic(ctx, reader, t)
|
||||
shardNum, err := publishToRepoRequestTopic(ctx, reader, t)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package data contains util fns for reading and writing data handled by the cron job.
|
||||
package data
|
||||
|
||||
import (
|
||||
@ -40,6 +41,7 @@ var (
|
||||
errParseBlobName = errors.New("error parsing input blob name")
|
||||
)
|
||||
|
||||
// GetBlobKeys returns all object keys for a given bucketURL.
|
||||
func GetBlobKeys(ctx context.Context, bucketURL string) ([]string, error) {
|
||||
bucket, err := blob.OpenBucket(ctx, bucketURL)
|
||||
if err != nil {
|
||||
@ -62,6 +64,7 @@ func GetBlobKeys(ctx context.Context, bucketURL string) ([]string, error) {
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// GetBlobContent returns the file content given a bucketURL and object key.
|
||||
func GetBlobContent(ctx context.Context, bucketURL, key string) ([]byte, error) {
|
||||
bucket, err := blob.OpenBucket(ctx, bucketURL)
|
||||
if err != nil {
|
||||
@ -76,6 +79,7 @@ func GetBlobContent(ctx context.Context, bucketURL, key string) ([]byte, error)
|
||||
return keyData, nil
|
||||
}
|
||||
|
||||
// BlobExists checks whether a given `bucketURL/key` blob exists.
|
||||
func BlobExists(ctx context.Context, bucketURL, key string) (bool, error) {
|
||||
bucket, err := blob.OpenBucket(ctx, bucketURL)
|
||||
if err != nil {
|
||||
@ -90,6 +94,7 @@ func BlobExists(ctx context.Context, bucketURL, key string) (bool, error) {
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// WriteToBlobStore creates and writes data to filename in bucketURL.
|
||||
func WriteToBlobStore(ctx context.Context, bucketURL, filename string, data []byte) error {
|
||||
bucket, err := blob.OpenBucket(ctx, bucketURL)
|
||||
if err != nil {
|
||||
@ -110,18 +115,22 @@ func WriteToBlobStore(ctx context.Context, bucketURL, filename string, data []by
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetBlobFilename returns a blob key for a shard. Takes Time object and filename as input.
|
||||
func GetBlobFilename(filename string, datetime time.Time) string {
|
||||
return datetime.Format(filePrefixFormat) + filename
|
||||
}
|
||||
|
||||
// GetShardNumFilename returns shard_num filename for a shard.
|
||||
func GetShardNumFilename(datetime time.Time) string {
|
||||
return GetBlobFilename(config.ShardNumFilename, datetime)
|
||||
}
|
||||
|
||||
// GetTransferStatusFilename returns transfer_status filename for a shard.
|
||||
func GetTransferStatusFilename(datetime time.Time) string {
|
||||
return GetBlobFilename(config.TransferStatusFilename, datetime)
|
||||
}
|
||||
|
||||
// ParseBlobFilename parses a blob key into a Time object.
|
||||
func ParseBlobFilename(key string) (time.Time, string, error) {
|
||||
if len(key) < len(filePrefixFormat) {
|
||||
return time.Now(), "", fmt.Errorf("%w: %s", errShortBlobName, key)
|
||||
|
@ -25,11 +25,14 @@ import (
|
||||
"github.com/ossf/scorecard/v2/repos"
|
||||
)
|
||||
|
||||
// Iterator interface is used to iterate through list of input repos for the cron job.
|
||||
type Iterator interface {
|
||||
HasNext() bool
|
||||
Next() (repos.RepoURL, error)
|
||||
}
|
||||
|
||||
// MakeIteratorFrom returns an implementation of Iterator interface.
|
||||
// Currently returns an instance of csvIterator.
|
||||
func MakeIteratorFrom(reader io.Reader) (Iterator, error) {
|
||||
csvReader := csv.NewReader(reader)
|
||||
csvReader.Comment = '#'
|
||||
|
@ -36,7 +36,7 @@ import (
|
||||
|
||||
var (
|
||||
// TODO = move them outside the sourcecode.
|
||||
bazelRepos = []RepositoryDepsURL{
|
||||
bazelRepos = []repositoryDepsURL{
|
||||
{
|
||||
Owner: "envoyproxy",
|
||||
Repo: "envoy",
|
||||
@ -54,7 +54,7 @@ var (
|
||||
},
|
||||
}
|
||||
// TODO = move them outside the sourcecode.
|
||||
gorepos = []RepositoryDepsURL{
|
||||
gorepos = []repositoryDepsURL{
|
||||
{
|
||||
Owner: "ossf",
|
||||
Repo: "scorecard",
|
||||
@ -71,14 +71,14 @@ var (
|
||||
}
|
||||
)
|
||||
|
||||
type RepositoryDepsURL struct {
|
||||
type repositoryDepsURL struct {
|
||||
Owner, Repo, File string
|
||||
Vendor bool
|
||||
}
|
||||
|
||||
// Programmatically gets Envoy's dependencies and add to projects.
|
||||
// Re-using a checker type.
|
||||
func getBazelDeps(repo RepositoryDepsURL) []repos.RepoURL {
|
||||
func getBazelDeps(repo repositoryDepsURL) []repos.RepoURL {
|
||||
client := github.NewClient(nil)
|
||||
ctx := context.Background()
|
||||
depRepos := []repos.RepoURL{}
|
||||
@ -112,7 +112,7 @@ func getBazelDeps(repo RepositoryDepsURL) []repos.RepoURL {
|
||||
}
|
||||
|
||||
// GetGoDeps returns go repo dependencies.
|
||||
func getGoDeps(repo RepositoryDepsURL) []repos.RepoURL {
|
||||
func getGoDeps(repo repositoryDepsURL) []repos.RepoURL {
|
||||
repoURLs := []repos.RepoURL{}
|
||||
pwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
|
@ -37,6 +37,7 @@ func repoFormatFromRepoURL(repoURLs []repos.RepoURL) []repoFormat {
|
||||
return repoentries
|
||||
}
|
||||
|
||||
// SortAndAppendTo appends `oldRepos` and `newRepos` before sorting and writing out the result to `out`.
|
||||
func SortAndAppendTo(out io.Writer, oldRepos, newRepos []repos.RepoURL) error {
|
||||
repoentries := repoFormatFromRepoURL(oldRepos)
|
||||
repoentries = append(repoentries, repoFormatFromRepoURL(newRepos)...)
|
||||
@ -53,6 +54,7 @@ func SortAndAppendTo(out io.Writer, oldRepos, newRepos []repos.RepoURL) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SortAndAppendFrom reads from `in`, appends to newRepos and writes the sorted output to `out`.
|
||||
func SortAndAppendFrom(in io.Reader, out io.Writer, newRepos []repos.RepoURL) error {
|
||||
iter, err := MakeIteratorFrom(in)
|
||||
if err != nil {
|
||||
|
@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package monitoring defines exporters to be used by opencensus package in the cron job.
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
@ -38,6 +39,7 @@ const (
|
||||
printer exporterType = "printer"
|
||||
)
|
||||
|
||||
// Exporter interface is a custom wrapper to represent an opencensus exporter.
|
||||
type Exporter interface {
|
||||
ExportView(viewData *view.Data)
|
||||
StartMetricsExporter() error
|
||||
@ -45,6 +47,7 @@ type Exporter interface {
|
||||
Flush()
|
||||
}
|
||||
|
||||
// GetExporter defines a factory for returning opencensus Exporter.
|
||||
func GetExporter() (Exporter, error) {
|
||||
exporter, err := config.GetMetricExporter()
|
||||
if err != nil {
|
||||
|
@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package pubsub handles interactions with PubSub framework.
|
||||
package pubsub
|
||||
|
||||
import (
|
||||
@ -33,11 +34,13 @@ import (
|
||||
|
||||
var errorPublish = errors.New("total errors when publishing")
|
||||
|
||||
// Publisher interface is used to publish cron job requests to PubSub.
|
||||
type Publisher interface {
|
||||
Publish(request *data.ScorecardBatchRequest) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
// CreatePublisher returns an implementation of the Publisher interface.
|
||||
func CreatePublisher(ctx context.Context, topicURL string) (Publisher, error) {
|
||||
ret := publisherImpl{}
|
||||
topic, err := pubsub.OpenTopic(ctx, topicURL)
|
||||
|
@ -27,6 +27,7 @@ import (
|
||||
// ErrorInParse indicates there was an error while unmarshalling the protocol buffer message.
|
||||
var ErrorInParse = errors.New("error during protojson.Unmarshal")
|
||||
|
||||
// Subscriber interface is used pull messages from PubSub.
|
||||
type Subscriber interface {
|
||||
SynchronousPull() (*data.ScorecardBatchRequest, error)
|
||||
Ack()
|
||||
@ -34,6 +35,8 @@ type Subscriber interface {
|
||||
Close() error
|
||||
}
|
||||
|
||||
// CreateSubscriber returns an implementation of Subscriber interface.
|
||||
// Currently returns an instance of gcsSubscriber.
|
||||
func CreateSubscriber(ctx context.Context, subscriptionURL string) (Subscriber, error) {
|
||||
return createGCSSubscriber(ctx, subscriptionURL)
|
||||
}
|
||||
|
@ -11,6 +11,8 @@
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package checks contains util fns for reading input YAML file.
|
||||
package checks
|
||||
|
||||
import (
|
||||
@ -23,16 +25,19 @@ import (
|
||||
//go:embed checks.yaml
|
||||
var checksYAML []byte
|
||||
|
||||
// Check defines expected check definition in checks.yaml.
|
||||
type Check struct {
|
||||
Risk string `yaml:"-"`
|
||||
Description string `yaml:"description"`
|
||||
Remediation []string `yaml:"remediation"`
|
||||
}
|
||||
|
||||
// Doc maps to checks.yaml file.
|
||||
type Doc struct {
|
||||
Checks map[string]Check
|
||||
}
|
||||
|
||||
// Read parses `checks.yaml` file and returns a `Doc` struct.
|
||||
func Read() (Doc, error) {
|
||||
var m Doc
|
||||
if err := yaml.Unmarshal(checksYAML, &m); err != nil {
|
||||
|
@ -32,6 +32,7 @@ var (
|
||||
errLowConfidence *ErrLowConfidence
|
||||
)
|
||||
|
||||
// GetErrorName returns the name of the error.
|
||||
func GetErrorName(err error) string {
|
||||
switch {
|
||||
case errors.As(err, &errRetry):
|
||||
|
@ -37,7 +37,7 @@ func Create(e error, msg string) error {
|
||||
return fmt.Errorf("%w", e)
|
||||
}
|
||||
|
||||
// Create an internal error, not using
|
||||
// CreateInternal creates an internal error, not using
|
||||
// any of the errors listed above.
|
||||
func CreateInternal(e error, msg string) error {
|
||||
return Create(e, msg)
|
||||
|
@ -19,10 +19,13 @@ import (
|
||||
)
|
||||
|
||||
type (
|
||||
ErrRetry struct{ wrappedError }
|
||||
// ErrRetry is returned when a check failed after maximum num_retries.
|
||||
ErrRetry struct{ wrappedError }
|
||||
// ErrLowConfidence is returned when check result is inconclusive.
|
||||
ErrLowConfidence struct{ wrappedError }
|
||||
)
|
||||
|
||||
// MakeRetryError returns a wrapped error of type ErrRetry.
|
||||
func MakeRetryError(err error) error {
|
||||
return &ErrRetry{
|
||||
wrappedError{
|
||||
@ -32,6 +35,7 @@ func MakeRetryError(err error) error {
|
||||
}
|
||||
}
|
||||
|
||||
// MakeLowConfidenceError returns a wrapped error of type ErrLowConfidence.
|
||||
func MakeLowConfidenceError(err error) error {
|
||||
return &ErrLowConfidence{
|
||||
wrappedError{
|
||||
|
@ -69,6 +69,7 @@ func runEnabledChecks(ctx context.Context,
|
||||
close(resultsCh)
|
||||
}
|
||||
|
||||
// RunScorecards runs enabled Scorecard checks on a RepoURL.
|
||||
func RunScorecards(ctx context.Context,
|
||||
repo repos.RepoURL,
|
||||
checksToRun checker.CheckNameToFnMap,
|
||||
|
@ -29,6 +29,7 @@ import (
|
||||
"github.com/ossf/scorecard/v2/checker"
|
||||
)
|
||||
|
||||
// ScorecardResult struct is returned on a successful Scorecard run.
|
||||
type ScorecardResult struct {
|
||||
Repo string
|
||||
Date string
|
||||
@ -67,6 +68,7 @@ func (r *ScorecardResult) AsJSON(showDetails bool, logLevel zapcore.Level, write
|
||||
return nil
|
||||
}
|
||||
|
||||
// AsCSV outputs ScorecardResult in CSV format.
|
||||
func (r *ScorecardResult) AsCSV(showDetails bool, logLevel zapcore.Level, writer io.Writer) error {
|
||||
w := csv.NewWriter(writer)
|
||||
record := []string{r.Repo}
|
||||
@ -93,6 +95,7 @@ func (r *ScorecardResult) AsCSV(showDetails bool, logLevel zapcore.Level, writer
|
||||
return nil
|
||||
}
|
||||
|
||||
// AsString returns ScorecardResult in string format.
|
||||
func (r *ScorecardResult) AsString(showDetails bool, logLevel zapcore.Level, writer io.Writer) error {
|
||||
data := make([][]string, len(r.Checks))
|
||||
//nolint
|
||||
|
@ -30,6 +30,7 @@ var (
|
||||
ErrorInvalidURL = errors.New("invalid repo flag")
|
||||
)
|
||||
|
||||
// RepoURL represents a URL of a repo.
|
||||
type RepoURL struct {
|
||||
Host, Owner, Repo string
|
||||
Metadata []string
|
||||
@ -40,14 +41,17 @@ func (r *RepoURL) Type() string {
|
||||
return "repo"
|
||||
}
|
||||
|
||||
// URL returns a valid url for RepoURL struct.
|
||||
func (r *RepoURL) URL() string {
|
||||
return fmt.Sprintf("%s/%s/%s", r.Host, r.Owner, r.Repo)
|
||||
}
|
||||
|
||||
// String returns a string representation of RepoURL struct.
|
||||
func (r *RepoURL) String() string {
|
||||
return fmt.Sprintf("%s-%s-%s", r.Host, r.Owner, r.Repo)
|
||||
}
|
||||
|
||||
// Set parses a URL string into RepoURL struct.
|
||||
func (r *RepoURL) Set(s string) error {
|
||||
// Allow skipping scheme for ease-of-use, default to https.
|
||||
if !strings.Contains(s, "://") {
|
||||
@ -69,6 +73,7 @@ func (r *RepoURL) Set(s string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidGitHubURL checks whether RepoURL represents a valid GitHub repo and returns errors otherwise.
|
||||
func (r *RepoURL) ValidGitHubURL() error {
|
||||
switch r.Host {
|
||||
case "github.com":
|
||||
|
@ -26,6 +26,7 @@ import (
|
||||
"github.com/ossf/scorecard/v2/stats"
|
||||
)
|
||||
|
||||
// MakeCensusTransport wraps input Roundtripper with monitoring logic.
|
||||
func MakeCensusTransport(innerTransport http.RoundTripper) http.RoundTripper {
|
||||
return &ochttp.Transport{
|
||||
Base: &censusTransport{
|
||||
|
@ -23,6 +23,7 @@ import (
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// MakeRateLimitedTransport returns a RoundTripper which rate limits GitHub requests.
|
||||
func MakeRateLimitedTransport(innerTransport http.RoundTripper, logger *zap.SugaredLogger) http.RoundTripper {
|
||||
return &rateLimitTransport{
|
||||
logger: logger,
|
||||
|
@ -43,10 +43,12 @@ func validateDetailTypes(messages []checker.CheckDetail, nw, ni, nd int) bool {
|
||||
end == nd
|
||||
}
|
||||
|
||||
// TestDetailLogger implements `checker.DetailLogger`.
|
||||
type TestDetailLogger struct {
|
||||
messages []checker.CheckDetail
|
||||
}
|
||||
|
||||
// TestReturn encapsulates expected CheckResult return values.
|
||||
type TestReturn struct {
|
||||
Errors []error
|
||||
Score int
|
||||
@ -55,22 +57,26 @@ type TestReturn struct {
|
||||
NumberOfDebug int
|
||||
}
|
||||
|
||||
// Info implements DetailLogger.Info.
|
||||
func (l *TestDetailLogger) Info(desc string, args ...interface{}) {
|
||||
cd := checker.CheckDetail{Type: checker.DetailInfo, Msg: fmt.Sprintf(desc, args...)}
|
||||
l.messages = append(l.messages, cd)
|
||||
}
|
||||
|
||||
// Warn implements DetailLogger.Warn.
|
||||
func (l *TestDetailLogger) Warn(desc string, args ...interface{}) {
|
||||
cd := checker.CheckDetail{Type: checker.DetailWarn, Msg: fmt.Sprintf(desc, args...)}
|
||||
l.messages = append(l.messages, cd)
|
||||
}
|
||||
|
||||
// Debug implements DetailLogger.Debug.
|
||||
func (l *TestDetailLogger) Debug(desc string, args ...interface{}) {
|
||||
cd := checker.CheckDetail{Type: checker.DetailDebug, Msg: fmt.Sprintf(desc, args...)}
|
||||
l.messages = append(l.messages, cd)
|
||||
}
|
||||
|
||||
//nolint
|
||||
// ValidateTestValues validates returned score and log values.
|
||||
// nolint: thelper
|
||||
func ValidateTestValues(t *testing.T, name string, te *TestReturn,
|
||||
score int, err error, dl *TestDetailLogger) bool {
|
||||
for _, we := range te.Errors {
|
||||
@ -97,7 +103,8 @@ func ValidateTestValues(t *testing.T, name string, te *TestReturn,
|
||||
return true
|
||||
}
|
||||
|
||||
//nolint
|
||||
// ValidateTestReturn validates expected TestReturn with actual checker.CheckResult values.
|
||||
// nolint: thelper
|
||||
func ValidateTestReturn(t *testing.T, name string, te *TestReturn,
|
||||
tr *checker.CheckResult, dl *TestDetailLogger) bool {
|
||||
return ValidateTestValues(t, name, te, tr.Score, tr.Error2, dl)
|
||||
|
Loading…
Reference in New Issue
Block a user