mirror of
https://github.com/ossf/scorecard.git
synced 2024-09-11 08:55:27 +03:00
Pass in specific commit-SHA in cron job (#1739)
Co-authored-by: Azeem Shaikh <azeems@google.com>
This commit is contained in:
parent
ba78d0aa59
commit
a3f4b05bbf
@ -24,12 +24,15 @@ import (
|
||||
"google.golang.org/protobuf/encoding/protojson"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/ossf/scorecard/v4/clients"
|
||||
"github.com/ossf/scorecard/v4/cron/config"
|
||||
"github.com/ossf/scorecard/v4/cron/data"
|
||||
"github.com/ossf/scorecard/v4/cron/pubsub"
|
||||
"github.com/ossf/scorecard/v4/pkg"
|
||||
)
|
||||
|
||||
var headSHA = clients.HeadSHA
|
||||
|
||||
func publishToRepoRequestTopic(iter data.Iterator, topicPublisher pubsub.Publisher,
|
||||
shardSize int, datetime time.Time) (int32, error) {
|
||||
var shardNum int32
|
||||
@ -48,7 +51,9 @@ func publishToRepoRequestTopic(iter data.Iterator, topicPublisher pubsub.Publish
|
||||
return shardNum, fmt.Errorf("error reading repoURL: %w", err)
|
||||
}
|
||||
request.Repos = append(request.GetRepos(), &data.Repo{
|
||||
Url: &repoURL.Repo,
|
||||
Url: &repoURL.Repo,
|
||||
// TODO(controller): pass in non-HEAD commitSHA here.
|
||||
Commit: &headSHA,
|
||||
Metadata: repoURL.Metadata.ToString(),
|
||||
})
|
||||
if len(request.GetRepos()) < shardSize {
|
||||
|
@ -21,12 +21,11 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
reflect "reflect"
|
||||
sync "sync"
|
||||
|
||||
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
|
||||
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
|
||||
timestamppb "google.golang.org/protobuf/types/known/timestamppb"
|
||||
reflect "reflect"
|
||||
sync "sync"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -42,6 +41,7 @@ type Repo struct {
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Url *string `protobuf:"bytes,1,opt,name=url,proto3,oneof" json:"url,omitempty"`
|
||||
Commit *string `protobuf:"bytes,3,opt,name=commit,proto3,oneof" json:"commit,omitempty"`
|
||||
Metadata []string `protobuf:"bytes,2,rep,name=metadata,proto3" json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
@ -84,6 +84,13 @@ func (x *Repo) GetUrl() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *Repo) GetCommit() string {
|
||||
if x != nil && x.Commit != nil {
|
||||
return *x.Commit
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *Repo) GetMetadata() []string {
|
||||
if x != nil {
|
||||
return x.Metadata
|
||||
@ -162,27 +169,29 @@ var file_cron_data_request_proto_rawDesc = []byte{
|
||||
0x73, 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, 0x72, 0x64, 0x2e, 0x63, 0x72, 0x6f, 0x6e, 0x2e, 0x64,
|
||||
0x61, 0x74, 0x61, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74,
|
||||
0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70,
|
||||
0x72, 0x6f, 0x74, 0x6f, 0x22, 0x41, 0x0a, 0x04, 0x52, 0x65, 0x70, 0x6f, 0x12, 0x15, 0x0a, 0x03,
|
||||
0x72, 0x6f, 0x74, 0x6f, 0x22, 0x69, 0x0a, 0x04, 0x52, 0x65, 0x70, 0x6f, 0x12, 0x15, 0x0a, 0x03,
|
||||
0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x03, 0x75, 0x72, 0x6c,
|
||||
0x88, 0x01, 0x01, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18,
|
||||
0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x42,
|
||||
0x06, 0x0a, 0x04, 0x5f, 0x75, 0x72, 0x6c, 0x22, 0xcc, 0x01, 0x0a, 0x15, 0x53, 0x63, 0x6f, 0x72,
|
||||
0x65, 0x63, 0x61, 0x72, 0x64, 0x42, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
|
||||
0x74, 0x12, 0x34, 0x0a, 0x05, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b,
|
||||
0x32, 0x1e, 0x2e, 0x6f, 0x73, 0x73, 0x66, 0x2e, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, 0x72,
|
||||
0x64, 0x2e, 0x63, 0x72, 0x6f, 0x6e, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x52, 0x65, 0x70, 0x6f,
|
||||
0x52, 0x05, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x20, 0x0a, 0x09, 0x73, 0x68, 0x61, 0x72, 0x64,
|
||||
0x5f, 0x6e, 0x75, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x48, 0x00, 0x52, 0x08, 0x73, 0x68,
|
||||
0x61, 0x72, 0x64, 0x4e, 0x75, 0x6d, 0x88, 0x01, 0x01, 0x12, 0x3a, 0x0a, 0x08, 0x6a, 0x6f, 0x62,
|
||||
0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f,
|
||||
0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69,
|
||||
0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x01, 0x52, 0x07, 0x6a, 0x6f, 0x62, 0x54, 0x69,
|
||||
0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f,
|
||||
0x6e, 0x75, 0x6d, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x6a, 0x6f, 0x62, 0x5f, 0x74, 0x69, 0x6d, 0x65,
|
||||
0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x42, 0x25, 0x5a, 0x23, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
|
||||
0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x73, 0x73, 0x66, 0x2f, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x63,
|
||||
0x61, 0x72, 0x64, 0x2f, 0x63, 0x72, 0x6f, 0x6e, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70,
|
||||
0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x88, 0x01, 0x01, 0x12, 0x1b, 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x03, 0x20,
|
||||
0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x88, 0x01, 0x01,
|
||||
0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x03,
|
||||
0x28, 0x09, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x42, 0x06, 0x0a, 0x04,
|
||||
0x5f, 0x75, 0x72, 0x6c, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x22,
|
||||
0xcc, 0x01, 0x0a, 0x15, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, 0x72, 0x64, 0x42, 0x61, 0x74,
|
||||
0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x34, 0x0a, 0x05, 0x72, 0x65, 0x70,
|
||||
0x6f, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x73, 0x73, 0x66, 0x2e,
|
||||
0x73, 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, 0x72, 0x64, 0x2e, 0x63, 0x72, 0x6f, 0x6e, 0x2e, 0x64,
|
||||
0x61, 0x74, 0x61, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x52, 0x05, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x12,
|
||||
0x20, 0x0a, 0x09, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x6e, 0x75, 0x6d, 0x18, 0x02, 0x20, 0x01,
|
||||
0x28, 0x05, 0x48, 0x00, 0x52, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x4e, 0x75, 0x6d, 0x88, 0x01,
|
||||
0x01, 0x12, 0x3a, 0x0a, 0x08, 0x6a, 0x6f, 0x62, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20,
|
||||
0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f,
|
||||
0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48,
|
||||
0x01, 0x52, 0x07, 0x6a, 0x6f, 0x62, 0x54, 0x69, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a,
|
||||
0x0a, 0x5f, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x6e, 0x75, 0x6d, 0x42, 0x0b, 0x0a, 0x09, 0x5f,
|
||||
0x6a, 0x6f, 0x62, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x42, 0x25,
|
||||
0x5a, 0x23, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x73, 0x73,
|
||||
0x66, 0x2f, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, 0x72, 0x64, 0x2f, 0x63, 0x72, 0x6f, 0x6e,
|
||||
0x2f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
@ -22,6 +22,7 @@ option go_package = "github.com/ossf/scorecard/cron/data";
|
||||
|
||||
message Repo {
|
||||
optional string url = 1;
|
||||
optional string commit = 3;
|
||||
repeated string metadata = 2;
|
||||
}
|
||||
|
||||
|
@ -27,7 +27,6 @@ import (
|
||||
"go.opencensus.io/stats/view"
|
||||
|
||||
"github.com/ossf/scorecard/v4/checker"
|
||||
"github.com/ossf/scorecard/v4/checks"
|
||||
"github.com/ossf/scorecard/v4/clients"
|
||||
"github.com/ossf/scorecard/v4/clients/githubrepo"
|
||||
githubstats "github.com/ossf/scorecard/v4/clients/githubrepo/stats"
|
||||
@ -40,14 +39,17 @@ import (
|
||||
sce "github.com/ossf/scorecard/v4/errors"
|
||||
"github.com/ossf/scorecard/v4/log"
|
||||
"github.com/ossf/scorecard/v4/pkg"
|
||||
"github.com/ossf/scorecard/v4/policy"
|
||||
"github.com/ossf/scorecard/v4/stats"
|
||||
)
|
||||
|
||||
var ignoreRuntimeErrors = flag.Bool("ignoreRuntimeErrors", false, "if set to true any runtime errors will be ignored")
|
||||
|
||||
// nolint: gocognit
|
||||
func processRequest(ctx context.Context,
|
||||
batchRequest *data.ScorecardBatchRequest, checksToRun checker.CheckNameToFnMap,
|
||||
bucketURL, bucketURL2 string, checkDocs docs.Doc,
|
||||
batchRequest *data.ScorecardBatchRequest,
|
||||
blacklistedChecks []string, bucketURL, bucketURL2 string,
|
||||
checkDocs docs.Doc,
|
||||
repoClient clients.RepoClient, ossFuzzRepoClient clients.RepoClient,
|
||||
ciiClient clients.CIIBestPracticesClient,
|
||||
vulnsClient clients.VulnerabilitiesClient,
|
||||
@ -74,16 +76,30 @@ func processRequest(ctx context.Context,
|
||||
var buffer bytes.Buffer
|
||||
var buffer2 bytes.Buffer
|
||||
// TODO: run Scorecard for each repo in a separate thread.
|
||||
for _, repo := range batchRequest.GetRepos() {
|
||||
logger.Info(fmt.Sprintf("Running Scorecard for repo: %s", *repo.Url))
|
||||
repo, err := githubrepo.MakeGithubRepo(*repo.Url)
|
||||
for _, repoReq := range batchRequest.GetRepos() {
|
||||
logger.Info(fmt.Sprintf("Running Scorecard for repo: %s", *repoReq.Url))
|
||||
repo, err := githubrepo.MakeGithubRepo(*repoReq.Url)
|
||||
if err != nil {
|
||||
// TODO(log): Previously Warn. Consider logging an error here.
|
||||
logger.Info(fmt.Sprintf("invalid GitHub URL: %v", err))
|
||||
continue
|
||||
}
|
||||
repo.AppendMetadata(repo.Metadata()...)
|
||||
result, err := pkg.RunScorecards(ctx, repo, clients.HeadSHA /*commitSHA*/, false /*raw*/, checksToRun,
|
||||
|
||||
commitSHA := clients.HeadSHA
|
||||
requiredRequestType := []checker.RequestType{}
|
||||
if repoReq.Commit != nil && *repoReq.Commit != clients.HeadSHA {
|
||||
commitSHA = *repoReq.Commit
|
||||
requiredRequestType = append(requiredRequestType, checker.CommitBased)
|
||||
}
|
||||
checksToRun, err := policy.GetEnabled(nil /*policy*/, nil /*checks*/, requiredRequestType)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error during policy.GetEnabled: %w", err)
|
||||
}
|
||||
for _, check := range blacklistedChecks {
|
||||
delete(checksToRun, check)
|
||||
}
|
||||
result, err := pkg.RunScorecards(ctx, repo, commitSHA, false /*raw*/, checksToRun,
|
||||
repoClient, ossFuzzRepoClient, ciiClient, vulnsClient)
|
||||
if errors.Is(err, sce.ErrRepoUnreachable) {
|
||||
// Not accessible repo - continue.
|
||||
@ -207,10 +223,6 @@ func main() {
|
||||
logger.Info(fmt.Sprintf("%v", http.ListenAndServe(":8080", nil)))
|
||||
}()
|
||||
|
||||
checksToRun := checks.AllChecks
|
||||
for _, check := range blacklistedChecks {
|
||||
delete(checksToRun, check)
|
||||
}
|
||||
for {
|
||||
req, err := subscriber.SynchronousPull()
|
||||
if err != nil {
|
||||
@ -223,7 +235,7 @@ func main() {
|
||||
logger.Info("subscription returned nil message during Receive, exiting")
|
||||
break
|
||||
}
|
||||
if err := processRequest(ctx, req, checksToRun,
|
||||
if err := processRequest(ctx, req, blacklistedChecks,
|
||||
bucketURL, bucketURL2, checkDocs,
|
||||
repoClient, ossFuzzRepoClient, ciiClient, vulnsClient, logger); err != nil {
|
||||
// TODO(log): Previously Warn. Consider logging an error here.
|
||||
|
Loading…
Reference in New Issue
Block a user