ossf/scorecard
1
1
Fork 0
mirror of https://github.com/ossf/scorecard.git synced 2024-09-11 08:55:27 +03:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

Remove repos package (#1191)

Browse Source 
Co-authored-by: Azeem Shaikh <azeems@google.com>
This commit is contained in:
Azeem Shaikh 2021-10-29 12:07:46 -04:00 committed by GitHub
parent 148446bb83
commit 83649a799e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 166 additions and 742 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -143,7 +143,7 @@ build-webhook: ## Runs go build on the cron webhook
build-add-script: ## Runs go build on the add script
build-add-script: cron/data/add/add
cron/data/add/add: cron/data/add/*.go cron/data/*.go repos/*.go cron/data/projects.csv
cron/data/add/add: cron/data/add/*.go cron/data/*.go cron/data/projects.csv
# Run go build on the add script
cd cron/data/add && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o add

34
cmd/root.go
View File

@ -39,11 +39,10 @@ import (
sce "github.com/ossf/scorecard/v3/errors"
"github.com/ossf/scorecard/v3/pkg"
spol "github.com/ossf/scorecard/v3/policy"
"github.com/ossf/scorecard/v3/repos"
)
var (
repo repos.RepoURI
repo string
checksToRun []string
metaData []string
// This one has to use goflag instead of pflag because it's defined by zap.
@ -139,23 +138,16 @@ func validateFormat(format string) bool {
}
}
func createRepoClient(ctx context.Context, uri *repos.RepoURI, logger *zap.Logger) (clients.RepoClient, error) {
var rc clients.RepoClient
switch uri.RepoType() {
// URL.
case repos.RepoTypeURL:
if err := repo.IsValidGitHubURL(); err != nil {
return rc, sce.WithMessage(sce.ErrScorecardInternal, err.Error())
}
rc = githubrepo.CreateGithubRepoClient(ctx, logger)
return rc, nil
// Local directory.
case repos.RepoTypeLocalDir:
return localdir.CreateLocalDirClient(ctx, logger), nil
func getRepoAccessors(ctx context.Context, uri string, logger *zap.Logger) (clients.Repo, clients.RepoClient, error) {
if repo, err := localdir.MakeLocalDirRepo(uri); err == nil {
// Local directory.
return repo, localdir.CreateLocalDirClient(ctx, logger), nil
}
return nil, sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("unspported URI: %v", uri.RepoType()))
if repo, err := githubrepo.MakeGithubRepo(uri); err == nil {
// GitHub URL.
return repo, githubrepo.CreateGithubRepoClient(ctx, logger), nil
}
return nil, nil, sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("unspported URI: %s", uri))
}
var rootCmd = &cobra.Command{
@ -223,7 +215,7 @@ var rootCmd = &cobra.Command{
// nolint
defer logger.Sync() // Flushes buffer, if any.
repoClient, err := createRepoClient(ctx, &repo, logger)
repoURI, repoClient, err := getRepoAccessors(ctx, repo, logger)
if err != nil {
log.Fatal(err)
}
@ -240,7 +232,7 @@ var rootCmd = &cobra.Command{
}
}
repoResult, err := pkg.RunScorecards(ctx, &repo, enabledChecks, repoClient)
repoResult, err := pkg.RunScorecards(ctx, repoURI, enabledChecks, repoClient)
if err != nil {
log.Fatal(err)
}
@ -408,7 +400,7 @@ func enableCheck(checkName string, enabledChecks *checker.CheckNameToFnMap) bool
func init() {
// Add the zap flag manually
rootCmd.PersistentFlags().AddGoFlagSet(goflag.CommandLine)
rootCmd.Flags().Var(&repo, "repo", "repository to check")
rootCmd.Flags().StringVar(&repo, "repo", "", "repository to check")
rootCmd.Flags().StringVar(
&npm, "npm", "",
"npm package to check, given that the npm package has a GitHub repository")

7
cmd/serve.go
View File

@ -27,7 +27,6 @@ import (
"github.com/ossf/scorecard/v3/checks"
"github.com/ossf/scorecard/v3/clients/githubrepo"
"github.com/ossf/scorecard/v3/pkg"
"github.com/ossf/scorecard/v3/repos"
)
//nolint:gochecknoinits
@ -59,13 +58,13 @@ var serveCmd = &cobra.Command{
if len(s) != length {
rw.WriteHeader(http.StatusBadRequest)
}
repo := repos.RepoURI{}
if err := repo.Set(repoParam); err != nil {
repo, err := githubrepo.MakeGithubRepo(repoParam)
if err != nil {
rw.WriteHeader(http.StatusBadRequest)
}
ctx := r.Context()
repoClient := githubrepo.CreateGithubRepoClient(ctx, logger)
repoResult, err := pkg.RunScorecards(ctx, &repo, checks.AllChecks, repoClient)
repoResult, err := pkg.RunScorecards(ctx, repo, checks.AllChecks, repoClient)
if err != nil {
sugar.Error(err)
rw.WriteHeader(http.StatusInternalServerError)

2
cron/controller/main.go
View File

@ -59,7 +59,7 @@ func publishToRepoRequestTopic(ctx context.Context, iter data.Iterator, datetime
if err != nil {
return shardNum, fmt.Errorf("error reading repoURL: %w", err)
}
request.Repos = append(request.GetRepos(), repoURL.URL())
request.Repos = append(request.GetRepos(), repoURL.Repo)
if len(request.GetRepos()) < shardSize {
continue
}

29
cron/data/add/main.go
View File

@ -21,7 +21,6 @@ import (
"os"
"github.com/ossf/scorecard/v3/cron/data"
"github.com/ossf/scorecard/v3/repos"
)
// Script to add new project repositories to the projects.csv file:
@ -64,34 +63,32 @@ func main() {
}
}
func getRepoURLs(iter data.Iterator) ([]repos.RepoURI, error) {
repoURLs := make(map[string]*repos.RepoURI)
func getRepoURLs(iter data.Iterator) ([]data.RepoFormat, error) {
repoURLs := make(map[string]*data.RepoFormat)
repoMap := make(map[string]map[string]bool)
for iter.HasNext() {
repo, err := iter.Next()
if err != nil {
return nil, fmt.Errorf("iter.Next: %w", err)
}
if _, ok := repoMap[repo.URL()]; !ok {
repoURLs[repo.URL()] = new(repos.RepoURI)
*repoURLs[repo.URL()] = repo
repoMap[repo.URL()] = make(map[string]bool)
for _, metadata := range repo.Metadata() {
repoMap[repo.URL()][metadata] = true
if _, ok := repoMap[repo.Repo]; !ok {
repoURLs[repo.Repo] = new(data.RepoFormat)
*repoURLs[repo.Repo] = repo
repoMap[repo.Repo] = make(map[string]bool)
for _, metadata := range repo.Metadata {
repoMap[repo.Repo][metadata] = true
}
continue
}
for _, metadata := range repo.Metadata() {
if _, ok := repoMap[repo.URL()][metadata]; !ok && metadata != "" {
if err := repoURLs[repo.URL()].AppendMetadata(metadata); err != nil {
return nil, fmt.Errorf("AppendMetadata: %w", err)
}
repoMap[repo.URL()][metadata] = true
for _, metadata := range repo.Metadata {
if _, ok := repoMap[repo.Repo][metadata]; !ok && metadata != "" {
repoURLs[repo.Repo].Metadata = append(repoURLs[repo.Repo].Metadata, metadata)
repoMap[repo.Repo][metadata] = true
}
}
}
newRepoURLs := make([]repos.RepoURI, 0)
newRepoURLs := make([]data.RepoFormat, 0)
for _, repoURL := range repoURLs {
newRepoURLs = append(newRepoURLs, *repoURL)
}

103
cron/data/add/main_test.go
View File

@ -17,25 +17,15 @@ package main
import (
"fmt"
"os"
"reflect"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/ossf/scorecard/v3/cron/data"
"github.com/ossf/scorecard/v3/repos"
)
type fields struct {
host string
owner string
repo string
metadata []string
}
//nolint: gocritic
func lessThanURI(x, y repos.RepoURI) bool {
func lessThanURI(x, y data.RepoFormat) bool {
return fmt.Sprintf("%+v", x) < fmt.Sprintf("%+v", y)
}
@ -43,93 +33,73 @@ func TestGetRepoURLs(t *testing.T) {
t.Parallel()
testcases := []struct {
name, filename string
outcome []fields
outcome []data.RepoFormat
}{
{
name: "NoChange",
filename: "testdata/no_change.csv",
outcome: []fields{
outcome: []data.RepoFormat{
{
host: "github.com",
owner: "owner1",
repo: "repo1",
metadata: []string{"meta1", "meta2"},
Repo: "github.com/owner1/repo1",
Metadata: []string{"meta1", "meta2"},
},
{
host: "github.com",
owner: "owner2",
repo: "repo2",
Repo: "github.com/owner2/repo2",
},
},
},
{
name: "AddMetadata",
filename: "testdata/add_metadata.csv",
outcome: []fields{
outcome: []data.RepoFormat{
{
host: "github.com",
owner: "owner1",
repo: "repo1",
metadata: []string{"meta1", "meta2"},
Repo: "github.com/owner1/repo1",
Metadata: []string{"meta1", "meta2"},
},
{
host: "github.com",
owner: "owner2",
repo: "repo2",
metadata: []string{"meta1"},
Repo: "github.com/owner2/repo2",
Metadata: []string{"meta1"},
},
},
},
{
name: "SkipLatest",
filename: "testdata/skip_latest.csv",
outcome: []fields{
outcome: []data.RepoFormat{
{
host: "github.com",
owner: "owner1",
repo: "repo1",
metadata: []string{"meta1", "meta2"},
Repo: "github.com/owner1/repo1",
Metadata: []string{"meta1", "meta2"},
},
{
host: "github.com",
owner: "owner2",
repo: "repo2",
Repo: "github.com/owner2/repo2",
},
},
},
{
name: "SkipEmpty",
filename: "testdata/skip_empty.csv",
outcome: []fields{
outcome: []data.RepoFormat{
{
host: "github.com",
owner: "owner1",
repo: "repo1",
metadata: []string{"meta1", "meta2"},
Repo: "github.com/owner1/repo1",
Metadata: []string{"meta1", "meta2"},
},
{
host: "github.com",
owner: "owner2",
repo: "repo2",
metadata: []string{"meta3"},
Repo: "github.com/owner2/repo2",
Metadata: []string{"meta3"},
},
},
},
{
name: "SkipEmpty_2",
filename: "testdata/skip_empty_2.csv",
outcome: []fields{
outcome: []data.RepoFormat{
{
host: "github.com",
owner: "owner1",
repo: "repo1",
metadata: []string{"meta1", "meta2"},
Repo: "github.com/owner1/repo1",
Metadata: []string{"meta1", "meta2"},
},
{
host: "github.com",
owner: "owner2",
repo: "repo2",
metadata: []string{"meta3"},
Repo: "github.com/owner2/repo2",
Metadata: []string{"meta3"},
},
},
},
@ -155,27 +125,8 @@ func TestGetRepoURLs(t *testing.T) {
t.Errorf("testcase failed: %v", err)
}
// Create the list of RepoURL from the outcome.
var rs []repos.RepoURI
for _, r := range testcase.outcome {
u := fmt.Sprintf("%s/%s/%s", r.host, r.owner, r.repo)
outcomeRepo, err := repos.NewFromURL(u)
if err != nil {
t.Errorf("repos.NewFromURL: %v", err)
}
if err := outcomeRepo.AppendMetadata(r.metadata...); err != nil {
t.Errorf("outcomeRepo.AppendMetadata: %v", err)
}
rs = append(rs, *outcomeRepo)
}
// Export all private fields for comparison.
exp := cmp.Exporter(func(t reflect.Type) bool {
return true
})
if !cmp.Equal(rs, repoURLs, exp, cmpopts.EquateEmpty(), cmpopts.SortSlices(lessThanURI)) {
t.Errorf("testcase failed. expected %+v, got %+v", rs, repoURLs)
if !cmp.Equal(testcase.outcome, repoURLs, cmpopts.EquateEmpty(), cmpopts.SortSlices(lessThanURI)) {
t.Errorf("testcase failed. expected equal, got diff: %s", cmp.Diff(testcase.outcome, repoURLs))
}
})
}

14
cron/data/format.go
View File

@ -16,13 +16,16 @@ package data
import "strings"
type csvStrings []string
// CSVStrings is []string with support for CSV formatting.
type CSVStrings []string
func (s csvStrings) MarshalCSV() ([]byte, error) {
// MarshalCSV implements []string -> []byte serialization.
func (s CSVStrings) MarshalCSV() ([]byte, error) {
return []byte(strings.Join(s, ",")), nil
}
func (s *csvStrings) UnmarshalCSV(input []byte) error {
// UnmarshalCSV implements []byte -> []string de-serializtion.
func (s *CSVStrings) UnmarshalCSV(input []byte) error {
if len(input) == 0 || string(input) == "" {
*s = nil
return nil
@ -31,7 +34,8 @@ func (s *csvStrings) UnmarshalCSV(input []byte) error {
return nil
}
type repoFormat struct {
// RepoFormat is used to read input repos.
type RepoFormat struct {
Repo string `csv:"repo"`
Metadata csvStrings `csv:"metadata"`
Metadata CSVStrings `csv:"metadata"`
}

4
cron/data/format_test.go
View File

@ -25,7 +25,7 @@ func TestUnmarshalCsv(t *testing.T) {
testcases := []struct {
name string
input []byte
output csvStrings
output CSVStrings
}{
{
name: "Basic",
@ -53,7 +53,7 @@ func TestUnmarshalCsv(t *testing.T) {
testcase := testcase
t.Run(testcase.name, func(t *testing.T) {
t.Parallel()
s := new(csvStrings)
s := new(CSVStrings)
if err := s.UnmarshalCSV(testcase.input); err != nil {
t.Errorf("testcase failed: %v", err)
}

26
cron/data/iterator.go
View File

@ -22,13 +22,13 @@ import (
"github.com/jszwec/csvutil"
"github.com/ossf/scorecard/v3/repos"
"github.com/ossf/scorecard/v3/clients/githubrepo"
)
// Iterator interface is used to iterate through list of input repos for the cron job.
type Iterator interface {
HasNext() bool
Next() (repos.RepoURI, error)
Next() (RepoFormat, error)
}
// MakeIteratorFrom returns an implementation of Iterator interface.
@ -46,7 +46,7 @@ func MakeIteratorFrom(reader io.Reader) (Iterator, error) {
type csvIterator struct {
decoder *csvutil.Decoder
err error
next repoFormat
next RepoFormat
}
func (reader *csvIterator) HasNext() bool {
@ -54,21 +54,13 @@ func (reader *csvIterator) HasNext() bool {
return !errors.Is(reader.err, io.EOF)
}
func (reader *csvIterator) Next() (repos.RepoURI, error) {
ret := repos.RepoURI{}
func (reader *csvIterator) Next() (RepoFormat, error) {
if reader.err != nil {
return ret, fmt.Errorf("reader has error: %w", reader.err)
return reader.next, fmt.Errorf("reader has error: %w", reader.err)
}
err := ret.SetMetadata(reader.next.Metadata)
if err != nil {
return ret, fmt.Errorf("error during SetMetadata: %w", err)
// Sanity check valid GitHub URL.
if _, err := githubrepo.MakeGithubRepo(reader.next.Repo); err != nil {
return reader.next, fmt.Errorf("invalid GitHub URL: %w", err)
}
if err := ret.Set(reader.next.Repo); err != nil {
return ret, fmt.Errorf("error during Set: %w", err)
}
if err := ret.IsValidGitHubURL(); err != nil {
return ret, fmt.Errorf("error during IsValidGitHubURL: %w", err)
}
return ret, nil
return reader.next, nil
}

104
cron/data/iterator_test.go
View File

@ -16,25 +16,17 @@ package data
import (
"errors"
"fmt"
"os"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/ossf/scorecard/v3/repos"
sce "github.com/ossf/scorecard/v3/errors"
)
type fields struct {
host string
owner string
repo string
metadata []string
}
type outcome struct {
expectedErr error
repoURI fields
repo RepoFormat
hasError bool
}
@ -53,27 +45,21 @@ func TestCsvIterator(t *testing.T) {
outcomes: []outcome{
{
hasError: false,
repoURI: fields{
host: "github.com",
owner: "owner1",
repo: "repo1",
repo: RepoFormat{
Repo: "github.com/owner1/repo1",
},
},
{
hasError: false,
repoURI: fields{
host: "github.com",
owner: "owner2",
repo: "repo2",
repo: RepoFormat{
Repo: "github.com/owner2/repo2",
},
},
{
hasError: false,
repoURI: fields{
host: "github.com",
owner: "owner3",
repo: "repo3",
metadata: []string{"meta"},
repo: RepoFormat{
Repo: "github.com/owner3/repo3",
Metadata: []string{"meta"},
},
},
},
@ -84,27 +70,21 @@ func TestCsvIterator(t *testing.T) {
outcomes: []outcome{
{
hasError: false,
repoURI: fields{
host: "github.com",
owner: "owner1",
repo: "repo1",
repo: RepoFormat{
Repo: "github.com/owner1/repo1",
},
},
{
hasError: false,
repoURI: fields{
host: "github.com",
owner: "owner2",
repo: "repo2",
repo: RepoFormat{
Repo: "github.com/owner2/repo2",
},
},
{
hasError: false,
repoURI: fields{
host: "github.com",
owner: "owner3",
repo: "repo3",
metadata: []string{"meta"},
repo: RepoFormat{
Repo: "github.com/owner3/repo3",
Metadata: []string{"meta"},
},
},
},
@ -115,15 +95,15 @@ func TestCsvIterator(t *testing.T) {
outcomes: []outcome{
{
hasError: true,
expectedErr: repos.ErrorUnsupportedhost,
expectedErr: sce.ErrorUnsupportedHost,
},
{
hasError: true,
expectedErr: repos.ErrorInvalidURL,
expectedErr: sce.ErrorInvalidURL,
},
{
hasError: true,
expectedErr: repos.ErrorInvalidURL,
expectedErr: sce.ErrorInvalidURL,
},
},
},
@ -133,27 +113,21 @@ func TestCsvIterator(t *testing.T) {
outcomes: []outcome{
{
hasError: false,
repoURI: fields{
host: "github.com",
owner: "owner1",
repo: "repo1",
repo: RepoFormat{
Repo: "github.com/owner1/repo1",
},
},
{
hasError: false,
repoURI: fields{
host: "github.com",
owner: "owner2",
repo: "repo2",
repo: RepoFormat{
Repo: "github.com/owner2/repo2",
},
},
{
hasError: false,
repoURI: fields{
host: "github.com",
owner: "owner3",
repo: "repo3",
metadata: []string{"meta"},
repo: RepoFormat{
Repo: "github.com/owner3/repo3",
Metadata: []string{"meta"},
},
},
},
@ -164,18 +138,14 @@ func TestCsvIterator(t *testing.T) {
outcomes: []outcome{
{
hasError: false,
repoURI: fields{
host: "github.com",
owner: "owner1",
repo: "repo1",
repo: RepoFormat{
Repo: "github.com/owner1/repo1",
},
},
{
hasError: false,
repoURI: fields{
host: "github.com",
owner: "owner2",
repo: "repo2",
repo: RepoFormat{
Repo: "github.com/owner2/repo2",
},
},
{
@ -209,18 +179,8 @@ func TestCsvIterator(t *testing.T) {
}
if !outcome.hasError {
u := fmt.Sprintf("%s/%s/%s",
outcome.repoURI.host, outcome.repoURI.owner, outcome.repoURI.repo)
outcomeRepo, err := repos.NewFromURL(u)
if err != nil {
t.Errorf("repos.NewFromURL: %v", err)
}
if err := outcomeRepo.AppendMetadata(outcome.repoURI.metadata...); err != nil {
t.Errorf("outcomeRepo.AppendMetadata: %v", err)
}
if !cmp.Equal(outcomeRepo, &repoURL) {
t.Errorf("expected repoURL: %+v, got %+v", *outcomeRepo, repoURL)
if !cmp.Equal(outcome.repo, repoURL) {
t.Errorf("expected equal, got diff: %s", cmp.Diff(outcome.repo, repoURL))
}
}
if outcome.hasError && outcome.expectedErr != nil && !errors.Is(err, outcome.expectedErr) {

38
cron/data/update/dependency.go
View File

@ -30,8 +30,8 @@ import (
"github.com/google/go-github/v38/github"
"golang.org/x/tools/go/vcs"
"github.com/ossf/scorecard/v3/clients/githubrepo"
"github.com/ossf/scorecard/v3/cron/data"
"github.com/ossf/scorecard/v3/repos"
)
var (
@ -78,10 +78,10 @@ type repositoryDepsURL struct {
// Programmatically gets Envoy's dependencies and add to projects.
// Re-using a checker type.
func getBazelDeps(repo repositoryDepsURL) []repos.RepoURI {
func getBazelDeps(repo repositoryDepsURL) []data.RepoFormat {
client := github.NewClient(nil)
ctx := context.Background()
depRepos := []repos.RepoURI{}
depRepos := []data.RepoFormat{}
fo, _, _, err := client.Repositories.GetContents(ctx, repo.Owner, repo.Repo, repo.File, nil)
if err != nil {
// If we can't get content, gracefully fail but alert.
@ -101,8 +101,9 @@ func getBazelDeps(repo repositoryDepsURL) []repos.RepoURI {
// TODO: Replace with a starlark interpreter that can be used for any project.
for _, match := range re.FindAllString(fc, -1) {
repo := repos.RepoURI{}
if err := repo.Set(strings.TrimSuffix(match, ".git")); err != nil {
repo := data.RepoFormat{}
repo.Repo = strings.TrimSuffix(match, ".git")
if _, err := githubrepo.MakeGithubRepo(repo.Repo); err != nil {
log.Panicf("error during repo.Set: %v", err)
return depRepos
}
@ -112,8 +113,8 @@ func getBazelDeps(repo repositoryDepsURL) []repos.RepoURI {
}
// GetGoDeps returns go repo dependencies.
func getGoDeps(repo repositoryDepsURL) []repos.RepoURI {
repoURLs := []repos.RepoURI{}
func getGoDeps(repo repositoryDepsURL) []data.RepoFormat {
repoURLs := []data.RepoFormat{}
pwd, err := os.Getwd()
if err != nil {
log.Default().Println(err)
@ -184,22 +185,23 @@ func getVanityRepoURL(u string) string {
return repo.Repo
}
func parseGoModURL(dependency string, repoURLs []repos.RepoURI) []repos.RepoURI {
repoURL := repos.RepoURI{}
func parseGoModURL(dependency string, repoURLs []data.RepoFormat) []data.RepoFormat {
repoURL := data.RepoFormat{}
splitURL := strings.Split(dependency, "/")
// nolint:gomnd
if len(splitURL) < 3 {
return repoURLs
}
u := fmt.Sprintf("%s/%s/%s", splitURL[0], splitURL[1], splitURL[2])
if err := repoURL.Set(u); err != nil {
if _, err := githubrepo.MakeGithubRepo(u); err != nil {
return repoURLs
}
repoURL.Repo = u
repoURLs = append(repoURLs, repoURL)
return repoURLs
}
func getDependencies(in io.Reader) (oldRepos, newRepos []repos.RepoURI, e error) {
func getDependencies(in io.Reader) (oldRepos, newRepos []data.RepoFormat, e error) {
iter, err := data.MakeIteratorFrom(in)
if err != nil {
return nil, nil, fmt.Errorf("error during data.MakeIterator: %w", err)
@ -207,7 +209,7 @@ func getDependencies(in io.Reader) (oldRepos, newRepos []repos.RepoURI, e error)
// Read all project repositores into a map.
m := make(map[string][]string)
oldRepos = make([]repos.RepoURI, 0)
oldRepos = make([]data.RepoFormat, 0)
for iter.HasNext() {
repo, err := iter.Next()
if err != nil {
@ -215,25 +217,25 @@ func getDependencies(in io.Reader) (oldRepos, newRepos []repos.RepoURI, e error)
}
oldRepos = append(oldRepos, repo)
// We do not handle duplicates.
m[repo.URL()] = repo.Metadata()
m[repo.Repo] = repo.Metadata
}
// Create a list of project dependencies that are not already present.
newRepos = []repos.RepoURI{}
newRepos = []data.RepoFormat{}
for _, repo := range bazelRepos {
for _, item := range getBazelDeps(repo) {
if _, ok := m[item.URL()]; !ok {
if _, ok := m[item.Repo]; !ok {
// Also add to m to avoid dupes.
m[item.URL()] = item.Metadata()
m[item.Repo] = item.Metadata
newRepos = append(newRepos, item)
}
}
}
for _, repo := range gorepos {
for _, item := range getGoDeps(repo) {
if _, ok := m[item.URL()]; !ok {
if _, ok := m[item.Repo]; !ok {
// Also add to m to avoid dupes.
m[item.URL()] = item.Metadata()
m[item.Repo] = item.Metadata
newRepos = append(newRepos, item)
}
}

BIN
cron/data/update/update Executable file
View File

Binary file not shown.

6
cron/data/validate/main.go
View File

@ -45,9 +45,9 @@ func main() {
if err != nil {
panic(err)
}
if _, ok := m[repo.URL()]; ok {
log.Panicf("Item already in the list %s", repo.URL())
if _, ok := m[repo.Repo]; ok {
log.Panicf("Item already in the list %s", repo.Repo)
}
m[repo.URL()] = true
m[repo.Repo] = true
}
}

29
cron/data/writer.go
View File

@ -21,33 +21,18 @@ import (
"sort"
"github.com/jszwec/csvutil"
"github.com/ossf/scorecard/v3/repos"
)
func repoFormatFromRepoURL(repoURLs []repos.RepoURI) []repoFormat {
repoentries := make([]repoFormat, 0)
for _, repoURL := range repoURLs {
repoentry := repoFormat{
Repo: repoURL.URL(),
Metadata: repoURL.Metadata(),
}
repoentries = append(repoentries, repoentry)
}
return repoentries
}
// SortAndAppendTo appends `oldRepos` and `newRepos` before sorting and writing out the result to `out`.
func SortAndAppendTo(out io.Writer, oldRepos, newRepos []repos.RepoURI) error {
repoentries := repoFormatFromRepoURL(oldRepos)
repoentries = append(repoentries, repoFormatFromRepoURL(newRepos)...)
func SortAndAppendTo(out io.Writer, oldRepos, newRepos []RepoFormat) error {
oldRepos = append(oldRepos, newRepos...)
sort.SliceStable(repoentries, func(i, j int) bool {
return repoentries[i].Repo < repoentries[j].Repo
sort.SliceStable(oldRepos, func(i, j int) bool {
return oldRepos[i].Repo < oldRepos[j].Repo
})
csvWriter := csv.NewWriter(out)
enc := csvutil.NewEncoder(csvWriter)
if err := enc.Encode(repoentries); err != nil {
if err := enc.Encode(oldRepos); err != nil {
return fmt.Errorf("error during Encode: %w", err)
}
csvWriter.Flush()
@ -55,13 +40,13 @@ func SortAndAppendTo(out io.Writer, oldRepos, newRepos []repos.RepoURI) error {
}
// SortAndAppendFrom reads from `in`, appends to newRepos and writes the sorted output to `out`.
func SortAndAppendFrom(in io.Reader, out io.Writer, newRepos []repos.RepoURI) error {
func SortAndAppendFrom(in io.Reader, out io.Writer, newRepos []RepoFormat) error {
iter, err := MakeIteratorFrom(in)
if err != nil {
return fmt.Errorf("error during MakeIterator: %w", err)
}
oldRepos := make([]repos.RepoURI, 0)
oldRepos := make([]RepoFormat, 0)
for iter.HasNext() {
repo, err := iter.Next()
if err != nil {

48
cron/data/writer_test.go
View File

@ -16,10 +16,7 @@ package data
import (
"bytes"
"fmt"
"testing"
"github.com/ossf/scorecard/v3/repos"
)
func TestCsvWriter(t *testing.T) {
@ -27,25 +24,21 @@ func TestCsvWriter(t *testing.T) {
testcases := []struct {
name string
out string
oldRepos []fields
newRepos []fields
oldRepos []RepoFormat
newRepos []RepoFormat
}{
{
name: "Basic",
oldRepos: []fields{
oldRepos: []RepoFormat{
{
host: "github.com",
owner: "owner1",
repo: "repo1",
metadata: []string{"meta1"},
Repo: "github.com/owner1/repo1",
Metadata: []string{"meta1"},
},
},
newRepos: []fields{
newRepos: []RepoFormat{
{
host: "github.com",
owner: "owner2",
repo: "repo2",
metadata: []string{"meta2"},
Repo: "github.com/owner2/repo2",
Metadata: []string{"meta2"},
},
},
out: `repo,metadata
@ -60,30 +53,7 @@ github.com/owner2/repo2,meta2
t.Run(testcase.name, func(t *testing.T) {
t.Parallel()
var buf bytes.Buffer
var oldRepos []repos.RepoURI
var newRepos []repos.RepoURI
for _, v := range testcase.oldRepos {
r, err := repos.NewFromURL(fmt.Sprintf("%s/%s/%s", v.host, v.owner, v.repo))
if err != nil {
t.Errorf("repos.NewFromURL: %v", err)
}
if err = r.AppendMetadata(v.metadata...); err != nil {
t.Errorf("r.AppendMetadata: %v", err)
}
oldRepos = append(oldRepos, *r)
}
for _, v := range testcase.newRepos {
r, err := repos.NewFromURL(fmt.Sprintf("%s/%s/%s", v.host, v.owner, v.repo))
if err != nil {
t.Errorf("repos.NewFromURL: %v", err)
}
if err = r.AppendMetadata(v.metadata...); err != nil {
t.Errorf("r.AppendMetadata: %v", err)
}
newRepos = append(newRepos, *r)
}
err := SortAndAppendTo(&buf, oldRepos, newRepos)
err := SortAndAppendTo(&buf, testcase.oldRepos, testcase.newRepos)
if err != nil {
t.Errorf("error while running testcase: %v", err)
}

25
cron/worker/main.go
View File

@ -43,7 +43,6 @@ import (
docs "github.com/ossf/scorecard/v3/docs/checks"
sce "github.com/ossf/scorecard/v3/errors"
"github.com/ossf/scorecard/v3/pkg"
"github.com/ossf/scorecard/v3/repos"
"github.com/ossf/scorecard/v3/stats"
)
@ -72,25 +71,17 @@ func processRequest(ctx context.Context,
return nil
}
repoURLs := make([]repos.RepoURI, 0, len(batchRequest.GetRepos()))
for _, repo := range batchRequest.GetRepos() {
repoURL := repos.RepoURI{}
if err := repoURL.Set(repo); err != nil {
return fmt.Errorf("error setting RepoURL: %w", err)
}
if err := repoURL.IsValidGitHubURL(); err != nil {
return fmt.Errorf("url is not a valid GitHub URL: %w", err)
}
repoURLs = append(repoURLs, repoURL)
}
var buffer bytes.Buffer
var buffer2 bytes.Buffer
// TODO: run Scorecard for each repo in a separate thread.
for i := range repoURLs {
repoURL := repoURLs[i]
logger.Info(fmt.Sprintf("Running Scorecard for repo: %s", repoURL.URL()))
result, err := pkg.RunScorecards(ctx, &repoURL, checksToRun, repoClient)
for _, repoURL := range batchRequest.GetRepos() {
logger.Info(fmt.Sprintf("Running Scorecard for repo: %s", repoURL))
repo, err := githubrepo.MakeGithubRepo(repoURL)
if err != nil {
logger.Warn(fmt.Sprintf("invalid GitHub URL: %v", err))
continue
}
result, err := pkg.RunScorecards(ctx, repo, checksToRun, repoClient)
if errors.Is(err, sce.ErrRepoUnreachable) {
// Not accessible repo - continue.
continue

60
pkg/scorecard.go
View File

@ -23,10 +23,7 @@ import (
"github.com/ossf/scorecard/v3/checker"
"github.com/ossf/scorecard/v3/clients"
"github.com/ossf/scorecard/v3/clients/githubrepo"
"github.com/ossf/scorecard/v3/clients/localdir"
sce "github.com/ossf/scorecard/v3/errors"
"github.com/ossf/scorecard/v3/repos"
)
func runEnabledChecks(ctx context.Context,
@ -56,62 +53,23 @@ func runEnabledChecks(ctx context.Context,
close(resultsCh)
}
func createRepo(uri *repos.RepoURI) (clients.Repo, error) {
var c clients.Repo
var e error
switch uri.RepoType() {
// URL.
case repos.RepoTypeURL:
c, e = githubrepo.MakeGithubRepo(uri.URL())
// LocalDir.
case repos.RepoTypeLocalDir:
c, e = localdir.MakeLocalDirRepo(uri.Path())
default:
return nil,
sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("unsupported URI type:%v", uri.RepoType()))
func getRepoCommitHash(r clients.RepoClient) (string, error) {
commits, err := r.ListCommits()
if err != nil {
return "", sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("ListCommits:%v", err.Error()))
}
if e != nil {
return c, sce.WithMessage(sce.ErrScorecardInternal, e.Error())
}
return c, nil
}
func getRepoCommitHash(r clients.RepoClient, uri *repos.RepoURI) (string, error) {
switch uri.RepoType() {
// URL.
case repos.RepoTypeURL:
commits, err := r.ListCommits()
if err != nil {
return "", sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("ListCommits:%v", err.Error()))
}
if len(commits) > 0 {
return commits[0].SHA, nil
}
return "no commits found", nil
// LocalDir.
case repos.RepoTypeLocalDir:
return "no commits for directory repo", nil
default:
return "",
sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("unsupported URI type:%v", uri.RepoType()))
if len(commits) > 0 {
return commits[0].SHA, nil
}
return "no commits found", nil
}
// RunScorecards runs enabled Scorecard checks on a Repo.
func RunScorecards(ctx context.Context,
repoURI *repos.RepoURI,
repo clients.Repo,
checksToRun checker.CheckNameToFnMap,
repoClient clients.RepoClient) (ScorecardResult, error) {
repo, err := createRepo(repoURI)
if err != nil {
return ScorecardResult{}, sce.WithMessage(err, "")
}
if err := repoClient.InitRepo(repo); err != nil {
// No need to call sce.WithMessage() since InitRepo will do that for us.
//nolint:wrapcheck
@ -119,7 +77,7 @@ func RunScorecards(ctx context.Context,
}
defer repoClient.Close()
commitSHA, err := getRepoCommitHash(repoClient, repoURI)
commitSHA, err := getRepoCommitHash(repoClient)
if err != nil {
return ScorecardResult{}, err
}

258
repos/repo_uri.go
View File

@ -1,258 +0,0 @@
// Copyright 2020 Security Scorecard Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package repos defines a generic repository.
package repos
import (
"errors"
"fmt"
"net/url"
"os"
"strings"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
sce "github.com/ossf/scorecard/v3/errors"
)
var (
// ErrorUnsupportedhost indicates the repo's host is unsupported.
ErrorUnsupportedhost = errors.New("unsupported host")
// ErrorInvalidGithubURL indicates the repo's GitHub URL is not in the proper format.
ErrorInvalidGithubURL = errors.New("invalid GitHub repo URL")
// ErrorInvalidURL indicates the repo's full GitHub URL was not passed.
ErrorInvalidURL = errors.New("invalid repo flag")
// errInvalidRepoType indicates the repo's type is invalid.
errInvalidRepoType = errors.New("invalid repo type")
)
// RepoURI represents the URI for a repo.
//nolint:govet
type RepoURI struct {
repoType RepoType
localDir repoLocalDir
url repoURL
metadata []string
}
type repoLocalDir struct {
path string
}
type repoURL struct {
host, owner, repo string
}
// RepoType is the type of a file.
type RepoType int
const (
// RepoTypeURL is for URLs.
RepoTypeURL RepoType = iota
// RepoTypeLocalDir is for source code in directories.
RepoTypeLocalDir
)
func (r repoLocalDir) Equal(o repoLocalDir) bool {
return r.path == o.path
}
func (r repoURL) Equal(o repoURL) bool {
return r.host == o.host &&
r.owner == o.owner &&
r.repo == o.repo
}
// NewFromURL creates a RepoURI from URL.
func NewFromURL(u string) (*RepoURI, error) {
r := &RepoURI{
repoType: RepoTypeURL,
}
if err := r.SetURL(u); err != nil {
return nil, fmt.Errorf("%w", err)
}
return r, nil
}
// NewFromLocalDirectory creates a RepoURI as a local directory.
func NewFromLocalDirectory(path string) *RepoURI {
return &RepoURI{
localDir: repoLocalDir{
path: path,
},
repoType: RepoTypeLocalDir,
}
}
// SetMetadata sets metadata.
func (r *RepoURI) SetMetadata(m []string) error {
r.metadata = m
return nil
}
// AppendMetadata appends metadata.
func (r *RepoURI) AppendMetadata(m ...string) error {
r.metadata = append(r.metadata, m...)
return nil
}
// SetURL sets the URL.
func (r *RepoURI) SetURL(u string) error {
if r.repoType != RepoTypeURL {
return fmt.Errorf("%w", errInvalidRepoType)
}
if err := r.Set(u); err != nil {
return fmt.Errorf("%w", err)
}
return nil
}
// Equal checks objects for equality.
func (r *RepoURI) Equal(o *RepoURI) bool {
return cmp.Equal(r.localDir, o.localDir) &&
cmp.Equal(r.url, o.url) &&
cmp.Equal(r.repoType, o.repoType) &&
cmp.Equal(r.metadata, o.metadata, cmpopts.SortSlices(func(x, y string) bool { return x < y }))
}
// Type method is needed so that this struct can be used as cmd flag.
func (r *RepoURI) Type() string {
return "repo"
}
// RepoType gives the type of URI.
func (r *RepoURI) RepoType() RepoType {
return r.repoType
}
// Path retusn the path for a local directory.
func (r *RepoURI) Path() string {
return r.localDir.path
}
// URL returns a valid url for Repo struct.
func (r *RepoURI) URL() string {
return fmt.Sprintf("%s/%s/%s", r.url.host, r.url.owner, r.url.repo)
}
// Metadata returns a valid url for Repo struct.
func (r *RepoURI) Metadata() []string {
return r.metadata
}
// String returns a string representation of Repo struct.
func (r *RepoURI) String() string {
return fmt.Sprintf("%s-%s-%s", r.url.host, r.url.owner, r.url.repo)
}
// setV4 for the v4 version.
func (r *RepoURI) setV4(s string) error {
const httpsPrefix = "https://"
const filePrefix = "file://"
// Validate the URI and scheme.
if !strings.HasPrefix(s, filePrefix) &&
!strings.HasPrefix(s, httpsPrefix) {
return sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("invalid URI: %v", s))
}
u, e := url.Parse(s)
if e != nil {
return sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("url.Parse: %v", e))
}
switch {
case strings.HasPrefix(s, httpsPrefix):
const splitLen = 2
split := strings.SplitN(strings.Trim(u.Path, "/"), "/", splitLen)
if len(split) != splitLen {
return sce.WithMessage(ErrorInvalidURL, fmt.Sprintf("%v. Expected full repository url", s))
}
r.url.host, r.url.owner, r.url.repo = u.Host, split[0], split[1]
case strings.HasPrefix(s, filePrefix):
r.localDir.path = s[len(filePrefix):]
r.repoType = RepoTypeLocalDir
default:
break
}
return nil
}
func (r *RepoURI) set(s string) error {
var t string
const two = 2
const three = 3
c := strings.Split(s, "/")
switch l := len(c); {
// This will takes care of repo/owner format.
// By default it will use github.com
case l == two:
t = "github.com/" + c[0] + "/" + c[1]
case l >= three:
t = s
}
// Allow skipping scheme for ease-of-use, default to https.
if !strings.Contains(t, "://") {
t = "https://" + t
}
u, e := url.Parse(t)
if e != nil {
return sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("url.Parse: %v", e))
}
const splitLen = 2
split := strings.SplitN(strings.Trim(u.Path, "/"), "/", splitLen)
if len(split) != splitLen {
return sce.WithMessage(ErrorInvalidURL, fmt.Sprintf("%v. Exepted full repository url", s))
}
r.url.host, r.url.owner, r.url.repo = u.Host, split[0], split[1]
return nil
}
// Set parses a URI string into Repo struct.
func (r *RepoURI) Set(s string) error {
var v4 bool
_, v4 = os.LookupEnv("SCORECARD_V4")
if v4 {
return r.setV4(s)
}
return r.set(s)
}
// IsValidGitHubURL checks whether Repo represents a valid GitHub repo and returns errors otherwise.
func (r *RepoURI) IsValidGitHubURL() error {
switch r.url.host {
case "github.com":
default:
return sce.WithMessage(ErrorUnsupportedhost, r.url.host)
}
if strings.TrimSpace(r.url.owner) == "" || strings.TrimSpace(r.url.repo) == "" {
return sce.WithMessage(ErrorInvalidGithubURL,
fmt.Sprintf("%v. Expected the full repository url", r.URL()))
}
return nil
}

119
repos/repo_uri_test.go
View File

@ -1,119 +0,0 @@
// Copyright 2020 Security Scorecard Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package repos
import (
"testing"
)
func TestRepoURI_ValidGitHubUrl(t *testing.T) {
t.Parallel()
type fields struct {
host string
owner string
repo string
}
type args struct {
s string
}
tests := []struct {
name string
fields fields
args args
wantErr bool
}{
{
name: "Valid http address",
fields: fields{
host: "github.com",
owner: "foo",
repo: "kubeflow",
},
args: args{s: "https://github.com/foo/kubeflow"},
wantErr: false,
},
{
name: "Valid http address with trailing slash",
fields: fields{
host: "github.com",
owner: "foo",
repo: "kubeflow",
},
args: args{s: "https://github.com/foo/kubeflow/"},
wantErr: false,
},
{
name: "Non github repository",
fields: fields{
host: "gitlab.com",
owner: "foo",
repo: "kubeflow",
},
args: args{s: "https://gitlab.com/foo/kubeflow"},
wantErr: true,
},
{
name: "github repository",
fields: fields{
host: "github.com",
owner: "foo",
repo: "kubeflow",
},
args: args{s: "foo/kubeflow"},
wantErr: false,
},
{
name: "github repository",
fields: fields{
host: "github.com",
owner: "foo",
repo: "kubeflow",
},
args: args{s: "https://github.com/foo/kubeflow"},
wantErr: false,
},
}
for _, tt := range tests {
tt := tt // Re-initializing variable so it is not changed while executing the closure below
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
r := &RepoURI{
url: repoURL{
host: tt.fields.host,
owner: tt.fields.owner,
repo: tt.fields.repo,
},
}
t.Log("Test")
if err := r.Set(tt.args.s); err != nil {
t.Errorf("repoURI.Set() error = %v", err)
}
if err := r.IsValidGitHubURL(); (err != nil) != tt.wantErr {
t.Errorf("repoURI.ValidGitHubUrl() error = %v, wantErr %v", err, tt.wantErr)
}
if !tt.wantErr {
if tt.fields.host != r.url.host {
t.Errorf("repo host expected to be %s but got %s", tt.fields.host, r.url.host)
}
if tt.fields.owner != r.url.owner {
t.Errorf("repo owner expected to be %s but got %s", tt.fields.owner, r.url.owner)
}
if tt.fields.repo != r.url.repo {
t.Errorf("repo expected to be %s but got %s", tt.fields.repo, r.url.repo)
}
}
})
}
}