mirror of
https://github.com/ossf/scorecard.git
synced 2024-09-11 08:55:27 +03:00
Remove repos
package (#1191)
Co-authored-by: Azeem Shaikh <azeems@google.com>
This commit is contained in:
parent
148446bb83
commit
83649a799e
2
Makefile
2
Makefile
@ -143,7 +143,7 @@ build-webhook: ## Runs go build on the cron webhook
|
||||
|
||||
build-add-script: ## Runs go build on the add script
|
||||
build-add-script: cron/data/add/add
|
||||
cron/data/add/add: cron/data/add/*.go cron/data/*.go repos/*.go cron/data/projects.csv
|
||||
cron/data/add/add: cron/data/add/*.go cron/data/*.go cron/data/projects.csv
|
||||
# Run go build on the add script
|
||||
cd cron/data/add && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o add
|
||||
|
||||
|
34
cmd/root.go
34
cmd/root.go
@ -39,11 +39,10 @@ import (
|
||||
sce "github.com/ossf/scorecard/v3/errors"
|
||||
"github.com/ossf/scorecard/v3/pkg"
|
||||
spol "github.com/ossf/scorecard/v3/policy"
|
||||
"github.com/ossf/scorecard/v3/repos"
|
||||
)
|
||||
|
||||
var (
|
||||
repo repos.RepoURI
|
||||
repo string
|
||||
checksToRun []string
|
||||
metaData []string
|
||||
// This one has to use goflag instead of pflag because it's defined by zap.
|
||||
@ -139,23 +138,16 @@ func validateFormat(format string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
func createRepoClient(ctx context.Context, uri *repos.RepoURI, logger *zap.Logger) (clients.RepoClient, error) {
|
||||
var rc clients.RepoClient
|
||||
switch uri.RepoType() {
|
||||
// URL.
|
||||
case repos.RepoTypeURL:
|
||||
if err := repo.IsValidGitHubURL(); err != nil {
|
||||
return rc, sce.WithMessage(sce.ErrScorecardInternal, err.Error())
|
||||
}
|
||||
rc = githubrepo.CreateGithubRepoClient(ctx, logger)
|
||||
return rc, nil
|
||||
|
||||
// Local directory.
|
||||
case repos.RepoTypeLocalDir:
|
||||
return localdir.CreateLocalDirClient(ctx, logger), nil
|
||||
func getRepoAccessors(ctx context.Context, uri string, logger *zap.Logger) (clients.Repo, clients.RepoClient, error) {
|
||||
if repo, err := localdir.MakeLocalDirRepo(uri); err == nil {
|
||||
// Local directory.
|
||||
return repo, localdir.CreateLocalDirClient(ctx, logger), nil
|
||||
}
|
||||
|
||||
return nil, sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("unspported URI: %v", uri.RepoType()))
|
||||
if repo, err := githubrepo.MakeGithubRepo(uri); err == nil {
|
||||
// GitHub URL.
|
||||
return repo, githubrepo.CreateGithubRepoClient(ctx, logger), nil
|
||||
}
|
||||
return nil, nil, sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("unspported URI: %s", uri))
|
||||
}
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
@ -223,7 +215,7 @@ var rootCmd = &cobra.Command{
|
||||
// nolint
|
||||
defer logger.Sync() // Flushes buffer, if any.
|
||||
|
||||
repoClient, err := createRepoClient(ctx, &repo, logger)
|
||||
repoURI, repoClient, err := getRepoAccessors(ctx, repo, logger)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@ -240,7 +232,7 @@ var rootCmd = &cobra.Command{
|
||||
}
|
||||
}
|
||||
|
||||
repoResult, err := pkg.RunScorecards(ctx, &repo, enabledChecks, repoClient)
|
||||
repoResult, err := pkg.RunScorecards(ctx, repoURI, enabledChecks, repoClient)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@ -408,7 +400,7 @@ func enableCheck(checkName string, enabledChecks *checker.CheckNameToFnMap) bool
|
||||
func init() {
|
||||
// Add the zap flag manually
|
||||
rootCmd.PersistentFlags().AddGoFlagSet(goflag.CommandLine)
|
||||
rootCmd.Flags().Var(&repo, "repo", "repository to check")
|
||||
rootCmd.Flags().StringVar(&repo, "repo", "", "repository to check")
|
||||
rootCmd.Flags().StringVar(
|
||||
&npm, "npm", "",
|
||||
"npm package to check, given that the npm package has a GitHub repository")
|
||||
|
@ -27,7 +27,6 @@ import (
|
||||
"github.com/ossf/scorecard/v3/checks"
|
||||
"github.com/ossf/scorecard/v3/clients/githubrepo"
|
||||
"github.com/ossf/scorecard/v3/pkg"
|
||||
"github.com/ossf/scorecard/v3/repos"
|
||||
)
|
||||
|
||||
//nolint:gochecknoinits
|
||||
@ -59,13 +58,13 @@ var serveCmd = &cobra.Command{
|
||||
if len(s) != length {
|
||||
rw.WriteHeader(http.StatusBadRequest)
|
||||
}
|
||||
repo := repos.RepoURI{}
|
||||
if err := repo.Set(repoParam); err != nil {
|
||||
repo, err := githubrepo.MakeGithubRepo(repoParam)
|
||||
if err != nil {
|
||||
rw.WriteHeader(http.StatusBadRequest)
|
||||
}
|
||||
ctx := r.Context()
|
||||
repoClient := githubrepo.CreateGithubRepoClient(ctx, logger)
|
||||
repoResult, err := pkg.RunScorecards(ctx, &repo, checks.AllChecks, repoClient)
|
||||
repoResult, err := pkg.RunScorecards(ctx, repo, checks.AllChecks, repoClient)
|
||||
if err != nil {
|
||||
sugar.Error(err)
|
||||
rw.WriteHeader(http.StatusInternalServerError)
|
||||
|
@ -59,7 +59,7 @@ func publishToRepoRequestTopic(ctx context.Context, iter data.Iterator, datetime
|
||||
if err != nil {
|
||||
return shardNum, fmt.Errorf("error reading repoURL: %w", err)
|
||||
}
|
||||
request.Repos = append(request.GetRepos(), repoURL.URL())
|
||||
request.Repos = append(request.GetRepos(), repoURL.Repo)
|
||||
if len(request.GetRepos()) < shardSize {
|
||||
continue
|
||||
}
|
||||
|
@ -21,7 +21,6 @@ import (
|
||||
"os"
|
||||
|
||||
"github.com/ossf/scorecard/v3/cron/data"
|
||||
"github.com/ossf/scorecard/v3/repos"
|
||||
)
|
||||
|
||||
// Script to add new project repositories to the projects.csv file:
|
||||
@ -64,34 +63,32 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
func getRepoURLs(iter data.Iterator) ([]repos.RepoURI, error) {
|
||||
repoURLs := make(map[string]*repos.RepoURI)
|
||||
func getRepoURLs(iter data.Iterator) ([]data.RepoFormat, error) {
|
||||
repoURLs := make(map[string]*data.RepoFormat)
|
||||
repoMap := make(map[string]map[string]bool)
|
||||
for iter.HasNext() {
|
||||
repo, err := iter.Next()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("iter.Next: %w", err)
|
||||
}
|
||||
if _, ok := repoMap[repo.URL()]; !ok {
|
||||
repoURLs[repo.URL()] = new(repos.RepoURI)
|
||||
*repoURLs[repo.URL()] = repo
|
||||
repoMap[repo.URL()] = make(map[string]bool)
|
||||
for _, metadata := range repo.Metadata() {
|
||||
repoMap[repo.URL()][metadata] = true
|
||||
if _, ok := repoMap[repo.Repo]; !ok {
|
||||
repoURLs[repo.Repo] = new(data.RepoFormat)
|
||||
*repoURLs[repo.Repo] = repo
|
||||
repoMap[repo.Repo] = make(map[string]bool)
|
||||
for _, metadata := range repo.Metadata {
|
||||
repoMap[repo.Repo][metadata] = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
for _, metadata := range repo.Metadata() {
|
||||
if _, ok := repoMap[repo.URL()][metadata]; !ok && metadata != "" {
|
||||
if err := repoURLs[repo.URL()].AppendMetadata(metadata); err != nil {
|
||||
return nil, fmt.Errorf("AppendMetadata: %w", err)
|
||||
}
|
||||
repoMap[repo.URL()][metadata] = true
|
||||
for _, metadata := range repo.Metadata {
|
||||
if _, ok := repoMap[repo.Repo][metadata]; !ok && metadata != "" {
|
||||
repoURLs[repo.Repo].Metadata = append(repoURLs[repo.Repo].Metadata, metadata)
|
||||
repoMap[repo.Repo][metadata] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
newRepoURLs := make([]repos.RepoURI, 0)
|
||||
newRepoURLs := make([]data.RepoFormat, 0)
|
||||
for _, repoURL := range repoURLs {
|
||||
newRepoURLs = append(newRepoURLs, *repoURL)
|
||||
}
|
||||
|
@ -17,25 +17,15 @@ package main
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/ossf/scorecard/v3/cron/data"
|
||||
"github.com/ossf/scorecard/v3/repos"
|
||||
)
|
||||
|
||||
type fields struct {
|
||||
host string
|
||||
owner string
|
||||
repo string
|
||||
metadata []string
|
||||
}
|
||||
|
||||
//nolint: gocritic
|
||||
func lessThanURI(x, y repos.RepoURI) bool {
|
||||
func lessThanURI(x, y data.RepoFormat) bool {
|
||||
return fmt.Sprintf("%+v", x) < fmt.Sprintf("%+v", y)
|
||||
}
|
||||
|
||||
@ -43,93 +33,73 @@ func TestGetRepoURLs(t *testing.T) {
|
||||
t.Parallel()
|
||||
testcases := []struct {
|
||||
name, filename string
|
||||
outcome []fields
|
||||
outcome []data.RepoFormat
|
||||
}{
|
||||
{
|
||||
name: "NoChange",
|
||||
filename: "testdata/no_change.csv",
|
||||
outcome: []fields{
|
||||
outcome: []data.RepoFormat{
|
||||
{
|
||||
host: "github.com",
|
||||
owner: "owner1",
|
||||
repo: "repo1",
|
||||
metadata: []string{"meta1", "meta2"},
|
||||
Repo: "github.com/owner1/repo1",
|
||||
Metadata: []string{"meta1", "meta2"},
|
||||
},
|
||||
{
|
||||
host: "github.com",
|
||||
owner: "owner2",
|
||||
repo: "repo2",
|
||||
Repo: "github.com/owner2/repo2",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "AddMetadata",
|
||||
filename: "testdata/add_metadata.csv",
|
||||
outcome: []fields{
|
||||
outcome: []data.RepoFormat{
|
||||
{
|
||||
host: "github.com",
|
||||
owner: "owner1",
|
||||
repo: "repo1",
|
||||
metadata: []string{"meta1", "meta2"},
|
||||
Repo: "github.com/owner1/repo1",
|
||||
Metadata: []string{"meta1", "meta2"},
|
||||
},
|
||||
{
|
||||
host: "github.com",
|
||||
owner: "owner2",
|
||||
repo: "repo2",
|
||||
metadata: []string{"meta1"},
|
||||
Repo: "github.com/owner2/repo2",
|
||||
Metadata: []string{"meta1"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "SkipLatest",
|
||||
filename: "testdata/skip_latest.csv",
|
||||
outcome: []fields{
|
||||
outcome: []data.RepoFormat{
|
||||
{
|
||||
host: "github.com",
|
||||
owner: "owner1",
|
||||
repo: "repo1",
|
||||
metadata: []string{"meta1", "meta2"},
|
||||
Repo: "github.com/owner1/repo1",
|
||||
Metadata: []string{"meta1", "meta2"},
|
||||
},
|
||||
{
|
||||
host: "github.com",
|
||||
owner: "owner2",
|
||||
repo: "repo2",
|
||||
Repo: "github.com/owner2/repo2",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "SkipEmpty",
|
||||
filename: "testdata/skip_empty.csv",
|
||||
outcome: []fields{
|
||||
outcome: []data.RepoFormat{
|
||||
{
|
||||
host: "github.com",
|
||||
owner: "owner1",
|
||||
repo: "repo1",
|
||||
metadata: []string{"meta1", "meta2"},
|
||||
Repo: "github.com/owner1/repo1",
|
||||
Metadata: []string{"meta1", "meta2"},
|
||||
},
|
||||
{
|
||||
host: "github.com",
|
||||
owner: "owner2",
|
||||
repo: "repo2",
|
||||
metadata: []string{"meta3"},
|
||||
Repo: "github.com/owner2/repo2",
|
||||
Metadata: []string{"meta3"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "SkipEmpty_2",
|
||||
filename: "testdata/skip_empty_2.csv",
|
||||
outcome: []fields{
|
||||
outcome: []data.RepoFormat{
|
||||
{
|
||||
host: "github.com",
|
||||
owner: "owner1",
|
||||
repo: "repo1",
|
||||
metadata: []string{"meta1", "meta2"},
|
||||
Repo: "github.com/owner1/repo1",
|
||||
Metadata: []string{"meta1", "meta2"},
|
||||
},
|
||||
{
|
||||
host: "github.com",
|
||||
owner: "owner2",
|
||||
repo: "repo2",
|
||||
metadata: []string{"meta3"},
|
||||
Repo: "github.com/owner2/repo2",
|
||||
Metadata: []string{"meta3"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -155,27 +125,8 @@ func TestGetRepoURLs(t *testing.T) {
|
||||
t.Errorf("testcase failed: %v", err)
|
||||
}
|
||||
|
||||
// Create the list of RepoURL from the outcome.
|
||||
var rs []repos.RepoURI
|
||||
for _, r := range testcase.outcome {
|
||||
u := fmt.Sprintf("%s/%s/%s", r.host, r.owner, r.repo)
|
||||
outcomeRepo, err := repos.NewFromURL(u)
|
||||
if err != nil {
|
||||
t.Errorf("repos.NewFromURL: %v", err)
|
||||
}
|
||||
if err := outcomeRepo.AppendMetadata(r.metadata...); err != nil {
|
||||
t.Errorf("outcomeRepo.AppendMetadata: %v", err)
|
||||
}
|
||||
rs = append(rs, *outcomeRepo)
|
||||
}
|
||||
|
||||
// Export all private fields for comparison.
|
||||
exp := cmp.Exporter(func(t reflect.Type) bool {
|
||||
return true
|
||||
})
|
||||
|
||||
if !cmp.Equal(rs, repoURLs, exp, cmpopts.EquateEmpty(), cmpopts.SortSlices(lessThanURI)) {
|
||||
t.Errorf("testcase failed. expected %+v, got %+v", rs, repoURLs)
|
||||
if !cmp.Equal(testcase.outcome, repoURLs, cmpopts.EquateEmpty(), cmpopts.SortSlices(lessThanURI)) {
|
||||
t.Errorf("testcase failed. expected equal, got diff: %s", cmp.Diff(testcase.outcome, repoURLs))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
@ -16,13 +16,16 @@ package data
|
||||
|
||||
import "strings"
|
||||
|
||||
type csvStrings []string
|
||||
// CSVStrings is []string with support for CSV formatting.
|
||||
type CSVStrings []string
|
||||
|
||||
func (s csvStrings) MarshalCSV() ([]byte, error) {
|
||||
// MarshalCSV implements []string -> []byte serialization.
|
||||
func (s CSVStrings) MarshalCSV() ([]byte, error) {
|
||||
return []byte(strings.Join(s, ",")), nil
|
||||
}
|
||||
|
||||
func (s *csvStrings) UnmarshalCSV(input []byte) error {
|
||||
// UnmarshalCSV implements []byte -> []string de-serializtion.
|
||||
func (s *CSVStrings) UnmarshalCSV(input []byte) error {
|
||||
if len(input) == 0 || string(input) == "" {
|
||||
*s = nil
|
||||
return nil
|
||||
@ -31,7 +34,8 @@ func (s *csvStrings) UnmarshalCSV(input []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type repoFormat struct {
|
||||
// RepoFormat is used to read input repos.
|
||||
type RepoFormat struct {
|
||||
Repo string `csv:"repo"`
|
||||
Metadata csvStrings `csv:"metadata"`
|
||||
Metadata CSVStrings `csv:"metadata"`
|
||||
}
|
||||
|
@ -25,7 +25,7 @@ func TestUnmarshalCsv(t *testing.T) {
|
||||
testcases := []struct {
|
||||
name string
|
||||
input []byte
|
||||
output csvStrings
|
||||
output CSVStrings
|
||||
}{
|
||||
{
|
||||
name: "Basic",
|
||||
@ -53,7 +53,7 @@ func TestUnmarshalCsv(t *testing.T) {
|
||||
testcase := testcase
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
s := new(csvStrings)
|
||||
s := new(CSVStrings)
|
||||
if err := s.UnmarshalCSV(testcase.input); err != nil {
|
||||
t.Errorf("testcase failed: %v", err)
|
||||
}
|
||||
|
@ -22,13 +22,13 @@ import (
|
||||
|
||||
"github.com/jszwec/csvutil"
|
||||
|
||||
"github.com/ossf/scorecard/v3/repos"
|
||||
"github.com/ossf/scorecard/v3/clients/githubrepo"
|
||||
)
|
||||
|
||||
// Iterator interface is used to iterate through list of input repos for the cron job.
|
||||
type Iterator interface {
|
||||
HasNext() bool
|
||||
Next() (repos.RepoURI, error)
|
||||
Next() (RepoFormat, error)
|
||||
}
|
||||
|
||||
// MakeIteratorFrom returns an implementation of Iterator interface.
|
||||
@ -46,7 +46,7 @@ func MakeIteratorFrom(reader io.Reader) (Iterator, error) {
|
||||
type csvIterator struct {
|
||||
decoder *csvutil.Decoder
|
||||
err error
|
||||
next repoFormat
|
||||
next RepoFormat
|
||||
}
|
||||
|
||||
func (reader *csvIterator) HasNext() bool {
|
||||
@ -54,21 +54,13 @@ func (reader *csvIterator) HasNext() bool {
|
||||
return !errors.Is(reader.err, io.EOF)
|
||||
}
|
||||
|
||||
func (reader *csvIterator) Next() (repos.RepoURI, error) {
|
||||
ret := repos.RepoURI{}
|
||||
func (reader *csvIterator) Next() (RepoFormat, error) {
|
||||
if reader.err != nil {
|
||||
return ret, fmt.Errorf("reader has error: %w", reader.err)
|
||||
return reader.next, fmt.Errorf("reader has error: %w", reader.err)
|
||||
}
|
||||
|
||||
err := ret.SetMetadata(reader.next.Metadata)
|
||||
if err != nil {
|
||||
return ret, fmt.Errorf("error during SetMetadata: %w", err)
|
||||
// Sanity check valid GitHub URL.
|
||||
if _, err := githubrepo.MakeGithubRepo(reader.next.Repo); err != nil {
|
||||
return reader.next, fmt.Errorf("invalid GitHub URL: %w", err)
|
||||
}
|
||||
if err := ret.Set(reader.next.Repo); err != nil {
|
||||
return ret, fmt.Errorf("error during Set: %w", err)
|
||||
}
|
||||
if err := ret.IsValidGitHubURL(); err != nil {
|
||||
return ret, fmt.Errorf("error during IsValidGitHubURL: %w", err)
|
||||
}
|
||||
return ret, nil
|
||||
return reader.next, nil
|
||||
}
|
||||
|
@ -16,25 +16,17 @@ package data
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
|
||||
"github.com/ossf/scorecard/v3/repos"
|
||||
sce "github.com/ossf/scorecard/v3/errors"
|
||||
)
|
||||
|
||||
type fields struct {
|
||||
host string
|
||||
owner string
|
||||
repo string
|
||||
metadata []string
|
||||
}
|
||||
|
||||
type outcome struct {
|
||||
expectedErr error
|
||||
repoURI fields
|
||||
repo RepoFormat
|
||||
hasError bool
|
||||
}
|
||||
|
||||
@ -53,27 +45,21 @@ func TestCsvIterator(t *testing.T) {
|
||||
outcomes: []outcome{
|
||||
{
|
||||
hasError: false,
|
||||
repoURI: fields{
|
||||
host: "github.com",
|
||||
owner: "owner1",
|
||||
repo: "repo1",
|
||||
repo: RepoFormat{
|
||||
Repo: "github.com/owner1/repo1",
|
||||
},
|
||||
},
|
||||
{
|
||||
hasError: false,
|
||||
repoURI: fields{
|
||||
host: "github.com",
|
||||
owner: "owner2",
|
||||
repo: "repo2",
|
||||
repo: RepoFormat{
|
||||
Repo: "github.com/owner2/repo2",
|
||||
},
|
||||
},
|
||||
{
|
||||
hasError: false,
|
||||
repoURI: fields{
|
||||
host: "github.com",
|
||||
owner: "owner3",
|
||||
repo: "repo3",
|
||||
metadata: []string{"meta"},
|
||||
repo: RepoFormat{
|
||||
Repo: "github.com/owner3/repo3",
|
||||
Metadata: []string{"meta"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -84,27 +70,21 @@ func TestCsvIterator(t *testing.T) {
|
||||
outcomes: []outcome{
|
||||
{
|
||||
hasError: false,
|
||||
repoURI: fields{
|
||||
host: "github.com",
|
||||
owner: "owner1",
|
||||
repo: "repo1",
|
||||
repo: RepoFormat{
|
||||
Repo: "github.com/owner1/repo1",
|
||||
},
|
||||
},
|
||||
{
|
||||
hasError: false,
|
||||
repoURI: fields{
|
||||
host: "github.com",
|
||||
owner: "owner2",
|
||||
repo: "repo2",
|
||||
repo: RepoFormat{
|
||||
Repo: "github.com/owner2/repo2",
|
||||
},
|
||||
},
|
||||
{
|
||||
hasError: false,
|
||||
repoURI: fields{
|
||||
host: "github.com",
|
||||
owner: "owner3",
|
||||
repo: "repo3",
|
||||
metadata: []string{"meta"},
|
||||
repo: RepoFormat{
|
||||
Repo: "github.com/owner3/repo3",
|
||||
Metadata: []string{"meta"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -115,15 +95,15 @@ func TestCsvIterator(t *testing.T) {
|
||||
outcomes: []outcome{
|
||||
{
|
||||
hasError: true,
|
||||
expectedErr: repos.ErrorUnsupportedhost,
|
||||
expectedErr: sce.ErrorUnsupportedHost,
|
||||
},
|
||||
{
|
||||
hasError: true,
|
||||
expectedErr: repos.ErrorInvalidURL,
|
||||
expectedErr: sce.ErrorInvalidURL,
|
||||
},
|
||||
{
|
||||
hasError: true,
|
||||
expectedErr: repos.ErrorInvalidURL,
|
||||
expectedErr: sce.ErrorInvalidURL,
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -133,27 +113,21 @@ func TestCsvIterator(t *testing.T) {
|
||||
outcomes: []outcome{
|
||||
{
|
||||
hasError: false,
|
||||
repoURI: fields{
|
||||
host: "github.com",
|
||||
owner: "owner1",
|
||||
repo: "repo1",
|
||||
repo: RepoFormat{
|
||||
Repo: "github.com/owner1/repo1",
|
||||
},
|
||||
},
|
||||
{
|
||||
hasError: false,
|
||||
repoURI: fields{
|
||||
host: "github.com",
|
||||
owner: "owner2",
|
||||
repo: "repo2",
|
||||
repo: RepoFormat{
|
||||
Repo: "github.com/owner2/repo2",
|
||||
},
|
||||
},
|
||||
{
|
||||
hasError: false,
|
||||
repoURI: fields{
|
||||
host: "github.com",
|
||||
owner: "owner3",
|
||||
repo: "repo3",
|
||||
metadata: []string{"meta"},
|
||||
repo: RepoFormat{
|
||||
Repo: "github.com/owner3/repo3",
|
||||
Metadata: []string{"meta"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -164,18 +138,14 @@ func TestCsvIterator(t *testing.T) {
|
||||
outcomes: []outcome{
|
||||
{
|
||||
hasError: false,
|
||||
repoURI: fields{
|
||||
host: "github.com",
|
||||
owner: "owner1",
|
||||
repo: "repo1",
|
||||
repo: RepoFormat{
|
||||
Repo: "github.com/owner1/repo1",
|
||||
},
|
||||
},
|
||||
{
|
||||
hasError: false,
|
||||
repoURI: fields{
|
||||
host: "github.com",
|
||||
owner: "owner2",
|
||||
repo: "repo2",
|
||||
repo: RepoFormat{
|
||||
Repo: "github.com/owner2/repo2",
|
||||
},
|
||||
},
|
||||
{
|
||||
@ -209,18 +179,8 @@ func TestCsvIterator(t *testing.T) {
|
||||
}
|
||||
|
||||
if !outcome.hasError {
|
||||
u := fmt.Sprintf("%s/%s/%s",
|
||||
outcome.repoURI.host, outcome.repoURI.owner, outcome.repoURI.repo)
|
||||
outcomeRepo, err := repos.NewFromURL(u)
|
||||
if err != nil {
|
||||
t.Errorf("repos.NewFromURL: %v", err)
|
||||
}
|
||||
if err := outcomeRepo.AppendMetadata(outcome.repoURI.metadata...); err != nil {
|
||||
t.Errorf("outcomeRepo.AppendMetadata: %v", err)
|
||||
}
|
||||
|
||||
if !cmp.Equal(outcomeRepo, &repoURL) {
|
||||
t.Errorf("expected repoURL: %+v, got %+v", *outcomeRepo, repoURL)
|
||||
if !cmp.Equal(outcome.repo, repoURL) {
|
||||
t.Errorf("expected equal, got diff: %s", cmp.Diff(outcome.repo, repoURL))
|
||||
}
|
||||
}
|
||||
if outcome.hasError && outcome.expectedErr != nil && !errors.Is(err, outcome.expectedErr) {
|
||||
|
@ -30,8 +30,8 @@ import (
|
||||
"github.com/google/go-github/v38/github"
|
||||
"golang.org/x/tools/go/vcs"
|
||||
|
||||
"github.com/ossf/scorecard/v3/clients/githubrepo"
|
||||
"github.com/ossf/scorecard/v3/cron/data"
|
||||
"github.com/ossf/scorecard/v3/repos"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -78,10 +78,10 @@ type repositoryDepsURL struct {
|
||||
|
||||
// Programmatically gets Envoy's dependencies and add to projects.
|
||||
// Re-using a checker type.
|
||||
func getBazelDeps(repo repositoryDepsURL) []repos.RepoURI {
|
||||
func getBazelDeps(repo repositoryDepsURL) []data.RepoFormat {
|
||||
client := github.NewClient(nil)
|
||||
ctx := context.Background()
|
||||
depRepos := []repos.RepoURI{}
|
||||
depRepos := []data.RepoFormat{}
|
||||
fo, _, _, err := client.Repositories.GetContents(ctx, repo.Owner, repo.Repo, repo.File, nil)
|
||||
if err != nil {
|
||||
// If we can't get content, gracefully fail but alert.
|
||||
@ -101,8 +101,9 @@ func getBazelDeps(repo repositoryDepsURL) []repos.RepoURI {
|
||||
|
||||
// TODO: Replace with a starlark interpreter that can be used for any project.
|
||||
for _, match := range re.FindAllString(fc, -1) {
|
||||
repo := repos.RepoURI{}
|
||||
if err := repo.Set(strings.TrimSuffix(match, ".git")); err != nil {
|
||||
repo := data.RepoFormat{}
|
||||
repo.Repo = strings.TrimSuffix(match, ".git")
|
||||
if _, err := githubrepo.MakeGithubRepo(repo.Repo); err != nil {
|
||||
log.Panicf("error during repo.Set: %v", err)
|
||||
return depRepos
|
||||
}
|
||||
@ -112,8 +113,8 @@ func getBazelDeps(repo repositoryDepsURL) []repos.RepoURI {
|
||||
}
|
||||
|
||||
// GetGoDeps returns go repo dependencies.
|
||||
func getGoDeps(repo repositoryDepsURL) []repos.RepoURI {
|
||||
repoURLs := []repos.RepoURI{}
|
||||
func getGoDeps(repo repositoryDepsURL) []data.RepoFormat {
|
||||
repoURLs := []data.RepoFormat{}
|
||||
pwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
log.Default().Println(err)
|
||||
@ -184,22 +185,23 @@ func getVanityRepoURL(u string) string {
|
||||
return repo.Repo
|
||||
}
|
||||
|
||||
func parseGoModURL(dependency string, repoURLs []repos.RepoURI) []repos.RepoURI {
|
||||
repoURL := repos.RepoURI{}
|
||||
func parseGoModURL(dependency string, repoURLs []data.RepoFormat) []data.RepoFormat {
|
||||
repoURL := data.RepoFormat{}
|
||||
splitURL := strings.Split(dependency, "/")
|
||||
// nolint:gomnd
|
||||
if len(splitURL) < 3 {
|
||||
return repoURLs
|
||||
}
|
||||
u := fmt.Sprintf("%s/%s/%s", splitURL[0], splitURL[1], splitURL[2])
|
||||
if err := repoURL.Set(u); err != nil {
|
||||
if _, err := githubrepo.MakeGithubRepo(u); err != nil {
|
||||
return repoURLs
|
||||
}
|
||||
repoURL.Repo = u
|
||||
repoURLs = append(repoURLs, repoURL)
|
||||
return repoURLs
|
||||
}
|
||||
|
||||
func getDependencies(in io.Reader) (oldRepos, newRepos []repos.RepoURI, e error) {
|
||||
func getDependencies(in io.Reader) (oldRepos, newRepos []data.RepoFormat, e error) {
|
||||
iter, err := data.MakeIteratorFrom(in)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("error during data.MakeIterator: %w", err)
|
||||
@ -207,7 +209,7 @@ func getDependencies(in io.Reader) (oldRepos, newRepos []repos.RepoURI, e error)
|
||||
|
||||
// Read all project repositores into a map.
|
||||
m := make(map[string][]string)
|
||||
oldRepos = make([]repos.RepoURI, 0)
|
||||
oldRepos = make([]data.RepoFormat, 0)
|
||||
for iter.HasNext() {
|
||||
repo, err := iter.Next()
|
||||
if err != nil {
|
||||
@ -215,25 +217,25 @@ func getDependencies(in io.Reader) (oldRepos, newRepos []repos.RepoURI, e error)
|
||||
}
|
||||
oldRepos = append(oldRepos, repo)
|
||||
// We do not handle duplicates.
|
||||
m[repo.URL()] = repo.Metadata()
|
||||
m[repo.Repo] = repo.Metadata
|
||||
}
|
||||
|
||||
// Create a list of project dependencies that are not already present.
|
||||
newRepos = []repos.RepoURI{}
|
||||
newRepos = []data.RepoFormat{}
|
||||
for _, repo := range bazelRepos {
|
||||
for _, item := range getBazelDeps(repo) {
|
||||
if _, ok := m[item.URL()]; !ok {
|
||||
if _, ok := m[item.Repo]; !ok {
|
||||
// Also add to m to avoid dupes.
|
||||
m[item.URL()] = item.Metadata()
|
||||
m[item.Repo] = item.Metadata
|
||||
newRepos = append(newRepos, item)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, repo := range gorepos {
|
||||
for _, item := range getGoDeps(repo) {
|
||||
if _, ok := m[item.URL()]; !ok {
|
||||
if _, ok := m[item.Repo]; !ok {
|
||||
// Also add to m to avoid dupes.
|
||||
m[item.URL()] = item.Metadata()
|
||||
m[item.Repo] = item.Metadata
|
||||
newRepos = append(newRepos, item)
|
||||
}
|
||||
}
|
||||
|
BIN
cron/data/update/update
Executable file
BIN
cron/data/update/update
Executable file
Binary file not shown.
@ -45,9 +45,9 @@ func main() {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if _, ok := m[repo.URL()]; ok {
|
||||
log.Panicf("Item already in the list %s", repo.URL())
|
||||
if _, ok := m[repo.Repo]; ok {
|
||||
log.Panicf("Item already in the list %s", repo.Repo)
|
||||
}
|
||||
m[repo.URL()] = true
|
||||
m[repo.Repo] = true
|
||||
}
|
||||
}
|
||||
|
@ -21,33 +21,18 @@ import (
|
||||
"sort"
|
||||
|
||||
"github.com/jszwec/csvutil"
|
||||
|
||||
"github.com/ossf/scorecard/v3/repos"
|
||||
)
|
||||
|
||||
func repoFormatFromRepoURL(repoURLs []repos.RepoURI) []repoFormat {
|
||||
repoentries := make([]repoFormat, 0)
|
||||
for _, repoURL := range repoURLs {
|
||||
repoentry := repoFormat{
|
||||
Repo: repoURL.URL(),
|
||||
Metadata: repoURL.Metadata(),
|
||||
}
|
||||
repoentries = append(repoentries, repoentry)
|
||||
}
|
||||
return repoentries
|
||||
}
|
||||
|
||||
// SortAndAppendTo appends `oldRepos` and `newRepos` before sorting and writing out the result to `out`.
|
||||
func SortAndAppendTo(out io.Writer, oldRepos, newRepos []repos.RepoURI) error {
|
||||
repoentries := repoFormatFromRepoURL(oldRepos)
|
||||
repoentries = append(repoentries, repoFormatFromRepoURL(newRepos)...)
|
||||
func SortAndAppendTo(out io.Writer, oldRepos, newRepos []RepoFormat) error {
|
||||
oldRepos = append(oldRepos, newRepos...)
|
||||
|
||||
sort.SliceStable(repoentries, func(i, j int) bool {
|
||||
return repoentries[i].Repo < repoentries[j].Repo
|
||||
sort.SliceStable(oldRepos, func(i, j int) bool {
|
||||
return oldRepos[i].Repo < oldRepos[j].Repo
|
||||
})
|
||||
csvWriter := csv.NewWriter(out)
|
||||
enc := csvutil.NewEncoder(csvWriter)
|
||||
if err := enc.Encode(repoentries); err != nil {
|
||||
if err := enc.Encode(oldRepos); err != nil {
|
||||
return fmt.Errorf("error during Encode: %w", err)
|
||||
}
|
||||
csvWriter.Flush()
|
||||
@ -55,13 +40,13 @@ func SortAndAppendTo(out io.Writer, oldRepos, newRepos []repos.RepoURI) error {
|
||||
}
|
||||
|
||||
// SortAndAppendFrom reads from `in`, appends to newRepos and writes the sorted output to `out`.
|
||||
func SortAndAppendFrom(in io.Reader, out io.Writer, newRepos []repos.RepoURI) error {
|
||||
func SortAndAppendFrom(in io.Reader, out io.Writer, newRepos []RepoFormat) error {
|
||||
iter, err := MakeIteratorFrom(in)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error during MakeIterator: %w", err)
|
||||
}
|
||||
|
||||
oldRepos := make([]repos.RepoURI, 0)
|
||||
oldRepos := make([]RepoFormat, 0)
|
||||
for iter.HasNext() {
|
||||
repo, err := iter.Next()
|
||||
if err != nil {
|
||||
|
@ -16,10 +16,7 @@ package data
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/ossf/scorecard/v3/repos"
|
||||
)
|
||||
|
||||
func TestCsvWriter(t *testing.T) {
|
||||
@ -27,25 +24,21 @@ func TestCsvWriter(t *testing.T) {
|
||||
testcases := []struct {
|
||||
name string
|
||||
out string
|
||||
oldRepos []fields
|
||||
newRepos []fields
|
||||
oldRepos []RepoFormat
|
||||
newRepos []RepoFormat
|
||||
}{
|
||||
{
|
||||
name: "Basic",
|
||||
oldRepos: []fields{
|
||||
oldRepos: []RepoFormat{
|
||||
{
|
||||
host: "github.com",
|
||||
owner: "owner1",
|
||||
repo: "repo1",
|
||||
metadata: []string{"meta1"},
|
||||
Repo: "github.com/owner1/repo1",
|
||||
Metadata: []string{"meta1"},
|
||||
},
|
||||
},
|
||||
newRepos: []fields{
|
||||
newRepos: []RepoFormat{
|
||||
{
|
||||
host: "github.com",
|
||||
owner: "owner2",
|
||||
repo: "repo2",
|
||||
metadata: []string{"meta2"},
|
||||
Repo: "github.com/owner2/repo2",
|
||||
Metadata: []string{"meta2"},
|
||||
},
|
||||
},
|
||||
out: `repo,metadata
|
||||
@ -60,30 +53,7 @@ github.com/owner2/repo2,meta2
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
var buf bytes.Buffer
|
||||
var oldRepos []repos.RepoURI
|
||||
var newRepos []repos.RepoURI
|
||||
for _, v := range testcase.oldRepos {
|
||||
r, err := repos.NewFromURL(fmt.Sprintf("%s/%s/%s", v.host, v.owner, v.repo))
|
||||
if err != nil {
|
||||
t.Errorf("repos.NewFromURL: %v", err)
|
||||
}
|
||||
if err = r.AppendMetadata(v.metadata...); err != nil {
|
||||
t.Errorf("r.AppendMetadata: %v", err)
|
||||
}
|
||||
oldRepos = append(oldRepos, *r)
|
||||
}
|
||||
for _, v := range testcase.newRepos {
|
||||
r, err := repos.NewFromURL(fmt.Sprintf("%s/%s/%s", v.host, v.owner, v.repo))
|
||||
if err != nil {
|
||||
t.Errorf("repos.NewFromURL: %v", err)
|
||||
}
|
||||
if err = r.AppendMetadata(v.metadata...); err != nil {
|
||||
t.Errorf("r.AppendMetadata: %v", err)
|
||||
}
|
||||
newRepos = append(newRepos, *r)
|
||||
}
|
||||
|
||||
err := SortAndAppendTo(&buf, oldRepos, newRepos)
|
||||
err := SortAndAppendTo(&buf, testcase.oldRepos, testcase.newRepos)
|
||||
if err != nil {
|
||||
t.Errorf("error while running testcase: %v", err)
|
||||
}
|
||||
|
@ -43,7 +43,6 @@ import (
|
||||
docs "github.com/ossf/scorecard/v3/docs/checks"
|
||||
sce "github.com/ossf/scorecard/v3/errors"
|
||||
"github.com/ossf/scorecard/v3/pkg"
|
||||
"github.com/ossf/scorecard/v3/repos"
|
||||
"github.com/ossf/scorecard/v3/stats"
|
||||
)
|
||||
|
||||
@ -72,25 +71,17 @@ func processRequest(ctx context.Context,
|
||||
return nil
|
||||
}
|
||||
|
||||
repoURLs := make([]repos.RepoURI, 0, len(batchRequest.GetRepos()))
|
||||
for _, repo := range batchRequest.GetRepos() {
|
||||
repoURL := repos.RepoURI{}
|
||||
if err := repoURL.Set(repo); err != nil {
|
||||
return fmt.Errorf("error setting RepoURL: %w", err)
|
||||
}
|
||||
if err := repoURL.IsValidGitHubURL(); err != nil {
|
||||
return fmt.Errorf("url is not a valid GitHub URL: %w", err)
|
||||
}
|
||||
repoURLs = append(repoURLs, repoURL)
|
||||
}
|
||||
|
||||
var buffer bytes.Buffer
|
||||
var buffer2 bytes.Buffer
|
||||
// TODO: run Scorecard for each repo in a separate thread.
|
||||
for i := range repoURLs {
|
||||
repoURL := repoURLs[i]
|
||||
logger.Info(fmt.Sprintf("Running Scorecard for repo: %s", repoURL.URL()))
|
||||
result, err := pkg.RunScorecards(ctx, &repoURL, checksToRun, repoClient)
|
||||
for _, repoURL := range batchRequest.GetRepos() {
|
||||
logger.Info(fmt.Sprintf("Running Scorecard for repo: %s", repoURL))
|
||||
repo, err := githubrepo.MakeGithubRepo(repoURL)
|
||||
if err != nil {
|
||||
logger.Warn(fmt.Sprintf("invalid GitHub URL: %v", err))
|
||||
continue
|
||||
}
|
||||
result, err := pkg.RunScorecards(ctx, repo, checksToRun, repoClient)
|
||||
if errors.Is(err, sce.ErrRepoUnreachable) {
|
||||
// Not accessible repo - continue.
|
||||
continue
|
||||
|
@ -23,10 +23,7 @@ import (
|
||||
|
||||
"github.com/ossf/scorecard/v3/checker"
|
||||
"github.com/ossf/scorecard/v3/clients"
|
||||
"github.com/ossf/scorecard/v3/clients/githubrepo"
|
||||
"github.com/ossf/scorecard/v3/clients/localdir"
|
||||
sce "github.com/ossf/scorecard/v3/errors"
|
||||
"github.com/ossf/scorecard/v3/repos"
|
||||
)
|
||||
|
||||
func runEnabledChecks(ctx context.Context,
|
||||
@ -56,62 +53,23 @@ func runEnabledChecks(ctx context.Context,
|
||||
close(resultsCh)
|
||||
}
|
||||
|
||||
func createRepo(uri *repos.RepoURI) (clients.Repo, error) {
|
||||
var c clients.Repo
|
||||
var e error
|
||||
switch uri.RepoType() {
|
||||
// URL.
|
||||
case repos.RepoTypeURL:
|
||||
c, e = githubrepo.MakeGithubRepo(uri.URL())
|
||||
// LocalDir.
|
||||
case repos.RepoTypeLocalDir:
|
||||
c, e = localdir.MakeLocalDirRepo(uri.Path())
|
||||
default:
|
||||
return nil,
|
||||
sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("unsupported URI type:%v", uri.RepoType()))
|
||||
func getRepoCommitHash(r clients.RepoClient) (string, error) {
|
||||
commits, err := r.ListCommits()
|
||||
if err != nil {
|
||||
return "", sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("ListCommits:%v", err.Error()))
|
||||
}
|
||||
|
||||
if e != nil {
|
||||
return c, sce.WithMessage(sce.ErrScorecardInternal, e.Error())
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func getRepoCommitHash(r clients.RepoClient, uri *repos.RepoURI) (string, error) {
|
||||
switch uri.RepoType() {
|
||||
// URL.
|
||||
case repos.RepoTypeURL:
|
||||
commits, err := r.ListCommits()
|
||||
if err != nil {
|
||||
return "", sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("ListCommits:%v", err.Error()))
|
||||
}
|
||||
|
||||
if len(commits) > 0 {
|
||||
return commits[0].SHA, nil
|
||||
}
|
||||
|
||||
return "no commits found", nil
|
||||
|
||||
// LocalDir.
|
||||
case repos.RepoTypeLocalDir:
|
||||
return "no commits for directory repo", nil
|
||||
|
||||
default:
|
||||
return "",
|
||||
sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("unsupported URI type:%v", uri.RepoType()))
|
||||
if len(commits) > 0 {
|
||||
return commits[0].SHA, nil
|
||||
}
|
||||
return "no commits found", nil
|
||||
}
|
||||
|
||||
// RunScorecards runs enabled Scorecard checks on a Repo.
|
||||
func RunScorecards(ctx context.Context,
|
||||
repoURI *repos.RepoURI,
|
||||
repo clients.Repo,
|
||||
checksToRun checker.CheckNameToFnMap,
|
||||
repoClient clients.RepoClient) (ScorecardResult, error) {
|
||||
repo, err := createRepo(repoURI)
|
||||
if err != nil {
|
||||
return ScorecardResult{}, sce.WithMessage(err, "")
|
||||
}
|
||||
if err := repoClient.InitRepo(repo); err != nil {
|
||||
// No need to call sce.WithMessage() since InitRepo will do that for us.
|
||||
//nolint:wrapcheck
|
||||
@ -119,7 +77,7 @@ func RunScorecards(ctx context.Context,
|
||||
}
|
||||
defer repoClient.Close()
|
||||
|
||||
commitSHA, err := getRepoCommitHash(repoClient, repoURI)
|
||||
commitSHA, err := getRepoCommitHash(repoClient)
|
||||
if err != nil {
|
||||
return ScorecardResult{}, err
|
||||
}
|
||||
|
@ -1,258 +0,0 @@
|
||||
// Copyright 2020 Security Scorecard Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package repos defines a generic repository.
|
||||
package repos
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
sce "github.com/ossf/scorecard/v3/errors"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrorUnsupportedhost indicates the repo's host is unsupported.
|
||||
ErrorUnsupportedhost = errors.New("unsupported host")
|
||||
// ErrorInvalidGithubURL indicates the repo's GitHub URL is not in the proper format.
|
||||
ErrorInvalidGithubURL = errors.New("invalid GitHub repo URL")
|
||||
// ErrorInvalidURL indicates the repo's full GitHub URL was not passed.
|
||||
ErrorInvalidURL = errors.New("invalid repo flag")
|
||||
// errInvalidRepoType indicates the repo's type is invalid.
|
||||
errInvalidRepoType = errors.New("invalid repo type")
|
||||
)
|
||||
|
||||
// RepoURI represents the URI for a repo.
|
||||
//nolint:govet
|
||||
type RepoURI struct {
|
||||
repoType RepoType
|
||||
localDir repoLocalDir
|
||||
url repoURL
|
||||
metadata []string
|
||||
}
|
||||
|
||||
type repoLocalDir struct {
|
||||
path string
|
||||
}
|
||||
|
||||
type repoURL struct {
|
||||
host, owner, repo string
|
||||
}
|
||||
|
||||
// RepoType is the type of a file.
|
||||
type RepoType int
|
||||
|
||||
const (
|
||||
// RepoTypeURL is for URLs.
|
||||
RepoTypeURL RepoType = iota
|
||||
// RepoTypeLocalDir is for source code in directories.
|
||||
RepoTypeLocalDir
|
||||
)
|
||||
|
||||
func (r repoLocalDir) Equal(o repoLocalDir) bool {
|
||||
return r.path == o.path
|
||||
}
|
||||
|
||||
func (r repoURL) Equal(o repoURL) bool {
|
||||
return r.host == o.host &&
|
||||
r.owner == o.owner &&
|
||||
r.repo == o.repo
|
||||
}
|
||||
|
||||
// NewFromURL creates a RepoURI from URL.
|
||||
func NewFromURL(u string) (*RepoURI, error) {
|
||||
r := &RepoURI{
|
||||
repoType: RepoTypeURL,
|
||||
}
|
||||
|
||||
if err := r.SetURL(u); err != nil {
|
||||
return nil, fmt.Errorf("%w", err)
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// NewFromLocalDirectory creates a RepoURI as a local directory.
|
||||
func NewFromLocalDirectory(path string) *RepoURI {
|
||||
return &RepoURI{
|
||||
localDir: repoLocalDir{
|
||||
path: path,
|
||||
},
|
||||
repoType: RepoTypeLocalDir,
|
||||
}
|
||||
}
|
||||
|
||||
// SetMetadata sets metadata.
|
||||
func (r *RepoURI) SetMetadata(m []string) error {
|
||||
r.metadata = m
|
||||
return nil
|
||||
}
|
||||
|
||||
// AppendMetadata appends metadata.
|
||||
func (r *RepoURI) AppendMetadata(m ...string) error {
|
||||
r.metadata = append(r.metadata, m...)
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetURL sets the URL.
|
||||
func (r *RepoURI) SetURL(u string) error {
|
||||
if r.repoType != RepoTypeURL {
|
||||
return fmt.Errorf("%w", errInvalidRepoType)
|
||||
}
|
||||
if err := r.Set(u); err != nil {
|
||||
return fmt.Errorf("%w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Equal checks objects for equality.
|
||||
func (r *RepoURI) Equal(o *RepoURI) bool {
|
||||
return cmp.Equal(r.localDir, o.localDir) &&
|
||||
cmp.Equal(r.url, o.url) &&
|
||||
cmp.Equal(r.repoType, o.repoType) &&
|
||||
cmp.Equal(r.metadata, o.metadata, cmpopts.SortSlices(func(x, y string) bool { return x < y }))
|
||||
}
|
||||
|
||||
// Type method is needed so that this struct can be used as cmd flag.
|
||||
func (r *RepoURI) Type() string {
|
||||
return "repo"
|
||||
}
|
||||
|
||||
// RepoType gives the type of URI.
|
||||
func (r *RepoURI) RepoType() RepoType {
|
||||
return r.repoType
|
||||
}
|
||||
|
||||
// Path retusn the path for a local directory.
|
||||
func (r *RepoURI) Path() string {
|
||||
return r.localDir.path
|
||||
}
|
||||
|
||||
// URL returns a valid url for Repo struct.
|
||||
func (r *RepoURI) URL() string {
|
||||
return fmt.Sprintf("%s/%s/%s", r.url.host, r.url.owner, r.url.repo)
|
||||
}
|
||||
|
||||
// Metadata returns a valid url for Repo struct.
|
||||
func (r *RepoURI) Metadata() []string {
|
||||
return r.metadata
|
||||
}
|
||||
|
||||
// String returns a string representation of Repo struct.
|
||||
func (r *RepoURI) String() string {
|
||||
return fmt.Sprintf("%s-%s-%s", r.url.host, r.url.owner, r.url.repo)
|
||||
}
|
||||
|
||||
// setV4 for the v4 version.
|
||||
func (r *RepoURI) setV4(s string) error {
|
||||
const httpsPrefix = "https://"
|
||||
const filePrefix = "file://"
|
||||
|
||||
// Validate the URI and scheme.
|
||||
if !strings.HasPrefix(s, filePrefix) &&
|
||||
!strings.HasPrefix(s, httpsPrefix) {
|
||||
return sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("invalid URI: %v", s))
|
||||
}
|
||||
|
||||
u, e := url.Parse(s)
|
||||
if e != nil {
|
||||
return sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("url.Parse: %v", e))
|
||||
}
|
||||
|
||||
switch {
|
||||
case strings.HasPrefix(s, httpsPrefix):
|
||||
const splitLen = 2
|
||||
split := strings.SplitN(strings.Trim(u.Path, "/"), "/", splitLen)
|
||||
if len(split) != splitLen {
|
||||
return sce.WithMessage(ErrorInvalidURL, fmt.Sprintf("%v. Expected full repository url", s))
|
||||
}
|
||||
r.url.host, r.url.owner, r.url.repo = u.Host, split[0], split[1]
|
||||
case strings.HasPrefix(s, filePrefix):
|
||||
r.localDir.path = s[len(filePrefix):]
|
||||
r.repoType = RepoTypeLocalDir
|
||||
default:
|
||||
break
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *RepoURI) set(s string) error {
|
||||
var t string
|
||||
|
||||
const two = 2
|
||||
const three = 3
|
||||
|
||||
c := strings.Split(s, "/")
|
||||
|
||||
switch l := len(c); {
|
||||
// This will takes care of repo/owner format.
|
||||
// By default it will use github.com
|
||||
case l == two:
|
||||
t = "github.com/" + c[0] + "/" + c[1]
|
||||
case l >= three:
|
||||
t = s
|
||||
}
|
||||
|
||||
// Allow skipping scheme for ease-of-use, default to https.
|
||||
if !strings.Contains(t, "://") {
|
||||
t = "https://" + t
|
||||
}
|
||||
|
||||
u, e := url.Parse(t)
|
||||
if e != nil {
|
||||
return sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("url.Parse: %v", e))
|
||||
}
|
||||
|
||||
const splitLen = 2
|
||||
split := strings.SplitN(strings.Trim(u.Path, "/"), "/", splitLen)
|
||||
if len(split) != splitLen {
|
||||
return sce.WithMessage(ErrorInvalidURL, fmt.Sprintf("%v. Exepted full repository url", s))
|
||||
}
|
||||
|
||||
r.url.host, r.url.owner, r.url.repo = u.Host, split[0], split[1]
|
||||
return nil
|
||||
}
|
||||
|
||||
// Set parses a URI string into Repo struct.
|
||||
func (r *RepoURI) Set(s string) error {
|
||||
var v4 bool
|
||||
_, v4 = os.LookupEnv("SCORECARD_V4")
|
||||
if v4 {
|
||||
return r.setV4(s)
|
||||
}
|
||||
|
||||
return r.set(s)
|
||||
}
|
||||
|
||||
// IsValidGitHubURL checks whether Repo represents a valid GitHub repo and returns errors otherwise.
|
||||
func (r *RepoURI) IsValidGitHubURL() error {
|
||||
switch r.url.host {
|
||||
case "github.com":
|
||||
default:
|
||||
return sce.WithMessage(ErrorUnsupportedhost, r.url.host)
|
||||
}
|
||||
|
||||
if strings.TrimSpace(r.url.owner) == "" || strings.TrimSpace(r.url.repo) == "" {
|
||||
return sce.WithMessage(ErrorInvalidGithubURL,
|
||||
fmt.Sprintf("%v. Expected the full repository url", r.URL()))
|
||||
}
|
||||
return nil
|
||||
}
|
@ -1,119 +0,0 @@
|
||||
// Copyright 2020 Security Scorecard Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package repos
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRepoURI_ValidGitHubUrl(t *testing.T) {
|
||||
t.Parallel()
|
||||
type fields struct {
|
||||
host string
|
||||
owner string
|
||||
repo string
|
||||
}
|
||||
type args struct {
|
||||
s string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
fields fields
|
||||
args args
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Valid http address",
|
||||
fields: fields{
|
||||
host: "github.com",
|
||||
owner: "foo",
|
||||
repo: "kubeflow",
|
||||
},
|
||||
args: args{s: "https://github.com/foo/kubeflow"},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Valid http address with trailing slash",
|
||||
fields: fields{
|
||||
host: "github.com",
|
||||
owner: "foo",
|
||||
repo: "kubeflow",
|
||||
},
|
||||
args: args{s: "https://github.com/foo/kubeflow/"},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Non github repository",
|
||||
fields: fields{
|
||||
host: "gitlab.com",
|
||||
owner: "foo",
|
||||
repo: "kubeflow",
|
||||
},
|
||||
args: args{s: "https://gitlab.com/foo/kubeflow"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "github repository",
|
||||
fields: fields{
|
||||
host: "github.com",
|
||||
owner: "foo",
|
||||
repo: "kubeflow",
|
||||
},
|
||||
args: args{s: "foo/kubeflow"},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "github repository",
|
||||
fields: fields{
|
||||
host: "github.com",
|
||||
owner: "foo",
|
||||
repo: "kubeflow",
|
||||
},
|
||||
args: args{s: "https://github.com/foo/kubeflow"},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
tt := tt // Re-initializing variable so it is not changed while executing the closure below
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
r := &RepoURI{
|
||||
url: repoURL{
|
||||
host: tt.fields.host,
|
||||
owner: tt.fields.owner,
|
||||
repo: tt.fields.repo,
|
||||
},
|
||||
}
|
||||
t.Log("Test")
|
||||
if err := r.Set(tt.args.s); err != nil {
|
||||
t.Errorf("repoURI.Set() error = %v", err)
|
||||
}
|
||||
if err := r.IsValidGitHubURL(); (err != nil) != tt.wantErr {
|
||||
t.Errorf("repoURI.ValidGitHubUrl() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
if !tt.wantErr {
|
||||
if tt.fields.host != r.url.host {
|
||||
t.Errorf("repo host expected to be %s but got %s", tt.fields.host, r.url.host)
|
||||
}
|
||||
if tt.fields.owner != r.url.owner {
|
||||
t.Errorf("repo owner expected to be %s but got %s", tt.fields.owner, r.url.owner)
|
||||
}
|
||||
if tt.fields.repo != r.url.repo {
|
||||
t.Errorf("repo expected to be %s but got %s", tt.fields.repo, r.url.repo)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user