mirror of
https://github.com/ossf/scorecard.git
synced 2024-09-11 08:55:27 +03:00
Script to add new projects to projects.csv file (#567)
Co-authored-by: Azeem Shaikh <azeems@google.com>
This commit is contained in:
parent
e7ea1a2b88
commit
c06f89af83
1
.gitignore
vendored
1
.gitignore
vendored
@ -1,6 +1,7 @@
|
||||
# binary.
|
||||
scorecard
|
||||
gitblobcache
|
||||
cron/data/add/add
|
||||
cron/data/validate/validate
|
||||
cron/data/update/projects-update
|
||||
cron/controller/controller
|
||||
|
20
Makefile
20
Makefile
@ -43,7 +43,7 @@ $(PROTOC):
|
||||
|
||||
################################## make all ###################################
|
||||
all: ## Runs build, test and verify
|
||||
all-targets = update-dependencies build check-linter unit-test validate-projects tree-status
|
||||
all-targets = update-dependencies build check-linter unit-test add-projects validate-projects tree-status
|
||||
.PHONY: all $(all-targets)
|
||||
all: $(all-targets)
|
||||
|
||||
@ -57,8 +57,13 @@ check-linter: $(GOLANGGCI_LINT)
|
||||
# Run golangci-lint linter
|
||||
golangci-lint run -c .golangci.yml
|
||||
|
||||
add-projects: ## Adds new projects to ./cron/data/projects.csv
|
||||
add-projects: ./cron/data/projects.csv | build-add-script
|
||||
# Add new projects to ./cron/data/projects.csv
|
||||
./cron/data/add/add ./cron/data/projects.csv
|
||||
|
||||
validate-projects: ## Validates ./cron/data/projects.csv
|
||||
validate-projects: build-validate-script
|
||||
validate-projects: ./cron/data/projects.csv | build-validate-script
|
||||
# Validate ./cron/data/projects.csv
|
||||
./cron/data/validate/validate
|
||||
|
||||
@ -69,8 +74,9 @@ tree-status: ## Verify tree is clean and all changes are committed
|
||||
|
||||
###############################################################################
|
||||
|
||||
############################### make build ################################
|
||||
build-targets = build-proto generate-docs build-scorecard build-pubsub build-validate-script build-update-script dockerbuild
|
||||
################################## make build #################################
|
||||
build-targets = build-proto generate-docs build-scorecard build-pubsub \
|
||||
build-add-script build-validate-script build-update-script dockerbuild
|
||||
.PHONY: build $(build-targets)
|
||||
build: ## Build all binaries and images in the reepo.
|
||||
build: $(build-targets)
|
||||
@ -95,6 +101,12 @@ build-pubsub: ## Runs go build on the PubSub cron job
|
||||
cd cron/controller && CGO_ENABLED=0 go build -a -ldflags '-w -extldflags "static"' -o controller
|
||||
cd cron/worker && CGO_ENABLED=0 go build -a -ldflags '-w -extldflags "static"' -o worker
|
||||
|
||||
build-add-script: ## Runs go build on the add script
|
||||
build-add-script: cron/data/add/add
|
||||
cron/data/add/add: cron/data/add/*.go cron/data/*.go
|
||||
# Run go build on the add script
|
||||
cd cron/data/add && CGO_ENABLED=0 go build -a -ldflags '-w -extldflags "-static"' -o add
|
||||
|
||||
build-validate-script: ## Runs go build on the validate script
|
||||
build-validate-script: cron/data/validate/validate
|
||||
cron/data/validate/validate: cron/data/validate/*.go cron/data/*.go
|
||||
|
63
cron/data/add/main.go
Normal file
63
cron/data/add/main.go
Normal file
@ -0,0 +1,63 @@
|
||||
// Copyright 2021 Security Scorecard Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
|
||||
"github.com/ossf/scorecard/cron/data"
|
||||
"github.com/ossf/scorecard/repos"
|
||||
)
|
||||
|
||||
// Script to add new project repositories to the projects.csv file:
|
||||
// * Removes any duplicates
|
||||
// * Repos are read in order and the first entry is honored in case of duplicates
|
||||
// * Sort and output all projects
|
||||
// Usage: add all new dependencies to the projects.csv file before running this script
|
||||
// Args:
|
||||
// path to output.csv file
|
||||
func main() {
|
||||
iter, err := data.MakeIterator()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
repoURLs := make([]repos.RepoURL, 0)
|
||||
repoMap := make(map[string]bool)
|
||||
for iter.HasNext() {
|
||||
repo, err := iter.Next()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if _, ok := repoMap[repo.URL()]; ok {
|
||||
continue
|
||||
}
|
||||
repoURLs = append(repoURLs, repo)
|
||||
repoMap[repo.URL()] = true
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := data.SortAndAppendTo(&buf, repoURLs, nil); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
projects, err := os.OpenFile(os.Args[1], os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if _, err := projects.Write(buf.Bytes()); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
@ -1543,7 +1543,6 @@ github.com/Kunena/Kunena-Forum,
|
||||
github.com/Kunzisoft/KeePassDX,
|
||||
github.com/LCTT/TranslateProject,
|
||||
github.com/LMMS/lmms,
|
||||
github.com/laurentsimon/scorecard-remediation-tests,
|
||||
github.com/Laravel-Backpack/CRUD,
|
||||
github.com/Laravel-Lang/lang,
|
||||
github.com/LemmyNet/lemmy,
|
||||
@ -10867,6 +10866,7 @@ github.com/laravel/telescope,
|
||||
github.com/laravel/valet,
|
||||
github.com/launchbadge/sqlx,
|
||||
github.com/laurent22/joplin,
|
||||
github.com/laurentsimon/scorecard-remediation-tests,
|
||||
github.com/lavabit/robox,
|
||||
github.com/lballabio/QuantLib,
|
||||
github.com/ldez/gomoddirectives,
|
||||
|
Can't render this file because it is too large.
|
@ -30,40 +30,47 @@ type repoEntry struct {
|
||||
Metadata string `csv:"metadata"`
|
||||
}
|
||||
|
||||
func repoEntryFromRepoURL(repoURLs []repos.RepoURL) []repoEntry {
|
||||
repoentries := make([]repoEntry, 0)
|
||||
for _, repoURL := range repoURLs {
|
||||
repoentry := repoEntry{
|
||||
Repo: repoURL.URL(),
|
||||
Metadata: repoURL.Metadata,
|
||||
}
|
||||
repoentries = append(repoentries, repoentry)
|
||||
}
|
||||
return repoentries
|
||||
}
|
||||
|
||||
func SortAndAppendTo(out io.Writer, oldRepos, newRepos []repos.RepoURL) error {
|
||||
repoentries := repoEntryFromRepoURL(oldRepos)
|
||||
repoentries = append(repoentries, repoEntryFromRepoURL(newRepos)...)
|
||||
|
||||
sort.SliceStable(repoentries, func(i, j int) bool {
|
||||
return repoentries[i].Repo < repoentries[j].Repo
|
||||
})
|
||||
csvWriter := csv.NewWriter(out)
|
||||
enc := csvutil.NewEncoder(csvWriter)
|
||||
if err := enc.Encode(repoentries); err != nil {
|
||||
return fmt.Errorf("error during Encode: %w", err)
|
||||
}
|
||||
csvWriter.Flush()
|
||||
return nil
|
||||
}
|
||||
|
||||
func SortAndAppend(out io.Writer, newRepos []repos.RepoURL) error {
|
||||
iter, err := MakeIterator()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error during MakeIterator: %w", err)
|
||||
}
|
||||
|
||||
oldRepos := make([]repoEntry, 0)
|
||||
oldRepos := make([]repos.RepoURL, 0)
|
||||
for iter.HasNext() {
|
||||
repo, err := iter.Next()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error during iter.Next: %w", err)
|
||||
}
|
||||
repoentry := repoEntry{
|
||||
Repo: repo.URL(),
|
||||
Metadata: repo.Metadata,
|
||||
}
|
||||
oldRepos = append(oldRepos, repoentry)
|
||||
oldRepos = append(oldRepos, repo)
|
||||
}
|
||||
for _, newRepo := range newRepos {
|
||||
repoentry := repoEntry{
|
||||
Repo: newRepo.URL(),
|
||||
Metadata: newRepo.Metadata,
|
||||
}
|
||||
oldRepos = append(oldRepos, repoentry)
|
||||
}
|
||||
sort.SliceStable(oldRepos, func(i, j int) bool {
|
||||
return oldRepos[i].Repo < oldRepos[j].Repo
|
||||
})
|
||||
|
||||
csvWriter := csv.NewWriter(out)
|
||||
enc := csvutil.NewEncoder(csvWriter)
|
||||
if err := enc.Encode(oldRepos); err != nil {
|
||||
return fmt.Errorf("error during Encode: %w", err)
|
||||
}
|
||||
csvWriter.Flush()
|
||||
return nil
|
||||
return SortAndAppendTo(out, oldRepos, newRepos)
|
||||
}
|
||||
|
71
cron/data/writer_test.go
Normal file
71
cron/data/writer_test.go
Normal file
@ -0,0 +1,71 @@
|
||||
// Copyright 2021 Security Scorecard Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package data
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/ossf/scorecard/repos"
|
||||
)
|
||||
|
||||
func TestCsvWriter(t *testing.T) {
|
||||
t.Parallel()
|
||||
testcases := []struct {
|
||||
name string
|
||||
out string
|
||||
oldRepos []repos.RepoURL
|
||||
newRepos []repos.RepoURL
|
||||
}{
|
||||
{
|
||||
name: "Basic",
|
||||
oldRepos: []repos.RepoURL{
|
||||
{
|
||||
Host: "github.com",
|
||||
Owner: "owner1",
|
||||
Repo: "repo1",
|
||||
Metadata: "meta1",
|
||||
},
|
||||
},
|
||||
newRepos: []repos.RepoURL{
|
||||
{
|
||||
Host: "github.com",
|
||||
Owner: "owner2",
|
||||
Repo: "repo2",
|
||||
Metadata: "meta2",
|
||||
},
|
||||
},
|
||||
out: `repo,metadata
|
||||
github.com/owner1/repo1,meta1
|
||||
github.com/owner2/repo2,meta2
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, testcase := range testcases {
|
||||
testcase := testcase
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
var buf bytes.Buffer
|
||||
err := SortAndAppendTo(&buf, testcase.oldRepos, testcase.newRepos)
|
||||
if err != nil {
|
||||
t.Errorf("error while running testcase: %v", err)
|
||||
}
|
||||
if buf.String() != testcase.out {
|
||||
t.Errorf("\nexpected: \n%s \ngot: \n%s", testcase.out, buf.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user