Script to add new projects to projects.csv file (#567)

Co-authored-by: Azeem Shaikh <azeems@google.com>

.gitignore

@@ -1,6 +1,7 @@
 # binary.
 scorecard
 gitblobcache
+cron/data/add/add
 cron/data/validate/validate
 cron/data/update/projects-update
 cron/controller/controller

Makefile

@@ -43,7 +43,7 @@ $(PROTOC):
 
 ################################## make all ###################################
 all:  ## Runs build, test and verify
-all-targets = update-dependencies build check-linter unit-test validate-projects tree-status
+all-targets = update-dependencies build check-linter unit-test add-projects validate-projects tree-status
 .PHONY: all $(all-targets)
 all: $(all-targets)
 
@@ -57,8 +57,13 @@ check-linter: $(GOLANGGCI_LINT)
 	# Run golangci-lint linter
 	golangci-lint run -c .golangci.yml
 
+add-projects: ## Adds new projects to ./cron/data/projects.csv
+add-projects: ./cron/data/projects.csv | build-add-script
+	# Add new projects to ./cron/data/projects.csv
+	./cron/data/add/add ./cron/data/projects.csv
+
 validate-projects: ## Validates ./cron/data/projects.csv
-validate-projects: build-validate-script
+validate-projects: ./cron/data/projects.csv | build-validate-script
 	# Validate ./cron/data/projects.csv
 	./cron/data/validate/validate
 
@@ -69,8 +74,9 @@ tree-status: ## Verify tree is clean and all changes are committed
 
 ###############################################################################
 
-############################### make build ################################
-build-targets = build-proto generate-docs build-scorecard build-pubsub build-validate-script build-update-script dockerbuild
+################################## make build #################################
+build-targets = build-proto generate-docs build-scorecard build-pubsub \
+	build-add-script build-validate-script build-update-script dockerbuild
 .PHONY: build $(build-targets)
 build: ## Build all binaries and images in the reepo.
 build: $(build-targets)
@@ -95,6 +101,12 @@ build-pubsub: ## Runs go build on the PubSub cron job
 	cd cron/controller && CGO_ENABLED=0 go build -a -ldflags '-w -extldflags "static"' -o controller
 	cd cron/worker && CGO_ENABLED=0 go build -a -ldflags '-w -extldflags "static"' -o worker
 
+build-add-script: ## Runs go build on the add script
+build-add-script: cron/data/add/add
+cron/data/add/add: cron/data/add/*.go cron/data/*.go
+	# Run go build on the add script
+	cd cron/data/add && CGO_ENABLED=0 go build -a -ldflags '-w -extldflags "-static"' -o add
+
 build-validate-script: ## Runs go build on the validate script
 build-validate-script: cron/data/validate/validate
 cron/data/validate/validate: cron/data/validate/*.go cron/data/*.go

cron/data/add/main.go

@@ -0,0 +1,63 @@
+// Copyright 2021 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"os"
+
+	"github.com/ossf/scorecard/cron/data"
+	"github.com/ossf/scorecard/repos"
+)
+
+// Script to add new project repositories to the projects.csv file:
+// * Removes any duplicates
+// * Repos are read in order and the first entry is honored in case of duplicates
+// * Sort and output all projects
+// Usage: add all new dependencies to the projects.csv file before running this script
+// Args:
+//     path to output.csv file
+func main() {
+	iter, err := data.MakeIterator()
+	if err != nil {
+		panic(err)
+	}
+
+	repoURLs := make([]repos.RepoURL, 0)
+	repoMap := make(map[string]bool)
+	for iter.HasNext() {
+		repo, err := iter.Next()
+		if err != nil {
+			panic(err)
+		}
+		if _, ok := repoMap[repo.URL()]; ok {
+			continue
+		}
+		repoURLs = append(repoURLs, repo)
+		repoMap[repo.URL()] = true
+	}
+
+	var buf bytes.Buffer
+	if err := data.SortAndAppendTo(&buf, repoURLs, nil); err != nil {
+		panic(err)
+	}
+	projects, err := os.OpenFile(os.Args[1], os.O_WRONLY, 0o644)
+	if err != nil {
+		panic(err)
+	}
+	if _, err := projects.Write(buf.Bytes()); err != nil {
+		panic(err)
+	}
+}

cron/data/projects.csv

@@ -1543,7 +1543,6 @@ github.com/Kunena/Kunena-Forum,
 github.com/Kunzisoft/KeePassDX,
 github.com/LCTT/TranslateProject,
 github.com/LMMS/lmms,
-github.com/laurentsimon/scorecard-remediation-tests,
 github.com/Laravel-Backpack/CRUD,
 github.com/Laravel-Lang/lang,
 github.com/LemmyNet/lemmy,
@@ -10867,6 +10866,7 @@ github.com/laravel/telescope,
 github.com/laravel/valet,
 github.com/launchbadge/sqlx,
 github.com/laurent22/joplin,
+github.com/laurentsimon/scorecard-remediation-tests,
 github.com/lavabit/robox,
 github.com/lballabio/QuantLib,
 github.com/ldez/gomoddirectives,

cron/data/writer.go

@@ -30,40 +30,47 @@ type repoEntry struct {
 	Metadata string `csv:"metadata"`
 }
 
+func repoEntryFromRepoURL(repoURLs []repos.RepoURL) []repoEntry {
+	repoentries := make([]repoEntry, 0)
+	for _, repoURL := range repoURLs {
+		repoentry := repoEntry{
+			Repo:     repoURL.URL(),
+			Metadata: repoURL.Metadata,
+		}
+		repoentries = append(repoentries, repoentry)
+	}
+	return repoentries
+}
+
+func SortAndAppendTo(out io.Writer, oldRepos, newRepos []repos.RepoURL) error {
+	repoentries := repoEntryFromRepoURL(oldRepos)
+	repoentries = append(repoentries, repoEntryFromRepoURL(newRepos)...)
+
+	sort.SliceStable(repoentries, func(i, j int) bool {
+		return repoentries[i].Repo < repoentries[j].Repo
+	})
+	csvWriter := csv.NewWriter(out)
+	enc := csvutil.NewEncoder(csvWriter)
+	if err := enc.Encode(repoentries); err != nil {
+		return fmt.Errorf("error during Encode: %w", err)
+	}
+	csvWriter.Flush()
+	return nil
+}
+
 func SortAndAppend(out io.Writer, newRepos []repos.RepoURL) error {
 	iter, err := MakeIterator()
 	if err != nil {
 		return fmt.Errorf("error during MakeIterator: %w", err)
 	}
 
-	oldRepos := make([]repoEntry, 0)
+	oldRepos := make([]repos.RepoURL, 0)
 	for iter.HasNext() {
 		repo, err := iter.Next()
 		if err != nil {
 			return fmt.Errorf("error during iter.Next: %w", err)
 		}
-		repoentry := repoEntry{
-			Repo:     repo.URL(),
-			Metadata: repo.Metadata,
-		}
-		oldRepos = append(oldRepos, repoentry)
+		oldRepos = append(oldRepos, repo)
 	}
-	for _, newRepo := range newRepos {
-		repoentry := repoEntry{
-			Repo:     newRepo.URL(),
-			Metadata: newRepo.Metadata,
-		}
-		oldRepos = append(oldRepos, repoentry)
-	}
-	sort.SliceStable(oldRepos, func(i, j int) bool {
-		return oldRepos[i].Repo < oldRepos[j].Repo
-	})
-
-	csvWriter := csv.NewWriter(out)
-	enc := csvutil.NewEncoder(csvWriter)
-	if err := enc.Encode(oldRepos); err != nil {
-		return fmt.Errorf("error during Encode: %w", err)
-	}
-	csvWriter.Flush()
-	return nil
+	return SortAndAppendTo(out, oldRepos, newRepos)
 }

cron/data/writer_test.go

@@ -0,0 +1,71 @@
+// Copyright 2021 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package data
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/ossf/scorecard/repos"
+)
+
+func TestCsvWriter(t *testing.T) {
+	t.Parallel()
+	testcases := []struct {
+		name     string
+		out      string
+		oldRepos []repos.RepoURL
+		newRepos []repos.RepoURL
+	}{
+		{
+			name: "Basic",
+			oldRepos: []repos.RepoURL{
+				{
+					Host:     "github.com",
+					Owner:    "owner1",
+					Repo:     "repo1",
+					Metadata: "meta1",
+				},
+			},
+			newRepos: []repos.RepoURL{
+				{
+					Host:     "github.com",
+					Owner:    "owner2",
+					Repo:     "repo2",
+					Metadata: "meta2",
+				},
+			},
+			out: `repo,metadata
+github.com/owner1/repo1,meta1
+github.com/owner2/repo2,meta2
+`,
+		},
+	}
+
+	for _, testcase := range testcases {
+		testcase := testcase
+		t.Run(testcase.name, func(t *testing.T) {
+			t.Parallel()
+			var buf bytes.Buffer
+			err := SortAndAppendTo(&buf, testcase.oldRepos, testcase.newRepos)
+			if err != nil {
+				t.Errorf("error while running testcase: %v", err)
+			}
+			if buf.String() != testcase.out {
+				t.Errorf("\nexpected: \n%s \ngot: \n%s", testcase.out, buf.String())
+			}
+		})
+	}
+}