Randomize the repos tested during release test (#1299)

Co-authored-by: Azeem Shaikh <azeems@google.com>

.gitignore

@@ -8,6 +8,7 @@ cron/data/update/projects-update
 cron/controller/controller
 cron/worker/worker
 cron/cii/cii-worker
+cron/shuffle/shuffle
 cron/webhook/webhook
 cron/bq/data-transfer
 

Makefile

@@ -93,8 +93,9 @@ tree-status: ## Verify tree is clean and all changes are committed
 
 ################################## make build #################################
 ## Build all cron-related targets
-build-cron: build-pubsub build-cii-worker build-bq-transfer build-github-server build-webhook build-add-script \
-	  build-validate-script build-update-script
+build-cron: build-controller build-worker build-cii-worker \
+	build-shuffler build-bq-transfer build-github-server \
+	build-webhook build-add-script build-validate-script build-update-script
 
 build-targets = generate-mocks generate-docs build-proto build-scorecard build-cron ko-build-everything dockerbuild
 .PHONY: build $(build-targets)
@@ -134,15 +135,22 @@ build-scorecard: ## Runs go build on repo
 	# Run go build and generate scorecard executable
 	CGO_ENABLED=0 go build -trimpath -a -tags netgo -ldflags '$(LDFLAGS)'
 
-build-pubsub: ## Runs go build on the PubSub cron job
-	# Run go build and the PubSub cron job
+build-controller: ## Runs go build on the cron PubSub controller
+	# Run go build on the cron PubSub controller
 	cd cron/controller && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o controller
+
+build-worker: ## Runs go build on the cron PubSub worker
+	# Run go build on the cron PubSub worker
 	cd cron/worker && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o worker
 
 build-cii-worker: ## Runs go build on the CII worker
 	# Run go build on the CII worker
 	cd cron/cii && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o cii-worker
 
+build-shuffler: ## Runs go build on the cron shuffle script
+	# Run go build on the cron shuffle script
+	cd cron/shuffle && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o shuffle
+
 build-bq-transfer: ## Runs go build on the BQ transfer cron job
 build-bq-transfer: ./cron/bq/*.go
 	# Run go build on the Copier cron job

cron/controller/Dockerfile

@@ -19,12 +19,19 @@ COPY go.* ./
 RUN go mod download
 COPY . ./
 
-FROM base AS pubsub
+FROM base AS shuffle
 ARG TARGETOS
 ARG TARGETARCH
-RUN CGO_ENABLED=0 make build-pubsub
+RUN CGO_ENABLED=0 make build-shuffler
+RUN ./cron/shuffle/shuffle 2500 cron/data/projects.csv cron/data/projects.release.csv
+
+FROM base AS controller
+ARG TARGETOS
+ARG TARGETARCH
+RUN CGO_ENABLED=0 make build-controller
 
 FROM gcr.io/distroless/base:nonroot@sha256:46d4514c17aca7a68559ee03975983339fc548e6d1014e2d7633f9123f2d3c59
 COPY ./cron/data/projects*csv cron/data/
-COPY --from=pubsub /src/cron/controller/controller cron/controller/controller
+COPY --from=shuffle /src/cron/data/projects.release.csv cron/data/projects.release.csv
+COPY --from=controller /src/cron/controller/controller cron/controller/controller
 ENTRYPOINT ["cron/controller/controller"]

cron/data/projects.release.csv

@@ -1,23 +0,0 @@
-repo,metadata
-#Basic
-github.com/ossf/scorecard,
-github.com/ossf/criticality_score,
-github.com/CircleCI-Public/circleci-dockerfiles,
-github.com/ChromeDevTools/devtools-protocol,
-github.com/kubernetes/kubernetes,
-# 300 redirects
-github.com/cocos2d/cocos2d-objc,
-github.com/dargullin/icheck,
-# Empty repos
-github.com/google/YouCompleteSubl,
-github.com/google-research-datasets/quest,
-# Does not exist
-github.com/does/not_exist,should_skip
-# Corrupted tarballs
-github.com/cdnjs/cdnjs,
-github.com/echen102/COVID-19-TweetIDs,
-# .github does not exist
-github.com/intel/intel-graphics-compiler,
-# Repository redirects
-github.com/chromium/ct-policy,
-github.com/flavioislima/HeroicGamesLauncher,

cron/data/writer.go

@@ -23,6 +23,17 @@ import (
 	"github.com/jszwec/csvutil"
 )
 
+// WriteTo writes `repos` to `out`.
+func WriteTo(out io.Writer, repos []RepoFormat) error {
+	csvWriter := csv.NewWriter(out)
+	enc := csvutil.NewEncoder(csvWriter)
+	if err := enc.Encode(repos); err != nil {
+		return fmt.Errorf("error during Encode: %w", err)
+	}
+	csvWriter.Flush()
+	return nil
+}
+
 // SortAndAppendTo appends `oldRepos` and `newRepos` before sorting and writing out the result to `out`.
 func SortAndAppendTo(out io.Writer, oldRepos, newRepos []RepoFormat) error {
 	oldRepos = append(oldRepos, newRepos...)
@@ -30,13 +41,7 @@ func SortAndAppendTo(out io.Writer, oldRepos, newRepos []RepoFormat) error {
 	sort.SliceStable(oldRepos, func(i, j int) bool {
 		return oldRepos[i].Repo < oldRepos[j].Repo
 	})
-	csvWriter := csv.NewWriter(out)
-	enc := csvutil.NewEncoder(csvWriter)
-	if err := enc.Encode(oldRepos); err != nil {
-		return fmt.Errorf("error during Encode: %w", err)
-	}
-	csvWriter.Flush()
-	return nil
+	return WriteTo(out, oldRepos)
 }
 
 // SortAndAppendFrom reads from `in`, appends to newRepos and writes the sorted output to `out`.

cron/shuffle/main.go

@@ -0,0 +1,66 @@
+// Copyright 2021 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package main implements the PubSub controller.
+package main
+
+import (
+	"math/rand"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/ossf/scorecard/v3/cron/data"
+)
+
+func main() {
+	if len(os.Args) != 4 {
+		panic("must provide exactly 3 arguments")
+	}
+
+	n, err := strconv.Atoi(os.Args[1])
+	if err != nil {
+		panic(err)
+	}
+
+	inFile, err := os.OpenFile(os.Args[2], os.O_RDONLY, 0o644)
+	if err != nil {
+		panic(err)
+	}
+	iter, err := data.MakeIteratorFrom(inFile)
+	if err != nil {
+		panic(err)
+	}
+
+	outFile, err := os.OpenFile(os.Args[3], os.O_WRONLY|os.O_CREATE, 0o755)
+	if err != nil {
+		panic(err)
+	}
+	var repoURLs []data.RepoFormat
+	for iter.HasNext() {
+		repo, err := iter.Next()
+		if err != nil {
+			panic(err)
+		}
+		repoURLs = append(repoURLs, repo)
+	}
+
+	rand.Seed(time.Now().UnixNano())
+	rand.Shuffle(len(repoURLs), func(i, j int) {
+		repoURLs[i], repoURLs[j] = repoURLs[j], repoURLs[i]
+	})
+	if err := data.WriteTo(outFile, repoURLs[:n]); err != nil {
+		panic(err)
+	}
+}

cron/worker/Dockerfile

@@ -19,11 +19,11 @@ COPY go.* ./
 RUN go mod download
 COPY . ./
 
-FROM base AS pubsub
+FROM base AS worker
 ARG TARGETOS
 ARG TARGETARCH
-RUN CGO_ENABLED=0 make build-pubsub
+RUN CGO_ENABLED=0 make build-worker
 
 FROM gcr.io/distroless/base:nonroot@sha256:46d4514c17aca7a68559ee03975983339fc548e6d1014e2d7633f9123f2d3c59
-COPY --from=pubsub /src/cron/worker/worker cron/worker/worker
+COPY --from=worker /src/cron/worker/worker cron/worker/worker
 ENTRYPOINT ["cron/worker/worker"]