✨ Refactor cron job

* Refactored cron job from shell script to go.
* Included metadata to the projects.txt for envoy
* Included checks for duplicate item in projects.txt
* Sorted the projects.txt so that it is easier for someone to look for a
project

.gitignore

@@ -1,6 +1,7 @@
 # binary.
 scorecard
 gitblobcache
+validate
 
 
 

Dockerfile.gsutil

@@ -26,7 +26,15 @@ ARG TARGETOS
 ARG TARGETARCH
 RUN CGO_ENABLED=0 go build -a -tags netgo -ldflags '-w -extldflags "-static"' -o /out/scorecard .
 
+
+FROM base AS cron
+ARG TARGETOS
+ARG TARGETARCH
+WORKDIR ./cron/
+RUN CGO_ENABLED=0 go build -a -tags netgo -ldflags '-w -extldflags "-static"' -o /out/cron .
+
 FROM gcr.io/google.com/cloudsdktool/cloud-sdk:slim 
+COPY ./cron/projects.txt /cron/projects.txt
 COPY --from=build /out/scorecard /
-COPY ./cron/ /cron/
-ENTRYPOINT [ "/scorecard" ] 
+COPY --from=cron /out/cron ./cron/cron
+ENTRYPOINT [ "/cron/cron", "/cron/projects.txt"] 

Makefile

@@ -24,7 +24,7 @@ help:  ## Display this help
 
 all:  ## Runs build, test and verify
 .PHONY: all
-all: build test verify 
+all: build test verify checkprojects
 
 .PHONY: build
 build: ## Runs go build and generates executable
@@ -107,3 +107,6 @@ dockerbuild: ## Runs docker build
 	docker build . --file Dockerfile --tag $(IMAGE_NAME) 
 	docker build . --file Dockerfile.gsutil --tag $(IMAGE_NAME)-gsutil
 
+checkprojects: ## Validates ./cron/projects.txt
+	cd ./scripts && go build  -o validate
+	./scripts/validate ./cron/projects.txt

cron/cron.sh

@@ -1,38 +0,0 @@
-#!/bin/bash
-# Copyright 2020 Security Scorecard Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http:#www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-SOURCE="${BASH_SOURCE[0]}"
-input=$(dirname "$SOURCE")/projects.txt
-output=$(date +"%m-%d-%Y").json
-touch "$output"
-echo "{ \"results\": [" >> "$output"
-
-# sort and uniqify these, in case there are duplicates
-# shellcheck disable=SC2002
-projects=$(cat "$input" | sort | uniq)
-while read -r proj; do
-    if [[ $proj =~ ^#.* ]]; then
-        continue
-    fi
-    echo "$proj"
-    ../scorecard --repo="$proj" --checks=Active,CI-Tests,CII-Best-Practices,Code-Review,Contributors,Frozen-Deps,Fuzzing,Packaging,Pull-Requests,SAST,Security-Policy,Signed-Releases,Signed-Tags --show-details --format=json >> "$output"
-    echo "," >> "$output"
-done <<< "$projects"
-sed -i '$d' "$output" # removing the trailing comma which will be last line.
-echo "]}" >> "$output"
-
-gsutil cp "$output" gs://"$GCS_BUCKET"
-# Also copy the most recent run into a "latest.json" file
-gsutil cp gs://"$GCS_BUCKET"/"$output" gs://"$GCS_BUCKET"/latest.json

cron/go.mod

@@ -0,0 +1,5 @@
+module github.com/ossf/scorecard/cron
+
+go 1.16
+
+require github.com/jszwec/csvutil v1.5.0

cron/go.sum

@@ -0,0 +1,2 @@
+github.com/jszwec/csvutil v1.5.0 h1:ErLnF1Qzzt9svk8CUY7CyLl/W9eET+KWPIZWkE1o6JM=
+github.com/jszwec/csvutil v1.5.0/go.mod h1:Rpu7Uu9giO9subDyMCIQfHVDuLrcaC36UA4YcJjGBkg=

cron/main.go

@@ -0,0 +1,95 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"sort"
+	"time"
+
+	"github.com/jszwec/csvutil"
+)
+
+type Repository struct {
+	Repo     string `csv:"repo"`
+	Metadata string `csv:"metadata,omitempty"`
+}
+
+func main() {
+	fileName := fmt.Sprintf("%d-%d-%d.json", time.Now().Month(), time.Now().Day(), time.Now().Year())
+	result, err := os.Create(fileName)
+	if err != nil {
+		panic(err)
+	}
+	defer result.Close()
+	projects, err := os.OpenFile(os.Args[1], os.O_RDONLY, 0644)
+	if err != nil {
+		panic(err)
+	}
+	defer projects.Close()
+
+	repos := []Repository{}
+	data, err := ioutil.ReadAll(projects)
+	if err != nil {
+		panic(err)
+	}
+	err = csvutil.Unmarshal(data, &repos)
+	if err != nil {
+		panic(err)
+	}
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].Repo < repos[j].Repo
+	})
+
+	//nolint
+	const checks string = "--checks=Active,CI-Tests,CII-Best-Practices,Code-Review,Contributors,Frozen-Deps,Fuzzing,Packaging,Pull-Requests,SAST,Security-Policy,Signed-Releases,Signed-Tags"
+
+	if _, err = result.Write([]byte(`{"results":[`)); err != nil {
+		panic(err)
+	}
+	for i, r := range repos {
+		fmt.Println(r.Repo)
+		//nolint
+		cmd := exec.Command("../scorecard", fmt.Sprintf("--repo=%s", r.Repo), fmt.Sprintf("--metadata=%s", r.Metadata), checks, "--show-details", "--format=json")
+		// passing the external github token the cmd
+		cmd.Env = append(cmd.Env, fmt.Sprintf("GITHUB_AUTH_TOKEN=%s", os.Getenv("GITHUB_AUTH_TOKEN")))
+		cmd.Stderr = io.Writer(os.Stderr)
+
+		// Execute the command
+		data, err := cmd.Output()
+		if err != nil {
+			fmt.Printf("error:failed for the repo %s and the error is %s", r.Repo, err.Error())
+			// continuing because this is just for that repo that failed.
+			continue
+		}
+
+		_, err = result.WriteString(string(data))
+		if err != nil {
+			panic(err)
+		}
+		if i < len(repos)-1 {
+			_, err = result.WriteString(",")
+			if err != nil {
+				panic(err)
+			}
+		}
+	}
+	if _, err := result.WriteString("\n]}\n"); err != nil {
+		panic(err)
+	}
+
+	// copying the file to the GCS bucket
+	if err := exec.Command("gsutil", "cp", fileName, fmt.Sprintf("gs://%s", os.Getenv("GCS_BUCKET"))).Run(); err != nil {
+		panic(err)
+	}
+	//copying the results to the latest.json
+	//nolint
+	if err := exec.Command("gsutil", "cp", fmt.Sprintf("gs://%s/%s", os.Getenv("GCS_BUCKET"), fileName),
+		fmt.Sprintf("gs://%s/latest.json", os.Getenv("GCS_BUCKET"))).Run(); err != nil {
+		panic(err)
+	}
+
+	fmt.Println("Finished")
+}

k8s/cron.yaml

@@ -12,10 +12,6 @@ spec:
           - name: run-score
             image: gcr.io/openssf/cron:latest
             imagePullPolicy: Always
-            command:
-            - /bin/sh
-            - -c
-            - ./cron/cron.sh
             env:
             - name: GITHUB_AUTH_TOKEN
               valueFrom:

scripts/go.mod

@@ -0,0 +1,5 @@
+module github.com/ossf/scorecard/scripts
+
+go 1.16
+
+require github.com/jszwec/csvutil v1.5.0

scripts/go.sum

@@ -0,0 +1,2 @@
+github.com/jszwec/csvutil v1.5.0 h1:ErLnF1Qzzt9svk8CUY7CyLl/W9eET+KWPIZWkE1o6JM=
+github.com/jszwec/csvutil v1.5.0/go.mod h1:Rpu7Uu9giO9subDyMCIQfHVDuLrcaC36UA4YcJjGBkg=

scripts/main.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+
+	"github.com/jszwec/csvutil"
+)
+
+type Repository struct {
+	Repo     string `csv:"repo"`
+	Metadata string `csv:"metadata,omitempty"`
+}
+
+// Checks for duplicate item in the projects.txt
+// This is used in the builds to validate there aren't duplicates in projects.txt.
+func main() {
+	projects, err := os.OpenFile(os.Args[1], os.O_RDONLY, 0644)
+	if err != nil {
+		panic(err)
+	}
+	defer projects.Close()
+
+	repos := []Repository{}
+	data, err := ioutil.ReadAll(projects)
+	if err != nil {
+		panic(err)
+	}
+	err = csvutil.Unmarshal(data, &repos)
+	if err != nil {
+		panic(err)
+	}
+
+	m := make(map[string]bool)
+	for _, item := range repos {
+		if _, ok := m[item.Repo]; ok {
+			log.Panicf("Item already in the list %s", item.Repo)
+		}
+		m[item.Repo] = true
+	}
+}