mirror of
https://github.com/ossf/scorecard.git
synced 2024-09-11 08:55:27 +03:00
✨ Refactor cron job
* Refactored cron job from shell script to go. * Included metadata to the projects.txt for envoy * Included checks for duplicate item in projects.txt * Sorted the projects.txt so that it is easier for someone to look for a project
This commit is contained in:
parent
52e742cce9
commit
688dc5e6c7
1
.gitignore
vendored
1
.gitignore
vendored
@ -1,6 +1,7 @@
|
||||
# binary.
|
||||
scorecard
|
||||
gitblobcache
|
||||
validate
|
||||
|
||||
|
||||
|
||||
|
@ -26,7 +26,15 @@ ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
RUN CGO_ENABLED=0 go build -a -tags netgo -ldflags '-w -extldflags "-static"' -o /out/scorecard .
|
||||
|
||||
|
||||
FROM base AS cron
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
WORKDIR ./cron/
|
||||
RUN CGO_ENABLED=0 go build -a -tags netgo -ldflags '-w -extldflags "-static"' -o /out/cron .
|
||||
|
||||
FROM gcr.io/google.com/cloudsdktool/cloud-sdk:slim
|
||||
COPY ./cron/projects.txt /cron/projects.txt
|
||||
COPY --from=build /out/scorecard /
|
||||
COPY ./cron/ /cron/
|
||||
ENTRYPOINT [ "/scorecard" ]
|
||||
COPY --from=cron /out/cron ./cron/cron
|
||||
ENTRYPOINT [ "/cron/cron", "/cron/projects.txt"]
|
||||
|
5
Makefile
5
Makefile
@ -24,7 +24,7 @@ help: ## Display this help
|
||||
|
||||
all: ## Runs build, test and verify
|
||||
.PHONY: all
|
||||
all: build test verify
|
||||
all: build test verify checkprojects
|
||||
|
||||
.PHONY: build
|
||||
build: ## Runs go build and generates executable
|
||||
@ -107,3 +107,6 @@ dockerbuild: ## Runs docker build
|
||||
docker build . --file Dockerfile --tag $(IMAGE_NAME)
|
||||
docker build . --file Dockerfile.gsutil --tag $(IMAGE_NAME)-gsutil
|
||||
|
||||
checkprojects: ## Validates ./cron/projects.txt
|
||||
cd ./scripts && go build -o validate
|
||||
./scripts/validate ./cron/projects.txt
|
||||
|
38
cron/cron.sh
38
cron/cron.sh
@ -1,38 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright 2020 Security Scorecard Authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http:#www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
SOURCE="${BASH_SOURCE[0]}"
|
||||
input=$(dirname "$SOURCE")/projects.txt
|
||||
output=$(date +"%m-%d-%Y").json
|
||||
touch "$output"
|
||||
echo "{ \"results\": [" >> "$output"
|
||||
|
||||
# sort and uniqify these, in case there are duplicates
|
||||
# shellcheck disable=SC2002
|
||||
projects=$(cat "$input" | sort | uniq)
|
||||
while read -r proj; do
|
||||
if [[ $proj =~ ^#.* ]]; then
|
||||
continue
|
||||
fi
|
||||
echo "$proj"
|
||||
../scorecard --repo="$proj" --checks=Active,CI-Tests,CII-Best-Practices,Code-Review,Contributors,Frozen-Deps,Fuzzing,Packaging,Pull-Requests,SAST,Security-Policy,Signed-Releases,Signed-Tags --show-details --format=json >> "$output"
|
||||
echo "," >> "$output"
|
||||
done <<< "$projects"
|
||||
sed -i '$d' "$output" # removing the trailing comma which will be last line.
|
||||
echo "]}" >> "$output"
|
||||
|
||||
gsutil cp "$output" gs://"$GCS_BUCKET"
|
||||
# Also copy the most recent run into a "latest.json" file
|
||||
gsutil cp gs://"$GCS_BUCKET"/"$output" gs://"$GCS_BUCKET"/latest.json
|
5
cron/go.mod
Normal file
5
cron/go.mod
Normal file
@ -0,0 +1,5 @@
|
||||
module github.com/ossf/scorecard/cron
|
||||
|
||||
go 1.16
|
||||
|
||||
require github.com/jszwec/csvutil v1.5.0
|
2
cron/go.sum
Normal file
2
cron/go.sum
Normal file
@ -0,0 +1,2 @@
|
||||
github.com/jszwec/csvutil v1.5.0 h1:ErLnF1Qzzt9svk8CUY7CyLl/W9eET+KWPIZWkE1o6JM=
|
||||
github.com/jszwec/csvutil v1.5.0/go.mod h1:Rpu7Uu9giO9subDyMCIQfHVDuLrcaC36UA4YcJjGBkg=
|
95
cron/main.go
Normal file
95
cron/main.go
Normal file
@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/jszwec/csvutil"
|
||||
)
|
||||
|
||||
type Repository struct {
|
||||
Repo string `csv:"repo"`
|
||||
Metadata string `csv:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
fileName := fmt.Sprintf("%d-%d-%d.json", time.Now().Month(), time.Now().Day(), time.Now().Year())
|
||||
result, err := os.Create(fileName)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer result.Close()
|
||||
projects, err := os.OpenFile(os.Args[1], os.O_RDONLY, 0644)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer projects.Close()
|
||||
|
||||
repos := []Repository{}
|
||||
data, err := ioutil.ReadAll(projects)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = csvutil.Unmarshal(data, &repos)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
sort.Slice(repos, func(i, j int) bool {
|
||||
return repos[i].Repo < repos[j].Repo
|
||||
})
|
||||
|
||||
//nolint
|
||||
const checks string = "--checks=Active,CI-Tests,CII-Best-Practices,Code-Review,Contributors,Frozen-Deps,Fuzzing,Packaging,Pull-Requests,SAST,Security-Policy,Signed-Releases,Signed-Tags"
|
||||
|
||||
if _, err = result.Write([]byte(`{"results":[`)); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for i, r := range repos {
|
||||
fmt.Println(r.Repo)
|
||||
//nolint
|
||||
cmd := exec.Command("../scorecard", fmt.Sprintf("--repo=%s", r.Repo), fmt.Sprintf("--metadata=%s", r.Metadata), checks, "--show-details", "--format=json")
|
||||
// passing the external github token the cmd
|
||||
cmd.Env = append(cmd.Env, fmt.Sprintf("GITHUB_AUTH_TOKEN=%s", os.Getenv("GITHUB_AUTH_TOKEN")))
|
||||
cmd.Stderr = io.Writer(os.Stderr)
|
||||
|
||||
// Execute the command
|
||||
data, err := cmd.Output()
|
||||
if err != nil {
|
||||
fmt.Printf("error:failed for the repo %s and the error is %s", r.Repo, err.Error())
|
||||
// continuing because this is just for that repo that failed.
|
||||
continue
|
||||
}
|
||||
|
||||
_, err = result.WriteString(string(data))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if i < len(repos)-1 {
|
||||
_, err = result.WriteString(",")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := result.WriteString("\n]}\n"); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// copying the file to the GCS bucket
|
||||
if err := exec.Command("gsutil", "cp", fileName, fmt.Sprintf("gs://%s", os.Getenv("GCS_BUCKET"))).Run(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
//copying the results to the latest.json
|
||||
//nolint
|
||||
if err := exec.Command("gsutil", "cp", fmt.Sprintf("gs://%s/%s", os.Getenv("GCS_BUCKET"), fileName),
|
||||
fmt.Sprintf("gs://%s/latest.json", os.Getenv("GCS_BUCKET"))).Run(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
fmt.Println("Finished")
|
||||
}
|
4628
cron/projects.txt
4628
cron/projects.txt
File diff suppressed because it is too large
Load Diff
@ -12,10 +12,6 @@ spec:
|
||||
- name: run-score
|
||||
image: gcr.io/openssf/cron:latest
|
||||
imagePullPolicy: Always
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- ./cron/cron.sh
|
||||
env:
|
||||
- name: GITHUB_AUTH_TOKEN
|
||||
valueFrom:
|
||||
|
5
scripts/go.mod
Normal file
5
scripts/go.mod
Normal file
@ -0,0 +1,5 @@
|
||||
module github.com/ossf/scorecard/scripts
|
||||
|
||||
go 1.16
|
||||
|
||||
require github.com/jszwec/csvutil v1.5.0
|
2
scripts/go.sum
Normal file
2
scripts/go.sum
Normal file
@ -0,0 +1,2 @@
|
||||
github.com/jszwec/csvutil v1.5.0 h1:ErLnF1Qzzt9svk8CUY7CyLl/W9eET+KWPIZWkE1o6JM=
|
||||
github.com/jszwec/csvutil v1.5.0/go.mod h1:Rpu7Uu9giO9subDyMCIQfHVDuLrcaC36UA4YcJjGBkg=
|
42
scripts/main.go
Normal file
42
scripts/main.go
Normal file
@ -0,0 +1,42 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/jszwec/csvutil"
|
||||
)
|
||||
|
||||
type Repository struct {
|
||||
Repo string `csv:"repo"`
|
||||
Metadata string `csv:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
// Checks for duplicate item in the projects.txt
|
||||
// This is used in the builds to validate there aren't duplicates in projects.txt.
|
||||
func main() {
|
||||
projects, err := os.OpenFile(os.Args[1], os.O_RDONLY, 0644)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer projects.Close()
|
||||
|
||||
repos := []Repository{}
|
||||
data, err := ioutil.ReadAll(projects)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = csvutil.Unmarshal(data, &repos)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
m := make(map[string]bool)
|
||||
for _, item := range repos {
|
||||
if _, ok := m[item.Repo]; ok {
|
||||
log.Panicf("Item already in the list %s", item.Repo)
|
||||
}
|
||||
m[item.Repo] = true
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user