ossf/scorecard
1
1
Fork 0
mirror of https://github.com/ossf/scorecard.git synced 2024-10-26 10:28:10 +03:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Embed data files into the go lib files (#475)

Browse Source 
Co-authored-by: Azeem Shaikh <azeems@google.com>
This commit is contained in:
Azeem Shaikh 2021-05-19 18:00:44 -07:00 committed by GitHub
parent abdfd23770
commit 09c44bd355
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 52 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
cron/config/config.go
View File

@ -15,9 +15,9 @@
package config
import (
_ "embed" // Used to embed config.yaml
"errors"
"fmt"
"io/ioutil"
"os"
"reflect"
"strconv"
@ -32,12 +32,13 @@ const (
requestSubscriptionURL string = "SCORECARD_REQUEST_SUBSCRIPTION_URL"
inputReposFile string = "SCORECARD_REPOS_FILE"
shardSize string = "SCORECARD_SHARD_SIZE"
configYAML string = "config.yaml"
)
var (
ErrorEmptyConfigValue = errors.New("config value set to empty")
ErrorValueConversion = errors.New("unexpected type, cannot convert value")
//go:embed config.yaml
configYAML []byte
)
type config struct {
@ -48,39 +49,28 @@ type config struct {
ShardSize int `yaml:"shard-size"`
}
func getParsedConfigFromFile(filename string) (config, error) {
yamlFile, err := os.Open(filename)
if err != nil {
return config{}, fmt.Errorf("error during os.Open: %w", err)
}
defer yamlFile.Close()
byteValue, err := ioutil.ReadAll(yamlFile)
if err != nil {
return config{}, fmt.Errorf("error during ioutil.ReadAll: %w", err)
}
func getParsedConfigFromFile(byteValue []byte) (config, error) {
var ret config
err = yaml.Unmarshal(byteValue, &ret)
err := yaml.Unmarshal(byteValue, &ret)
if err != nil {
return config{}, fmt.Errorf("error during yaml.Unmarshal: %w", err)
}
return ret, nil
}
func getConfigValue(envVar, filename, fieldName string) (reflect.Value, error) {
func getConfigValue(envVar string, byteValue []byte, fieldName string) (reflect.Value, error) {
if val, present := os.LookupEnv(envVar); present {
return reflect.ValueOf(val), nil
}
parsedConfig, err := getParsedConfigFromFile(filename)
parsedConfig, err := getParsedConfigFromFile(byteValue)
if err != nil {
return reflect.ValueOf(parsedConfig), fmt.Errorf("error parsing config file: %w", err)
}
return reflect.ValueOf(parsedConfig).FieldByName(fieldName), nil
}
func getStringConfigValue(envVar, filename, fieldName, configName string) (string, error) {
value, err := getConfigValue(envVar, filename, fieldName)
func getStringConfigValue(envVar string, byteValue []byte, fieldName, configName string) (string, error) {
value, err := getConfigValue(envVar, byteValue, fieldName)
if err != nil {
return "", fmt.Errorf("error getting config value %s: %w", configName, err)
}
@ -93,8 +83,8 @@ func getStringConfigValue(envVar, filename, fieldName, configName string) (strin
return value.String(), fmt.Errorf("%w: %s", ErrorEmptyConfigValue, configName)
}
func getIntConfigValue(envVar, filename, fieldName, configName string) (int, error) {
value, err := getConfigValue(envVar, filename, fieldName)
func getIntConfigValue(envVar string, byteValue []byte, fieldName, configName string) (int, error) {
value, err := getConfigValue(envVar, byteValue, fieldName)
if err != nil {
return 0, fmt.Errorf("error getting config value %s: %w", configName, err)
}

27
cron/config/config_test.go
View File

@ -16,6 +16,7 @@ package config
import (
"errors"
"io/ioutil"
"os"
"testing"
)
@ -29,6 +30,14 @@ const (
prodShardSize int = 250
)
func getByteValueFromFile(filename string) ([]byte, error) {
if filename == "" {
return nil, nil
}
//nolint
return ioutil.ReadFile(filename)
}
func TestYAMLParsing(t *testing.T) {
t.Parallel()
testcases := []struct {
@ -72,7 +81,11 @@ func TestYAMLParsing(t *testing.T) {
testcase := testcase
t.Run(testcase.name, func(t *testing.T) {
t.Parallel()
parsedConfig, err := getParsedConfigFromFile(testcase.filename)
byteValue, err := getByteValueFromFile(testcase.filename)
if err != nil {
t.Errorf("test failed to parse input file: %v", err)
}
parsedConfig, err := getParsedConfigFromFile(byteValue)
if err != nil {
t.Errorf("failed to parse test file: %v", err)
}
@ -124,7 +137,11 @@ func TestGetStringConfigValue(t *testing.T) {
os.Setenv(testEnvVar, testcase.envVal)
}
actual, err := getStringConfigValue(testEnvVar, testcase.filename, testcase.fieldName, "test-config" /*configName*/)
byteValue, err := getByteValueFromFile(testcase.filename)
if err != nil {
t.Errorf("test failed during input parsing: %v", err)
}
actual, err := getStringConfigValue(testEnvVar, byteValue, testcase.fieldName, "test-config" /*configName*/)
if testcase.hasError {
if err == nil || !errors.Is(err, testcase.expectedErr) {
t.Errorf("test failed: expectedErr - %v, got - %v", testcase.expectedErr, err)
@ -175,7 +192,11 @@ func TestGetIntConfigValue(t *testing.T) {
os.Setenv(testEnvVar, testcase.envVal)
}
actual, err := getIntConfigValue(testEnvVar, testcase.filename, testcase.fieldName, "test-config" /*configName*/)
byteValue, err := getByteValueFromFile(testcase.filename)
if err != nil {
t.Errorf("test failed during input parsing: %v", err)
}
actual, err := getIntConfigValue(testEnvVar, byteValue, testcase.fieldName, "test-config" /*configName*/)
if testcase.hasError {
if err == nil || !errors.Is(err, testcase.expectedErr) {
t.Errorf("test failed: expectedErr - %v, got - %v", testcase.expectedErr, err)

2
cron/controller/Dockerfile
View File

@ -27,6 +27,6 @@ ARG TARGETARCH
RUN CGO_ENABLED=0 make build-pubsub
FROM gcr.io/distroless/base:nonroot@sha256:bc84925113289d139a9ef2f309f0dd7ac46ea7b786f172ba9084ffdb4cbd9490
COPY ./cron/config/projects.csv /cron/config/projects.csv
COPY ./cron/data/projects.csv /cron/data/projects.csv
COPY --from=pubsub /src/cron/controller/controller ./cron/controller/controller
ENTRYPOINT ["/cron/controller/controller"]

11
cron/controller/main.go
View File

@ -17,7 +17,6 @@ package main
import (
"context"
"fmt"
"os"
"strconv"
"time"
@ -82,15 +81,7 @@ func PublishToRepoRequestTopic(ctx context.Context, iter data.Iterator, datetime
func main() {
ctx := context.Background()
t := time.Now()
inputReposFile, err := config.GetInputReposFile()
if err != nil {
panic(err)
}
reposFile, err := os.OpenFile(inputReposFile, os.O_RDONLY, 0o644)
if err != nil {
panic(err)
}
reader, err := data.MakeIterator(reposFile)
reader, err := data.MakeIterator()
if err != nil {
panic(err)
}

16
cron/data/iterator.go
View File

@ -15,6 +15,7 @@
package data
import (
"embed"
"encoding/csv"
"errors"
"fmt"
@ -24,15 +25,26 @@ import (
"github.com/ossf/scorecard/repos"
)
//go:embed projects.csv
var csvFile embed.FS
type Iterator interface {
HasNext() bool
Next() (repos.RepoURL, error)
}
func MakeIterator(reader io.Reader) (Iterator, error) {
func MakeIterator() (Iterator, error) {
csvReader, err := csvFile.Open("projects.csv")
if err != nil {
return nil, fmt.Errorf("error in Open: %w", err)
}
return MakeIteratorFrom(csvReader)
}
func MakeIteratorFrom(reader io.Reader) (Iterator, error) {
dec, err := csvutil.NewDecoder(csv.NewReader(reader))
if err != nil {
return &csvIterator{}, fmt.Errorf("error in csvutil.NewDecoder: %w", err)
return nil, fmt.Errorf("error in csvutil.NewDecoder: %w", err)
}
return &csvIterator{decoder: dec}, nil
}

2
cron/data/iterator_test.go
View File

@ -151,7 +151,7 @@ func TestCsvIterator(t *testing.T) {
}
defer testFile.Close()
testReader, err := MakeIterator(testFile)
testReader, err := MakeIteratorFrom(testFile)
if err != nil {
t.Errorf("failed to create reader: %v", err)
}

0
cron/config/projects.csv → cron/data/projects.csv
View File