mirror of
https://github.com/ossf/scorecard.git
synced 2024-10-26 10:28:10 +03:00
Embed data files into the go lib files (#475)
Co-authored-by: Azeem Shaikh <azeems@google.com>
This commit is contained in:
parent
abdfd23770
commit
09c44bd355
@ -15,9 +15,9 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
_ "embed" // Used to embed config.yaml
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"reflect"
|
||||
"strconv"
|
||||
@ -32,12 +32,13 @@ const (
|
||||
requestSubscriptionURL string = "SCORECARD_REQUEST_SUBSCRIPTION_URL"
|
||||
inputReposFile string = "SCORECARD_REPOS_FILE"
|
||||
shardSize string = "SCORECARD_SHARD_SIZE"
|
||||
configYAML string = "config.yaml"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrorEmptyConfigValue = errors.New("config value set to empty")
|
||||
ErrorValueConversion = errors.New("unexpected type, cannot convert value")
|
||||
//go:embed config.yaml
|
||||
configYAML []byte
|
||||
)
|
||||
|
||||
type config struct {
|
||||
@ -48,39 +49,28 @@ type config struct {
|
||||
ShardSize int `yaml:"shard-size"`
|
||||
}
|
||||
|
||||
func getParsedConfigFromFile(filename string) (config, error) {
|
||||
yamlFile, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return config{}, fmt.Errorf("error during os.Open: %w", err)
|
||||
}
|
||||
defer yamlFile.Close()
|
||||
|
||||
byteValue, err := ioutil.ReadAll(yamlFile)
|
||||
if err != nil {
|
||||
return config{}, fmt.Errorf("error during ioutil.ReadAll: %w", err)
|
||||
}
|
||||
|
||||
func getParsedConfigFromFile(byteValue []byte) (config, error) {
|
||||
var ret config
|
||||
err = yaml.Unmarshal(byteValue, &ret)
|
||||
err := yaml.Unmarshal(byteValue, &ret)
|
||||
if err != nil {
|
||||
return config{}, fmt.Errorf("error during yaml.Unmarshal: %w", err)
|
||||
}
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func getConfigValue(envVar, filename, fieldName string) (reflect.Value, error) {
|
||||
func getConfigValue(envVar string, byteValue []byte, fieldName string) (reflect.Value, error) {
|
||||
if val, present := os.LookupEnv(envVar); present {
|
||||
return reflect.ValueOf(val), nil
|
||||
}
|
||||
parsedConfig, err := getParsedConfigFromFile(filename)
|
||||
parsedConfig, err := getParsedConfigFromFile(byteValue)
|
||||
if err != nil {
|
||||
return reflect.ValueOf(parsedConfig), fmt.Errorf("error parsing config file: %w", err)
|
||||
}
|
||||
return reflect.ValueOf(parsedConfig).FieldByName(fieldName), nil
|
||||
}
|
||||
|
||||
func getStringConfigValue(envVar, filename, fieldName, configName string) (string, error) {
|
||||
value, err := getConfigValue(envVar, filename, fieldName)
|
||||
func getStringConfigValue(envVar string, byteValue []byte, fieldName, configName string) (string, error) {
|
||||
value, err := getConfigValue(envVar, byteValue, fieldName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error getting config value %s: %w", configName, err)
|
||||
}
|
||||
@ -93,8 +83,8 @@ func getStringConfigValue(envVar, filename, fieldName, configName string) (strin
|
||||
return value.String(), fmt.Errorf("%w: %s", ErrorEmptyConfigValue, configName)
|
||||
}
|
||||
|
||||
func getIntConfigValue(envVar, filename, fieldName, configName string) (int, error) {
|
||||
value, err := getConfigValue(envVar, filename, fieldName)
|
||||
func getIntConfigValue(envVar string, byteValue []byte, fieldName, configName string) (int, error) {
|
||||
value, err := getConfigValue(envVar, byteValue, fieldName)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("error getting config value %s: %w", configName, err)
|
||||
}
|
||||
|
@ -16,6 +16,7 @@ package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
@ -29,6 +30,14 @@ const (
|
||||
prodShardSize int = 250
|
||||
)
|
||||
|
||||
func getByteValueFromFile(filename string) ([]byte, error) {
|
||||
if filename == "" {
|
||||
return nil, nil
|
||||
}
|
||||
//nolint
|
||||
return ioutil.ReadFile(filename)
|
||||
}
|
||||
|
||||
func TestYAMLParsing(t *testing.T) {
|
||||
t.Parallel()
|
||||
testcases := []struct {
|
||||
@ -72,7 +81,11 @@ func TestYAMLParsing(t *testing.T) {
|
||||
testcase := testcase
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
parsedConfig, err := getParsedConfigFromFile(testcase.filename)
|
||||
byteValue, err := getByteValueFromFile(testcase.filename)
|
||||
if err != nil {
|
||||
t.Errorf("test failed to parse input file: %v", err)
|
||||
}
|
||||
parsedConfig, err := getParsedConfigFromFile(byteValue)
|
||||
if err != nil {
|
||||
t.Errorf("failed to parse test file: %v", err)
|
||||
}
|
||||
@ -124,7 +137,11 @@ func TestGetStringConfigValue(t *testing.T) {
|
||||
os.Setenv(testEnvVar, testcase.envVal)
|
||||
}
|
||||
|
||||
actual, err := getStringConfigValue(testEnvVar, testcase.filename, testcase.fieldName, "test-config" /*configName*/)
|
||||
byteValue, err := getByteValueFromFile(testcase.filename)
|
||||
if err != nil {
|
||||
t.Errorf("test failed during input parsing: %v", err)
|
||||
}
|
||||
actual, err := getStringConfigValue(testEnvVar, byteValue, testcase.fieldName, "test-config" /*configName*/)
|
||||
if testcase.hasError {
|
||||
if err == nil || !errors.Is(err, testcase.expectedErr) {
|
||||
t.Errorf("test failed: expectedErr - %v, got - %v", testcase.expectedErr, err)
|
||||
@ -175,7 +192,11 @@ func TestGetIntConfigValue(t *testing.T) {
|
||||
os.Setenv(testEnvVar, testcase.envVal)
|
||||
}
|
||||
|
||||
actual, err := getIntConfigValue(testEnvVar, testcase.filename, testcase.fieldName, "test-config" /*configName*/)
|
||||
byteValue, err := getByteValueFromFile(testcase.filename)
|
||||
if err != nil {
|
||||
t.Errorf("test failed during input parsing: %v", err)
|
||||
}
|
||||
actual, err := getIntConfigValue(testEnvVar, byteValue, testcase.fieldName, "test-config" /*configName*/)
|
||||
if testcase.hasError {
|
||||
if err == nil || !errors.Is(err, testcase.expectedErr) {
|
||||
t.Errorf("test failed: expectedErr - %v, got - %v", testcase.expectedErr, err)
|
||||
|
@ -27,6 +27,6 @@ ARG TARGETARCH
|
||||
RUN CGO_ENABLED=0 make build-pubsub
|
||||
|
||||
FROM gcr.io/distroless/base:nonroot@sha256:bc84925113289d139a9ef2f309f0dd7ac46ea7b786f172ba9084ffdb4cbd9490
|
||||
COPY ./cron/config/projects.csv /cron/config/projects.csv
|
||||
COPY ./cron/data/projects.csv /cron/data/projects.csv
|
||||
COPY --from=pubsub /src/cron/controller/controller ./cron/controller/controller
|
||||
ENTRYPOINT ["/cron/controller/controller"]
|
||||
|
@ -17,7 +17,6 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
@ -82,15 +81,7 @@ func PublishToRepoRequestTopic(ctx context.Context, iter data.Iterator, datetime
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
t := time.Now()
|
||||
inputReposFile, err := config.GetInputReposFile()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
reposFile, err := os.OpenFile(inputReposFile, os.O_RDONLY, 0o644)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
reader, err := data.MakeIterator(reposFile)
|
||||
reader, err := data.MakeIterator()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
@ -15,6 +15,7 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"encoding/csv"
|
||||
"errors"
|
||||
"fmt"
|
||||
@ -24,15 +25,26 @@ import (
|
||||
"github.com/ossf/scorecard/repos"
|
||||
)
|
||||
|
||||
//go:embed projects.csv
|
||||
var csvFile embed.FS
|
||||
|
||||
type Iterator interface {
|
||||
HasNext() bool
|
||||
Next() (repos.RepoURL, error)
|
||||
}
|
||||
|
||||
func MakeIterator(reader io.Reader) (Iterator, error) {
|
||||
func MakeIterator() (Iterator, error) {
|
||||
csvReader, err := csvFile.Open("projects.csv")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error in Open: %w", err)
|
||||
}
|
||||
return MakeIteratorFrom(csvReader)
|
||||
}
|
||||
|
||||
func MakeIteratorFrom(reader io.Reader) (Iterator, error) {
|
||||
dec, err := csvutil.NewDecoder(csv.NewReader(reader))
|
||||
if err != nil {
|
||||
return &csvIterator{}, fmt.Errorf("error in csvutil.NewDecoder: %w", err)
|
||||
return nil, fmt.Errorf("error in csvutil.NewDecoder: %w", err)
|
||||
}
|
||||
return &csvIterator{decoder: dec}, nil
|
||||
}
|
||||
|
@ -151,7 +151,7 @@ func TestCsvIterator(t *testing.T) {
|
||||
}
|
||||
defer testFile.Close()
|
||||
|
||||
testReader, err := MakeIterator(testFile)
|
||||
testReader, err := MakeIteratorFrom(testFile)
|
||||
if err != nil {
|
||||
t.Errorf("failed to create reader: %v", err)
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user