mirror of
https://github.com/ossf/scorecard.git
synced 2024-09-11 08:55:27 +03:00
🌱 cron: make CSV header optional (3/n) (#2261)
* Make CSV header optional. Signed-off-by: Spencer Schrock <sschrock@google.com> * Appease linter. Signed-off-by: Spencer Schrock <sschrock@google.com> * Address PR feedback. Signed-off-by: Spencer Schrock <sschrock@google.com> Signed-off-by: Spencer Schrock <sschrock@google.com>
This commit is contained in:
parent
bde0ae166a
commit
2231d1f722
@ -19,6 +19,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
|
||||
"github.com/jszwec/csvutil"
|
||||
|
||||
@ -36,7 +37,11 @@ type Iterator interface {
|
||||
func MakeIteratorFrom(reader io.Reader) (Iterator, error) {
|
||||
csvReader := csv.NewReader(reader)
|
||||
csvReader.Comment = '#'
|
||||
dec, err := csvutil.NewDecoder(csvReader)
|
||||
header, err := csvutil.Header(RepoFormat{}, "csv")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error in csvutil.Header: %w", err)
|
||||
}
|
||||
dec, err := csvutil.NewDecoder(csvReader, header...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error in csvutil.NewDecoder: %w", err)
|
||||
}
|
||||
@ -44,13 +49,33 @@ func MakeIteratorFrom(reader io.Reader) (Iterator, error) {
|
||||
}
|
||||
|
||||
type csvIterator struct {
|
||||
decoder *csvutil.Decoder
|
||||
err error
|
||||
next RepoFormat
|
||||
decoder *csvutil.Decoder
|
||||
err error
|
||||
next RepoFormat
|
||||
afterHeader bool
|
||||
}
|
||||
|
||||
// returns true on the first call if the most recently decoded record is a header.
|
||||
// always returns false on subsequent calls, as this is only intended to evaluate the first line.
|
||||
func (reader *csvIterator) isHeader() bool {
|
||||
if reader.afterHeader {
|
||||
return false
|
||||
}
|
||||
header, err := csvutil.Header(RepoFormat{}, "csv")
|
||||
if err != nil {
|
||||
reader.err = err
|
||||
return false
|
||||
}
|
||||
lastRead := reader.decoder.Record()
|
||||
reader.afterHeader = true
|
||||
return reflect.DeepEqual(header, lastRead)
|
||||
}
|
||||
|
||||
func (reader *csvIterator) HasNext() bool {
|
||||
reader.err = reader.decoder.Decode(&reader.next)
|
||||
if reader.isHeader() {
|
||||
reader.err = reader.decoder.Decode(&reader.next)
|
||||
}
|
||||
return !errors.Is(reader.err, io.EOF)
|
||||
}
|
||||
|
||||
|
@ -153,6 +153,42 @@ func TestCsvIterator(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Ignore First Header Row",
|
||||
filename: "testdata/ignore_header.csv",
|
||||
outcomes: []outcome{
|
||||
{
|
||||
hasError: false,
|
||||
repo: RepoFormat{
|
||||
Repo: "github.com/owner1/repo1",
|
||||
},
|
||||
},
|
||||
{
|
||||
// will error due to GitHub URL sanity check
|
||||
hasError: true,
|
||||
repo: RepoFormat{
|
||||
Repo: "repo",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "No Header Row",
|
||||
filename: "testdata/no_header.csv",
|
||||
outcomes: []outcome{
|
||||
{
|
||||
hasError: false,
|
||||
repo: RepoFormat{
|
||||
Repo: "github.com/owner1/repo1",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Only Header Row",
|
||||
filename: "testdata/only_header.csv",
|
||||
outcomes: []outcome{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, testcase := range testcases {
|
||||
|
3
cron/internal/data/testdata/ignore_header.csv
vendored
Normal file
3
cron/internal/data/testdata/ignore_header.csv
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
repo,metadata
|
||||
github.com/owner1/repo1,
|
||||
repo,metadata
|
|
1
cron/internal/data/testdata/no_header.csv
vendored
Normal file
1
cron/internal/data/testdata/no_header.csv
vendored
Normal file
@ -0,0 +1 @@
|
||||
github.com/owner1/repo1,
|
|
1
cron/internal/data/testdata/only_header.csv
vendored
Normal file
1
cron/internal/data/testdata/only_header.csv
vendored
Normal file
@ -0,0 +1 @@
|
||||
repo,metadata
|
|
Loading…
Reference in New Issue
Block a user