diff --git a/format/csv/csv.go b/format/csv/csv.go index 21cd4434..b1501592 100644 --- a/format/csv/csv.go +++ b/format/csv/csv.go @@ -23,7 +23,7 @@ func init() { interp.RegisterFormat(decode.Format{ Name: format.CSV, Description: "Comma separated values", - ProbeOrder: format.ProbeOrderText, + ProbeOrder: format.ProbeOrderTextFuzzy, DecodeFn: decodeCSV, DecodeInArg: format.CSVLIn{ Comma: ",", diff --git a/format/format.go b/format/format.go index 504aceb3..01812953 100644 --- a/format/format.go +++ b/format/format.go @@ -3,8 +3,9 @@ package format // TODO: do before-format somehow and topology sort? const ( ProbeOrderBinUnique = 0 // binary with unlikely overlap - ProbeOrderBinFuzzy = 50 // binary with possible overlap - ProbeOrderText = 100 // text format + ProbeOrderBinFuzzy = 100 // binary with possible overlap + ProbeOrderTextJSON = 200 // text json has prio as yaml overlap + ProbeOrderTextFuzzy = 300 // text with possible overlap ) // TODO: change to CamelCase? diff --git a/format/json/json.go b/format/json/json.go index 04ea9753..7fb43181 100644 --- a/format/json/json.go +++ b/format/json/json.go @@ -25,7 +25,7 @@ func init() { interp.RegisterFormat(decode.Format{ Name: format.JSON, Description: "JavaScript Object Notation", - ProbeOrder: format.ProbeOrderText, + ProbeOrder: format.ProbeOrderTextJSON, Groups: []string{format.PROBE}, DecodeFn: decodeJSON, Functions: []string{"_todisplay"}, diff --git a/format/json/testdata/json.fqtest b/format/json/testdata/json.fqtest index ff7ae9aa..cc0dee9d 100644 --- a/format/json/testdata/json.fqtest +++ b/format/json/testdata/json.fqtest @@ -8,6 +8,18 @@ $ fq . /probe.json } $ fq . /probe_scalar.json 123 +# make sure all value types parse to json format +$ fq -rRs 'fromjson[] | tojson | fromjson | format' variants.json +json +json +json +json +json +json +json +json +json +json $ fq -rRs 'fromjson[] | (tojson | ., fromjson), "----", (tojson({indent:2}) | ., fromjson), "----"' variants.json null null diff --git a/format/json/testdata/trailing.fqtest b/format/json/testdata/trailing.fqtest index 937f2e06..9adbbb41 100644 --- a/format/json/testdata/trailing.fqtest +++ b/format/json/testdata/trailing.fqtest @@ -2,3 +2,7 @@ $ fq -n '"123 trailing" | fromjson._error.error' exitcode: 5 stderr: error: error at position 0xc: trialing data after top-level value +$ fq -n '`{"a":123}{"b":444}` | fromjson' +exitcode: 5 +stderr: +error: error at position 0x12: trialing data after top-level value diff --git a/format/toml/toml.go b/format/toml/toml.go index bc3fba54..c8e6751d 100644 --- a/format/toml/toml.go +++ b/format/toml/toml.go @@ -20,7 +20,7 @@ func init() { interp.RegisterFormat(decode.Format{ Name: format.TOML, Description: "Tom's Obvious, Minimal Language", - ProbeOrder: format.ProbeOrderText, + ProbeOrder: format.ProbeOrderTextFuzzy, Groups: []string{format.PROBE}, DecodeFn: decodeTOML, Functions: []string{"_todisplay"}, diff --git a/format/xml/xml.go b/format/xml/xml.go index 1cde33b3..a62a3ce9 100644 --- a/format/xml/xml.go +++ b/format/xml/xml.go @@ -34,7 +34,7 @@ func init() { interp.RegisterFormat(decode.Format{ Name: format.XML, Description: "Extensible Markup Language", - ProbeOrder: format.ProbeOrderText, + ProbeOrder: format.ProbeOrderTextFuzzy, Groups: []string{format.PROBE}, DecodeFn: decodeXML, DecodeInArg: format.XMLIn{ diff --git a/format/yaml/testdata/trailing.fqtest b/format/yaml/testdata/trailing.fqtest index 3b137dda..a0ad25aa 100644 --- a/format/yaml/testdata/trailing.fqtest +++ b/format/yaml/testdata/trailing.fqtest @@ -2,3 +2,7 @@ $ fq -n '"- a\ntrailing" | fromyaml._error.error' exitcode: 5 stderr: error: error at position 0xc: yaml: line 2: could not find expected ':' +$ fq -n '`{"a":123}{"b":444}` | fromyaml' +exitcode: 5 +stderr: +error: error at position 0x12: trialing data after top-level value diff --git a/format/yaml/yaml.go b/format/yaml/yaml.go index b10fd498..df9ea60b 100644 --- a/format/yaml/yaml.go +++ b/format/yaml/yaml.go @@ -4,6 +4,8 @@ package yaml import ( "embed" + "errors" + "io" "github.com/wader/fq/format" "github.com/wader/fq/internal/gojqex" @@ -21,7 +23,7 @@ func init() { interp.RegisterFormat(decode.Format{ Name: format.YAML, Description: "YAML Ain't Markup Language", - ProbeOrder: format.ProbeOrderText, + ProbeOrder: format.ProbeOrderTextFuzzy, Groups: []string{format.PROBE}, DecodeFn: decodeYAML, Functions: []string{"_todisplay"}, @@ -34,9 +36,14 @@ func decodeYAML(d *decode.D, _ any) any { br := d.RawLen(d.Len()) var r any - if err := yaml.NewDecoder(bitio.NewIOReader(br)).Decode(&r); err != nil { + yd := yaml.NewDecoder(bitio.NewIOReader(br)) + if err := yd.Decode(&r); err != nil { d.Fatalf("%s", err) } + if err := yd.Decode(new(any)); !errors.Is(err, io.EOF) { + d.Fatalf("trialing data after top-level value") + } + var s scalar.S s.Actual = r