wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-12-25 06:12:30 +03:00
Code Issues Projects Releases Wiki Activity

decode,interp: Don't shadow _key and error on missing _key

Browse Source 
For fromjson and other "value" decode values fq make then behave both like
a normal jq value and decode value. This is to make tobytes, format etc work.
Before all _* would be treated as special keys. Now they are first looked up in
the wrapped value and then as decode values.

Also now ._key that don't exist reutrn null instead of throw error.

$ fq -n '`{"_format": 123}` | fromjson | ._format'
Now:
123
Before:
"json"

$ fq -n '`{}` | fromjson | ._missing'
Now:
null
Before
error
This commit is contained in:
Mattias Wadman 2023-09-02 20:07:52 +02:00
parent d97876caa6
commit c0352f2fd1
10 changed files with 159 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
pkg/interp/decode.go
View File

@ -7,7 +7,6 @@ import (
"io"
"math/big"
"reflect"
"strings"
"time"
"github.com/mitchellh/copystructure"
@ -27,14 +26,6 @@ func init() {
RegisterFunc2("_decode", (*Interp)._decode)
}
type expectedExtkeyError struct {
Key string
}
func (err expectedExtkeyError) Error() string {
return "expected a extkey but got: " + err.Key
}
// TODO: redo/rename
// used by _isDecodeValue
type DecodeValue interface {
@ -294,21 +285,19 @@ func (i *Interp) _decode(c any, format string, opts decodeOpts) any {
return makeDecodeValueOut(dv, decodeValueValue, formatOutMap)
}
func valueKey(name string, a, b func(name string) any) any {
if strings.HasPrefix(name, "_") {
return a(name)
func valueOrFallbackKey(name string, baseKey func(name string) any, valueHas func(key any) any, valueKey func(name string) any) any {
v := valueHas(name)
if b, ok := v.(bool); ok && b {
return valueKey(name)
}
return b(name)
return baseKey(name)
}
func valueHas(key any, a func(name string) any, b func(key any) any) any {
stringKey, ok := key.(string)
if ok && strings.HasPrefix(stringKey, "_") {
if err, ok := a(stringKey).(error); ok {
return err
func valueOrFallbackHas(key any, baseHas func(key any) any, valueHas func(key any) any) any {
v := valueHas(key)
if b, ok := v.(bool); ok && !b {
return baseHas(key)
}
return true
}
return b(key)
return v
}
// TODO: make more efficient somehow? shallow values but might be hard
@ -461,16 +450,20 @@ func (dvb decodeValueBase) ToBinary() (Binary, error) {
}
func (decodeValueBase) ExtType() string { return "decode_value" }
func (dvb decodeValueBase) ExtKeys() []string {
kv := []string{
return []string{
"_actual",
"_bits",
"_buffer_root",
"_bytes",
"_description",
"_error",
"_format_root",
"_format",
"_gap",
"_index",
"_len",
"_name",
"_out",
"_parent",
"_path",
"_root",
@ -478,20 +471,38 @@ func (dvb decodeValueBase) ExtKeys() []string {
"_stop",
"_sym",
}
}
if _, ok := dvb.dv.V.(*decode.Compound); ok {
kv = append(kv,
func (dvb decodeValueBase) JQValueHas(key any) any {
name, ok := key.(string)
if !ok {
return false
}
switch name {
case "_actual",
"_bits",
"_buffer_root",
"_bytes",
"_description",
"_error",
"_format_root",
"_format",
"_gap",
"_index",
"_len",
"_name",
"_out",
)
if dvb.dv.Index != -1 {
kv = append(kv, "_index")
}
"_parent",
"_path",
"_root",
"_start",
"_stop",
"_sym":
return true
}
return kv
return false
}
func (dvb decodeValueBase) JQValueKey(name string) any {
@ -591,7 +602,7 @@ func (dvb decodeValueBase) JQValueKey(name string) any {
}
}
return expectedExtkeyError{Key: name}
return nil
}
var _ DecodeValue = decodeValue{}
@ -603,10 +614,10 @@ type decodeValue struct {
}
func (v decodeValue) JQValueKey(name string) any {
return valueKey(name, v.decodeValueBase.JQValueKey, v.JQValue.JQValueKey)
return valueOrFallbackKey(name, v.decodeValueBase.JQValueKey, v.JQValue.JQValueHas, v.JQValue.JQValueKey)
}
func (v decodeValue) JQValueHas(key any) any {
return valueHas(key, v.decodeValueBase.JQValueKey, v.JQValue.JQValueHas)
return valueOrFallbackHas(key, v.decodeValueBase.JQValueHas, v.JQValue.JQValueHas)
}
func (v decodeValue) JQValueToGoJQEx(optsFn func() (*Options, error)) any {
if !v.isRaw {
@ -641,7 +652,7 @@ func NewArrayDecodeValue(dv *decode.Value, out any, c *decode.Compound) ArrayDec
}
func (v ArrayDecodeValue) JQValueKey(name string) any {
return valueKey(name, v.decodeValueBase.JQValueKey, v.Base.JQValueKey)
return valueOrFallbackKey(name, v.decodeValueBase.JQValueKey, v.Base.JQValueHas, v.Base.JQValueKey)
}
func (v ArrayDecodeValue) JQValueSliceLen() any { return len(v.Compound.Children) }
func (v ArrayDecodeValue) JQValueLength() any { return len(v.Compound.Children) }
@ -674,9 +685,9 @@ func (v ArrayDecodeValue) JQValueKeys() any {
return vs
}
func (v ArrayDecodeValue) JQValueHas(key any) any {
return valueHas(
return valueOrFallbackHas(
key,
v.decodeValueBase.JQValueKey,
v.decodeValueBase.JQValueHas,
func(key any) any {
intKey, ok := key.(int)
if !ok {
@ -730,9 +741,22 @@ func NewStructDecodeValue(dv *decode.Value, out any, c *decode.Compound) StructD
func (v StructDecodeValue) JQValueLength() any { return len(v.Compound.Children) }
func (v StructDecodeValue) JQValueSliceLen() any { return len(v.Compound.Children) }
func (v StructDecodeValue) JQValueKey(name string) any {
if strings.HasPrefix(name, "_") {
return v.decodeValueBase.JQValueKey(name)
return valueOrFallbackKey(
name,
v.decodeValueBase.JQValueKey,
func(key any) any {
stringKey, ok := key.(string)
if !ok {
return false
}
if v.Compound.ByName != nil {
if _, ok := v.Compound.ByName[stringKey]; ok {
return true
}
}
return false
},
func(name string) any {
if v.Compound.ByName != nil {
if f, ok := v.Compound.ByName[name]; ok {
return makeDecodeValue(f, decodeValueValue)
@ -740,6 +764,8 @@ func (v StructDecodeValue) JQValueKey(name string) any {
}
return nil
},
)
}
func (v StructDecodeValue) JQValueEach() any {
props := make([]gojq.PathValue, len(v.Compound.Children))
@ -756,19 +782,21 @@ func (v StructDecodeValue) JQValueKeys() any {
return vs
}
func (v StructDecodeValue) JQValueHas(key any) any {
return valueHas(
return valueOrFallbackHas(
key,
v.decodeValueBase.JQValueKey,
v.decodeValueBase.JQValueHas,
func(key any) any {
stringKey, ok := key.(string)
if !ok {
return gojqex.HasKeyTypeError{L: gojq.JQTypeObject, R: fmt.Sprintf("%v", key)}
}
for _, f := range v.Compound.Children {
if f.Name == stringKey {
if v.Compound.ByName != nil {
if _, ok := v.Compound.ByName[stringKey]; ok {
return true
}
}
return false
},
)

1
pkg/interp/testdata/completion.fqtest vendored
View File

@ -46,6 +46,7 @@ _error
_format
_format_root
_gap
_index
_len
_name
_out

76
pkg/interp/testdata/value.fqtest vendored
View File

@ -127,77 +127,77 @@ true
"_format"
true
"_abc"
"expected a extkey but got: _abc"
false
mp3> .headers as $c | ("_start", "_stop", "_len", "_name", "_root", "_buffer_root", "_format_root", "_parent", "_sym", "_description", "_path", "_bits", "_bytes", "_error", "_gap", "_format", "_abc") as $n | $n, try ($c | has($n)) catch .
"_start"
true
"cannot check whether array has a key: _start"
"_stop"
true
"cannot check whether array has a key: _stop"
"_len"
true
"cannot check whether array has a key: _len"
"_name"
true
"cannot check whether array has a key: _name"
"_root"
true
"cannot check whether array has a key: _root"
"_buffer_root"
true
"cannot check whether array has a key: _buffer_root"
"_format_root"
true
"cannot check whether array has a key: _format_root"
"_parent"
true
"cannot check whether array has a key: _parent"
"_sym"
true
"cannot check whether array has a key: _sym"
"_description"
true
"cannot check whether array has a key: _description"
"_path"
true
"cannot check whether array has a key: _path"
"_bits"
true
"cannot check whether array has a key: _bits"
"_bytes"
true
"cannot check whether array has a key: _bytes"
"_error"
true
"cannot check whether array has a key: _error"
"_gap"
true
"cannot check whether array has a key: _gap"
"_format"
true
"cannot check whether array has a key: _format"
"_abc"
"expected a extkey but got: _abc"
"cannot check whether array has a key: _abc"
mp3> .headers[0].header.magic as $c | ("_start", "_stop", "_len", "_name", "_root", "_buffer_root", "_format_root", "_parent", "_sym", "_description", "_path", "_bits", "_bytes", "_error", "_gap", "_format", "_abc") as $n | $n, try ($c | has($n)) catch .
"_start"
true
"has cannot be applied to: string"
"_stop"
true
"has cannot be applied to: string"
"_len"
true
"has cannot be applied to: string"
"_name"
true
"has cannot be applied to: string"
"_root"
true
"has cannot be applied to: string"
"_buffer_root"
true
"has cannot be applied to: string"
"_format_root"
true
"has cannot be applied to: string"
"_parent"
true
"has cannot be applied to: string"
"_sym"
true
"has cannot be applied to: string"
"_description"
true
"has cannot be applied to: string"
"_path"
true
"has cannot be applied to: string"
"_bits"
true
"has cannot be applied to: string"
"_bytes"
true
"has cannot be applied to: string"
"_error"
true
"has cannot be applied to: string"
"_gap"
true
"has cannot be applied to: string"
"_format"
true
"has cannot be applied to: string"
"_abc"
"expected a extkey but got: _abc"
"has cannot be applied to: string"
mp3> ^D
$ fq -d mp3 -i . test.mp3
mp3> ._start
@ -255,13 +255,13 @@ false
mp3> ._format
"mp3"
mp3> ._abc
error: expected a extkey but got: _abc
null
mp3> fgrep("toc") | arrays[10]._index
10
mp3> .headers._index
error: expected a extkey but got: _index
null
mp3> ._index
error: expected a extkey but got: _index
null
mp3> .headers[0].header.version as $r | {a:12} | tojson({indent: $r}) | println
{
"a": 12

6
pkg/interp/testdata/value_array.fqtest vendored
View File

@ -113,7 +113,9 @@ mp3> .headers[-1000:2000] | ., type, length?
"array"
1
mp3> .headers["test"] | ., type, length?
error: expected an object with key "test" but got: array
null
"null"
0
mp3> [.headers[]] | type, length?
"array"
1
@ -522,7 +524,7 @@ mp3> .headers[0] = 1
]
}
mp3> .headers.a |= empty
error: expected an object with key "a" but got: array
error: delpaths([["headers","a"]]) cannot be applied to {"footers":[],"frames":[{" ...: expected an object but got: array ([{"frames":[{"flags":{"co ...])
mp3> .headers[0] |= empty
{
"footers": [],

6
pkg/interp/testdata/value_boolean.fqtest vendored
View File

@ -19,7 +19,9 @@ error: expected an array but got: boolean
mp3> .headers[0].header.flags.unsynchronisation[-1000:2000] | ., type, length?
error: expected an array but got: boolean
mp3> .headers[0].header.flags.unsynchronisation["test"] | ., type, length?
error: expected an object but got: boolean
null
"null"
0
mp3> [.headers[0].header.flags.unsynchronisation[]] | type, length?
error: cannot iterate over: boolean
mp3> .headers[0].header.flags.unsynchronisation | keys
@ -99,7 +101,7 @@ error: setpath(["headers",0,"header","fl ...]; 1) cannot be applied to {"footers
mp3> .headers[0].header.flags.unsynchronisation[0] = 1
error: setpath(["headers",0,"header","fl ...]; 1) cannot be applied to {"footers":[],"frames":[{" ...: expected an array but got: boolean (false)
mp3> .headers[0].header.flags.unsynchronisation.a |= empty
error: expected an object but got: boolean
error: delpaths([["headers",0,"header","f ...]) cannot be applied to {"footers":[],"frames":[{" ...: expected an object but got: boolean (false)
mp3> .headers[0].header.flags.unsynchronisation[0] |= empty
error: expected an array but got: boolean
mp3> .headers[0].header.flags.unsynchronisation | setpath(["a"]; 1)

8
pkg/interp/testdata/value_json_array.fqtest vendored
View File

@ -31,7 +31,9 @@ json> (.)[-1000:2000] | ., type, length?
"array"
0
json> (.)["test"] | ., type, length?
error: expected an object but got: array
null
"null"
0
json> [(.)[]] | type, length?
"array"
0
@ -101,13 +103,13 @@ json> (.)._gap | ., type, length?
false
"boolean"
json> (.).a = 1
error: expected an object but got: array
error: setpath(["a"]; 1) cannot be applied to []: expected an object but got: array ([])
json> (.)[0] = 1
[
1
]
json> (.).a |= empty
error: expected an object but got: array
error: delpaths([["a"]]) cannot be applied to []: expected an object but got: array ([])
json> (.)[0] |= empty
[]
json> (.) | setpath(["a"]; 1)

6
pkg/interp/testdata/value_null.fqtest vendored
View File

@ -32,7 +32,9 @@ mp3> .headers[0].padding[-1000:2000] | ., type, length?
"string"
10
mp3> .headers[0].padding["test"] | ., type, length?
error: expected an object with key "test" but got: string
null
"null"
0
mp3> [.headers[0].padding[]] | type, length?
error: cannot iterate over: string
mp3> .headers[0].padding | keys
@ -111,7 +113,7 @@ error: setpath(["headers",0,"padding","a"]; 1) cannot be applied to {"footers":[
mp3> .headers[0].padding[0] = 1
error: setpath(["headers",0,"padding",0]; 1) cannot be applied to {"footers":[],"frames":[{" ...: expected an array but got: string ("\u0000\u0000\u0000\u0000 ...")
mp3> .headers[0].padding.a |= empty
error: expected an object with key "a" but got: string
error: delpaths([["headers",0,"padding","a"]]) cannot be applied to {"footers":[],"frames":[{" ...: expected an object but got: string ("\u0000\u0000\u0000\u0000 ...")
mp3> .headers[0].padding[0] |= empty
error: delpaths([["headers",0,"padding",0]]) cannot be applied to {"footers":[],"frames":[{" ...: expected an array but got: string ("\u0000\u0000\u0000\u0000 ...")
mp3> .headers[0].padding | setpath(["a"]; 1)

6
pkg/interp/testdata/value_number.fqtest vendored
View File

@ -20,7 +20,9 @@ error: expected an array but got: number
mp3> .headers[0].header.version[-1000:2000] | ., type, length?
error: expected an array but got: number
mp3> .headers[0].header.version["test"] | ., type, length?
error: expected an object but got: number
null
"null"
0
mp3> [.headers[0].header.version[]] | type, length?
error: cannot iterate over: number
mp3> .headers[0].header.version | keys
@ -100,7 +102,7 @@ error: setpath(["headers",0,"header","ve ...]; 1) cannot be applied to {"footers
mp3> .headers[0].header.version[0] = 1
error: setpath(["headers",0,"header","ve ...]; 1) cannot be applied to {"footers":[],"frames":[{" ...: expected an array but got: number (4)
mp3> .headers[0].header.version.a |= empty
error: expected an object but got: number
error: delpaths([["headers",0,"header","v ...]) cannot be applied to {"footers":[],"frames":[{" ...: expected an object but got: number (4)
mp3> .headers[0].header.version[0] |= empty
error: expected an array but got: number
mp3> .headers[0].header.version | setpath(["a"]; 1)

15
pkg/interp/testdata/value_shadow.fqtest vendored Normal file
View File

@ -0,0 +1,15 @@
$ fq -i
null> (`{"_format": "format", "_a": "a"}`, `{"_a":"a"}`) | fromjson | ._format, ._a, ._b, has("_format", "_a", "_b")
"format"
"a"
null
true
true
false
"json"
"a"
null
true
true
false
null> ^D

6
pkg/interp/testdata/value_string.fqtest vendored
View File

@ -32,7 +32,9 @@ mp3> .headers[0].header.magic[-1000:2000] | ., type, length?
"string"
3
mp3> .headers[0].header.magic["test"] | ., type, length?
error: expected an object but got: string
null
"null"
0
mp3> [.headers[0].header.magic[]] | type, length?
error: cannot iterate over: string
mp3> .headers[0].header.magic | keys
@ -112,7 +114,7 @@ error: setpath(["headers",0,"header","ma ...]; 1) cannot be applied to {"footers
mp3> .headers[0].header.magic[0] = 1
error: setpath(["headers",0,"header","ma ...]; 1) cannot be applied to {"footers":[],"frames":[{" ...: expected an array but got: string ("ID3")
mp3> .headers[0].header.magic.a |= empty
error: expected an object but got: string
error: delpaths([["headers",0,"header","m ...]) cannot be applied to {"footers":[],"frames":[{" ...: expected an object but got: string ("ID3")
mp3> .headers[0].header.magic[0] |= empty
error: delpaths([["headers",0,"header","m ...]) cannot be applied to {"footers":[],"frames":[{" ...: expected an array but got: string ("ID3")
mp3> .headers[0].header.magic | setpath(["a"]; 1)