mirror of
https://github.com/wader/fq.git
synced 2024-08-17 07:50:29 +03:00
Merge pull request #649 from wader/tovalue-skip-gaps
interp: Add skip_gaps option for tovalue/-V
This commit is contained in:
commit
94834c0378
26
doc/usage.md
26
doc/usage.md
@ -796,16 +796,36 @@ fq has some general options in addition to decode and decoders specific options.
|
||||
|
||||
`<value>` is fuzzily parsed based on the type of the option. Ex: a string can be specified as `-o name=string` or `-o name="string"`.
|
||||
|
||||
### `bits_format`
|
||||
### `-o bits_format=<string>`
|
||||
|
||||
How to represent raw bits as JSON.
|
||||
How to represent raw binary as JSON.
|
||||
|
||||
- `-o bits_foramt=string` String with raw bytes (zero bit padded). The string is binary safe internally in fq but bytes not representable as UTF-8 will be lost if turn to JSON.
|
||||
- `-o bits_foramt=string` String with raw bytes (zero bit padded if size is not byte aligned). The string is binary safe internally in fq but bytes not representable as UTF-8 will be lost if turn to JSON.
|
||||
- `-o bits_format=md5` MD5 hex string (zero bit padded).
|
||||
- `-o bits_format=base64` Base64 string.
|
||||
- `-p bits_foramt=truncate` Truncated string.
|
||||
- `-o bits_format=snippet` Truncated Base64 string prefixed with bit length.
|
||||
|
||||
```sh
|
||||
$ fq -V -o bits_format=base64 . file`
|
||||
```
|
||||
In query
|
||||
```jq
|
||||
tovalue({bits_format: "md5"})
|
||||
```
|
||||
|
||||
### `-o skip_gaps=<boolean>`
|
||||
|
||||
Skip gaps fields (`gap0` etc) when using `tovalue` or `-V`. Note that this might affect array indexes if one more more gaps fields are skipped in an array.
|
||||
|
||||
```sh
|
||||
$ fq -V -o skip_gaps=true . file`
|
||||
```
|
||||
In query
|
||||
```jq
|
||||
tovalue({skip_gaps: true})
|
||||
```
|
||||
|
||||
## Color and unicode output
|
||||
|
||||
fq by default tries to use colors if possible, this can be disabled with `-M`. You can also
|
||||
|
@ -163,9 +163,12 @@ func (i *Interp) _registry(c any) any {
|
||||
}
|
||||
}
|
||||
|
||||
func (i *Interp) _toValue(c any, opts map[string]any) any {
|
||||
func (i *Interp) _toValue(c any, om map[string]any) any {
|
||||
return toValue(
|
||||
func() Options { return OptionsFromValue(opts) },
|
||||
func() *Options {
|
||||
opts := OptionsFromValue(om)
|
||||
return &opts
|
||||
},
|
||||
c,
|
||||
)
|
||||
}
|
||||
@ -307,7 +310,7 @@ func valueHas(key any, a func(name string) any, b func(key any) any) any {
|
||||
|
||||
// TODO: make more efficient somehow? shallow values but might be hard
|
||||
// when things like tovalue.key should behave like a jq value and not a decode value etc
|
||||
func toValue(optsFn func() Options, v any) any {
|
||||
func toValue(optsFn func() *Options, v any) any {
|
||||
nv, _ := gojqex.ToGoJQValueFn(v, func(v any) (any, bool) {
|
||||
switch v := v.(type) {
|
||||
case JQValueEx:
|
||||
@ -540,7 +543,7 @@ func (dvb decodeValueBase) JQValueKey(name string) any {
|
||||
case "_gap":
|
||||
switch vv := dv.V.(type) {
|
||||
case Scalarable:
|
||||
return vv.ScalarGap()
|
||||
return vv.ScalarIsGap()
|
||||
default:
|
||||
return false
|
||||
}
|
||||
@ -606,7 +609,7 @@ func (v decodeValue) JQValueKey(name string) any {
|
||||
func (v decodeValue) JQValueHas(key any) any {
|
||||
return valueHas(key, v.decodeValueBase.JQValueKey, v.JQValue.JQValueHas)
|
||||
}
|
||||
func (v decodeValue) JQValueToGoJQEx(optsFn func() Options) any {
|
||||
func (v decodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
|
||||
if !v.bitsFormat {
|
||||
return v.JQValueToGoJQ()
|
||||
}
|
||||
@ -695,13 +698,26 @@ func (v ArrayDecodeValue) JQValueHas(key any) any {
|
||||
return intKey >= 0 && intKey < len(v.Compound.Children)
|
||||
})
|
||||
}
|
||||
func (v ArrayDecodeValue) JQValueToGoJQ() any {
|
||||
vs := make([]any, len(v.Compound.Children))
|
||||
for i, f := range v.Compound.Children {
|
||||
vs[i] = makeDecodeValue(f, decodeValueValue)
|
||||
func (v ArrayDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
|
||||
opts := optsFn()
|
||||
|
||||
vs := make([]any, 0, len(v.Compound.Children))
|
||||
for _, f := range v.Compound.Children {
|
||||
switch s := f.V.(type) {
|
||||
case Scalarable:
|
||||
if s.ScalarIsGap() && opts.SkipGaps {
|
||||
// skip, note for arrays this will affect indexes
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
vs = append(vs, makeDecodeValue(f, decodeValueValue))
|
||||
}
|
||||
return vs
|
||||
}
|
||||
func (v ArrayDecodeValue) JQValueToGoJQ() any {
|
||||
return v.JQValueToGoJQEx(func() *Options { return &Options{} })
|
||||
}
|
||||
|
||||
// decode value struct
|
||||
|
||||
@ -767,10 +783,22 @@ func (v StructDecodeValue) JQValueHas(key any) any {
|
||||
},
|
||||
)
|
||||
}
|
||||
func (v StructDecodeValue) JQValueToGoJQ() any {
|
||||
func (v StructDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
|
||||
opts := optsFn()
|
||||
|
||||
vm := make(map[string]any, len(v.Compound.Children))
|
||||
for _, f := range v.Compound.Children {
|
||||
switch s := f.V.(type) {
|
||||
case Scalarable:
|
||||
if s.ScalarIsGap() && opts.SkipGaps {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
vm[f.Name] = makeDecodeValue(f, decodeValueValue)
|
||||
}
|
||||
return vm
|
||||
}
|
||||
func (v StructDecodeValue) JQValueToGoJQ() any {
|
||||
return v.JQValueToGoJQEx(func() *Options { return &Options{} })
|
||||
}
|
||||
|
@ -82,7 +82,7 @@ type Scalarable interface {
|
||||
ScalarValue() any
|
||||
ScalarSym() any
|
||||
ScalarDescription() string
|
||||
ScalarGap() bool
|
||||
ScalarIsGap() bool
|
||||
ScalarDisplayFormat() scalar.DisplayFormat
|
||||
}
|
||||
|
||||
@ -210,7 +210,7 @@ type Display interface {
|
||||
|
||||
type JQValueEx interface {
|
||||
gojq.JQValue
|
||||
JQValueToGoJQEx(optsFn func() Options) any
|
||||
JQValueToGoJQEx(optsFn func() *Options) any
|
||||
}
|
||||
|
||||
func valuePath(v *decode.Value) []any {
|
||||
@ -682,7 +682,7 @@ func (i *Interp) _printColorJSON(c any, v any) gojq.Iter {
|
||||
Color: opts.Color,
|
||||
Tab: false,
|
||||
Indent: indent,
|
||||
ValueFn: func(v any) any { return toValue(func() Options { return opts }, v) },
|
||||
ValueFn: func(v any) any { return toValue(func() *Options { return &opts }, v) },
|
||||
Colors: colorjson.Colors{
|
||||
Reset: []byte(ansi.Reset.SetString),
|
||||
Null: []byte(opts.Decorator.Null.SetString),
|
||||
@ -1034,6 +1034,7 @@ type Options struct {
|
||||
DisplayBytes int
|
||||
Addrbase int
|
||||
Sizebase int
|
||||
SkipGaps bool
|
||||
|
||||
Decorator Decorator
|
||||
BitsFormatFn func(br bitio.ReaderAtSeeker) (any, error)
|
||||
|
@ -59,6 +59,7 @@ def _opt_build_default_fixed:
|
||||
raw_output: ($stdout.is_terminal | not),
|
||||
raw_string: false,
|
||||
repl: false,
|
||||
skip_gaps: false,
|
||||
sizebase: 10,
|
||||
show_formats: false,
|
||||
show_help: false,
|
||||
@ -101,9 +102,10 @@ def _opt_options:
|
||||
raw_output: "boolean",
|
||||
raw_string: "boolean",
|
||||
repl: "boolean",
|
||||
sizebase: "number",
|
||||
show_formats: "boolean",
|
||||
show_help: "boolean",
|
||||
sizebase: "number",
|
||||
skip_gaps: "boolean",
|
||||
slurp: "boolean",
|
||||
string_input: "boolean",
|
||||
unicode: "boolean",
|
||||
|
1
pkg/interp/testdata/args.fqtest
vendored
1
pkg/interp/testdata/args.fqtest
vendored
@ -99,6 +99,7 @@ repl false
|
||||
show_formats false
|
||||
show_help options
|
||||
sizebase 10
|
||||
skip_gaps false
|
||||
slurp false
|
||||
string_input false
|
||||
unicode false
|
||||
|
1
pkg/interp/testdata/options.fqtest
vendored
1
pkg/interp/testdata/options.fqtest
vendored
@ -83,6 +83,7 @@ $ fq -n options
|
||||
"show_formats": false,
|
||||
"show_help": false,
|
||||
"sizebase": 10,
|
||||
"skip_gaps": false,
|
||||
"slurp": false,
|
||||
"string_input": false,
|
||||
"unicode": false,
|
||||
|
22
pkg/interp/testdata/tovalue.fqtest
vendored
22
pkg/interp/testdata/tovalue.fqtest
vendored
@ -2,9 +2,21 @@
|
||||
$ fq -i . test.mp3
|
||||
mp3> .headers[0] | tovalue.header.magic
|
||||
"ID3"
|
||||
mp3> [tobytes, "abc"] | mp3 | tovalue, tovalue({skip_gaps: true}) | keys
|
||||
[
|
||||
"footers",
|
||||
"frames",
|
||||
"gap0",
|
||||
"headers"
|
||||
]
|
||||
[
|
||||
"footers",
|
||||
"frames",
|
||||
"headers"
|
||||
]
|
||||
mp3> "abc" | mpeg_ts | tovalue, tovalue({skip_gaps: true}) | keys
|
||||
[
|
||||
"gap0"
|
||||
]
|
||||
[]
|
||||
mp3> ^D
|
||||
$ fq -i
|
||||
null> "aaa" | mp3_frame | .gap0 | tovalue, tovalue({sizebase: 2})
|
||||
"aaa"
|
||||
"aaa"
|
||||
null> ^D
|
||||
|
@ -27,7 +27,7 @@ func (s Any) ScalarValue() any {
|
||||
}
|
||||
func (s Any) ScalarSym() any { return s.Sym }
|
||||
func (s Any) ScalarDescription() string { return s.Description }
|
||||
func (s Any) ScalarGap() bool { return s.Gap }
|
||||
func (s Any) ScalarIsGap() bool { return s.Gap }
|
||||
func (s Any) ScalarDisplayFormat() DisplayFormat { return 0 }
|
||||
|
||||
func AnyActual(v any) AnyMapper {
|
||||
@ -224,7 +224,7 @@ func (s BigInt) ScalarValue() any {
|
||||
}
|
||||
func (s BigInt) ScalarSym() any { return s.Sym }
|
||||
func (s BigInt) ScalarDescription() string { return s.Description }
|
||||
func (s BigInt) ScalarGap() bool { return s.Gap }
|
||||
func (s BigInt) ScalarIsGap() bool { return s.Gap }
|
||||
func (s BigInt) ScalarDisplayFormat() DisplayFormat { return s.DisplayFormat }
|
||||
|
||||
func BigIntActual(v *big.Int) BigIntMapper {
|
||||
@ -420,7 +420,7 @@ func (s BitBuf) ScalarValue() any {
|
||||
}
|
||||
func (s BitBuf) ScalarSym() any { return s.Sym }
|
||||
func (s BitBuf) ScalarDescription() string { return s.Description }
|
||||
func (s BitBuf) ScalarGap() bool { return s.Gap }
|
||||
func (s BitBuf) ScalarIsGap() bool { return s.Gap }
|
||||
func (s BitBuf) ScalarDisplayFormat() DisplayFormat { return 0 }
|
||||
|
||||
func BitBufActual(v bitio.ReaderAtSeeker) BitBufMapper {
|
||||
@ -616,7 +616,7 @@ func (s Bool) ScalarValue() any {
|
||||
}
|
||||
func (s Bool) ScalarSym() any { return s.Sym }
|
||||
func (s Bool) ScalarDescription() string { return s.Description }
|
||||
func (s Bool) ScalarGap() bool { return s.Gap }
|
||||
func (s Bool) ScalarIsGap() bool { return s.Gap }
|
||||
func (s Bool) ScalarDisplayFormat() DisplayFormat { return 0 }
|
||||
|
||||
func BoolActual(v bool) BoolMapper {
|
||||
@ -812,7 +812,7 @@ func (s Flt) ScalarValue() any {
|
||||
}
|
||||
func (s Flt) ScalarSym() any { return s.Sym }
|
||||
func (s Flt) ScalarDescription() string { return s.Description }
|
||||
func (s Flt) ScalarGap() bool { return s.Gap }
|
||||
func (s Flt) ScalarIsGap() bool { return s.Gap }
|
||||
func (s Flt) ScalarDisplayFormat() DisplayFormat { return 0 }
|
||||
|
||||
func FltActual(v float64) FltMapper {
|
||||
@ -1009,7 +1009,7 @@ func (s Sint) ScalarValue() any {
|
||||
}
|
||||
func (s Sint) ScalarSym() any { return s.Sym }
|
||||
func (s Sint) ScalarDescription() string { return s.Description }
|
||||
func (s Sint) ScalarGap() bool { return s.Gap }
|
||||
func (s Sint) ScalarIsGap() bool { return s.Gap }
|
||||
func (s Sint) ScalarDisplayFormat() DisplayFormat { return s.DisplayFormat }
|
||||
|
||||
func SintActual(v int64) SintMapper {
|
||||
@ -1205,7 +1205,7 @@ func (s Str) ScalarValue() any {
|
||||
}
|
||||
func (s Str) ScalarSym() any { return s.Sym }
|
||||
func (s Str) ScalarDescription() string { return s.Description }
|
||||
func (s Str) ScalarGap() bool { return s.Gap }
|
||||
func (s Str) ScalarIsGap() bool { return s.Gap }
|
||||
func (s Str) ScalarDisplayFormat() DisplayFormat { return 0 }
|
||||
|
||||
func StrActual(v string) StrMapper {
|
||||
@ -1402,7 +1402,7 @@ func (s Uint) ScalarValue() any {
|
||||
}
|
||||
func (s Uint) ScalarSym() any { return s.Sym }
|
||||
func (s Uint) ScalarDescription() string { return s.Description }
|
||||
func (s Uint) ScalarGap() bool { return s.Gap }
|
||||
func (s Uint) ScalarIsGap() bool { return s.Gap }
|
||||
func (s Uint) ScalarDisplayFormat() DisplayFormat { return s.DisplayFormat }
|
||||
|
||||
func UintActual(v uint64) UintMapper {
|
||||
|
@ -31,7 +31,7 @@ import (
|
||||
}
|
||||
func (s {{$name}}) ScalarSym() any { return s.Sym }
|
||||
func (s {{$name}}) ScalarDescription() string { return s.Description }
|
||||
func (s {{$name}}) ScalarGap() bool { return s.Gap }
|
||||
func (s {{$name}}) ScalarIsGap() bool { return s.Gap }
|
||||
{{- if $t.display_format}}
|
||||
func (s {{$name}}) ScalarDisplayFormat() DisplayFormat { return s.DisplayFormat }
|
||||
{{- else }}
|
||||
|
Loading…
Reference in New Issue
Block a user