mirror of
https://github.com/wader/fq.git
synced 2024-08-16 15:30:36 +03:00
Merge pull request #673 from wader/bit-format-hex
interp: Add hex bits format
This commit is contained in:
commit
93423d0da6
@ -802,6 +802,7 @@ How to represent raw binary as JSON.
|
||||
|
||||
- `-o bits_format=string` String with raw bytes (zero bit padded if size is not byte aligned). The string is binary safe internally in fq but bytes not representable as UTF-8 will be lost if turn to JSON.
|
||||
- `-o bits_format=md5` MD5 hex string (zero bit padded).
|
||||
- `-o bits_format=hex` Hex string.
|
||||
- `-o bits_format=base64` Base64 string.
|
||||
- `-p bits_format=truncate` Truncated string.
|
||||
- `-o bits_format=snippet` Truncated Base64 string prefixed with bit length.
|
||||
|
@ -94,12 +94,12 @@ func makeEncoder(opts ToJSONOpts) *colorjson.Encoder {
|
||||
Color: false,
|
||||
Tab: false,
|
||||
Indent: opts.Indent,
|
||||
ValueFn: func(v any) any {
|
||||
ValueFn: func(v any) (any, error) {
|
||||
switch v := v.(type) {
|
||||
case gojq.JQValue:
|
||||
return v.JQValueToGoJQ()
|
||||
return v.JQValueToGoJQ(), nil
|
||||
default:
|
||||
return v
|
||||
return v, nil
|
||||
}
|
||||
},
|
||||
Colors: colorjson.Colors{},
|
||||
|
@ -50,7 +50,7 @@ type Options struct {
|
||||
Color bool
|
||||
Tab bool
|
||||
Indent int
|
||||
ValueFn func(v any) any
|
||||
ValueFn func(v any) (any, error)
|
||||
Colors Colors
|
||||
}
|
||||
|
||||
@ -120,7 +120,11 @@ func (e *Encoder) encode(v any) error {
|
||||
if e.opts.ValueFn == nil {
|
||||
panic(fmt.Sprintf("unknown type and to ValueFn set: %[1]T (%[1]v)", v))
|
||||
}
|
||||
return e.encode(e.opts.ValueFn(v))
|
||||
vv, err := e.opts.ValueFn(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return e.encode(vv)
|
||||
}
|
||||
if e.w.Len() > 8*1024 {
|
||||
return e.flush()
|
||||
|
@ -6,6 +6,7 @@
|
||||
package gojqex
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"math/big"
|
||||
|
||||
@ -23,75 +24,77 @@ func IsNull(x any) bool {
|
||||
}
|
||||
}
|
||||
|
||||
func ToGoJQValue(v any) (any, bool) {
|
||||
return ToGoJQValueFn(v, func(v any) (any, bool) {
|
||||
func ToGoJQValue(v any) (any, error) {
|
||||
return ToGoJQValueFn(v, func(v any) (any, error) {
|
||||
switch v := v.(type) {
|
||||
case gojq.JQValue:
|
||||
return v.JQValueToGoJQ(), true
|
||||
return v.JQValueToGoJQ(), nil
|
||||
default:
|
||||
return nil, false
|
||||
return nil, fmt.Errorf("not a JQValue")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func ToGoJQValueFn(v any, valueFn func(v any) (any, bool)) (any, bool) {
|
||||
func ToGoJQValueFn(v any, valueFn func(v any) (any, error)) (any, error) {
|
||||
switch vv := v.(type) {
|
||||
case nil:
|
||||
return vv, true
|
||||
return vv, nil
|
||||
case bool:
|
||||
return vv, true
|
||||
return vv, nil
|
||||
case int:
|
||||
return vv, true
|
||||
return vv, nil
|
||||
case int64:
|
||||
if vv >= math.MinInt && vv <= math.MaxInt {
|
||||
return int(vv), true
|
||||
return int(vv), nil
|
||||
}
|
||||
return big.NewInt(vv), true
|
||||
return big.NewInt(vv), nil
|
||||
case uint64:
|
||||
if vv <= math.MaxInt {
|
||||
return int(vv), true
|
||||
return int(vv), nil
|
||||
}
|
||||
return new(big.Int).SetUint64(vv), true
|
||||
return new(big.Int).SetUint64(vv), nil
|
||||
case float32:
|
||||
return float64(vv), true
|
||||
return float64(vv), nil
|
||||
case float64:
|
||||
return vv, true
|
||||
return vv, nil
|
||||
case *big.Int:
|
||||
if vv.IsInt64() {
|
||||
vv := vv.Int64()
|
||||
if vv >= math.MinInt && vv <= math.MaxInt {
|
||||
return int(vv), true
|
||||
return int(vv), nil
|
||||
}
|
||||
}
|
||||
return vv, true
|
||||
return vv, nil
|
||||
case string:
|
||||
return vv, true
|
||||
return vv, nil
|
||||
case []byte:
|
||||
return string(vv), true
|
||||
return string(vv), nil
|
||||
case []any:
|
||||
vvs := make([]any, len(vv))
|
||||
for i, v := range vv {
|
||||
v, ok := ToGoJQValueFn(v, valueFn)
|
||||
if !ok {
|
||||
return nil, false
|
||||
v, err := ToGoJQValueFn(v, valueFn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
vvs[i] = v
|
||||
}
|
||||
return vvs, true
|
||||
return vvs, nil
|
||||
case map[string]any:
|
||||
vvs := make(map[string]any, len(vv))
|
||||
for k, v := range vv {
|
||||
v, ok := ToGoJQValueFn(v, valueFn)
|
||||
if !ok {
|
||||
return nil, false
|
||||
v, err := ToGoJQValueFn(v, valueFn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
vvs[k] = v
|
||||
}
|
||||
return vvs, true
|
||||
return vvs, nil
|
||||
default:
|
||||
if nv, ok := valueFn(vv); ok {
|
||||
return ToGoJQValueFn(nv, valueFn)
|
||||
nv, err := valueFn(vv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, false
|
||||
|
||||
return ToGoJQValueFn(nv, valueFn)
|
||||
}
|
||||
}
|
||||
|
@ -125,14 +125,14 @@ func CastFn[T any](v any, structFn func(input any, result any) error) (T, bool)
|
||||
ft := reflect.TypeOf(&t)
|
||||
if ft.Elem().Kind() == reflect.Struct {
|
||||
// TODO: some way to allow decode value passthru?
|
||||
m, ok := ToGoJQValue(v)
|
||||
if !ok {
|
||||
m, err := ToGoJQValue(v)
|
||||
if err != nil {
|
||||
return t, false
|
||||
}
|
||||
if structFn == nil {
|
||||
panic("structFn nil")
|
||||
}
|
||||
err := structFn(m, &t)
|
||||
err = structFn(m, &t)
|
||||
if err != nil {
|
||||
return t, false
|
||||
}
|
||||
|
@ -153,7 +153,7 @@ type openFile struct {
|
||||
var _ Value = (*openFile)(nil)
|
||||
var _ ToBinary = (*openFile)(nil)
|
||||
|
||||
func (of *openFile) Display(w io.Writer, opts Options) error {
|
||||
func (of *openFile) Display(w io.Writer, opts *Options) error {
|
||||
_, err := fmt.Fprintf(w, "<openfile %q>\n", of.filename)
|
||||
return err
|
||||
}
|
||||
@ -405,7 +405,7 @@ func (b Binary) JQValueToGoJQ() any {
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
func (b Binary) Display(w io.Writer, opts Options) error {
|
||||
func (b Binary) Display(w io.Writer, opts *Options) error {
|
||||
if opts.RawOutput {
|
||||
br, err := b.toReader()
|
||||
if err != nil {
|
||||
|
@ -164,13 +164,16 @@ func (i *Interp) _registry(c any) any {
|
||||
}
|
||||
|
||||
func (i *Interp) _toValue(c any, om map[string]any) any {
|
||||
return toValue(
|
||||
func() *Options {
|
||||
opts := OptionsFromValue(om)
|
||||
return &opts
|
||||
},
|
||||
c,
|
||||
)
|
||||
opts, err := OptionsFromValue(om)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
v, err := toValue(func() (*Options, error) { return opts, nil }, c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
type decodeOpts struct {
|
||||
@ -310,21 +313,20 @@ func valueHas(key any, a func(name string) any, b func(key any) any) any {
|
||||
|
||||
// TODO: make more efficient somehow? shallow values but might be hard
|
||||
// when things like tovalue.key should behave like a jq value and not a decode value etc
|
||||
func toValue(optsFn func() *Options, v any) any {
|
||||
nv, _ := gojqex.ToGoJQValueFn(v, func(v any) (any, bool) {
|
||||
func toValue(optsFn func() (*Options, error), v any) (any, error) {
|
||||
return gojqex.ToGoJQValueFn(v, func(v any) (any, error) {
|
||||
switch v := v.(type) {
|
||||
case JQValueEx:
|
||||
if optsFn == nil {
|
||||
return v.JQValueToGoJQ(), true
|
||||
return v.JQValueToGoJQ(), nil
|
||||
}
|
||||
return v.JQValueToGoJQEx(optsFn), true
|
||||
return v.JQValueToGoJQEx(optsFn), nil
|
||||
case gojq.JQValue:
|
||||
return v.JQValueToGoJQ(), true
|
||||
return v.JQValueToGoJQ(), nil
|
||||
default:
|
||||
return v, true
|
||||
return v, nil
|
||||
}
|
||||
})
|
||||
return nv
|
||||
}
|
||||
|
||||
type decodeValueKind int
|
||||
@ -453,7 +455,7 @@ func (dvb decodeValueBase) DecodeValue() *decode.Value {
|
||||
return dvb.dv
|
||||
}
|
||||
|
||||
func (dvb decodeValueBase) Display(w io.Writer, opts Options) error { return dump(dvb.dv, w, opts) }
|
||||
func (dvb decodeValueBase) Display(w io.Writer, opts *Options) error { return dump(dvb.dv, w, opts) }
|
||||
func (dvb decodeValueBase) ToBinary() (Binary, error) {
|
||||
return Binary{br: dvb.dv.RootReader, r: dvb.dv.InnerRange(), unit: 8}, nil
|
||||
}
|
||||
@ -606,7 +608,7 @@ func (v decodeValue) JQValueKey(name string) any {
|
||||
func (v decodeValue) JQValueHas(key any) any {
|
||||
return valueHas(key, v.decodeValueBase.JQValueKey, v.JQValue.JQValueHas)
|
||||
}
|
||||
func (v decodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
|
||||
func (v decodeValue) JQValueToGoJQEx(optsFn func() (*Options, error)) any {
|
||||
if !v.bitsFormat {
|
||||
return v.JQValueToGoJQ()
|
||||
}
|
||||
@ -625,7 +627,12 @@ func (v decodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
|
||||
return err
|
||||
}
|
||||
|
||||
s, err := optsFn().BitsFormatFn(brC)
|
||||
opts, err := optsFn()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s, err := opts.BitsFormatFn(brC)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -695,8 +702,11 @@ func (v ArrayDecodeValue) JQValueHas(key any) any {
|
||||
return intKey >= 0 && intKey < len(v.Compound.Children)
|
||||
})
|
||||
}
|
||||
func (v ArrayDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
|
||||
opts := optsFn()
|
||||
func (v ArrayDecodeValue) JQValueToGoJQEx(optsFn func() (*Options, error)) any {
|
||||
opts, err := optsFn()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
vs := make([]any, 0, len(v.Compound.Children))
|
||||
for _, f := range v.Compound.Children {
|
||||
@ -713,7 +723,7 @@ func (v ArrayDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
|
||||
return vs
|
||||
}
|
||||
func (v ArrayDecodeValue) JQValueToGoJQ() any {
|
||||
return v.JQValueToGoJQEx(func() *Options { return &Options{} })
|
||||
return v.JQValueToGoJQEx(func() (*Options, error) { return &Options{}, nil })
|
||||
}
|
||||
|
||||
// decode value struct
|
||||
@ -780,8 +790,11 @@ func (v StructDecodeValue) JQValueHas(key any) any {
|
||||
},
|
||||
)
|
||||
}
|
||||
func (v StructDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
|
||||
opts := optsFn()
|
||||
func (v StructDecodeValue) JQValueToGoJQEx(optsFn func() (*Options, error)) any {
|
||||
opts, err := optsFn()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
vm := make(map[string]any, len(v.Compound.Children))
|
||||
for _, f := range v.Compound.Children {
|
||||
@ -797,5 +810,5 @@ func (v StructDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
|
||||
return vm
|
||||
}
|
||||
func (v StructDecodeValue) JQValueToGoJQ() any {
|
||||
return v.JQValueToGoJQEx(func() *Options { return &Options{} })
|
||||
return v.JQValueToGoJQEx(func() (*Options, error) { return &Options{}, nil })
|
||||
}
|
||||
|
@ -46,7 +46,7 @@ func isCompound(v *decode.Value) bool {
|
||||
}
|
||||
|
||||
type dumpCtx struct {
|
||||
opts Options
|
||||
opts *Options
|
||||
buf []byte
|
||||
cw *columnwriter.Writer
|
||||
hexHeader string
|
||||
@ -336,7 +336,7 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD
|
||||
return nil
|
||||
}
|
||||
|
||||
func dump(v *decode.Value, w io.Writer, opts Options) error {
|
||||
func dump(v *decode.Value, w io.Writer, opts *Options) error {
|
||||
maxAddrIndentWidth := 0
|
||||
makeWalkFn := func(fn decode.WalkFn) decode.WalkFn {
|
||||
return func(v *decode.Value, rootV *decode.Value, depth int, rootDepth int) error {
|
||||
@ -409,7 +409,7 @@ func dump(v *decode.Value, w io.Writer, opts Options) error {
|
||||
}))
|
||||
}
|
||||
|
||||
func hexdump(w io.Writer, bv Binary, opts Options) error {
|
||||
func hexdump(w io.Writer, bv Binary, opts *Options) error {
|
||||
br, err := bitioex.Range(bv.br, bv.r.Start, bv.r.Len)
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -205,12 +205,12 @@ type Value interface {
|
||||
}
|
||||
|
||||
type Display interface {
|
||||
Display(w io.Writer, opts Options) error
|
||||
Display(w io.Writer, opts *Options) error
|
||||
}
|
||||
|
||||
type JQValueEx interface {
|
||||
gojq.JQValue
|
||||
JQValueToGoJQEx(optsFn func() *Options) any
|
||||
JQValueToGoJQEx(optsFn func() (*Options, error)) any
|
||||
}
|
||||
|
||||
func valuePath(v *decode.Value) []any {
|
||||
@ -640,7 +640,10 @@ func (i *Interp) history(c any) any {
|
||||
}
|
||||
|
||||
func (i *Interp) _display(c any, v any) gojq.Iter {
|
||||
opts := OptionsFromValue(v)
|
||||
opts, err := OptionsFromValue(v)
|
||||
if err != nil {
|
||||
return gojq.NewIter(err)
|
||||
}
|
||||
|
||||
switch v := c.(type) {
|
||||
case Display:
|
||||
@ -659,7 +662,11 @@ func (i *Interp) _canDisplay(c any) any {
|
||||
}
|
||||
|
||||
func (i *Interp) _hexdump(c any, v any) gojq.Iter {
|
||||
opts := OptionsFromValue(v)
|
||||
opts, err := OptionsFromValue(v)
|
||||
if err != nil {
|
||||
return gojq.NewIter(err)
|
||||
}
|
||||
|
||||
bv, err := toBinary(c)
|
||||
if err != nil {
|
||||
return gojq.NewIter(err)
|
||||
@ -672,17 +679,22 @@ func (i *Interp) _hexdump(c any, v any) gojq.Iter {
|
||||
}
|
||||
|
||||
func (i *Interp) _printColorJSON(c any, v any) gojq.Iter {
|
||||
opts := OptionsFromValue(v)
|
||||
opts, err := OptionsFromValue(v)
|
||||
if err != nil {
|
||||
return gojq.NewIter(err)
|
||||
}
|
||||
|
||||
indent := 2
|
||||
if opts.Compact {
|
||||
indent = 0
|
||||
}
|
||||
|
||||
cj := colorjson.NewEncoder(colorjson.Options{
|
||||
Color: opts.Color,
|
||||
Tab: false,
|
||||
Indent: indent,
|
||||
ValueFn: func(v any) any { return toValue(func() *Options { return &opts }, v) },
|
||||
Color: opts.Color,
|
||||
Tab: false,
|
||||
Indent: indent,
|
||||
// uses a function to cache OptionsFromValue
|
||||
ValueFn: func(v any) (any, error) { return toValue(func() (*Options, error) { return opts, nil }, v) },
|
||||
Colors: colorjson.Colors{
|
||||
Reset: []byte(ansi.Reset.SetString),
|
||||
Null: []byte(opts.Decorator.Null.SetString),
|
||||
@ -1040,7 +1052,7 @@ type Options struct {
|
||||
BitsFormatFn func(br bitio.ReaderAtSeeker) (any, error)
|
||||
}
|
||||
|
||||
func OptionsFromValue(v any) Options {
|
||||
func OptionsFromValue(v any) (*Options, error) {
|
||||
var opts Options
|
||||
_ = mapstruct.ToStruct(v, &opts)
|
||||
opts.ArrayTruncate = mathex.Max(0, opts.ArrayTruncate)
|
||||
@ -1050,12 +1062,16 @@ func OptionsFromValue(v any) Options {
|
||||
opts.LineBytes = mathex.Max(0, opts.LineBytes)
|
||||
opts.DisplayBytes = mathex.Max(0, opts.DisplayBytes)
|
||||
opts.Decorator = decoratorFromOptions(opts)
|
||||
opts.BitsFormatFn = bitsFormatFnFromOptions(opts)
|
||||
if fn, err := bitsFormatFnFromOptions(opts); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
opts.BitsFormatFn = fn
|
||||
}
|
||||
|
||||
return opts
|
||||
return &opts, nil
|
||||
}
|
||||
|
||||
func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, error) {
|
||||
func bitsFormatFnFromOptions(opts Options) (func(br bitio.ReaderAtSeeker) (any, error), error) {
|
||||
switch opts.BitsFormat {
|
||||
case "md5":
|
||||
return func(br bitio.ReaderAtSeeker) (any, error) {
|
||||
@ -1064,7 +1080,16 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(d.Sum(nil)), nil
|
||||
}
|
||||
}, nil
|
||||
case "hex":
|
||||
return func(br bitio.ReaderAtSeeker) (any, error) {
|
||||
b := &bytes.Buffer{}
|
||||
e := hex.NewEncoder(b)
|
||||
if _, err := bitioex.CopyBits(e, br); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return b.String(), nil
|
||||
}, nil
|
||||
case "base64":
|
||||
return func(br bitio.ReaderAtSeeker) (any, error) {
|
||||
b := &bytes.Buffer{}
|
||||
@ -1074,7 +1099,7 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
|
||||
}
|
||||
e.Close()
|
||||
return b.String(), nil
|
||||
}
|
||||
}, nil
|
||||
case "truncate":
|
||||
// TODO: configure
|
||||
return func(br bitio.ReaderAtSeeker) (any, error) {
|
||||
@ -1083,7 +1108,7 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
|
||||
return "", err
|
||||
}
|
||||
return b.String(), nil
|
||||
}
|
||||
}, nil
|
||||
case "string":
|
||||
return func(br bitio.ReaderAtSeeker) (any, error) {
|
||||
b := &bytes.Buffer{}
|
||||
@ -1091,10 +1116,8 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
|
||||
return "", err
|
||||
}
|
||||
return b.String(), nil
|
||||
}
|
||||
}, nil
|
||||
case "snippet":
|
||||
fallthrough
|
||||
default:
|
||||
return func(br bitio.ReaderAtSeeker) (any, error) {
|
||||
b := &bytes.Buffer{}
|
||||
e := base64.NewEncoder(base64.StdEncoding, b)
|
||||
@ -1102,14 +1125,15 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
|
||||
return "", err
|
||||
}
|
||||
e.Close()
|
||||
|
||||
brLen, err := bitioex.Len(br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return fmt.Sprintf("<%s>%s", mathex.Bits(brLen).StringByteBits(opts.Sizebase), b.String()), nil
|
||||
}
|
||||
}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("invalid bits format %q", opts.BitsFormat)
|
||||
}
|
||||
}
|
||||
|
||||
|
5
pkg/interp/testdata/binary.fqtest
vendored
5
pkg/interp/testdata/binary.fqtest
vendored
@ -35,9 +35,10 @@ $ fq -d mp3 '.frames[]._bits[0:12] | tonumber' test.mp3
|
||||
4095
|
||||
$ fq -d mp3 '.headers[0].header.magic._bits[0:24] | tostring' test.mp3
|
||||
"ID3"
|
||||
$ fq -d mp3 '.frames[0].audio_data | ("", "md5", "base64", "snippet") as $f | tovalue({bits_format: $f})' test.mp3
|
||||
"<5>AAAAAAA="
|
||||
$ fq -d mp3 '.frames[0].audio_data | ("", "md5", "hex", "base64", "snippet") as $f | try tovalue({bits_format: $f}) catch .' test.mp3
|
||||
"invalid bits format \"\""
|
||||
"ca9c491ac66b2c62500882e93f3719a8"
|
||||
"0000000000"
|
||||
"AAAAAAA="
|
||||
"<5>AAAAAAA="
|
||||
$ fq -d mp3 -i . test.mp3
|
||||
|
Loading…
Reference in New Issue
Block a user