diff --git a/doc/usage.md b/doc/usage.md index 5139b960..71757321 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -802,6 +802,7 @@ How to represent raw binary as JSON. - `-o bits_format=string` String with raw bytes (zero bit padded if size is not byte aligned). The string is binary safe internally in fq but bytes not representable as UTF-8 will be lost if turn to JSON. - `-o bits_format=md5` MD5 hex string (zero bit padded). +- `-o bits_format=hex` Hex string. - `-o bits_format=base64` Base64 string. - `-p bits_format=truncate` Truncated string. - `-o bits_format=snippet` Truncated Base64 string prefixed with bit length. diff --git a/format/json/json.go b/format/json/json.go index f6cb208c..868ef34f 100644 --- a/format/json/json.go +++ b/format/json/json.go @@ -94,12 +94,12 @@ func makeEncoder(opts ToJSONOpts) *colorjson.Encoder { Color: false, Tab: false, Indent: opts.Indent, - ValueFn: func(v any) any { + ValueFn: func(v any) (any, error) { switch v := v.(type) { case gojq.JQValue: - return v.JQValueToGoJQ() + return v.JQValueToGoJQ(), nil default: - return v + return v, nil } }, Colors: colorjson.Colors{}, diff --git a/internal/colorjson/encoder.go b/internal/colorjson/encoder.go index 01fb81a4..6b529bd8 100644 --- a/internal/colorjson/encoder.go +++ b/internal/colorjson/encoder.go @@ -50,7 +50,7 @@ type Options struct { Color bool Tab bool Indent int - ValueFn func(v any) any + ValueFn func(v any) (any, error) Colors Colors } @@ -120,7 +120,11 @@ func (e *Encoder) encode(v any) error { if e.opts.ValueFn == nil { panic(fmt.Sprintf("unknown type and to ValueFn set: %[1]T (%[1]v)", v)) } - return e.encode(e.opts.ValueFn(v)) + vv, err := e.opts.ValueFn(v) + if err != nil { + return err + } + return e.encode(vv) } if e.w.Len() > 8*1024 { return e.flush() diff --git a/internal/gojqex/totype.go b/internal/gojqex/totype.go index 3fd6f27a..b407421c 100644 --- a/internal/gojqex/totype.go +++ b/internal/gojqex/totype.go @@ -6,6 +6,7 @@ package gojqex import ( + "fmt" "math" "math/big" @@ -23,75 +24,77 @@ func IsNull(x any) bool { } } -func ToGoJQValue(v any) (any, bool) { - return ToGoJQValueFn(v, func(v any) (any, bool) { +func ToGoJQValue(v any) (any, error) { + return ToGoJQValueFn(v, func(v any) (any, error) { switch v := v.(type) { case gojq.JQValue: - return v.JQValueToGoJQ(), true + return v.JQValueToGoJQ(), nil default: - return nil, false + return nil, fmt.Errorf("not a JQValue") } }) } -func ToGoJQValueFn(v any, valueFn func(v any) (any, bool)) (any, bool) { +func ToGoJQValueFn(v any, valueFn func(v any) (any, error)) (any, error) { switch vv := v.(type) { case nil: - return vv, true + return vv, nil case bool: - return vv, true + return vv, nil case int: - return vv, true + return vv, nil case int64: if vv >= math.MinInt && vv <= math.MaxInt { - return int(vv), true + return int(vv), nil } - return big.NewInt(vv), true + return big.NewInt(vv), nil case uint64: if vv <= math.MaxInt { - return int(vv), true + return int(vv), nil } - return new(big.Int).SetUint64(vv), true + return new(big.Int).SetUint64(vv), nil case float32: - return float64(vv), true + return float64(vv), nil case float64: - return vv, true + return vv, nil case *big.Int: if vv.IsInt64() { vv := vv.Int64() if vv >= math.MinInt && vv <= math.MaxInt { - return int(vv), true + return int(vv), nil } } - return vv, true + return vv, nil case string: - return vv, true + return vv, nil case []byte: - return string(vv), true + return string(vv), nil case []any: vvs := make([]any, len(vv)) for i, v := range vv { - v, ok := ToGoJQValueFn(v, valueFn) - if !ok { - return nil, false + v, err := ToGoJQValueFn(v, valueFn) + if err != nil { + return nil, err } vvs[i] = v } - return vvs, true + return vvs, nil case map[string]any: vvs := make(map[string]any, len(vv)) for k, v := range vv { - v, ok := ToGoJQValueFn(v, valueFn) - if !ok { - return nil, false + v, err := ToGoJQValueFn(v, valueFn) + if err != nil { + return nil, err } vvs[k] = v } - return vvs, true + return vvs, nil default: - if nv, ok := valueFn(vv); ok { - return ToGoJQValueFn(nv, valueFn) + nv, err := valueFn(vv) + if err != nil { + return nil, err } - return nil, false + + return ToGoJQValueFn(nv, valueFn) } } diff --git a/internal/gojqex/types.go b/internal/gojqex/types.go index 67344fa7..ad190810 100644 --- a/internal/gojqex/types.go +++ b/internal/gojqex/types.go @@ -125,14 +125,14 @@ func CastFn[T any](v any, structFn func(input any, result any) error) (T, bool) ft := reflect.TypeOf(&t) if ft.Elem().Kind() == reflect.Struct { // TODO: some way to allow decode value passthru? - m, ok := ToGoJQValue(v) - if !ok { + m, err := ToGoJQValue(v) + if err != nil { return t, false } if structFn == nil { panic("structFn nil") } - err := structFn(m, &t) + err = structFn(m, &t) if err != nil { return t, false } diff --git a/pkg/interp/binary.go b/pkg/interp/binary.go index 9f7d309a..4d0baffc 100644 --- a/pkg/interp/binary.go +++ b/pkg/interp/binary.go @@ -153,7 +153,7 @@ type openFile struct { var _ Value = (*openFile)(nil) var _ ToBinary = (*openFile)(nil) -func (of *openFile) Display(w io.Writer, opts Options) error { +func (of *openFile) Display(w io.Writer, opts *Options) error { _, err := fmt.Fprintf(w, "\n", of.filename) return err } @@ -405,7 +405,7 @@ func (b Binary) JQValueToGoJQ() any { return buf.String() } -func (b Binary) Display(w io.Writer, opts Options) error { +func (b Binary) Display(w io.Writer, opts *Options) error { if opts.RawOutput { br, err := b.toReader() if err != nil { diff --git a/pkg/interp/decode.go b/pkg/interp/decode.go index 51e34efe..fe725276 100644 --- a/pkg/interp/decode.go +++ b/pkg/interp/decode.go @@ -164,13 +164,16 @@ func (i *Interp) _registry(c any) any { } func (i *Interp) _toValue(c any, om map[string]any) any { - return toValue( - func() *Options { - opts := OptionsFromValue(om) - return &opts - }, - c, - ) + opts, err := OptionsFromValue(om) + if err != nil { + return err + } + + v, err := toValue(func() (*Options, error) { return opts, nil }, c) + if err != nil { + return err + } + return v } type decodeOpts struct { @@ -310,21 +313,20 @@ func valueHas(key any, a func(name string) any, b func(key any) any) any { // TODO: make more efficient somehow? shallow values but might be hard // when things like tovalue.key should behave like a jq value and not a decode value etc -func toValue(optsFn func() *Options, v any) any { - nv, _ := gojqex.ToGoJQValueFn(v, func(v any) (any, bool) { +func toValue(optsFn func() (*Options, error), v any) (any, error) { + return gojqex.ToGoJQValueFn(v, func(v any) (any, error) { switch v := v.(type) { case JQValueEx: if optsFn == nil { - return v.JQValueToGoJQ(), true + return v.JQValueToGoJQ(), nil } - return v.JQValueToGoJQEx(optsFn), true + return v.JQValueToGoJQEx(optsFn), nil case gojq.JQValue: - return v.JQValueToGoJQ(), true + return v.JQValueToGoJQ(), nil default: - return v, true + return v, nil } }) - return nv } type decodeValueKind int @@ -453,7 +455,7 @@ func (dvb decodeValueBase) DecodeValue() *decode.Value { return dvb.dv } -func (dvb decodeValueBase) Display(w io.Writer, opts Options) error { return dump(dvb.dv, w, opts) } +func (dvb decodeValueBase) Display(w io.Writer, opts *Options) error { return dump(dvb.dv, w, opts) } func (dvb decodeValueBase) ToBinary() (Binary, error) { return Binary{br: dvb.dv.RootReader, r: dvb.dv.InnerRange(), unit: 8}, nil } @@ -606,7 +608,7 @@ func (v decodeValue) JQValueKey(name string) any { func (v decodeValue) JQValueHas(key any) any { return valueHas(key, v.decodeValueBase.JQValueKey, v.JQValue.JQValueHas) } -func (v decodeValue) JQValueToGoJQEx(optsFn func() *Options) any { +func (v decodeValue) JQValueToGoJQEx(optsFn func() (*Options, error)) any { if !v.bitsFormat { return v.JQValueToGoJQ() } @@ -625,7 +627,12 @@ func (v decodeValue) JQValueToGoJQEx(optsFn func() *Options) any { return err } - s, err := optsFn().BitsFormatFn(brC) + opts, err := optsFn() + if err != nil { + return err + } + + s, err := opts.BitsFormatFn(brC) if err != nil { return err } @@ -695,8 +702,11 @@ func (v ArrayDecodeValue) JQValueHas(key any) any { return intKey >= 0 && intKey < len(v.Compound.Children) }) } -func (v ArrayDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any { - opts := optsFn() +func (v ArrayDecodeValue) JQValueToGoJQEx(optsFn func() (*Options, error)) any { + opts, err := optsFn() + if err != nil { + return err + } vs := make([]any, 0, len(v.Compound.Children)) for _, f := range v.Compound.Children { @@ -713,7 +723,7 @@ func (v ArrayDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any { return vs } func (v ArrayDecodeValue) JQValueToGoJQ() any { - return v.JQValueToGoJQEx(func() *Options { return &Options{} }) + return v.JQValueToGoJQEx(func() (*Options, error) { return &Options{}, nil }) } // decode value struct @@ -780,8 +790,11 @@ func (v StructDecodeValue) JQValueHas(key any) any { }, ) } -func (v StructDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any { - opts := optsFn() +func (v StructDecodeValue) JQValueToGoJQEx(optsFn func() (*Options, error)) any { + opts, err := optsFn() + if err != nil { + return err + } vm := make(map[string]any, len(v.Compound.Children)) for _, f := range v.Compound.Children { @@ -797,5 +810,5 @@ func (v StructDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any { return vm } func (v StructDecodeValue) JQValueToGoJQ() any { - return v.JQValueToGoJQEx(func() *Options { return &Options{} }) + return v.JQValueToGoJQEx(func() (*Options, error) { return &Options{}, nil }) } diff --git a/pkg/interp/dump.go b/pkg/interp/dump.go index aebeb10a..ef78167c 100644 --- a/pkg/interp/dump.go +++ b/pkg/interp/dump.go @@ -46,7 +46,7 @@ func isCompound(v *decode.Value) bool { } type dumpCtx struct { - opts Options + opts *Options buf []byte cw *columnwriter.Writer hexHeader string @@ -336,7 +336,7 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD return nil } -func dump(v *decode.Value, w io.Writer, opts Options) error { +func dump(v *decode.Value, w io.Writer, opts *Options) error { maxAddrIndentWidth := 0 makeWalkFn := func(fn decode.WalkFn) decode.WalkFn { return func(v *decode.Value, rootV *decode.Value, depth int, rootDepth int) error { @@ -409,7 +409,7 @@ func dump(v *decode.Value, w io.Writer, opts Options) error { })) } -func hexdump(w io.Writer, bv Binary, opts Options) error { +func hexdump(w io.Writer, bv Binary, opts *Options) error { br, err := bitioex.Range(bv.br, bv.r.Start, bv.r.Len) if err != nil { return err diff --git a/pkg/interp/interp.go b/pkg/interp/interp.go index 1f12e599..700674ec 100644 --- a/pkg/interp/interp.go +++ b/pkg/interp/interp.go @@ -205,12 +205,12 @@ type Value interface { } type Display interface { - Display(w io.Writer, opts Options) error + Display(w io.Writer, opts *Options) error } type JQValueEx interface { gojq.JQValue - JQValueToGoJQEx(optsFn func() *Options) any + JQValueToGoJQEx(optsFn func() (*Options, error)) any } func valuePath(v *decode.Value) []any { @@ -640,7 +640,10 @@ func (i *Interp) history(c any) any { } func (i *Interp) _display(c any, v any) gojq.Iter { - opts := OptionsFromValue(v) + opts, err := OptionsFromValue(v) + if err != nil { + return gojq.NewIter(err) + } switch v := c.(type) { case Display: @@ -659,7 +662,11 @@ func (i *Interp) _canDisplay(c any) any { } func (i *Interp) _hexdump(c any, v any) gojq.Iter { - opts := OptionsFromValue(v) + opts, err := OptionsFromValue(v) + if err != nil { + return gojq.NewIter(err) + } + bv, err := toBinary(c) if err != nil { return gojq.NewIter(err) @@ -672,17 +679,22 @@ func (i *Interp) _hexdump(c any, v any) gojq.Iter { } func (i *Interp) _printColorJSON(c any, v any) gojq.Iter { - opts := OptionsFromValue(v) + opts, err := OptionsFromValue(v) + if err != nil { + return gojq.NewIter(err) + } + indent := 2 if opts.Compact { indent = 0 } cj := colorjson.NewEncoder(colorjson.Options{ - Color: opts.Color, - Tab: false, - Indent: indent, - ValueFn: func(v any) any { return toValue(func() *Options { return &opts }, v) }, + Color: opts.Color, + Tab: false, + Indent: indent, + // uses a function to cache OptionsFromValue + ValueFn: func(v any) (any, error) { return toValue(func() (*Options, error) { return opts, nil }, v) }, Colors: colorjson.Colors{ Reset: []byte(ansi.Reset.SetString), Null: []byte(opts.Decorator.Null.SetString), @@ -1040,7 +1052,7 @@ type Options struct { BitsFormatFn func(br bitio.ReaderAtSeeker) (any, error) } -func OptionsFromValue(v any) Options { +func OptionsFromValue(v any) (*Options, error) { var opts Options _ = mapstruct.ToStruct(v, &opts) opts.ArrayTruncate = mathex.Max(0, opts.ArrayTruncate) @@ -1050,12 +1062,16 @@ func OptionsFromValue(v any) Options { opts.LineBytes = mathex.Max(0, opts.LineBytes) opts.DisplayBytes = mathex.Max(0, opts.DisplayBytes) opts.Decorator = decoratorFromOptions(opts) - opts.BitsFormatFn = bitsFormatFnFromOptions(opts) + if fn, err := bitsFormatFnFromOptions(opts); err != nil { + return nil, err + } else { + opts.BitsFormatFn = fn + } - return opts + return &opts, nil } -func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, error) { +func bitsFormatFnFromOptions(opts Options) (func(br bitio.ReaderAtSeeker) (any, error), error) { switch opts.BitsFormat { case "md5": return func(br bitio.ReaderAtSeeker) (any, error) { @@ -1064,7 +1080,16 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e return "", err } return hex.EncodeToString(d.Sum(nil)), nil - } + }, nil + case "hex": + return func(br bitio.ReaderAtSeeker) (any, error) { + b := &bytes.Buffer{} + e := hex.NewEncoder(b) + if _, err := bitioex.CopyBits(e, br); err != nil { + return "", err + } + return b.String(), nil + }, nil case "base64": return func(br bitio.ReaderAtSeeker) (any, error) { b := &bytes.Buffer{} @@ -1074,7 +1099,7 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e } e.Close() return b.String(), nil - } + }, nil case "truncate": // TODO: configure return func(br bitio.ReaderAtSeeker) (any, error) { @@ -1083,7 +1108,7 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e return "", err } return b.String(), nil - } + }, nil case "string": return func(br bitio.ReaderAtSeeker) (any, error) { b := &bytes.Buffer{} @@ -1091,10 +1116,8 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e return "", err } return b.String(), nil - } + }, nil case "snippet": - fallthrough - default: return func(br bitio.ReaderAtSeeker) (any, error) { b := &bytes.Buffer{} e := base64.NewEncoder(base64.StdEncoding, b) @@ -1102,14 +1125,15 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e return "", err } e.Close() - brLen, err := bitioex.Len(br) if err != nil { return nil, err } return fmt.Sprintf("<%s>%s", mathex.Bits(brLen).StringByteBits(opts.Sizebase), b.String()), nil - } + }, nil + default: + return nil, fmt.Errorf("invalid bits format %q", opts.BitsFormat) } } diff --git a/pkg/interp/testdata/binary.fqtest b/pkg/interp/testdata/binary.fqtest index f5302c00..de50c02c 100644 --- a/pkg/interp/testdata/binary.fqtest +++ b/pkg/interp/testdata/binary.fqtest @@ -35,9 +35,10 @@ $ fq -d mp3 '.frames[]._bits[0:12] | tonumber' test.mp3 4095 $ fq -d mp3 '.headers[0].header.magic._bits[0:24] | tostring' test.mp3 "ID3" -$ fq -d mp3 '.frames[0].audio_data | ("", "md5", "base64", "snippet") as $f | tovalue({bits_format: $f})' test.mp3 -"<5>AAAAAAA=" +$ fq -d mp3 '.frames[0].audio_data | ("", "md5", "hex", "base64", "snippet") as $f | try tovalue({bits_format: $f}) catch .' test.mp3 +"invalid bits format \"\"" "ca9c491ac66b2c62500882e93f3719a8" +"0000000000" "AAAAAAA=" "<5>AAAAAAA=" $ fq -d mp3 -i . test.mp3