Merge pull request #673 from wader/bit-format-hex

interp: Add hex bits format

doc/usage.md

@@ -802,6 +802,7 @@ How to represent raw binary as JSON.
 
 - `-o bits_format=string` String with raw bytes (zero bit padded if size is not byte aligned). The string is binary safe internally in fq but bytes not representable as UTF-8 will be lost if turn to JSON.
 - `-o bits_format=md5` MD5 hex string (zero bit padded).
+- `-o bits_format=hex` Hex string.
 - `-o bits_format=base64` Base64 string.
 - `-p bits_format=truncate` Truncated string.
 - `-o bits_format=snippet` Truncated Base64 string prefixed with bit length.

format/json/json.go

@@ -94,12 +94,12 @@ func makeEncoder(opts ToJSONOpts) *colorjson.Encoder {
 		Color:  false,
 		Tab:    false,
 		Indent: opts.Indent,
-		ValueFn: func(v any) any {
+		ValueFn: func(v any) (any, error) {
 			switch v := v.(type) {
 			case gojq.JQValue:
-				return v.JQValueToGoJQ()
+				return v.JQValueToGoJQ(), nil
 			default:
-				return v
+				return v, nil
 			}
 		},
 		Colors: colorjson.Colors{},

internal/colorjson/encoder.go

@@ -50,7 +50,7 @@ type Options struct {
 	Color   bool
 	Tab     bool
 	Indent  int
-	ValueFn func(v any) any
+	ValueFn func(v any) (any, error)
 	Colors  Colors
 }
 
@@ -120,7 +120,11 @@ func (e *Encoder) encode(v any) error {
 		if e.opts.ValueFn == nil {
 			panic(fmt.Sprintf("unknown type and to ValueFn set: %[1]T (%[1]v)", v))
 		}
-		return e.encode(e.opts.ValueFn(v))
+		vv, err := e.opts.ValueFn(v)
+		if err != nil {
+			return err
+		}
+		return e.encode(vv)
 	}
 	if e.w.Len() > 8*1024 {
 		return e.flush()

internal/gojqex/totype.go

@@ -6,6 +6,7 @@
 package gojqex
 
 import (
+	"fmt"
 	"math"
 	"math/big"
 
@@ -23,75 +24,77 @@ func IsNull(x any) bool {
 	}
 }
 
-func ToGoJQValue(v any) (any, bool) {
-	return ToGoJQValueFn(v, func(v any) (any, bool) {
+func ToGoJQValue(v any) (any, error) {
+	return ToGoJQValueFn(v, func(v any) (any, error) {
 		switch v := v.(type) {
 		case gojq.JQValue:
-			return v.JQValueToGoJQ(), true
+			return v.JQValueToGoJQ(), nil
 		default:
-			return nil, false
+			return nil, fmt.Errorf("not a JQValue")
 		}
 	})
 }
 
-func ToGoJQValueFn(v any, valueFn func(v any) (any, bool)) (any, bool) {
+func ToGoJQValueFn(v any, valueFn func(v any) (any, error)) (any, error) {
 	switch vv := v.(type) {
 	case nil:
-		return vv, true
+		return vv, nil
 	case bool:
-		return vv, true
+		return vv, nil
 	case int:
-		return vv, true
+		return vv, nil
 	case int64:
 		if vv >= math.MinInt && vv <= math.MaxInt {
-			return int(vv), true
+			return int(vv), nil
 		}
-		return big.NewInt(vv), true
+		return big.NewInt(vv), nil
 	case uint64:
 		if vv <= math.MaxInt {
-			return int(vv), true
+			return int(vv), nil
 		}
-		return new(big.Int).SetUint64(vv), true
+		return new(big.Int).SetUint64(vv), nil
 	case float32:
-		return float64(vv), true
+		return float64(vv), nil
 	case float64:
-		return vv, true
+		return vv, nil
 	case *big.Int:
 		if vv.IsInt64() {
 			vv := vv.Int64()
 			if vv >= math.MinInt && vv <= math.MaxInt {
-				return int(vv), true
+				return int(vv), nil
 			}
 		}
-		return vv, true
+		return vv, nil
 	case string:
-		return vv, true
+		return vv, nil
 	case []byte:
-		return string(vv), true
+		return string(vv), nil
 	case []any:
 		vvs := make([]any, len(vv))
 		for i, v := range vv {
-			v, ok := ToGoJQValueFn(v, valueFn)
-			if !ok {
-				return nil, false
+			v, err := ToGoJQValueFn(v, valueFn)
+			if err != nil {
+				return nil, err
 			}
 			vvs[i] = v
 		}
-		return vvs, true
+		return vvs, nil
 	case map[string]any:
 		vvs := make(map[string]any, len(vv))
 		for k, v := range vv {
-			v, ok := ToGoJQValueFn(v, valueFn)
-			if !ok {
-				return nil, false
+			v, err := ToGoJQValueFn(v, valueFn)
+			if err != nil {
+				return nil, err
 			}
 			vvs[k] = v
 		}
-		return vvs, true
+		return vvs, nil
 	default:
-		if nv, ok := valueFn(vv); ok {
-			return ToGoJQValueFn(nv, valueFn)
+		nv, err := valueFn(vv)
+		if err != nil {
+			return nil, err
 		}
-		return nil, false
+
+		return ToGoJQValueFn(nv, valueFn)
 	}
 }

internal/gojqex/types.go

@@ -125,14 +125,14 @@ func CastFn[T any](v any, structFn func(input any, result any) error) (T, bool)
 		ft := reflect.TypeOf(&t)
 		if ft.Elem().Kind() == reflect.Struct {
 			// TODO: some way to allow decode value passthru?
-			m, ok := ToGoJQValue(v)
-			if !ok {
+			m, err := ToGoJQValue(v)
+			if err != nil {
 				return t, false
 			}
 			if structFn == nil {
 				panic("structFn nil")
 			}
-			err := structFn(m, &t)
+			err = structFn(m, &t)
 			if err != nil {
 				return t, false
 			}

pkg/interp/binary.go

@@ -153,7 +153,7 @@ type openFile struct {
 var _ Value = (*openFile)(nil)
 var _ ToBinary = (*openFile)(nil)
 
-func (of *openFile) Display(w io.Writer, opts Options) error {
+func (of *openFile) Display(w io.Writer, opts *Options) error {
 	_, err := fmt.Fprintf(w, "<openfile %q>\n", of.filename)
 	return err
 }
@@ -405,7 +405,7 @@ func (b Binary) JQValueToGoJQ() any {
 	return buf.String()
 }
 
-func (b Binary) Display(w io.Writer, opts Options) error {
+func (b Binary) Display(w io.Writer, opts *Options) error {
 	if opts.RawOutput {
 		br, err := b.toReader()
 		if err != nil {

pkg/interp/decode.go

@@ -164,13 +164,16 @@ func (i *Interp) _registry(c any) any {
 }
 
 func (i *Interp) _toValue(c any, om map[string]any) any {
-	return toValue(
-		func() *Options {
-			opts := OptionsFromValue(om)
-			return &opts
-		},
-		c,
-	)
+	opts, err := OptionsFromValue(om)
+	if err != nil {
+		return err
+	}
+
+	v, err := toValue(func() (*Options, error) { return opts, nil }, c)
+	if err != nil {
+		return err
+	}
+	return v
 }
 
 type decodeOpts struct {
@@ -310,21 +313,20 @@ func valueHas(key any, a func(name string) any, b func(key any) any) any {
 
 // TODO: make more efficient somehow? shallow values but might be hard
 // when things like tovalue.key should behave like a jq value and not a decode value etc
-func toValue(optsFn func() *Options, v any) any {
-	nv, _ := gojqex.ToGoJQValueFn(v, func(v any) (any, bool) {
+func toValue(optsFn func() (*Options, error), v any) (any, error) {
+	return gojqex.ToGoJQValueFn(v, func(v any) (any, error) {
 		switch v := v.(type) {
 		case JQValueEx:
 			if optsFn == nil {
-				return v.JQValueToGoJQ(), true
+				return v.JQValueToGoJQ(), nil
 			}
-			return v.JQValueToGoJQEx(optsFn), true
+			return v.JQValueToGoJQEx(optsFn), nil
 		case gojq.JQValue:
-			return v.JQValueToGoJQ(), true
+			return v.JQValueToGoJQ(), nil
 		default:
-			return v, true
+			return v, nil
 		}
 	})
-	return nv
 }
 
 type decodeValueKind int
@@ -453,7 +455,7 @@ func (dvb decodeValueBase) DecodeValue() *decode.Value {
 	return dvb.dv
 }
 
-func (dvb decodeValueBase) Display(w io.Writer, opts Options) error { return dump(dvb.dv, w, opts) }
+func (dvb decodeValueBase) Display(w io.Writer, opts *Options) error { return dump(dvb.dv, w, opts) }
 func (dvb decodeValueBase) ToBinary() (Binary, error) {
 	return Binary{br: dvb.dv.RootReader, r: dvb.dv.InnerRange(), unit: 8}, nil
 }
@@ -606,7 +608,7 @@ func (v decodeValue) JQValueKey(name string) any {
 func (v decodeValue) JQValueHas(key any) any {
 	return valueHas(key, v.decodeValueBase.JQValueKey, v.JQValue.JQValueHas)
 }
-func (v decodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
+func (v decodeValue) JQValueToGoJQEx(optsFn func() (*Options, error)) any {
 	if !v.bitsFormat {
 		return v.JQValueToGoJQ()
 	}
@@ -625,7 +627,12 @@ func (v decodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
 		return err
 	}
 
-	s, err := optsFn().BitsFormatFn(brC)
+	opts, err := optsFn()
+	if err != nil {
+		return err
+	}
+
+	s, err := opts.BitsFormatFn(brC)
 	if err != nil {
 		return err
 	}
@@ -695,8 +702,11 @@ func (v ArrayDecodeValue) JQValueHas(key any) any {
 			return intKey >= 0 && intKey < len(v.Compound.Children)
 		})
 }
-func (v ArrayDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
-	opts := optsFn()
+func (v ArrayDecodeValue) JQValueToGoJQEx(optsFn func() (*Options, error)) any {
+	opts, err := optsFn()
+	if err != nil {
+		return err
+	}
 
 	vs := make([]any, 0, len(v.Compound.Children))
 	for _, f := range v.Compound.Children {
@@ -713,7 +723,7 @@ func (v ArrayDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
 	return vs
 }
 func (v ArrayDecodeValue) JQValueToGoJQ() any {
-	return v.JQValueToGoJQEx(func() *Options { return &Options{} })
+	return v.JQValueToGoJQEx(func() (*Options, error) { return &Options{}, nil })
 }
 
 // decode value struct
@@ -780,8 +790,11 @@ func (v StructDecodeValue) JQValueHas(key any) any {
 		},
 	)
 }
-func (v StructDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
-	opts := optsFn()
+func (v StructDecodeValue) JQValueToGoJQEx(optsFn func() (*Options, error)) any {
+	opts, err := optsFn()
+	if err != nil {
+		return err
+	}
 
 	vm := make(map[string]any, len(v.Compound.Children))
 	for _, f := range v.Compound.Children {
@@ -797,5 +810,5 @@ func (v StructDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
 	return vm
 }
 func (v StructDecodeValue) JQValueToGoJQ() any {
-	return v.JQValueToGoJQEx(func() *Options { return &Options{} })
+	return v.JQValueToGoJQEx(func() (*Options, error) { return &Options{}, nil })
 }

pkg/interp/dump.go

@@ -46,7 +46,7 @@ func isCompound(v *decode.Value) bool {
 }
 
 type dumpCtx struct {
-	opts        Options
+	opts        *Options
 	buf         []byte
 	cw          *columnwriter.Writer
 	hexHeader   string
@@ -336,7 +336,7 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD
 	return nil
 }
 
-func dump(v *decode.Value, w io.Writer, opts Options) error {
+func dump(v *decode.Value, w io.Writer, opts *Options) error {
 	maxAddrIndentWidth := 0
 	makeWalkFn := func(fn decode.WalkFn) decode.WalkFn {
 		return func(v *decode.Value, rootV *decode.Value, depth int, rootDepth int) error {
@@ -409,7 +409,7 @@ func dump(v *decode.Value, w io.Writer, opts Options) error {
 	}))
 }
 
-func hexdump(w io.Writer, bv Binary, opts Options) error {
+func hexdump(w io.Writer, bv Binary, opts *Options) error {
 	br, err := bitioex.Range(bv.br, bv.r.Start, bv.r.Len)
 	if err != nil {
 		return err

pkg/interp/interp.go

@@ -205,12 +205,12 @@ type Value interface {
 }
 
 type Display interface {
-	Display(w io.Writer, opts Options) error
+	Display(w io.Writer, opts *Options) error
 }
 
 type JQValueEx interface {
 	gojq.JQValue
-	JQValueToGoJQEx(optsFn func() *Options) any
+	JQValueToGoJQEx(optsFn func() (*Options, error)) any
 }
 
 func valuePath(v *decode.Value) []any {
@@ -640,7 +640,10 @@ func (i *Interp) history(c any) any {
 }
 
 func (i *Interp) _display(c any, v any) gojq.Iter {
-	opts := OptionsFromValue(v)
+	opts, err := OptionsFromValue(v)
+	if err != nil {
+		return gojq.NewIter(err)
+	}
 
 	switch v := c.(type) {
 	case Display:
@@ -659,7 +662,11 @@ func (i *Interp) _canDisplay(c any) any {
 }
 
 func (i *Interp) _hexdump(c any, v any) gojq.Iter {
-	opts := OptionsFromValue(v)
+	opts, err := OptionsFromValue(v)
+	if err != nil {
+		return gojq.NewIter(err)
+	}
+
 	bv, err := toBinary(c)
 	if err != nil {
 		return gojq.NewIter(err)
@@ -672,17 +679,22 @@ func (i *Interp) _hexdump(c any, v any) gojq.Iter {
 }
 
 func (i *Interp) _printColorJSON(c any, v any) gojq.Iter {
-	opts := OptionsFromValue(v)
+	opts, err := OptionsFromValue(v)
+	if err != nil {
+		return gojq.NewIter(err)
+	}
+
 	indent := 2
 	if opts.Compact {
 		indent = 0
 	}
 
 	cj := colorjson.NewEncoder(colorjson.Options{
-		Color:   opts.Color,
-		Tab:     false,
-		Indent:  indent,
-		ValueFn: func(v any) any { return toValue(func() *Options { return &opts }, v) },
+		Color:  opts.Color,
+		Tab:    false,
+		Indent: indent,
+		// uses a function to cache OptionsFromValue
+		ValueFn: func(v any) (any, error) { return toValue(func() (*Options, error) { return opts, nil }, v) },
 		Colors: colorjson.Colors{
 			Reset:     []byte(ansi.Reset.SetString),
 			Null:      []byte(opts.Decorator.Null.SetString),
@@ -1040,7 +1052,7 @@ type Options struct {
 	BitsFormatFn func(br bitio.ReaderAtSeeker) (any, error)
 }
 
-func OptionsFromValue(v any) Options {
+func OptionsFromValue(v any) (*Options, error) {
 	var opts Options
 	_ = mapstruct.ToStruct(v, &opts)
 	opts.ArrayTruncate = mathex.Max(0, opts.ArrayTruncate)
@@ -1050,12 +1062,16 @@ func OptionsFromValue(v any) Options {
 	opts.LineBytes = mathex.Max(0, opts.LineBytes)
 	opts.DisplayBytes = mathex.Max(0, opts.DisplayBytes)
 	opts.Decorator = decoratorFromOptions(opts)
-	opts.BitsFormatFn = bitsFormatFnFromOptions(opts)
+	if fn, err := bitsFormatFnFromOptions(opts); err != nil {
+		return nil, err
+	} else {
+		opts.BitsFormatFn = fn
+	}
 
-	return opts
+	return &opts, nil
 }
 
-func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, error) {
+func bitsFormatFnFromOptions(opts Options) (func(br bitio.ReaderAtSeeker) (any, error), error) {
 	switch opts.BitsFormat {
 	case "md5":
 		return func(br bitio.ReaderAtSeeker) (any, error) {
@@ -1064,7 +1080,16 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
 				return "", err
 			}
 			return hex.EncodeToString(d.Sum(nil)), nil
-		}
+		}, nil
+	case "hex":
+		return func(br bitio.ReaderAtSeeker) (any, error) {
+			b := &bytes.Buffer{}
+			e := hex.NewEncoder(b)
+			if _, err := bitioex.CopyBits(e, br); err != nil {
+				return "", err
+			}
+			return b.String(), nil
+		}, nil
 	case "base64":
 		return func(br bitio.ReaderAtSeeker) (any, error) {
 			b := &bytes.Buffer{}
@@ -1074,7 +1099,7 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
 			}
 			e.Close()
 			return b.String(), nil
-		}
+		}, nil
 	case "truncate":
 		// TODO: configure
 		return func(br bitio.ReaderAtSeeker) (any, error) {
@@ -1083,7 +1108,7 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
 				return "", err
 			}
 			return b.String(), nil
-		}
+		}, nil
 	case "string":
 		return func(br bitio.ReaderAtSeeker) (any, error) {
 			b := &bytes.Buffer{}
@@ -1091,10 +1116,8 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
 				return "", err
 			}
 			return b.String(), nil
-		}
+		}, nil
 	case "snippet":
-		fallthrough
-	default:
 		return func(br bitio.ReaderAtSeeker) (any, error) {
 			b := &bytes.Buffer{}
 			e := base64.NewEncoder(base64.StdEncoding, b)
@@ -1102,14 +1125,15 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
 				return "", err
 			}
 			e.Close()
-
 			brLen, err := bitioex.Len(br)
 			if err != nil {
 				return nil, err
 			}
 
 			return fmt.Sprintf("<%s>%s", mathex.Bits(brLen).StringByteBits(opts.Sizebase), b.String()), nil
-		}
+		}, nil
+	default:
+		return nil, fmt.Errorf("invalid bits format %q", opts.BitsFormat)
 	}
 }
 

pkg/interp/testdata/binary.fqtest

@@ -35,9 +35,10 @@ $ fq -d mp3 '.frames[]._bits[0:12] | tonumber' test.mp3
 4095
 $ fq -d mp3 '.headers[0].header.magic._bits[0:24] | tostring' test.mp3
 "ID3"
-$ fq -d mp3 '.frames[0].audio_data | ("", "md5", "base64", "snippet") as $f | tovalue({bits_format: $f})' test.mp3
-"<5>AAAAAAA="
+$ fq -d mp3 '.frames[0].audio_data | ("", "md5", "hex", "base64", "snippet") as $f | try tovalue({bits_format: $f}) catch .' test.mp3
+"invalid bits format \"\""
 "ca9c491ac66b2c62500882e93f3719a8"
+"0000000000"
 "AAAAAAA="
 "<5>AAAAAAA="
 $ fq -d mp3 -i . test.mp3