1
1
mirror of https://github.com/wader/fq.git synced 2024-12-24 22:05:31 +03:00

interp: Add hex bits format

$ cat random.bin.gz | fq -rV -o bits_format=hex .uncompressed
f6f2074cf77d449d

Also made unknown bit formats an error.
This commit is contained in:
Mattias Wadman 2023-05-15 17:23:59 +02:00
parent 070d5f43ab
commit 8a468f45df
10 changed files with 134 additions and 88 deletions

View File

@ -802,6 +802,7 @@ How to represent raw binary as JSON.
- `-o bits_foramt=string` String with raw bytes (zero bit padded if size is not byte aligned). The string is binary safe internally in fq but bytes not representable as UTF-8 will be lost if turn to JSON.
- `-o bits_format=md5` MD5 hex string (zero bit padded).
- `-o bits_format=hex` Hex string.
- `-o bits_format=base64` Base64 string.
- `-p bits_foramt=truncate` Truncated string.
- `-o bits_format=snippet` Truncated Base64 string prefixed with bit length.

View File

@ -94,12 +94,12 @@ func makeEncoder(opts ToJSONOpts) *colorjson.Encoder {
Color: false,
Tab: false,
Indent: opts.Indent,
ValueFn: func(v any) any {
ValueFn: func(v any) (any, error) {
switch v := v.(type) {
case gojq.JQValue:
return v.JQValueToGoJQ()
return v.JQValueToGoJQ(), nil
default:
return v
return v, nil
}
},
Colors: colorjson.Colors{},

View File

@ -50,7 +50,7 @@ type Options struct {
Color bool
Tab bool
Indent int
ValueFn func(v any) any
ValueFn func(v any) (any, error)
Colors Colors
}
@ -120,7 +120,11 @@ func (e *Encoder) encode(v any) error {
if e.opts.ValueFn == nil {
panic(fmt.Sprintf("unknown type and to ValueFn set: %[1]T (%[1]v)", v))
}
return e.encode(e.opts.ValueFn(v))
vv, err := e.opts.ValueFn(v)
if err != nil {
return err
}
return e.encode(vv)
}
if e.w.Len() > 8*1024 {
return e.flush()

View File

@ -6,6 +6,7 @@
package gojqex
import (
"fmt"
"math"
"math/big"
@ -23,75 +24,77 @@ func IsNull(x any) bool {
}
}
func ToGoJQValue(v any) (any, bool) {
return ToGoJQValueFn(v, func(v any) (any, bool) {
func ToGoJQValue(v any) (any, error) {
return ToGoJQValueFn(v, func(v any) (any, error) {
switch v := v.(type) {
case gojq.JQValue:
return v.JQValueToGoJQ(), true
return v.JQValueToGoJQ(), nil
default:
return nil, false
return nil, fmt.Errorf("not a JQValue")
}
})
}
func ToGoJQValueFn(v any, valueFn func(v any) (any, bool)) (any, bool) {
func ToGoJQValueFn(v any, valueFn func(v any) (any, error)) (any, error) {
switch vv := v.(type) {
case nil:
return vv, true
return vv, nil
case bool:
return vv, true
return vv, nil
case int:
return vv, true
return vv, nil
case int64:
if vv >= math.MinInt && vv <= math.MaxInt {
return int(vv), true
return int(vv), nil
}
return big.NewInt(vv), true
return big.NewInt(vv), nil
case uint64:
if vv <= math.MaxInt {
return int(vv), true
return int(vv), nil
}
return new(big.Int).SetUint64(vv), true
return new(big.Int).SetUint64(vv), nil
case float32:
return float64(vv), true
return float64(vv), nil
case float64:
return vv, true
return vv, nil
case *big.Int:
if vv.IsInt64() {
vv := vv.Int64()
if vv >= math.MinInt && vv <= math.MaxInt {
return int(vv), true
return int(vv), nil
}
}
return vv, true
return vv, nil
case string:
return vv, true
return vv, nil
case []byte:
return string(vv), true
return string(vv), nil
case []any:
vvs := make([]any, len(vv))
for i, v := range vv {
v, ok := ToGoJQValueFn(v, valueFn)
if !ok {
return nil, false
v, err := ToGoJQValueFn(v, valueFn)
if err != nil {
return nil, err
}
vvs[i] = v
}
return vvs, true
return vvs, nil
case map[string]any:
vvs := make(map[string]any, len(vv))
for k, v := range vv {
v, ok := ToGoJQValueFn(v, valueFn)
if !ok {
return nil, false
v, err := ToGoJQValueFn(v, valueFn)
if err != nil {
return nil, err
}
vvs[k] = v
}
return vvs, true
return vvs, nil
default:
if nv, ok := valueFn(vv); ok {
return ToGoJQValueFn(nv, valueFn)
nv, err := valueFn(vv)
if err != nil {
return nil, err
}
return nil, false
return ToGoJQValueFn(nv, valueFn)
}
}

View File

@ -125,14 +125,14 @@ func CastFn[T any](v any, structFn func(input any, result any) error) (T, bool)
ft := reflect.TypeOf(&t)
if ft.Elem().Kind() == reflect.Struct {
// TODO: some way to allow decode value passthru?
m, ok := ToGoJQValue(v)
if !ok {
m, err := ToGoJQValue(v)
if err != nil {
return t, false
}
if structFn == nil {
panic("structFn nil")
}
err := structFn(m, &t)
err = structFn(m, &t)
if err != nil {
return t, false
}

View File

@ -153,7 +153,7 @@ type openFile struct {
var _ Value = (*openFile)(nil)
var _ ToBinary = (*openFile)(nil)
func (of *openFile) Display(w io.Writer, opts Options) error {
func (of *openFile) Display(w io.Writer, opts *Options) error {
_, err := fmt.Fprintf(w, "<openfile %q>\n", of.filename)
return err
}
@ -405,7 +405,7 @@ func (b Binary) JQValueToGoJQ() any {
return buf.String()
}
func (b Binary) Display(w io.Writer, opts Options) error {
func (b Binary) Display(w io.Writer, opts *Options) error {
if opts.RawOutput {
br, err := b.toReader()
if err != nil {

View File

@ -164,13 +164,16 @@ func (i *Interp) _registry(c any) any {
}
func (i *Interp) _toValue(c any, om map[string]any) any {
return toValue(
func() *Options {
opts := OptionsFromValue(om)
return &opts
},
c,
)
opts, err := OptionsFromValue(om)
if err != nil {
return err
}
v, err := toValue(func() (*Options, error) { return opts, nil }, c)
if err != nil {
return err
}
return v
}
type decodeOpts struct {
@ -310,21 +313,20 @@ func valueHas(key any, a func(name string) any, b func(key any) any) any {
// TODO: make more efficient somehow? shallow values but might be hard
// when things like tovalue.key should behave like a jq value and not a decode value etc
func toValue(optsFn func() *Options, v any) any {
nv, _ := gojqex.ToGoJQValueFn(v, func(v any) (any, bool) {
func toValue(optsFn func() (*Options, error), v any) (any, error) {
return gojqex.ToGoJQValueFn(v, func(v any) (any, error) {
switch v := v.(type) {
case JQValueEx:
if optsFn == nil {
return v.JQValueToGoJQ(), true
return v.JQValueToGoJQ(), nil
}
return v.JQValueToGoJQEx(optsFn), true
return v.JQValueToGoJQEx(optsFn), nil
case gojq.JQValue:
return v.JQValueToGoJQ(), true
return v.JQValueToGoJQ(), nil
default:
return v, true
return v, nil
}
})
return nv
}
type decodeValueKind int
@ -453,7 +455,7 @@ func (dvb decodeValueBase) DecodeValue() *decode.Value {
return dvb.dv
}
func (dvb decodeValueBase) Display(w io.Writer, opts Options) error { return dump(dvb.dv, w, opts) }
func (dvb decodeValueBase) Display(w io.Writer, opts *Options) error { return dump(dvb.dv, w, opts) }
func (dvb decodeValueBase) ToBinary() (Binary, error) {
return Binary{br: dvb.dv.RootReader, r: dvb.dv.InnerRange(), unit: 8}, nil
}
@ -606,7 +608,7 @@ func (v decodeValue) JQValueKey(name string) any {
func (v decodeValue) JQValueHas(key any) any {
return valueHas(key, v.decodeValueBase.JQValueKey, v.JQValue.JQValueHas)
}
func (v decodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
func (v decodeValue) JQValueToGoJQEx(optsFn func() (*Options, error)) any {
if !v.bitsFormat {
return v.JQValueToGoJQ()
}
@ -625,7 +627,12 @@ func (v decodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
return err
}
s, err := optsFn().BitsFormatFn(brC)
opts, err := optsFn()
if err != nil {
return err
}
s, err := opts.BitsFormatFn(brC)
if err != nil {
return err
}
@ -695,8 +702,11 @@ func (v ArrayDecodeValue) JQValueHas(key any) any {
return intKey >= 0 && intKey < len(v.Compound.Children)
})
}
func (v ArrayDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
opts := optsFn()
func (v ArrayDecodeValue) JQValueToGoJQEx(optsFn func() (*Options, error)) any {
opts, err := optsFn()
if err != nil {
return err
}
vs := make([]any, 0, len(v.Compound.Children))
for _, f := range v.Compound.Children {
@ -713,7 +723,7 @@ func (v ArrayDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
return vs
}
func (v ArrayDecodeValue) JQValueToGoJQ() any {
return v.JQValueToGoJQEx(func() *Options { return &Options{} })
return v.JQValueToGoJQEx(func() (*Options, error) { return &Options{}, nil })
}
// decode value struct
@ -780,8 +790,11 @@ func (v StructDecodeValue) JQValueHas(key any) any {
},
)
}
func (v StructDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
opts := optsFn()
func (v StructDecodeValue) JQValueToGoJQEx(optsFn func() (*Options, error)) any {
opts, err := optsFn()
if err != nil {
return err
}
vm := make(map[string]any, len(v.Compound.Children))
for _, f := range v.Compound.Children {
@ -797,5 +810,5 @@ func (v StructDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
return vm
}
func (v StructDecodeValue) JQValueToGoJQ() any {
return v.JQValueToGoJQEx(func() *Options { return &Options{} })
return v.JQValueToGoJQEx(func() (*Options, error) { return &Options{}, nil })
}

View File

@ -46,7 +46,7 @@ func isCompound(v *decode.Value) bool {
}
type dumpCtx struct {
opts Options
opts *Options
buf []byte
cw *columnwriter.Writer
hexHeader string
@ -336,7 +336,7 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD
return nil
}
func dump(v *decode.Value, w io.Writer, opts Options) error {
func dump(v *decode.Value, w io.Writer, opts *Options) error {
maxAddrIndentWidth := 0
makeWalkFn := func(fn decode.WalkFn) decode.WalkFn {
return func(v *decode.Value, rootV *decode.Value, depth int, rootDepth int) error {
@ -409,7 +409,7 @@ func dump(v *decode.Value, w io.Writer, opts Options) error {
}))
}
func hexdump(w io.Writer, bv Binary, opts Options) error {
func hexdump(w io.Writer, bv Binary, opts *Options) error {
br, err := bitioex.Range(bv.br, bv.r.Start, bv.r.Len)
if err != nil {
return err

View File

@ -205,12 +205,12 @@ type Value interface {
}
type Display interface {
Display(w io.Writer, opts Options) error
Display(w io.Writer, opts *Options) error
}
type JQValueEx interface {
gojq.JQValue
JQValueToGoJQEx(optsFn func() *Options) any
JQValueToGoJQEx(optsFn func() (*Options, error)) any
}
func valuePath(v *decode.Value) []any {
@ -640,7 +640,10 @@ func (i *Interp) history(c any) any {
}
func (i *Interp) _display(c any, v any) gojq.Iter {
opts := OptionsFromValue(v)
opts, err := OptionsFromValue(v)
if err != nil {
return gojq.NewIter(err)
}
switch v := c.(type) {
case Display:
@ -659,7 +662,11 @@ func (i *Interp) _canDisplay(c any) any {
}
func (i *Interp) _hexdump(c any, v any) gojq.Iter {
opts := OptionsFromValue(v)
opts, err := OptionsFromValue(v)
if err != nil {
return gojq.NewIter(err)
}
bv, err := toBinary(c)
if err != nil {
return gojq.NewIter(err)
@ -672,17 +679,22 @@ func (i *Interp) _hexdump(c any, v any) gojq.Iter {
}
func (i *Interp) _printColorJSON(c any, v any) gojq.Iter {
opts := OptionsFromValue(v)
opts, err := OptionsFromValue(v)
if err != nil {
return gojq.NewIter(err)
}
indent := 2
if opts.Compact {
indent = 0
}
cj := colorjson.NewEncoder(colorjson.Options{
Color: opts.Color,
Tab: false,
Indent: indent,
ValueFn: func(v any) any { return toValue(func() *Options { return &opts }, v) },
Color: opts.Color,
Tab: false,
Indent: indent,
// uses a function to cache OptionsFromValue
ValueFn: func(v any) (any, error) { return toValue(func() (*Options, error) { return opts, nil }, v) },
Colors: colorjson.Colors{
Reset: []byte(ansi.Reset.SetString),
Null: []byte(opts.Decorator.Null.SetString),
@ -1040,7 +1052,7 @@ type Options struct {
BitsFormatFn func(br bitio.ReaderAtSeeker) (any, error)
}
func OptionsFromValue(v any) Options {
func OptionsFromValue(v any) (*Options, error) {
var opts Options
_ = mapstruct.ToStruct(v, &opts)
opts.ArrayTruncate = mathex.Max(0, opts.ArrayTruncate)
@ -1050,12 +1062,16 @@ func OptionsFromValue(v any) Options {
opts.LineBytes = mathex.Max(0, opts.LineBytes)
opts.DisplayBytes = mathex.Max(0, opts.DisplayBytes)
opts.Decorator = decoratorFromOptions(opts)
opts.BitsFormatFn = bitsFormatFnFromOptions(opts)
if fn, err := bitsFormatFnFromOptions(opts); err != nil {
return nil, err
} else {
opts.BitsFormatFn = fn
}
return opts
return &opts, nil
}
func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, error) {
func bitsFormatFnFromOptions(opts Options) (func(br bitio.ReaderAtSeeker) (any, error), error) {
switch opts.BitsFormat {
case "md5":
return func(br bitio.ReaderAtSeeker) (any, error) {
@ -1064,7 +1080,16 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
return "", err
}
return hex.EncodeToString(d.Sum(nil)), nil
}
}, nil
case "hex":
return func(br bitio.ReaderAtSeeker) (any, error) {
b := &bytes.Buffer{}
e := hex.NewEncoder(b)
if _, err := bitioex.CopyBits(e, br); err != nil {
return "", err
}
return b.String(), nil
}, nil
case "base64":
return func(br bitio.ReaderAtSeeker) (any, error) {
b := &bytes.Buffer{}
@ -1074,7 +1099,7 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
}
e.Close()
return b.String(), nil
}
}, nil
case "truncate":
// TODO: configure
return func(br bitio.ReaderAtSeeker) (any, error) {
@ -1083,7 +1108,7 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
return "", err
}
return b.String(), nil
}
}, nil
case "string":
return func(br bitio.ReaderAtSeeker) (any, error) {
b := &bytes.Buffer{}
@ -1091,10 +1116,8 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
return "", err
}
return b.String(), nil
}
}, nil
case "snippet":
fallthrough
default:
return func(br bitio.ReaderAtSeeker) (any, error) {
b := &bytes.Buffer{}
e := base64.NewEncoder(base64.StdEncoding, b)
@ -1102,14 +1125,15 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
return "", err
}
e.Close()
brLen, err := bitioex.Len(br)
if err != nil {
return nil, err
}
return fmt.Sprintf("<%s>%s", mathex.Bits(brLen).StringByteBits(opts.Sizebase), b.String()), nil
}
}, nil
default:
return nil, fmt.Errorf("invalid bits format %q", opts.BitsFormat)
}
}

View File

@ -35,9 +35,10 @@ $ fq -d mp3 '.frames[]._bits[0:12] | tonumber' test.mp3
4095
$ fq -d mp3 '.headers[0].header.magic._bits[0:24] | tostring' test.mp3
"ID3"
$ fq -d mp3 '.frames[0].audio_data | ("", "md5", "base64", "snippet") as $f | tovalue({bits_format: $f})' test.mp3
"<5>AAAAAAA="
$ fq -d mp3 '.frames[0].audio_data | ("", "md5", "hex", "base64", "snippet") as $f | try tovalue({bits_format: $f}) catch .' test.mp3
"invalid bits format \"\""
"ca9c491ac66b2c62500882e93f3719a8"
"0000000000"
"AAAAAAA="
"<5>AAAAAAA="
$ fq -d mp3 -i . test.mp3