interp: Extract to/from map/struct to own package

internal/gojqextra/types.go

@@ -16,7 +16,7 @@ import (
 
 // Cast gojq value to go value
 //nolint: forcetypeassert, unconvert
-func CastFn[T any](v any, structFn func(input map[string]any, result any) error) (T, bool) {
+func CastFn[T any](v any, structFn func(input any, result any) error) (T, bool) {
 	var t T
 	switch any(t).(type) {
 	case bool:
@@ -130,6 +130,9 @@ func CastFn[T any](v any, structFn func(input map[string]any, result any) error)
 				return t, false
 			}
 
+			if structFn == nil {
+				panic("structFn nil")
+			}
 			err := structFn(m, &t)
 			if err != nil {
 				return t, false

internal/mapstruct/mapstruct.go

@@ -0,0 +1,65 @@
+// Package mapstruct maps struct <-> JSON using came case <-> snake case
+// also set default values based on struct tags
+package mapstruct
+
+// TODO: implement own version as we don't need much?
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/mitchellh/mapstructure"
+)
+
+var camelToSnakeRe = regexp.MustCompile(`[[:lower:]][[:upper:]]`)
+
+// "AaaBbb" -> "aaa_bbb"
+func CamelToSnake(s string) string {
+	return strings.ToLower(camelToSnakeRe.ReplaceAllStringFunc(s, func(s string) string {
+		return s[0:1] + "_" + s[1:2]
+	}))
+}
+
+func ToStruct(m any, v any) error {
+	ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
+		MatchName: func(mapKey, fieldName string) bool {
+			return CamelToSnake(fieldName) == mapKey
+		},
+		TagName: "mapstruct",
+		Result:  v,
+	})
+	if err != nil {
+		return err
+	}
+	if err := ms.Decode(m); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func camelCaseMap(m map[string]any) map[string]any {
+	nm := map[string]any{}
+	for k, v := range m {
+		if vm, ok := v.(map[string]any); ok {
+			v = camelCaseMap(vm)
+		}
+		nm[CamelToSnake(k)] = v
+	}
+	return nm
+}
+
+func ToMap(v any) (map[string]any, error) {
+	m := map[string]any{}
+	ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
+		Result: &m,
+	})
+	if err != nil {
+		return nil, err
+	}
+	if err := ms.Decode(v); err != nil {
+		return nil, err
+	}
+
+	return camelCaseMap(m), nil
+}

pkg/interp/decode.go

@@ -12,10 +12,10 @@ import (
 	"time"
 
 	"github.com/mitchellh/copystructure"
-	"github.com/mitchellh/mapstructure"
 	"github.com/wader/fq/internal/bitioextra"
 	"github.com/wader/fq/internal/gojqextra"
 	"github.com/wader/fq/internal/ioextra"
+	"github.com/wader/fq/internal/mapstruct"
 	"github.com/wader/fq/pkg/bitio"
 	"github.com/wader/fq/pkg/decode"
 	"github.com/wader/fq/pkg/scalar"
@@ -103,12 +103,12 @@ func (i *Interp) _registry(c any, a []any) any {
 			for i := 0; i < st.NumField(); i++ {
 				f := st.Field(i)
 				if v, ok := f.Tag.Lookup("doc"); ok {
-					doc[camelToSnake(f.Name)] = v
+					doc[mapstruct.CamelToSnake(f.Name)] = v
 				}
 			}
 			vf["decode_in_arg_doc"] = doc
 
-			args, err := structToMap(f.DecodeInArg)
+			args, err := mapstruct.ToMap(f.DecodeInArg)
 			if err != nil {
 				return err
 			}
@@ -119,7 +119,7 @@ func (i *Interp) _registry(c any, a []any) any {
 					delete(args, k)
 				}
 			}
-			vf["decode_in_arg"] = args
+			vf["decode_in_arg"] = norm(args)
 		}
 
 		if f.Files != nil {
@@ -174,13 +174,14 @@ func (i *Interp) _toValue(c any, a []any) any {
 	return v
 }
 
+type decodeOpts struct {
+	Force    bool
+	Progress string
+	Remain   map[string]any `mapstruct:",remain"`
+}
+
 func (i *Interp) _decode(c any, a []any) any {
-	var opts struct {
+	opts, _ := gojqextra.CastFn[decodeOpts](a[1], mapstruct.ToStruct)
-		Force    bool           `mapstructure:"force"`
-		Progress string         `mapstructure:"_progress"`
-		Remain   map[string]any `mapstructure:",remain"`
-	}
-	_ = mapstructure.Decode(a[1], &opts)
 
 	var filename string
 
@@ -259,7 +260,7 @@ func (i *Interp) _decode(c any, a []any) any {
 				}
 
 				if len(opts.Remain) > 0 {
-					if err := mapToStruct(opts.Remain, &inArg); err != nil {
+					if err := mapstruct.ToStruct(opts.Remain, &inArg); err != nil {
 						// TODO: currently ignores failed struct mappings
 						return f.DecodeInArg, nil
 					}
@@ -286,7 +287,7 @@ func (i *Interp) _decode(c any, a []any) any {
 	var formatOutMap any
 
 	if formatOut != nil {
-		formatOutMap, err = structToMap(formatOut)
+		formatOutMap, err = mapstruct.ToMap(formatOut)
 		if err != nil {
 			return err
 		}

pkg/interp/decode.jq

@@ -38,7 +38,7 @@ def decode($name; $decode_opts):
   | _decode(
       $name;
       ( {
-          _progress: (
+          progress: (
             if $opts.decode_progress and $opts.repl and stdout_tty.is_terminal then
               "_decode_progress"
             else null

pkg/interp/dump.go

@@ -180,7 +180,7 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD
 
 	if opts.Verbose {
 		cfmt(colField, " %s (%s)",
-			mathextra.BitRange(innerRange).StringByteBits(opts.AddrBase), mathextra.Bits(innerRange.Len).StringByteBits(opts.SizeBase))
+			mathextra.BitRange(innerRange).StringByteBits(opts.Addrbase), mathextra.Bits(innerRange.Len).StringByteBits(opts.Sizebase))
 	}
 
 	cprint(colField, "\n")
@@ -270,7 +270,7 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD
 	// has length and is not compound or a collapsed struct/array (max depth)
 	if innerRange.Len > 0 && (!isCompound(v) || (opts.Depth != 0 && opts.Depth == depth)) {
 		cfmt(colAddr, "%s%s\n",
-			rootIndent, deco.DumpAddr.F(mathextra.PadFormatInt(startLineByte, opts.AddrBase, true, addrWidth)))
+			rootIndent, deco.DumpAddr.F(mathextra.PadFormatInt(startLineByte, opts.Addrbase, true, addrWidth)))
 
 		vBR, err := bitioextra.Range(rootV.RootReader, startByte*8, displaySizeBits)
 		if err != nil {
@@ -306,7 +306,7 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD
 		for i := int64(1); i < addrLines; i++ {
 			lineStartByte := startLineByte + i*int64(opts.LineBytes)
 			columns()
-			cfmt(colAddr, "%s%s\n", rootIndent, deco.DumpAddr.F(mathextra.PadFormatInt(lineStartByte, opts.AddrBase, true, addrWidth)))
+			cfmt(colAddr, "%s%s\n", rootIndent, deco.DumpAddr.F(mathextra.PadFormatInt(lineStartByte, opts.Addrbase, true, addrWidth)))
 		}
 		// TODO: correct? should rethink columnwriter api maybe?
 		lastLineStopByte := startLineByte + addrLines*int64(opts.LineBytes) - 1
@@ -327,9 +327,9 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD
 			cprint(colHex, "\n")
 			// TODO: truncate if display_bytes is small?
 			cfmt(colHex, "until %s%s (%s)",
-				mathextra.Bits(stopBit).StringByteBits(opts.AddrBase),
+				mathextra.Bits(stopBit).StringByteBits(opts.Addrbase),
 				isEnd,
-				mathextra.PadFormatInt(bitio.BitsByteCount(sizeBits), opts.SizeBase, true, 0))
+				mathextra.PadFormatInt(bitio.BitsByteCount(sizeBits), opts.Sizebase, true, 0))
 			// TODO: dump last line?
 		}
 	}
@@ -356,7 +356,7 @@ func dump(v *decode.Value, w io.Writer, opts Options) error {
 	_ = v.WalkPreOrder(makeWalkFn(func(v *decode.Value, rootV *decode.Value, depth int, rootDepth int) error {
 		maxAddrIndentWidth = mathextra.MaxInt(
 			maxAddrIndentWidth,
-			rootDepth+mathextra.DigitsInBase(bitio.BitsByteCount(v.InnerRange().Stop()), true, opts.AddrBase),
+			rootDepth+mathextra.DigitsInBase(bitio.BitsByteCount(v.InnerRange().Stop()), true, opts.Addrbase),
 		)
 		return nil
 	}))
@@ -382,7 +382,7 @@ func dump(v *decode.Value, w io.Writer, opts Options) error {
 	var hexHeader string
 	var asciiHeader string
 	for i := 0; i < opts.LineBytes; i++ {
-		s := mathextra.PadFormatInt(int64(i), opts.AddrBase, false, 2)
+		s := mathextra.PadFormatInt(int64(i), opts.Addrbase, false, 2)
 		hexHeader += s
 		if i < opts.LineBytes-1 {
 			hexHeader += " "

pkg/interp/encoding.go

@@ -23,6 +23,7 @@ import (
 	"github.com/BurntSushi/toml"
 	"github.com/wader/fq/internal/colorjson"
 	"github.com/wader/fq/internal/gojqextra"
+	"github.com/wader/fq/internal/mapstruct"
 	"github.com/wader/fq/internal/proxysort"
 	"github.com/wader/fq/pkg/bitio"
 
@@ -93,7 +94,7 @@ func addFunc[Tc any](name string, fn func(c Tc) any) {
 		func(i *Interp) []Function {
 			return []Function{{
 				name, 0, 0, func(c any, a []any) any {
-					cv, ok := gojqextra.CastFn[Tc](c, mapToStruct)
+					cv, ok := gojqextra.CastFn[Tc](c, mapstruct.ToStruct)
 					if !ok {
 						return gojqextra.FuncTypeError{Name: name[1:], V: c}
 					}
@@ -114,11 +115,11 @@ func addFunc1[Tc any, Ta0 any](name string, fn func(c Tc, a0 Ta0) any) {
 		func(i *Interp) []Function {
 			return []Function{{
 				name, 1, 1, func(c any, a []any) any {
-					cv, ok := gojqextra.CastFn[Tc](c, mapToStruct)
+					cv, ok := gojqextra.CastFn[Tc](c, mapstruct.ToStruct)
 					if !ok {
 						return gojqextra.FuncTypeError{Name: name[1:], V: c}
 					}
-					a0, ok := gojqextra.CastFn[Ta0](a[0], mapToStruct)
+					a0, ok := gojqextra.CastFn[Ta0](a[0], mapstruct.ToStruct)
 					if !ok {
 						return gojqextra.FuncArgTypeError{Name: name[1:], ArgName: "first", V: c}
 					}
@@ -184,7 +185,7 @@ func init() {
 				if v, ok := toValue(nil, v); ok {
 					return v
 				}
-				panic(fmt.Sprintf("toValue not a JQValue value: %#v", v))
+				panic(fmt.Sprintf("toValue not a JQValue value: %#v %T", v, v))
 			},
 			colorjson.Colors{},
 		)

pkg/interp/interp.go

@@ -14,19 +14,17 @@ import (
 	"io/ioutil"
 	"math/big"
 	"path"
-	"reflect"
-	"regexp"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/mitchellh/mapstructure"
 	"github.com/wader/fq/internal/ansi"
 	"github.com/wader/fq/internal/bitioextra"
 	"github.com/wader/fq/internal/colorjson"
 	"github.com/wader/fq/internal/ctxstack"
 	"github.com/wader/fq/internal/gojqextra"
 	"github.com/wader/fq/internal/ioextra"
+	"github.com/wader/fq/internal/mapstruct"
 	"github.com/wader/fq/internal/mathextra"
 	"github.com/wader/fq/internal/pos"
 	"github.com/wader/fq/pkg/bitio"
@@ -62,7 +60,7 @@ var functionRegisterFns []func(i *Interp) []Function
 func init() {
 	functionRegisterFns = append(functionRegisterFns, func(i *Interp) []Function {
 		return []Function{
-			{"_readline", 0, 1, nil, i._readline},
+			{"_readline", 1, 1, nil, i._readline},
 			{"_eval", 1, 2, nil, i._eval},
 			{"_stdin", 0, 1, nil, i.makeStdioFn("stdin", i.os.Stdin())},
 			{"_stdout", 0, 0, nil, i.makeStdioFn("stdout", i.os.Stdout())},
@@ -431,23 +429,26 @@ func (i *Interp) Main(ctx context.Context, output Output, versionStr string) err
 	return nil
 }
 
+type completionResult struct {
+	Names  []string
+	Prefix string
+}
+
+type readlineOpts struct {
+	Prompt   string
+	Complete string
+	Timeout  float64
+}
+
 func (i *Interp) _readline(c any, a []any) gojq.Iter {
 	if i.evalInstance.isCompleting {
 		return gojq.NewIter()
 	}
 
-	var opts struct {
+	opts, _ := gojqextra.CastFn[readlineOpts](a[0], mapstruct.ToStruct)
-		Promopt  string  `mapstructure:"prompt"`
-		Complete string  `mapstructure:"complete"`
-		Timeout  float64 `mapstructure:"timeout"`
-	}
-
-	if len(a) > 0 {
-		_ = mapstructure.Decode(a[0], &opts)
-	}
 
 	expr, err := i.os.Readline(ReadlineOpts{
-		Prompt: opts.Promopt,
+		Prompt: opts.Prompt,
 		CompleteFn: func(line string, pos int) (newLine []string, shared int) {
 			completeCtx := i.evalInstance.ctx
 			if opts.Timeout > 0 {
@@ -480,20 +481,14 @@ func (i *Interp) _readline(c any, a []any) gojq.Iter {
 				}
 
 				// {abc: 123, abd: 123} | complete(".ab"; 3) will return {prefix: "ab", names: ["abc", "abd"]}
-
+				r, ok := gojqextra.CastFn[completionResult](v, mapstruct.ToStruct)
-				var result struct {
+				if !ok {
-					Names  []string `mapstructure:"names"`
+					return nil, pos, fmt.Errorf("completion result not a map")
-					Prefix string   `mapstructure:"prefix"`
 				}
 
-				_ = mapstructure.Decode(v, &result)
+				sharedLen := len(r.Prefix)
-				if len(result.Names) == 0 {
-					return nil, pos, nil
-				}
 
-				sharedLen := len(result.Prefix)
+				return r.Names, sharedLen, nil
-
-				return result.Names, sharedLen, nil
 			}()
 
 			// TODO: how to report err?
@@ -975,28 +970,28 @@ func (i *Interp) EvalFuncValues(ctx context.Context, c any, name string, args []
 }
 
 type Options struct {
-	Depth          int               `mapstructure:"depth"`
+	Depth          int
-	ArrayTruncate  int               `mapstructure:"array_truncate"`
+	ArrayTruncate  int
-	Verbose        bool              `mapstructure:"verbose"`
+	Verbose        bool
-	Width          int               `mapstructure:"width"`
+	Width          int
-	DecodeProgress bool              `mapstructure:"decode_progress"`
+	DecodeProgress bool
-	Color          bool              `mapstructure:"color"`
+	Color          bool
-	Colors         map[string]string `mapstructure:"colors"`
+	Colors         map[string]string
 	ByteColors     []struct {
-		Ranges [][2]int `mapstructure:"ranges"`
+		Ranges [][2]int
-		Value  string   `mapstructure:"value"`
+		Value  string
-	} `mapstructure:"byte_colors"`
+	}
-	Unicode      bool   `mapstructure:"unicode"`
+	Unicode      bool
-	RawOutput    bool   `mapstructure:"raw_output"`
+	RawOutput    bool
-	REPL         bool   `mapstructure:"repl"`
+	REPL         bool
-	RawString    bool   `mapstructure:"raw_string"`
+	RawString    bool
-	JoinString   string `mapstructure:"join_string"`
+	JoinString   string
-	Compact      bool   `mapstructure:"compact"`
+	Compact      bool
-	BitsFormat   string `mapstructure:"bits_format"`
+	BitsFormat   string
-	LineBytes    int    `mapstructure:"line_bytes"`
+	LineBytes    int
-	DisplayBytes int    `mapstructure:"display_bytes"`
+	DisplayBytes int
-	AddrBase     int    `mapstructure:"addrbase"`
+	Addrbase     int
-	SizeBase     int    `mapstructure:"sizebase"`
+	Sizebase     int
 
 	Decorator    Decorator
 	BitsFormatFn func(br bitio.ReaderAtSeeker) (any, error)
@@ -1055,7 +1050,7 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
 				return nil, err
 			}
 
-			return fmt.Sprintf("<%s>%s", mathextra.Bits(brLen).StringByteBits(opts.SizeBase), b.String()), nil
+			return fmt.Sprintf("<%s>%s", mathextra.Bits(brLen).StringByteBits(opts.Sizebase), b.String()), nil
 		}
 	}
 }
@@ -1091,11 +1086,11 @@ func (i *Interp) slurps() map[string]any {
 
 func (i *Interp) Options(v any) Options {
 	var opts Options
-	_ = mapstructure.Decode(v, &opts)
+	_ = mapstruct.ToStruct(v, &opts)
 	opts.ArrayTruncate = mathextra.MaxInt(0, opts.ArrayTruncate)
 	opts.Depth = mathextra.MaxInt(0, opts.Depth)
-	opts.AddrBase = mathextra.ClampInt(2, 36, opts.AddrBase)
+	opts.Addrbase = mathextra.ClampInt(2, 36, opts.Addrbase)
-	opts.SizeBase = mathextra.ClampInt(2, 36, opts.SizeBase)
+	opts.Sizebase = mathextra.ClampInt(2, 36, opts.Sizebase)
 	opts.LineBytes = mathextra.MaxInt(0, opts.LineBytes)
 	opts.DisplayBytes = mathextra.MaxInt(0, opts.DisplayBytes)
 	opts.Decorator = decoratorFromOptions(opts)
@@ -1133,68 +1128,3 @@ func (i *Interp) NewColorJSON(opts Options) (*colorjson.Encoder, error) {
 		},
 	), nil
 }
-
-var camelToSnakeRe = regexp.MustCompile(`[[:lower:]][[:upper:]]`)
-
-// "AaaBbb" -> "aaa_bbb"
-func camelToSnake(s string) string {
-	return strings.ToLower(camelToSnakeRe.ReplaceAllStringFunc(s, func(s string) string {
-		return s[0:1] + "_" + s[1:2]
-	}))
-}
-func mapToStruct(m map[string]any, v any) error {
-	ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
-		MatchName: func(mapKey, fieldName string) bool {
-			return camelToSnake(fieldName) == mapKey
-		},
-		DecodeHook: func(
-			f reflect.Value,
-			t reflect.Value) (any, error) {
-
-			// log.Printf("f: %#+v -> t: %#+v\n", f, t)
-
-			return f.Interface(), nil
-		},
-		Result: v,
-	})
-	if err != nil {
-		return err
-	}
-
-	if err := ms.Decode(m); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func camelCaseMap(m map[string]any) map[string]any {
-	nm := map[string]any{}
-	for k, v := range m {
-		sk := camelToSnake(k)
-		if vm, ok := v.(map[string]any); ok {
-			v = camelCaseMap(vm)
-		} else {
-			// TODO: error
-			v, _ = gojqextra.ToGoJQValue(v)
-		}
-		nm[sk] = v
-	}
-
-	return nm
-}
-
-func structToMap(v any) (map[string]any, error) {
-	m := map[string]any{}
-	ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
-		Result: &m,
-	})
-	if err != nil {
-		return nil, err
-	}
-	if err := ms.Decode(v); err != nil {
-		return nil, err
-	}
-
-	return camelCaseMap(m), nil
-}