wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-11-28 03:02:55 +03:00
Code Issues Projects Releases Wiki Activity

Merge pull request #298 from wader/interp-extract-mapstruct

Browse Source 
interp: Extract to/from map/struct to own package
This commit is contained in:
Mattias Wadman 2022-06-20 21:34:16 +02:00 committed by GitHub
commit 6f92751690
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 140 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
internal/gojqextra/types.go
View File

@ -16,7 +16,7 @@ import (
// Cast gojq value to go value
//nolint: forcetypeassert, unconvert
func CastFn[T any](v any, structFn func(input map[string]any, result any) error) (T, bool) {
func CastFn[T any](v any, structFn func(input any, result any) error) (T, bool) {
var t T
switch any(t).(type) {
case bool:
@ -130,6 +130,9 @@ func CastFn[T any](v any, structFn func(input map[string]any, result any) error)
return t, false
}
if structFn == nil {
panic("structFn nil")
}
err := structFn(m, &t)
if err != nil {
return t, false

65
internal/mapstruct/mapstruct.go Normal file
View File

@ -0,0 +1,65 @@
// Package mapstruct maps struct <-> JSON using came case <-> snake case
// also set default values based on struct tags
package mapstruct
// TODO: implement own version as we don't need much?
import (
"regexp"
"strings"
"github.com/mitchellh/mapstructure"
)
var camelToSnakeRe = regexp.MustCompile(`[[:lower:]][[:upper:]]`)
// "AaaBbb" -> "aaa_bbb"
func CamelToSnake(s string) string {
return strings.ToLower(camelToSnakeRe.ReplaceAllStringFunc(s, func(s string) string {
return s[0:1] + "_" + s[1:2]
}))
}
func ToStruct(m any, v any) error {
ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
MatchName: func(mapKey, fieldName string) bool {
return CamelToSnake(fieldName) == mapKey
},
TagName: "mapstruct",
Result: v,
})
if err != nil {
return err
}
if err := ms.Decode(m); err != nil {
return err
}
return nil
}
func camelCaseMap(m map[string]any) map[string]any {
nm := map[string]any{}
for k, v := range m {
if vm, ok := v.(map[string]any); ok {
v = camelCaseMap(vm)
}
nm[CamelToSnake(k)] = v
}
return nm
}
func ToMap(v any) (map[string]any, error) {
m := map[string]any{}
ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
Result: &m,
})
if err != nil {
return nil, err
}
if err := ms.Decode(v); err != nil {
return nil, err
}
return camelCaseMap(m), nil
}

25
pkg/interp/decode.go
View File

@ -12,10 +12,10 @@ import (
"time"
"github.com/mitchellh/copystructure"
"github.com/mitchellh/mapstructure"
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/ioextra"
"github.com/wader/fq/internal/mapstruct"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/scalar"
@ -103,12 +103,12 @@ func (i *Interp) _registry(c any, a []any) any {
for i := 0; i < st.NumField(); i++ {
f := st.Field(i)
if v, ok := f.Tag.Lookup("doc"); ok {
doc[camelToSnake(f.Name)] = v
doc[mapstruct.CamelToSnake(f.Name)] = v
}
}
vf["decode_in_arg_doc"] = doc
args, err := structToMap(f.DecodeInArg)
args, err := mapstruct.ToMap(f.DecodeInArg)
if err != nil {
return err
}
@ -119,7 +119,7 @@ func (i *Interp) _registry(c any, a []any) any {
delete(args, k)
}
}
vf["decode_in_arg"] = args
vf["decode_in_arg"] = norm(args)
}
if f.Files != nil {
@ -174,13 +174,14 @@ func (i *Interp) _toValue(c any, a []any) any {
return v
}
type decodeOpts struct {
Force bool
Progress string
Remain map[string]any `mapstruct:",remain"`
}
func (i *Interp) _decode(c any, a []any) any {
var opts struct {
Force bool `mapstructure:"force"`
Progress string `mapstructure:"_progress"`
Remain map[string]any `mapstructure:",remain"`
}
_ = mapstructure.Decode(a[1], &opts)
opts, _ := gojqextra.CastFn[decodeOpts](a[1], mapstruct.ToStruct)
var filename string
@ -259,7 +260,7 @@ func (i *Interp) _decode(c any, a []any) any {
}
if len(opts.Remain) > 0 {
if err := mapToStruct(opts.Remain, &inArg); err != nil {
if err := mapstruct.ToStruct(opts.Remain, &inArg); err != nil {
// TODO: currently ignores failed struct mappings
return f.DecodeInArg, nil
}
@ -286,7 +287,7 @@ func (i *Interp) _decode(c any, a []any) any {
var formatOutMap any
if formatOut != nil {
formatOutMap, err = structToMap(formatOut)
formatOutMap, err = mapstruct.ToMap(formatOut)
if err != nil {
return err
}

2
pkg/interp/decode.jq
View File

@ -38,7 +38,7 @@ def decode($name; $decode_opts):
| _decode(
$name;
( {
_progress: (
progress: (
if $opts.decode_progress and $opts.repl and stdout_tty.is_terminal then
"_decode_progress"
else null

14
pkg/interp/dump.go
View File

@ -180,7 +180,7 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD
if opts.Verbose {
cfmt(colField, " %s (%s)",
mathextra.BitRange(innerRange).StringByteBits(opts.AddrBase), mathextra.Bits(innerRange.Len).StringByteBits(opts.SizeBase))
mathextra.BitRange(innerRange).StringByteBits(opts.Addrbase), mathextra.Bits(innerRange.Len).StringByteBits(opts.Sizebase))
}
cprint(colField, "\n")
@ -270,7 +270,7 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD
// has length and is not compound or a collapsed struct/array (max depth)
if innerRange.Len > 0 && (!isCompound(v) || (opts.Depth != 0 && opts.Depth == depth)) {
cfmt(colAddr, "%s%s\n",
rootIndent, deco.DumpAddr.F(mathextra.PadFormatInt(startLineByte, opts.AddrBase, true, addrWidth)))
rootIndent, deco.DumpAddr.F(mathextra.PadFormatInt(startLineByte, opts.Addrbase, true, addrWidth)))
vBR, err := bitioextra.Range(rootV.RootReader, startByte*8, displaySizeBits)
if err != nil {
@ -306,7 +306,7 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD
for i := int64(1); i < addrLines; i++ {
lineStartByte := startLineByte + i*int64(opts.LineBytes)
columns()
cfmt(colAddr, "%s%s\n", rootIndent, deco.DumpAddr.F(mathextra.PadFormatInt(lineStartByte, opts.AddrBase, true, addrWidth)))
cfmt(colAddr, "%s%s\n", rootIndent, deco.DumpAddr.F(mathextra.PadFormatInt(lineStartByte, opts.Addrbase, true, addrWidth)))
}
// TODO: correct? should rethink columnwriter api maybe?
lastLineStopByte := startLineByte + addrLines*int64(opts.LineBytes) - 1
@ -327,9 +327,9 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD
cprint(colHex, "\n")
// TODO: truncate if display_bytes is small?
cfmt(colHex, "until %s%s (%s)",
mathextra.Bits(stopBit).StringByteBits(opts.AddrBase),
mathextra.Bits(stopBit).StringByteBits(opts.Addrbase),
isEnd,
mathextra.PadFormatInt(bitio.BitsByteCount(sizeBits), opts.SizeBase, true, 0))
mathextra.PadFormatInt(bitio.BitsByteCount(sizeBits), opts.Sizebase, true, 0))
// TODO: dump last line?
}
}
@ -356,7 +356,7 @@ func dump(v *decode.Value, w io.Writer, opts Options) error {
_ = v.WalkPreOrder(makeWalkFn(func(v *decode.Value, rootV *decode.Value, depth int, rootDepth int) error {
maxAddrIndentWidth = mathextra.MaxInt(
maxAddrIndentWidth,
rootDepth+mathextra.DigitsInBase(bitio.BitsByteCount(v.InnerRange().Stop()), true, opts.AddrBase),
rootDepth+mathextra.DigitsInBase(bitio.BitsByteCount(v.InnerRange().Stop()), true, opts.Addrbase),
)
return nil
}))
@ -382,7 +382,7 @@ func dump(v *decode.Value, w io.Writer, opts Options) error {
var hexHeader string
var asciiHeader string
for i := 0; i < opts.LineBytes; i++ {
s := mathextra.PadFormatInt(int64(i), opts.AddrBase, false, 2)
s := mathextra.PadFormatInt(int64(i), opts.Addrbase, false, 2)
hexHeader += s
if i < opts.LineBytes-1 {
hexHeader += " "

9
pkg/interp/encoding.go
View File

@ -23,6 +23,7 @@ import (
"github.com/BurntSushi/toml"
"github.com/wader/fq/internal/colorjson"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/mapstruct"
"github.com/wader/fq/internal/proxysort"
"github.com/wader/fq/pkg/bitio"
@ -93,7 +94,7 @@ func addFunc[Tc any](name string, fn func(c Tc) any) {
func(i *Interp) []Function {
return []Function{{
name, 0, 0, func(c any, a []any) any {
cv, ok := gojqextra.CastFn[Tc](c, mapToStruct)
cv, ok := gojqextra.CastFn[Tc](c, mapstruct.ToStruct)
if !ok {
return gojqextra.FuncTypeError{Name: name[1:], V: c}
}
@ -114,11 +115,11 @@ func addFunc1[Tc any, Ta0 any](name string, fn func(c Tc, a0 Ta0) any) {
func(i *Interp) []Function {
return []Function{{
name, 1, 1, func(c any, a []any) any {
cv, ok := gojqextra.CastFn[Tc](c, mapToStruct)
cv, ok := gojqextra.CastFn[Tc](c, mapstruct.ToStruct)
if !ok {
return gojqextra.FuncTypeError{Name: name[1:], V: c}
}
a0, ok := gojqextra.CastFn[Ta0](a[0], mapToStruct)
a0, ok := gojqextra.CastFn[Ta0](a[0], mapstruct.ToStruct)
if !ok {
return gojqextra.FuncArgTypeError{Name: name[1:], ArgName: "first", V: c}
}
@ -184,7 +185,7 @@ func init() {
if v, ok := toValue(nil, v); ok {
return v
}
panic(fmt.Sprintf("toValue not a JQValue value: %#v", v))
panic(fmt.Sprintf("toValue not a JQValue value: %#v %T", v, v))
},
colorjson.Colors{},
)

160
pkg/interp/interp.go
View File

@ -14,19 +14,17 @@ import (
"io/ioutil"
"math/big"
"path"
"reflect"
"regexp"
"strconv"
"strings"
"time"
"github.com/mitchellh/mapstructure"
"github.com/wader/fq/internal/ansi"
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/internal/colorjson"
"github.com/wader/fq/internal/ctxstack"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/ioextra"
"github.com/wader/fq/internal/mapstruct"
"github.com/wader/fq/internal/mathextra"
"github.com/wader/fq/internal/pos"
"github.com/wader/fq/pkg/bitio"
@ -62,7 +60,7 @@ var functionRegisterFns []func(i *Interp) []Function
func init() {
functionRegisterFns = append(functionRegisterFns, func(i *Interp) []Function {
return []Function{
{"_readline", 0, 1, nil, i._readline},
{"_readline", 1, 1, nil, i._readline},
{"_eval", 1, 2, nil, i._eval},
{"_stdin", 0, 1, nil, i.makeStdioFn("stdin", i.os.Stdin())},
{"_stdout", 0, 0, nil, i.makeStdioFn("stdout", i.os.Stdout())},
@ -431,23 +429,26 @@ func (i *Interp) Main(ctx context.Context, output Output, versionStr string) err
return nil
}
type completionResult struct {
Names []string
Prefix string
}
type readlineOpts struct {
Prompt string
Complete string
Timeout float64
}
func (i *Interp) _readline(c any, a []any) gojq.Iter {
if i.evalInstance.isCompleting {
return gojq.NewIter()
}
var opts struct {
Promopt string `mapstructure:"prompt"`
Complete string `mapstructure:"complete"`
Timeout float64 `mapstructure:"timeout"`
}
if len(a) > 0 {
_ = mapstructure.Decode(a[0], &opts)
}
opts, _ := gojqextra.CastFn[readlineOpts](a[0], mapstruct.ToStruct)
expr, err := i.os.Readline(ReadlineOpts{
Prompt: opts.Promopt,
Prompt: opts.Prompt,
CompleteFn: func(line string, pos int) (newLine []string, shared int) {
completeCtx := i.evalInstance.ctx
if opts.Timeout > 0 {
@ -480,20 +481,14 @@ func (i *Interp) _readline(c any, a []any) gojq.Iter {
}
// {abc: 123, abd: 123} | complete(".ab"; 3) will return {prefix: "ab", names: ["abc", "abd"]}
var result struct {
Names []string `mapstructure:"names"`
Prefix string `mapstructure:"prefix"`
r, ok := gojqextra.CastFn[completionResult](v, mapstruct.ToStruct)
if !ok {
return nil, pos, fmt.Errorf("completion result not a map")
}
_ = mapstructure.Decode(v, &result)
if len(result.Names) == 0 {
return nil, pos, nil
}
sharedLen := len(r.Prefix)
sharedLen := len(result.Prefix)
return result.Names, sharedLen, nil
return r.Names, sharedLen, nil
}()
// TODO: how to report err?
@ -975,28 +970,28 @@ func (i *Interp) EvalFuncValues(ctx context.Context, c any, name string, args []
}
type Options struct {
Depth int `mapstructure:"depth"`
ArrayTruncate int `mapstructure:"array_truncate"`
Verbose bool `mapstructure:"verbose"`
Width int `mapstructure:"width"`
DecodeProgress bool `mapstructure:"decode_progress"`
Color bool `mapstructure:"color"`
Colors map[string]string `mapstructure:"colors"`
Depth int
ArrayTruncate int
Verbose bool
Width int
DecodeProgress bool
Color bool
Colors map[string]string
ByteColors []struct {
Ranges [][2]int `mapstructure:"ranges"`
Value string `mapstructure:"value"`
} `mapstructure:"byte_colors"`
Unicode bool `mapstructure:"unicode"`
RawOutput bool `mapstructure:"raw_output"`
REPL bool `mapstructure:"repl"`
RawString bool `mapstructure:"raw_string"`
JoinString string `mapstructure:"join_string"`
Compact bool `mapstructure:"compact"`
BitsFormat string `mapstructure:"bits_format"`
LineBytes int `mapstructure:"line_bytes"`
DisplayBytes int `mapstructure:"display_bytes"`
AddrBase int `mapstructure:"addrbase"`
SizeBase int `mapstructure:"sizebase"`
Ranges [][2]int
Value string
}
Unicode bool
RawOutput bool
REPL bool
RawString bool
JoinString string
Compact bool
BitsFormat string
LineBytes int
DisplayBytes int
Addrbase int
Sizebase int
Decorator Decorator
BitsFormatFn func(br bitio.ReaderAtSeeker) (any, error)
@ -1055,7 +1050,7 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
return nil, err
}
return fmt.Sprintf("<%s>%s", mathextra.Bits(brLen).StringByteBits(opts.SizeBase), b.String()), nil
return fmt.Sprintf("<%s>%s", mathextra.Bits(brLen).StringByteBits(opts.Sizebase), b.String()), nil
}
}
}
@ -1091,11 +1086,11 @@ func (i *Interp) slurps() map[string]any {
func (i *Interp) Options(v any) Options {
var opts Options
_ = mapstructure.Decode(v, &opts)
_ = mapstruct.ToStruct(v, &opts)
opts.ArrayTruncate = mathextra.MaxInt(0, opts.ArrayTruncate)
opts.Depth = mathextra.MaxInt(0, opts.Depth)
opts.AddrBase = mathextra.ClampInt(2, 36, opts.AddrBase)
opts.SizeBase = mathextra.ClampInt(2, 36, opts.SizeBase)
opts.Addrbase = mathextra.ClampInt(2, 36, opts.Addrbase)
opts.Sizebase = mathextra.ClampInt(2, 36, opts.Sizebase)
opts.LineBytes = mathextra.MaxInt(0, opts.LineBytes)
opts.DisplayBytes = mathextra.MaxInt(0, opts.DisplayBytes)
opts.Decorator = decoratorFromOptions(opts)
@ -1133,68 +1128,3 @@ func (i *Interp) NewColorJSON(opts Options) (*colorjson.Encoder, error) {
},
), nil
}
var camelToSnakeRe = regexp.MustCompile(`[[:lower:]][[:upper:]]`)
// "AaaBbb" -> "aaa_bbb"
func camelToSnake(s string) string {
return strings.ToLower(camelToSnakeRe.ReplaceAllStringFunc(s, func(s string) string {
return s[0:1] + "_" + s[1:2]
}))
}
func mapToStruct(m map[string]any, v any) error {
ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
MatchName: func(mapKey, fieldName string) bool {
return camelToSnake(fieldName) == mapKey
},
DecodeHook: func(
f reflect.Value,
t reflect.Value) (any, error) {
// log.Printf("f: %#+v -> t: %#+v\n", f, t)
return f.Interface(), nil
},
Result: v,
})
if err != nil {
return err
}
if err := ms.Decode(m); err != nil {
return err
}
return nil
}
func camelCaseMap(m map[string]any) map[string]any {
nm := map[string]any{}
for k, v := range m {
sk := camelToSnake(k)
if vm, ok := v.(map[string]any); ok {
v = camelCaseMap(vm)
} else {
// TODO: error
v, _ = gojqextra.ToGoJQValue(v)
}
nm[sk] = v
}
return nm
}
func structToMap(v any) (map[string]any, error) {
m := map[string]any{}
ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
Result: &m,
})
if err != nil {
return nil, err
}
if err := ms.Decode(v); err != nil {
return nil, err
}
return camelCaseMap(m), nil
}