From 528e6b91abc4798ac378c5e9e3c5ad991a0f2ee5 Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Wed, 22 Sep 2021 01:35:27 +0200 Subject: [PATCH] interp: Refactor and use mapstructure --- go.mod | 3 + go.sum | 2 + pkg/interp/funcs.go | 78 ++++++++------------ pkg/interp/interp.go | 169 +++++++++---------------------------------- pkg/interp/interp.jq | 2 +- 5 files changed, 72 insertions(+), 182 deletions(-) diff --git a/go.mod b/go.mod index 980ff697..4ea52821 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,9 @@ module github.com/wader/fq go 1.17 require ( + // bump: mapstructure /github.com\/mitchellh\/mapstructure v(.*)/ git://github.com/mitchellh/mapstructure|^1 + // bump: mapstructure command go get -d github.com/mitchellh/mapstructure@v$LATEST && go mod tidy + github.com/mitchellh/mapstructure v1.4.2 // bump: go-difflib /github.com\/pmezard\/go-difflib v(.*)/ git://github.com/pmezard/go-difflib|^1 // bump: go-difflib command go get -d github.com/pmezard/go-difflib@v$LATEST && go mod tidy github.com/pmezard/go-difflib v1.0.0 diff --git a/go.sum b/go.sum index 8c5c76fe..b4dac14d 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921i github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= +github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354/go.mod h1:KSVJerMDfblTH7p5MZaTt+8zaT2iEk3AkVb9PQdZuE8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/pkg/interp/funcs.go b/pkg/interp/funcs.go index 9a0620bc..52d5e564 100644 --- a/pkg/interp/funcs.go +++ b/pkg/interp/funcs.go @@ -20,6 +20,7 @@ import ( "net/url" "time" + "github.com/mitchellh/mapstructure" "github.com/wader/fq/internal/aheadreadseeker" "github.com/wader/fq/internal/ctxreadseeker" "github.com/wader/fq/internal/gojqextra" @@ -196,8 +197,12 @@ func makeHashFn(fn func() (hash.Hash, error)) func(c interface{}, a []interface{ } func (i *Interp) readline(c interface{}, a []interface{}) interface{} { + var opts struct { + Complete string `mapstructure:"complete"` + Timeout float64 `mapstructure:"timeout"` + } + var err error - completeFn := "" prompt := "" if len(a) > 0 { @@ -207,21 +212,22 @@ func (i *Interp) readline(c interface{}, a []interface{}) interface{} { } } if len(a) > 1 { - completeFn, err = toString(a[1]) - if err != nil { - return fmt.Errorf("complete function: %w", err) - } + _ = mapstructure.Decode(a[1], &opts) } src, err := i.os.Readline( prompt, func(line string, pos int) (newLine []string, shared int) { - completeCtx, completeCtxCancelFn := context.WithTimeout(i.evalContext.ctx, 1*time.Second) - defer completeCtxCancelFn() + completeCtx := i.evalContext.ctx + if opts.Timeout > 0 { + var completeCtxCancelFn context.CancelFunc + completeCtx, completeCtxCancelFn = context.WithTimeout(i.evalContext.ctx, time.Duration(opts.Timeout*float64(time.Second))) + defer completeCtxCancelFn() + } names, shared, err := func() (newLine []string, shared int, err error) { vs, err := i.EvalFuncValues( - completeCtx, CompletionMode, c, completeFn, []interface{}{line, pos}, DiscardCtxWriter{Ctx: completeCtx}, + completeCtx, c, opts.Complete, []interface{}{line, pos}, DiscardCtxWriter{Ctx: completeCtx}, ) if err != nil { return nil, pos, err @@ -236,32 +242,19 @@ func (i *Interp) readline(c interface{}, a []interface{}) interface{} { // {abc: 123, abd: 123} | complete(".ab"; 3) will return {prefix: "ab", names: ["abc", "abd"]} - var names []string - var prefix string - cm, ok := v.(map[string]interface{}) - if !ok { - return nil, pos, fmt.Errorf("%v: complete function return value not an object", cm) - } - if namesV, ok := cm["names"].([]interface{}); ok { - for _, name := range namesV { - names = append(names, name.(string)) - } - } else { - return nil, pos, fmt.Errorf("%v: names missing in complete return object", cm) - } - if prefixV, ok := cm["prefix"]; ok { - prefix, _ = prefixV.(string) - } else { - return nil, pos, fmt.Errorf("%v: prefix missing in complete return object", cm) + var result struct { + Names []string `mapstructure:"names"` + Prefix string `mapstructure:"prefix"` } - if len(names) == 0 { + _ = mapstructure.Decode(v, &result) + if len(result.Names) == 0 { return nil, pos, nil } - sharedLen := len(prefix) + sharedLen := len(result.Prefix) - return names, sharedLen, nil + return result.Names, sharedLen, nil }() // TODO: how to report err? @@ -296,7 +289,7 @@ func (i *Interp) eval(c interface{}, a []interface{}) gojq.Iter { } } - iter, err := i.Eval(i.evalContext.ctx, ScriptMode, c, src, filenameHint, i.evalContext.output) + iter, err := i.Eval(i.evalContext.ctx, c, src, filenameHint, i.evalContext.output) if err != nil { return gojq.NewIter(err) } @@ -585,33 +578,26 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} { } func (i *Interp) _decode(c interface{}, a []interface{}) interface{} { - filename := "" - - opts := map[string]interface{}{} - if m, ok := a[1].(map[string]interface{}); ok { - opts = m - } - - // TODO: structmap - var progress string - if progressV, ok := opts["_progress"]; ok { - progress, _ = progressV.(string) + var opts struct { + Filename string `mapstructure:"filename"` + Progress string `mapstructure:"_progress"` + Remain map[string]interface{} `mapstructure:",remain"` } + _ = mapstructure.Decode(a[1], &opts) // TODO: progress hack // would be nice to move all progress code into decode but it might be // tricky to keep track of absolute positions in the underlaying readers // when it uses BitBuf slices, maybe only in Pos()? if bbf, ok := c.(*bitBufFile); ok { - filename = bbf.filename + opts.Filename = bbf.filename - if progress != "" { + if opts.Progress != "" { evalProgress := func(c interface{}) { _, _ = i.EvalFuncValues( i.evalContext.ctx, - CompletionMode, c, - progress, + opts.Progress, nil, DiscardCtxWriter{Ctx: i.evalContext.ctx}, ) @@ -644,8 +630,8 @@ func (i *Interp) _decode(c interface{}, a []interface{}) interface{} { dv, _, err := decode.Decode(i.evalContext.ctx, bb, decodeFormats, decode.DecodeOptions{ - Description: filename, - FormatOptions: opts, + Description: opts.Filename, + FormatOptions: opts.Remain, }, ) if dv == nil { diff --git a/pkg/interp/interp.go b/pkg/interp/interp.go index 7202f962..6a09cdb2 100644 --- a/pkg/interp/interp.go +++ b/pkg/interp/interp.go @@ -17,6 +17,7 @@ import ( "strings" "time" + "github.com/mitchellh/mapstructure" "github.com/wader/fq/format/registry" "github.com/wader/fq/internal/ansi" "github.com/wader/fq/internal/colorjson" @@ -241,44 +242,6 @@ type loadModule struct { func (l loadModule) LoadInitModules() ([]*gojq.Query, error) { return l.init() } func (l loadModule) LoadModule(name string) (*gojq.Query, error) { return l.load(name) } -func toBool(v interface{}) (bool, error) { - switch v := v.(type) { - case bool: - return v, nil - case *big.Int: - return v.Int64() != 0, nil - case int: - return v != 0, nil - case float64: - return v != 0, nil - default: - return false, fmt.Errorf("value is not a number") - } -} - -func toBoolZ(v interface{}) bool { - b, _ := toBool(v) - return b -} - -func toInt(v interface{}) (int, error) { - switch v := v.(type) { - case *big.Int: - return int(v.Int64()), nil - case int: - return v, nil - case float64: - return int(v), nil - default: - return 0, fmt.Errorf("value is not a number") - } -} - -func toIntZ(v interface{}) int { - n, _ := toInt(v) - return n -} - func toString(v interface{}) (string, error) { switch v := v.(type) { case string: @@ -295,11 +258,6 @@ func toString(v interface{}) (string, error) { } } -func toStringZ(v interface{}) string { - s, _ := toString(v) - return s -} - func toBigInt(v interface{}) (*big.Int, error) { switch v := v.(type) { case int: @@ -451,7 +409,6 @@ type evalContext struct { // structcheck has problems with embedding https://gitlab.com/opennota/check#known-limitations ctx context.Context output io.Writer - mode RunMode } type Interp struct { @@ -500,8 +457,6 @@ func (i *Interp) Stop() { } func (i *Interp) Main(ctx context.Context, output Output, version string) error { - runMode := ScriptMode - var args []interface{} for _, a := range i.os.Args() { args = append(args, a) @@ -512,7 +467,7 @@ func (i *Interp) Main(ctx context.Context, output Output, version string) error "version": version, } - iter, err := i.EvalFunc(ctx, runMode, input, "_main", nil, output) + iter, err := i.EvalFunc(ctx, input, "_main", nil, output) if err != nil { fmt.Fprintln(i.os.Stderr(), err) return err @@ -542,7 +497,7 @@ func (i *Interp) Main(ctx context.Context, output Output, version string) error return nil } -func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src string, srcFilename string, output io.Writer) (gojq.Iter, error) { +func (i *Interp) Eval(ctx context.Context, c interface{}, src string, srcFilename string, output io.Writer) (gojq.Iter, error) { gq, err := gojq.Parse(src) if err != nil { p := queryErrorPosition(src, err) @@ -557,10 +512,7 @@ func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src stri // make copy of interp ci := *i ni := &ci - - ni.evalContext = evalContext{ - mode: mode, - } + ni.evalContext = evalContext{} var variableNames []string var variableValues []interface{} @@ -767,7 +719,7 @@ func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src stri return iterWrapper, nil } -func (i *Interp) EvalFunc(ctx context.Context, mode RunMode, c interface{}, name string, args []interface{}, output io.Writer) (gojq.Iter, error) { +func (i *Interp) EvalFunc(ctx context.Context, c interface{}, name string, args []interface{}, output io.Writer) (gojq.Iter, error) { var argsExpr []string for i := range args { argsExpr = append(argsExpr, fmt.Sprintf("$_args[%d]", i)) @@ -784,15 +736,15 @@ func (i *Interp) EvalFunc(ctx context.Context, mode RunMode, c interface{}, name // _args to mark variable as internal and hide it from completion // {input: ..., args: [...]} | .args as {args: $_args} | .input | name[($_args[0]; ...)] trampolineExpr := fmt.Sprintf(". as {args: $_args} | .input | %s%s", name, argExpr) - iter, err := i.Eval(ctx, mode, trampolineInput, trampolineExpr, "", output) + iter, err := i.Eval(ctx, trampolineInput, trampolineExpr, "", output) if err != nil { return nil, err } return iter, nil } -func (i *Interp) EvalFuncValues(ctx context.Context, mode RunMode, c interface{}, name string, args []interface{}, output io.Writer) ([]interface{}, error) { - iter, err := i.EvalFunc(ctx, mode, c, name, args, output) +func (i *Interp) EvalFuncValues(ctx context.Context, c interface{}, name string, args []interface{}, output io.Writer) ([]interface{}, error) { + iter, err := i.EvalFunc(ctx, c, name, args, output) if err != nil { return nil, err } @@ -810,88 +762,29 @@ func (i *Interp) EvalFuncValues(ctx context.Context, mode RunMode, c interface{} } type Options struct { - Depth int - ArrayTruncate int - Verbose bool - DecodeProgress bool - Color bool - Colors string - ByteColors string - Unicode bool - RawOutput bool - REPL bool - RawString bool - JoinString string - Compact bool - BitsFormat string - - LineBytes int - DisplayBytes int - AddrBase int - SizeBase int + Depth int `mapstructure:"depth"` + ArrayTruncate int `mapstructure:"array_truncate"` + Verbose bool `mapstructure:"verbose"` + DecodeProgress bool `mapstructure:"decode_progress"` + Color bool `mapstructure:"color"` + Colors string `mapstructure:"colors"` + ByteColors string `mapstructure:"byte_colors"` + Unicode bool `mapstructure:"unicode"` + RawOutput bool `mapstructure:"raw_output"` + REPL bool `mapstructure:"repl"` + RawString bool `mapstructure:"raw_string"` + JoinString string `mapstructure:"join_string"` + Compact bool `mapstructure:"compact"` + BitsFormat string `mapstructure:"bits_format"` + LineBytes int `mapstructure:"line_bytes"` + DisplayBytes int `mapstructure:"display_bytes"` + AddrBase int `mapstructure:"addrbase"` + SizeBase int `mapstructure:"sizebase"` Decorator Decorator BitsFormatFn func(bb *bitio.Buffer) (interface{}, error) } -func mapSetOptions(d *Options, m map[string]interface{}) { - if v, ok := m["depth"]; ok { - d.Depth = num.MaxInt(0, toIntZ(v)) - } - if v, ok := m["array_truncate"]; ok { - d.ArrayTruncate = num.MaxInt(0, toIntZ(v)) - } - if v, ok := m["verbose"]; ok { - d.Verbose = toBoolZ(v) - } - if v, ok := m["decode_progress"]; ok { - d.DecodeProgress = toBoolZ(v) - } - if v, ok := m["color"]; ok { - d.Color = toBoolZ(v) - } - if v, ok := m["colors"]; ok { - d.Colors = toStringZ(v) - } - if v, ok := m["byte_colors"]; ok { - d.ByteColors = toStringZ(v) - } - if v, ok := m["unicode"]; ok { - d.Unicode = toBoolZ(v) - } - if v, ok := m["raw_output"]; ok { - d.RawOutput = toBoolZ(v) - } - if v, ok := m["repl"]; ok { - d.REPL = toBoolZ(v) - } - if v, ok := m["raw_string"]; ok { - d.RawString = toBoolZ(v) - } - if v, ok := m["join_string"]; ok { - d.JoinString = toStringZ(v) - } - if v, ok := m["compact"]; ok { - d.Compact = toBoolZ(v) - } - if v, ok := m["bits_format"]; ok { - d.BitsFormat = toStringZ(v) - } - - if v, ok := m["line_bytes"]; ok { - d.LineBytes = num.MaxInt(0, toIntZ(v)) - } - if v, ok := m["display_bytes"]; ok { - d.DisplayBytes = num.MaxInt(0, toIntZ(v)) - } - if v, ok := m["addrbase"]; ok { - d.AddrBase = num.ClampInt(2, 36, toIntZ(v)) - } - if v, ok := m["sizebase"]; ok { - d.SizeBase = num.ClampInt(2, 36, toIntZ(v)) - } -} - func bitsFormatFnFromOptions(opts Options) func(bb *bitio.Buffer) (interface{}, error) { switch opts.BitsFormat { case "md5": @@ -970,7 +863,7 @@ func (i *Interp) variables() map[string]interface{} { } func (i *Interp) Options(fnOptsV ...interface{}) (Options, error) { - vs, err := i.EvalFuncValues(i.evalContext.ctx, ScriptMode, nil, "options", []interface{}{fnOptsV}, DiscardCtxWriter{Ctx: i.evalContext.ctx}) + vs, err := i.EvalFuncValues(i.evalContext.ctx, nil, "options", []interface{}{fnOptsV}, DiscardCtxWriter{Ctx: i.evalContext.ctx}) if err != nil { return Options{}, err } @@ -987,7 +880,13 @@ func (i *Interp) Options(fnOptsV ...interface{}) (Options, error) { } var opts Options - mapSetOptions(&opts, m) + _ = mapstructure.Decode(m, &opts) + opts.Depth = num.MaxInt(0, opts.Depth) + opts.ArrayTruncate = num.MaxInt(0, opts.ArrayTruncate) + opts.AddrBase = num.ClampInt(2, 36, opts.AddrBase) + opts.SizeBase = num.ClampInt(2, 36, opts.SizeBase) + opts.LineBytes = num.MaxInt(0, opts.LineBytes) + opts.DisplayBytes = num.MaxInt(0, opts.DisplayBytes) opts.Decorator = decoratorFromOptions(opts) opts.BitsFormatFn = bitsFormatFnFromOptions(opts) diff --git a/pkg/interp/interp.jq b/pkg/interp/interp.jq index 2d8622e8..bfb93905 100644 --- a/pkg/interp/interp.jq +++ b/pkg/interp/interp.jq @@ -327,7 +327,7 @@ def _repl($opts): #:: a|(Opts) => @ def _read_expr: # both _prompt and _complete want arrays ( . as $c - | _readline(_prompt; "_complete") + | _readline(_prompt; {complete: "_complete", timeout: 0.5}) | if trim == "" then $c | _read_expr end