1
1
mirror of https://github.com/wader/fq.git synced 2024-12-23 05:13:30 +03:00

interp: Eval options in jq instead of calling jq from go

Simpler and causes less weird performance issues
This commit is contained in:
Mattias Wadman 2021-11-01 16:50:28 +01:00
parent 13fae09172
commit 96cc1283cd
10 changed files with 55 additions and 61 deletions

View File

@ -33,6 +33,7 @@
"Errorer",
"errorln",
"esds",
"eval",
"Exif",
"Exiter",
"FALLID",

View File

@ -14,6 +14,7 @@
- `format/0` overlap with jq builtin `format/1`. What to rename it to? `decode_format`?
- repl expression returning a value that produced lots of output can't be interrupted. This is becaus ctrl-C currently only interrupts the evaluation of the expression, outputted value is printed (`display`) by parent.
- Rework cli/repl user interrupt (context cancel via ctrl-c), see comment in Interp.Main
- Optimize `Interp.Options` calls, now called per display. Cache per eval? needs to handle nested evals.
### TODO and ideas

View File

@ -61,7 +61,7 @@ func (i *Interp) makeFunctions() []Function {
{[]string{"_tobitsrange"}, 0, 2, i._toBitsRange, nil},
{[]string{"tovalue"}, 0, 1, i.toValue, nil},
{[]string{"_tovalue"}, 1, 1, i._toValue, nil},
{[]string{"hex"}, 0, 0, makeStringBitBufTransformFn(
func(r io.Reader) (io.Reader, error) { return hex.NewDecoder(r), nil },
@ -664,7 +664,7 @@ func (i *Interp) format(c interface{}, a []interface{}) interface{} {
}
func (i *Interp) _display(c interface{}, a []interface{}) gojq.Iter {
opts := i.OptionsEx(a...)
opts := i.Options(a[0])
switch v := c.(type) {
case Display:
@ -734,9 +734,9 @@ func (i *Interp) _toBitsRange(c interface{}, a []interface{}) interface{} {
return bv
}
func (i *Interp) toValue(c interface{}, a []interface{}) interface{} {
func (i *Interp) _toValue(c interface{}, a []interface{}) interface{} {
v, _ := toValue(
func() Options { return i.OptionsEx(append([]interface{}{}, a...)...) },
func() Options { return i.Options(a[0]) },
c,
)
return v
@ -921,7 +921,7 @@ func (i *Interp) _bitsMatch(c interface{}, a []interface{}) gojq.Iter {
}
func (i *Interp) _hexdump(c interface{}, a []interface{}) gojq.Iter {
opts := i.OptionsEx(a...)
opts := i.Options(a[0])
bv, err := toBufferView(c)
if err != nil {
return gojq.NewIter(err)

View File

@ -54,22 +54,26 @@ def decode($name; $opts):
def decode($name): decode($name; {});
def decode: decode(options.decode_format; {});
def tovalue($opts): _tovalue(options($opts));
def tovalue: _tovalue({});
def display($opts): _display($opts);
def display: _display({});
def d($opts): _display($opts);
def d: _display({});
def full($opts): _display({array_truncate: 0} + $opts);
def display($opts): _display(options($opts));
def display: display({});
def d($opts): display($opts);
def d: display({});
def full($opts): display({array_truncate: 0} + $opts);
# TODO: rename, gets mixed up with f args often
def full: full({});
def f($opts): full($opts);
def f: full;
def verbose($opts): _display({verbose: true, array_truncate: 0} + $opts);
def verbose($opts): display({verbose: true, array_truncate: 0} + $opts);
def verbose: verbose({});
def v($opts): verbose($opts);
def v: verbose;
def hexdump($opts): _hexdump({display_bytes: 0} + $opts);
def hexdump: _hexdump({display_bytes: 0});
def hexdump($opts): _hexdump(options({display_bytes: 0} + $opts));
def hexdump: hexdump({display_bytes: 0});
def hd($opts): hexdump($opts);
def hd: hexdump;

View File

@ -1,3 +1,12 @@
# TODO: remove once symbolic value is done properly
def _grep_tovalue:
( if _is_decode_value then
( ._symbol as $s
| if $s != "" then $s end
)
end
);
def _grep($v; filter_cond; string_cond; other_cond):
if $v | type == "string" then
( ..
@ -10,14 +19,14 @@ def _grep($v; filter_cond; string_cond; other_cond):
end;
def _value_grep_string_cond($v; $flags):
( _tovalue
( _grep_tovalue
| if type == "string" then test($v; $flags)
else false
end
)? // false;
def _value_grep_other_cond($v; $flags):
( _tovalue
( _grep_tovalue
| . == $v
)? // false;

View File

@ -34,7 +34,9 @@ def _input_decode_errors(f): _global_var("input_decode_errors"; f);
def _variables: _global_var("variables");
def _variables(f): _global_var("variables"; f);
# eval f and finally eval fin even on empty or error
# eval f and finally eval fin even on empty or error.
# note that if f outputs more than one value fin will be called
# for each value.
def _finally(f; fin):
( try f // (fin | empty)
catch (fin as $_ | error)
@ -100,15 +102,6 @@ def _eval($expr; $filename; f; on_error; on_compile_error):
else on_error
end;
# TODO: remove one symbolic value is done properly?
def _tovalue:
( if _is_decode_value then
( ._symbol as $s
| if $s != "" then $s end
)
end
);
def _is_scalar:
type |. != "array" and . != "object";

View File

@ -871,31 +871,11 @@ func (i *Interp) variables() map[string]interface{} {
return variablesAny
}
func (i *Interp) OptionsEx(fnOptsV ...interface{}) Options {
opts := func() Options {
vs, err := i.EvalFuncValues(i.evalContext.ctx, nil, "options", []interface{}{fnOptsV}, DiscardCtxWriter{Ctx: i.evalContext.ctx})
if err != nil {
return Options{}
}
if len(vs) < 1 {
return Options{}
}
v := vs[0]
if _, ok := v.(error); ok {
return Options{}
}
m, ok := v.(map[string]interface{})
if !ok {
return Options{}
}
var opts Options
_ = mapstructure.Decode(m, &opts)
opts.Depth = num.MaxInt(0, opts.Depth)
return opts
}()
func (i *Interp) Options(v interface{}) Options {
var opts Options
_ = mapstructure.Decode(v, &opts)
opts.ArrayTruncate = num.MaxInt(0, opts.ArrayTruncate)
opts.Depth = num.MaxInt(0, opts.Depth)
opts.AddrBase = num.ClampInt(2, 36, opts.AddrBase)
opts.SizeBase = num.ClampInt(2, 36, opts.SizeBase)
opts.LineBytes = num.MaxInt(0, opts.LineBytes)
@ -906,10 +886,6 @@ func (i *Interp) OptionsEx(fnOptsV ...interface{}) Options {
return opts
}
func (i *Interp) Options() Options {
return i.OptionsEx(nil)
}
func (i *Interp) NewColorJSON(opts Options) (*colorjson.Encoder, error) {
indent := 2
if opts.Compact {

View File

@ -135,8 +135,6 @@ def _to_options:
| with_entries(select(.value != null))
);
# . will have additional array of options taking priority
# NOTE: is called from go *interp.Interp Options()
def options($opts):
[_build_default_dynamic_options] + _options_stack + $opts | add;
def options: options([{}]);
[_build_default_dynamic_options] + _options_stack + [$opts] | add;
def options: options({});

View File

@ -153,16 +153,29 @@ def _prompt:
, _values
] | join(" ") + "> ";
def _repl_display: _display({depth: 1});
# _repl_display takes a opts arg to make it possible for repl_eval to
# just call options/0 once per eval even if it was multiple outputs
def _repl_display_opts: options({depth: 1});
def _repl_display($opts): _display($opts);
def _repl_display: _display(_repl_display_opts);
def _repl_on_error:
( if _eval_is_compile_error then _eval_compile_error_tostring
# was interrupte by user, just ignore
# was interrupted by user, just ignore
elif _is_context_canceled_error then empty
end
| (_error_str | println)
);
def _repl_on_compile_error: _repl_on_error;
def _repl_eval($expr): _eval($expr; "repl"; _repl_display; _repl_on_error; _repl_on_compile_error);
def _repl_eval($expr):
( _repl_display_opts as $opts
| _eval(
$expr;
"repl";
_repl_display($opts);
_repl_on_error;
_repl_on_compile_error
)
);
# run read-eval-print-loop
def _repl($opts): #:: a|(Opts) => @

View File

@ -118,7 +118,6 @@ vpx_ccr VPX Codec Configuration Record
wav WAV file
webp WebP image
xing Xing header
zip ZIP archive
$ fq -X
exitcode: 2
stderr: