wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-12-23 05:13:30 +03:00
Code Issues Projects Releases Wiki Activity

repl,interp: Refactor repl and slurp

Browse Source 
Now repl, slurp and help implemented using same query rewrite.
Include filename context in error if possible.
Add spew function that does opposite of slurp.
Start of help infra, not done or documented yet.
Show error pointer on parse error.
Rename internal eval to _eval and make eval be wrapper that
does rewrite and has various eror handling etc.
Nicer repl, slupr and help errors.
This commit is contained in:
Mattias Wadman 2022-02-20 22:25:46 +01:00
parent 47b3c64bf9
commit 0a043f9096
26 changed files with 805 additions and 311 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitattributes vendored
View File

@ -1,4 +1,4 @@
# for windows: fqtest uses \n and asserts output so should be exact
# for windows: fqtest, jq and json uses \n and uses it in output so must be exact
*.fqtest eol=lf
# for windows: there are fqtest using json that need to be exact
*.json eol=lf
*.jq eol=lf

1
doc/TODO.md
View File

@ -36,7 +36,6 @@
#### Language
- Nicer variables somehow? `... | var($VAR)`? make slurp and rewrite `$var` to `$var[]`?
- Cleanup/Make binary buffers make sense.
- gojq uses golang `int` for slice indexes, might be issue for non-64bit cpus

4
doc/usage.md
View File

@ -417,7 +417,9 @@ you currently have to do `fq -d raw 'mp3({force: true})' file`.
- `ddv`/`ddv($opts)` verbosely display value and don't truncate arrays or binaries
- `p`/`preview` show preview of field tree
- `hd`/`hexdump` hexdump value
- `repl` nested REPL, must be last in a pipeline. `1 | repl`, can "slurp" outputs `1, 2, 3 | repl`.
- `repl`/`repl($opts)` nested REPL, must be last in a pipeline. `1 | repl`, can "slurp" outputs. Ex: `1, 2, 3 | repl`, `[1,2,3] | repl({compact: true})`.
- `slurp($name)` slurp outputs and save them to `$name`, must be last in pipeline. Will be available as global array `$name`. Ex `1,2,3 | slurp("a")`, `$a[]` same as `spew("a")`.
- `spew`/`spew($name)` output previously slurped values. `spew` outputs all slurps as an object, `spew($name)` outouts one slurp. Ex: `spew("a")`.
- `paste` read string from stdin until ^D. Useful for pasting text.
- Ex: `paste | frompem | asn1_ber | repl` read from stdin then decode and start a new sub-REPL with result.

4
pkg/interp/.jq-lsp.jq
View File

@ -2,6 +2,8 @@
def _can_display: empty;
def _decode($format; $opts): empty;
def _display($opts): empty;
def _eval($expr; $filename): empty;
def _eval($expr): empty;
def _extkeys: empty;
def _exttype: empty;
def _global_state: empty;
@ -26,7 +28,5 @@ def _tobits($bits; $is_range; $pad): empty;
def _tovalue: empty;
def _tovalue($opts): empty;
def base64: empty;
def eval($expr; $filename): empty;
def eval($expr): empty;
def open: empty;
def scope: empty;

7
pkg/interp/binary.go
View File

@ -2,6 +2,7 @@ package interp
import (
"bytes"
"errors"
"fmt"
"io"
"io/fs"
@ -173,7 +174,6 @@ func (of *openFile) ToBinary() (Binary, error) {
return newBinaryFromBitReader(of.br, 8, 0)
}
// def open: #:: string| => binary
// opens a file for reading from filesystem
// TODO: when to close? when br loses all refs? need to use finalizer somehow?
func (i *Interp) _open(c interface{}, a []interface{}) gojq.Iter {
@ -196,6 +196,11 @@ func (i *Interp) _open(c interface{}, a []interface{}) gojq.Iter {
}
f, err = i.os.FS().Open(path)
if err != nil {
// path context added in jq error code
var pe *fs.PathError
if errors.As(err, &pe) {
return gojq.NewIter(pe.Err)
}
return gojq.NewIter(err)
}
}

98
pkg/interp/eval.jq Normal file
View File

@ -0,0 +1,98 @@
include "internal";
include "query";
def _eval_error($what; $error):
error({
what: $what,
error: $error,
column: 0,
line: 1,
filename: ""
});
def _eval_error_function_not_defined($name; $args):
_eval_error(
"compile";
"function not defined: \($name)/\($args | length)"
);
def _eval_query_rewrite($opts):
_query_fromtostring(
( . as $orig_query
| _query_pipe_last as $last
| ( $last
| if _query_is_func then [_query_func_name, _query_func_args]
else ["", []]
end
) as [$last_func_name, $last_func_args]
| $opts.slurps[$last_func_name] as $slurp
| if $slurp then
_query_transform_pipe_last(_query_ident)
end
| if $opts.catch_query then
_query_try(.; $opts.catch_query)
end
| _query_pipe(
$opts.input_query // _query_ident;
.
)
| if $slurp then
_query_func(
$slurp;
[ # pass original, rewritten and args queries as query ast trees
( { slurp: _query_string($last_func_name)
, slurp_args:
( $last_func_args
| if . then
( map(_query_toquery)
| _query_commas
| _query_array
)
else (null | _query_array)
end
)
, orig: ($orig_query | _query_toquery)
, rewrite: _query_toquery
}
| _query_object
)
]
)
end
)
);
# TODO: better way? what about nested eval errors?
def _eval_is_compile_error:
type == "object" and .error != null and .what != null;
def _eval_compile_error_tostring:
[ (.filename // "expr")
, if .line != 1 or .column != 0 then "\(.line):\(.column)"
else empty
end
, " \(.error)"
] | join(":");
def eval($expr; $opts; on_error; on_compile_error):
( . as $c
| ($opts.filename // "expr") as $filename
| try
_eval(
$expr | _eval_query_rewrite($opts);
$filename
)
catch
if _eval_is_compile_error then
# rewrite parse error will not have filename
( .filename = $filename
| {error: ., input: $c}
| on_compile_error
)
else
( {error: ., input: $c}
| on_error
)
end
);
def eval($expr): eval($expr; {}; .error | error; .error | error);

6
pkg/interp/funcs.jq
View File

@ -2,6 +2,9 @@ include "internal";
include "options";
include "binary";
def _display_default_opts:
options({depth: 1});
def display($opts):
( options($opts) as $opts
| if _can_display then _display($opts)
@ -18,6 +21,7 @@ def display($opts):
);
def display: display({});
def hexdump($opts): _hexdump(options({display_bytes: 0} + $opts));
def hexdump: hexdump({display_bytes: 0});
def hd($opts): hexdump($opts);
@ -55,7 +59,7 @@ def path_to_expr:
# TODO: don't use eval? should support '.a.b[1]."c.c"' and escapes?
def expr_to_path:
( if type != "string" then error("require string argument") end
| eval("null | path(\(.))")
| _eval("null | path(\(.))")
);
def trim: capture("^\\s*(?<str>.*?)\\s*$"; "").str;

130
pkg/interp/help.jq Normal file
View File

@ -0,0 +1,130 @@
include "internal";
include "query";
include "eval";
include "repl";
# TODO: variants, values, keywords?
# TODO: store some other way?
def _help_functions:
{ length: {
summary: "Length of string, array, object, etc",
doc:
"- For string number of unicode codepoints
- For array number of elements in array
- For object number of key-value pairs
- For null zero
- For number the number itself
- For boolean is an error
",
examples:
[ [[1,2,3], "length"]
, ["abc", "length"]
, [{a: 1, b: 2}, "length"]
, [null, "length"]
, [123, "length"]
, [true, "length"]
]
},
"..": {
summary: "Recursive descent of .",
doc:
"Recursively descend . and output each value.
Same as recurse without argument.
",
examples:
[ ["a", ".."]
, [[1,2,3], ".."]
, [{a: 1, b: {c: 3}}, ".."]
]
},
empty: {
summary: "Output nothing",
doc:
"Output no value, not even null, and cause backtrack.
",
examples:
[ ["empty"]
, ["[1,empty,2]"]
]
}
};
def help($_): error("help must be alone or last in pipeline. ex: help(length) or ... | help");
def help: help(null);
# TODO: refactor
def _help_slurp($query):
def _name:
if _query_is_func then _query_func_name
else _query_tostring
end;
if $query.orig | _query_is_func then
( ($query.orig | _query_func_args) as $args
| ($args | length) as $argc
| if $args == null then
# help
( "Type expression to evaluate"
, "\\t Completion"
, "Up/Down History"
, "^C Interrupt execution"
, "... | repl Start a new REPL"
, "^D Exit REPL"
) | println
elif $argc == 1 then
# help(...)
( ($args[0] | _name) as $name
| _help_functions[$name] as $hf
| if $hf then
# help(name)
( "\($name): \($hf.summary)"
, $hf.doc
, "Examples:"
, ( $hf.examples[]
| . as $e
| if length == 1 then
( "> \($e[0])"
, (null | try _eval($e[0]) | tojson catch "error: \(.)")
)
else
( "> \($e[0] | tojson) | \($e[1])"
, ($e[0] | try _eval($e[1]) | tojson catch "error: \(.)")
)
end
)
) | println
else
# help(unknown)
# TODO: check builtin
( ( . # TODO: extract
| builtins
| map(split("/") | {key: .[0], value: true})
| from_entries
) as $builtins
| ( . # TODO: extract
| scope
| map({key: ., value: true})
| from_entries
) as $scope
| if $builtins | has($name) then
"\($name) is builtin function"
elif $scope | has($name) then
"\($name) is a function or variable"
else
"don't know what \($name) is "
end
| println
)
end
)
else
_eval_error("compile"; "help must be last in pipeline. ex: help(length) or ... | help")
end
)
else
# ... | help
# TODO: check builtin
( _repl_slurp_eval($query.rewrite) as $outputs
| "value help"
, $outputs
)
end;

36
pkg/interp/internal.jq
View File

@ -9,12 +9,15 @@ def println: ., "\n" | print;
def printerr: tostring | _stderr;
def printerrln: ., "\n" | printerr;
# jq compat
def debug:
( ((["DEBUG", .] | tojson) | printerrln)
def _debug($name):
( (([$name, .] | tojson) | printerrln)
, .
);
# jq compat
def debug: _debug("DEBUG");
def debug(f): . as $c | f | debug | $c;
# jq compat, output to compact json to stderr and let input thru
def stderr:
( (tojson | printerr)
@ -45,9 +48,6 @@ def _include_paths(f): _global_var("include_paths"; f);
def _options_stack: _global_var("options_stack");
def _options_stack(f): _global_var("options_stack"; f);
def _options_cache: _global_var("options_cache");
def _options_cache(f): _global_var("options_cache"; f);
def _cli_last_expr_error: _global_var("cli_last_expr_error");
def _cli_last_expr_error(f): _global_var("cli_last_expr_error"; f);
@ -69,10 +69,10 @@ def _input_io_errors(f): _global_var("input_io_errors"; f);
def _input_decode_errors: _global_var("input_decode_errors");
def _input_decode_errors(f): _global_var("input_decode_errors"; f);
def _variables: _global_var("variables");
def _variables(f): _global_var("variables"; f);
def _slurps: _global_var("slurps");
def _slurps(f): _global_var("slurps"; f);
# eval f and finally eval fin even if empty or error.
# call f and finally eval fin even if empty or error.
# _finally(1; debug)
# _finally(null; debug)
# _finally(error("a"); debug)
@ -135,24 +135,10 @@ def _recurse_break(f):
else error
end;
# TODO: better way? what about nested eval errors?
def _eval_is_compile_error: type == "object" and .error != null and .what != null;
def _eval_compile_error_tostring:
[ (.filename | if . == "" then "expr" end)
, if .line != 1 or .column != 0 then "\(.line):\(.column)" else empty end
, " \(.error)"
] | join(":");
def _eval($expr; $filename; f; on_error; on_compile_error):
try
eval($expr; $filename) | f
catch
if _eval_is_compile_error then on_compile_error
else on_error
end;
def _is_scalar:
type |. != "array" and . != "object";
def _is_context_canceled_error: . == "context canceled";
def _error_str: "error: \(.)";
def _error_str($contexts): (["error"] + $contexts + [.]) | join(": ");
def _error_str: _error_str([]);

16
pkg/interp/interp.go
View File

@ -36,6 +36,7 @@ import (
//go:embed interp.jq
//go:embed internal.jq
//go:embed eval.jq
//go:embed options.jq
//go:embed binary.jq
//go:embed decode.jq
@ -45,6 +46,7 @@ import (
//go:embed args.jq
//go:embed query.jq
//go:embed repl.jq
//go:embed help.jq
//go:embed formats.jq
var builtinFS embed.FS
@ -56,7 +58,7 @@ func init() {
functionRegisterFns = append(functionRegisterFns, func(i *Interp) []Function {
return []Function{
{"_readline", 0, 1, nil, i._readline},
{"eval", 1, 2, nil, i.eval},
{"_eval", 1, 2, nil, i._eval},
{"_stdin", 0, 1, nil, i.makeStdioFn("stdin", i.os.Stdin())},
{"_stdout", 0, 0, nil, i.makeStdioFn("stdout", i.os.Stdout())},
{"_stderr", 0, 0, nil, i.makeStdioFn("stderr", i.os.Stderr())},
@ -507,7 +509,7 @@ func (i *Interp) _readline(c interface{}, a []interface{}) gojq.Iter {
return gojq.NewIter(expr)
}
func (i *Interp) eval(c interface{}, a []interface{}) gojq.Iter {
func (i *Interp) _eval(c interface{}, a []interface{}) gojq.Iter {
var err error
expr, err := toString(a[0])
if err != nil {
@ -742,7 +744,7 @@ func (i *Interp) Eval(ctx context.Context, c interface{}, expr string, opts Eval
var variableNames []string
var variableValues []interface{}
for k, v := range i.variables() {
for k, v := range i.slurps() {
variableNames = append(variableNames, "$"+k)
variableValues = append(variableValues, v)
}
@ -794,7 +796,7 @@ func (i *Interp) Eval(ctx context.Context, c interface{}, expr string, opts Eval
}
ni.evalInstance.includeSeen[filename] = struct{}{}
// return cached version if file has already been compiled
// return cached version if file has already been parsed
if q, ok := ni.includeCache[filename]; ok {
return q, nil
}
@ -1074,9 +1076,9 @@ func (i *Interp) includePaths() []string {
return paths
}
func (i *Interp) variables() map[string]interface{} {
variablesAny, _ := i.lookupState("variables").(map[string]interface{})
return variablesAny
func (i *Interp) slurps() map[string]interface{} {
slurpsAny, _ := i.lookupState("slurps").(map[string]interface{})
return slurpsAny
}
func (i *Interp) Options(v interface{}) Options {

167
pkg/interp/interp.jq
View File

@ -6,8 +6,11 @@ include "match";
include "funcs";
include "grep";
include "args";
include "eval";
include "query";
include "repl";
# generated decode functions per format and format helpers
include "help";
# generate torepr, format decode helpers and include format specific functions
include "formats";
# optional user init
include "@config/init?";
@ -44,20 +47,20 @@ def input:
( . as $err
| _input_io_errors(. += {($name): $err}) as $_
| $err
| (_error_str | printerrln)
| (_error_str([$name]) | printerrln)
, _input($opts; f)
)
| try f
catch
( . as $err
| _input_decode_errors(. += {($name): $err}) as $_
| [ "\($name): \($opts.decode_format)"
| [ $opts.decode_format
, if $err | type == "string" then ": \($err)"
# TODO: if not string assume decode itself failed for now
else ": failed to decode (try -d FORMAT)"
end
] | join("")
| (_error_str | printerrln)
| (_error_str([$name]) | printerrln)
, _input($opts; f)
)
);
@ -116,30 +119,46 @@ def inputs: _repeat_break(input);
def input_filename: _input_filename;
def var: _variables;
def var($k; f):
( . as $c
| if ($k | _is_ident | not) then error("invalid variable name: \($k)") end
| _variables(.[$k] |= f)
| empty
);
def var($k): . as $c | var($k; $c);
def _cli_expr_on_error:
( . as $err
# user expr error, report and continue
def _cli_eval_on_expr_error:
( if type == "object" then
if .error | _eval_is_compile_error then .error | _eval_compile_error_tostring
elif .error then .error
end
else tostring
end
| . as $err
| _cli_last_expr_error($err) as $_
| (_error_str | printerrln)
| (_error_str([input_filename // empty]) | printerrln)
);
def _cli_expr_on_compile_error:
( _eval_compile_error_tostring
# other expr error, should not happen, report and halt
def _cli_eval_on_error:
halt_error(_exit_code_expr_error);
# could not compile expr, report and halt
def _cli_eval_on_compile_error:
( .error
| _eval_compile_error_tostring
| halt_error(_exit_code_compile_error)
);
# _cli_expr_eval halts on compile errors
def _cli_expr_eval($expr; $filename; f):
_eval($expr; $filename; f; _cli_expr_on_error; _cli_expr_on_compile_error);
def _cli_expr_eval($expr; $filename):
_eval($expr; $filename; .; _cli_expr_on_error; _cli_expr_on_compile_error);
def _cli_repl_error($_):
_eval_error("compile"; "repl can only be used from interactive repl");
def _cli_slurp_error(_):
_eval_error("compile"; "slurp can only be used from interactive repl");
# _cli_eval halts on compile errors
def _cli_eval($expr; $opts):
eval(
$expr;
$opts + {
slurps: {
help: "_help_slurp",
repl: "_cli_repl_error",
slurp: "_cli_slurp_error"
},
catch_query: _query_func("_cli_eval_on_expr_error")
};
_cli_eval_on_error;
_cli_eval_on_compile_error
);
def _main:
@ -166,20 +185,22 @@ def _main:
, args_help_text(_opt_cli_opts)
);
def _formats_list:
[ ( formats
( [ formats
| to_entries[]
| [(.key+" "), .value.description]
)
]
]
| table(
.;
map(
( . as $rc
| .string
| if $rc.column != 1 then rpad(" "; $rc.maxwidth) end
# right pad format name to align description
| if .column == 0 then .string | rpad(" "; $rc.maxwidth)
else $rc.string
end
)
) | join("")
);
)
);
def _map_decode_file:
map(
( . as $a
@ -228,12 +249,10 @@ def _main:
, null | halt_error(_exit_code_args_error)
)
else
# use _finally as display etc prints and outputs empty
_finally(
# store some globals
( # store some global state
( _include_paths($opts.include_path) as $_
| _input_filenames($opts.filenames) as $_
| _variables(
| _slurps(
( $opts.arg +
$opts.argjson +
$opts.raw_file +
@ -242,45 +261,51 @@ def _main:
| from_entries
)
)
# for inputs a, b, c:
# repl: [a,b,c] | repl
# repl slurp: [[a, b, c]] | repl
# cli a, b, c | expr
# cli slurp [a ,b c] | expr
| ( def _inputs:
( if $opts.null_input then null
# note that jq --slurp --raw-input (string_input) is special, will concat
# all files into one string instead of iterating lines
elif $opts.string_input then inputs
elif $opts.slurp then [inputs]
else inputs
end
);
if $opts.repl then
( [_inputs]
| map(_cli_expr_eval($opts.expr; $opts.expr_eval_path))
| _repl({})
)
else
( _inputs
# iterate all inputs
| _cli_last_expr_error(null) as $_
| _cli_expr_eval($opts.expr; $opts.expr_eval_path; _repl_display)
)
end
) as $_
| { filename: $opts.expr_eval_path
} as $eval_opts
# use _finally as display etc prints and outputs empty
| _finally(
if $opts.repl then
# TODO: share input_query but first have to figure out how to handle
# context/interrupts better as open will happen in a sub repl which
# context will be cancelled.
( def _inputs:
if $opts.null_input then null
elif $opts.string_input then inputs
elif $opts.slurp then [inputs]
else inputs
end;
[_inputs]
| map(_cli_eval($opts.expr; $eval_opts))
| _repl({})
)
)
; # finally
( if _input_io_errors then
null | halt_error(_exit_code_input_io_error)
end
| if _input_decode_errors then
null | halt_error(_exit_code_input_decode_error)
end
| if _cli_last_expr_error then
null | halt_error(_exit_code_expr_error)
end
else
( _cli_last_expr_error(null) as $_
| _display_default_opts as $default_opts
| _cli_eval(
$opts.expr;
( $eval_opts
| .input_query =
( if $opts.null_input then _query_null
# note that jq --slurp --raw-input (string_input) is special, will concat
# all files into one string instead of iterating lines
elif $opts.string_input then _query_func("inputs")
elif $opts.slurp then _query_func("inputs") | _query_array
else _query_func("inputs")
end
)
)
)
| display($default_opts)
)
end;
# finally
( if _input_io_errors then null | halt_error(_exit_code_input_io_error) end
| if _input_decode_errors then null | halt_error(_exit_code_input_decode_error) end
| if _cli_last_expr_error then null | halt_error(_exit_code_expr_error) end
)
)
)
end
);

9
pkg/interp/options.jq
View File

@ -86,9 +86,10 @@ def _opt_eval($rest):
# if -f was used, all rest non-args are filenames
# otherwise first is expr rest is filesnames
( .expr_file
| . as $expr_file
| if . then
try (open | tobytes | tostring)
catch halt_error(_exit_code_args_error)
catch ("\($expr_file): \(.)" | halt_error(_exit_code_args_error))
else $rest[0] // null
end
)
@ -123,8 +124,10 @@ def _opt_eval($rest):
( .raw_file
| if . then
( map(.[1] |=
try (open | tobytes | tostring)
catch halt_error(_exit_code_args_error)
( . as $f
| try (open | tobytes | tostring)
catch ("\($f): \(.)" | halt_error(_exit_code_args_error))
)
)
)
end

176
pkg/interp/query.jq
View File

@ -1,45 +1,123 @@
# a() -> b()
# null
def _query_null:
{term: {type: "TermTypeNull"}};
# string
def _query_string($str):
{ term: {
type: "TermTypeString",
str: {
str: $str
}
}
};
# .
def _query_ident:
{term: {type: "TermTypeIdentity"}};
def _query_is_ident:
.term.type == "TermTypeIdentity";
# a($args...) -> b($args...)
def _query_func_rename(name):
.term.func.name = name;
# $name($args)
def _query_func($name; $args):
{ term: {
type: "TermTypeFunc",
func: {
args: $args,
name: $name
}
}
};
def _query_func($name):
_query_func($name; null);
# . | r
def _query_pipe(r):
def _query_func_name:
.term.func.name;
def _query_func_args:
.term.func.args;
def _query_is_func:
.term.type == "TermTypeFunc";
def _query_is_func($name):
_query_is_func and _query_func_name == $name;
def _query_empty:
_query_func("empty");
# l | r
def _query_pipe(l; r):
{ op: "|",
left: .,
left: l,
right: r
};
# . -> [.]
def _query_array:
( . as $q
| { term: {
type: "TermTypeArray",
array: {}
}
}
| if $q then .term.array.query = $q end
);
# {} -> {}
def _query_object:
{ term: {
object: {
key_vals:
( to_entries
| map(
{
key: .key,
val: {
queries: [.value]
}
}
)
)
},
type: "TermTypeObject"
}
};
# l,r
def _query_comma(l; r):
{ left: l,
op: ",",
right: r
};
# [1,2,3] -> 1,2,3
# output each query in array
def _query_commas:
if length == 0 then _query_empty
else
reduce .[1:][] as $q (
.[0];
_query_comma(.; $q)
)
end;
# . -> .[]
def _query_iter:
.term.suffix_list = [{iter: true}];
def _query_ident:
{term: {type: "TermTypeIdentity"}};
def _query_try(f):
# try b catch c
def _query_try(b; c):
{ term: {
type: "TermTypeTry",
try: {
body: f,
},
type: "TermTypeTry"
body: b,
catch: c
}
}
};
def _query_func($name; $args):
{ term: {
func: {
args: $args,
name: $name
},
type: "TermTypeFunc"
}
};
def _query_func($name):
_query_func($name; null);
def _query_is_func(name):
.term.func.name == name;
def _query_try(b):
_query_try(b; null);
# last query in pipeline
def _query_pipe_last:
@ -162,14 +240,9 @@ def _query_completion(f):
| if . then
( .query |=
( _query_func("map"; [
_query_pipe(
_query_try(
_query_func($c | f)
)
)
_query_pipe(.; _query_try(_query_func($c | f)))
])
| _query_pipe(
_query_func("add")
| _query_pipe(.; _query_func("add")
)
| .meta = $meta
| .imports = $imports
@ -185,32 +258,21 @@ def _query_completion(f):
catch {type: "error", name: "", error: .}
);
# <filter...> | <slurp_func> ->
# map(<filter...> | .) | (<slurp_func> | f)
def _query_slurp_wrap(f):
# save and move directives to new root query
( . as {$meta, $imports}
# query ast to ast of quey itself, used by query rewrite/slurp
def _query_toquery:
( tojson
| _query_fromstring
);
# query rewrite helper, takes care of from/to and directives
def _query_fromtostring(f):
( _query_fromstring
# save and move directives to possible new root query
| . as {$meta, $imports}
| del(.meta)
| del(.imports)
| _query_pipe_last as $lq
| _query_transform_pipe_last(_query_ident) as $pipe
| _query_func("map"; [$pipe])
| _query_pipe($lq | f)
| f
| .meta = $meta
| .imports = $imports
);
# filter -> .[] | filter
def _query_iter_wrap:
( . as $q
| _query_ident
| _query_iter
| _query_pipe($q)
);
# query rewrite helper, takes care of from/to
def _query_fromto(f):
( _query_fromstring
| f
| _query_tostring
);

21
pkg/interp/query_test.jq
View File

@ -21,25 +21,4 @@ include "query";
.[0] | _query_fromstring | _query_pipe_last | _query_tostring
)
)
,
([
["", "map(.) | ."],
[".", "map(.) | ."],
["a", "map(.) | a"],
["1, 2", "map(.) | 1, 2"],
["1 | 2", "map(1 | .) | 2"],
["1 | 2 | 3", "map(1 | 2 | .) | 3"],
["(1 | 2) | 3", "map((1 | 2) | .) | 3"],
["1 | (2 | 3)", "map(1 | .) | (2 | 3)"],
["1 as $_ | 2", "map(1 as $_ | .) | 2"],
["def f: 1; 1", "map(.) | def f: 1; 1"],
["def f: 1; 1 | 2", "map(def f: 1; 1 | .) | 2"],
["module {a:1};\ninclude \"a\";\n1", "module { a: 1 };\ninclude \"a\";\nmap(.) | 1"],
empty
][] | assert(
"\(.) | _query_slurp_wrap";
.[1];
.[0] | _query_fromstring | _query_slurp_wrap(.) | _query_tostring
)
)
)

168
pkg/interp/repl.jq
View File

@ -1,10 +1,11 @@
include "internal";
include "options";
include "eval";
include "query";
include "decode";
include "funcs";
# TODO: currently only make sense to allow keywords start start a term or directive
# TODO: currently only make sense to allow keywords starting a term or directive
def _complete_keywords:
[
"and",
@ -46,7 +47,7 @@ def _complete($line; $cursor_pos):
def _is_separator: . as $c | " .;[]()|=" | contains($c);
def _is_internal: startswith("_") or startswith("$_");
def _query_index_or_key($q):
( ([.[] | eval($q) | type]) as $n
( ([.[] | _eval($q) | type]) as $n
| if ($n | all(. == "object")) then "."
elif ($n | all(. == "array")) then "[]"
else null
@ -75,7 +76,7 @@ def _complete($line; $cursor_pos):
)
else
( $c
| eval($query)
| _eval($query)
| ($prefix | _is_internal) as $prefix_is_internal
| map(
select(
@ -156,33 +157,56 @@ def _prompt:
, _values
] | join(" ") + "> ";
# _repl_display takes a opts arg to make it possible for repl_eval to
# just call options/0 once per eval even if it was multiple outputs
def _repl_display_opts: options({depth: 1});
def _repl_display($opts): display($opts);
def _repl_display: display(_repl_display_opts);
def _repl_on_error:
# user expr error
def _repl_on_expr_error:
( if _eval_is_compile_error then _eval_compile_error_tostring
# was interrupted by user, just ignore
elif _is_context_canceled_error then empty
else tostring
end
| (_error_str | println)
| _error_str
| println
);
def _repl_on_compile_error: _repl_on_error;
def _repl_eval($expr):
( _repl_display_opts as $opts
| _eval(
$expr;
"";
_repl_display($opts);
_repl_on_error;
_repl_on_compile_error
)
# other expr error, should not happen
def _repl_on_error:
halt_error(_exit_code_expr_error);
# compile error
def _repl_on_compile_error:
( if .error | _eval_is_compile_error then
( # TODO: move, redo as: def _symbols: if unicode then {...} else {...} end?
def _arrow_up: if options.unicode then "⬆" else "^" end;
if .error.column != 0 then
( ((.input | _prompt | length) + .error.column-1) as $pos
| " " * $pos + "\(_arrow_up) \(.error.error)"
)
else
( .error
| _eval_compile_error_tostring
| _error_str
)
end
)
# was interrupted by user, just ignore
elif .error | _is_context_canceled_error then empty
else .error | _error_str
end
| println
);
def _repl_eval($expr; on_error; on_compile_error; $slurps):
eval(
$expr;
{ slurps: $slurps,
input_query: (_query_ident | _query_iter), # .[]
catch_query: _query_func("_repl_on_expr_error")
};
on_error;
on_compile_error
);
def _repl_eval($expr; on_error; on_compile_error):
_repl_eval($expr; on_error; on_compile_error; null);
# run read-eval-print-loop
# input is array of inputs to iterate
def _repl($opts): #:: a|(Opts) => @
def _repl($opts):
def _read_expr:
_repeat_break(
# both _prompt and _complete want input arrays
@ -198,18 +222,17 @@ def _repl($opts): #:: a|(Opts) => @
);
def _repl_loop:
try
_repl_eval(
( _read_expr
| _query_fromto(
if _query_pipe_last | _query_is_func("repl") then
# "... | repl" -> "map(... | .) | _repl_slurp"
_query_slurp_wrap(_query_func_rename("_repl_slurp"))
else
# "..." to -> ".[] | ..."
_query_iter_wrap
end
)
( _display_default_opts as $default_opts
| _repl_eval(
_read_expr;
_repl_on_error;
_repl_on_compile_error;
{ repl: "_repl_slurp",
help: "_help_slurp",
slurp: "_slurp"
}
)
| display($default_opts)
)
catch
if . == "interrupt" then empty
@ -217,7 +240,9 @@ def _repl($opts): #:: a|(Opts) => @
elif _eval_is_compile_error then _repl_on_error
else error
end;
if _is_completing | not then
if $opts | type != "object" then
error("options must be an object")
elif _is_completing | not then
( _options_stack(. + [$opts]) as $_
| _finally(
_repeat_break(_repl_loop);
@ -227,22 +252,65 @@ def _repl($opts): #:: a|(Opts) => @
else empty
end;
def _repl_slurp($opts): _repl($opts);
def _repl_slurp: _repl({});
def _repl_slurp_eval($query):
try
[ eval(
$query | _query_tostring;
{};
_repl_on_expr_error;
error
)
]
catch
error(.error);
# TODO: introspect and show doc, reflection somehow?
def help:
( "Type expression to evaluate"
, "\\t Completion"
, "Up/Down History"
, "^C Interrupt execution"
, "... | repl Start a new REPL"
, "^D Exit REPL"
) | println;
def _repl_slurp($query):
if ($query.slurp_args | length) > 1 then
_eval_error("compile"; "repl requires none or one options argument. ex: ... | repl or ... | repl({compact: true})")
else
# only allow one output for args, multiple would be confusing i think (would start multiples repl:s)
( ( if ($query.slurp_args | length) > 0 then
first(_repl_slurp_eval($query.slurp_args[0])[])
else {}
end
) as $opts
| if $opts | type != "object" then
_eval_error("compile"; "options must be an object")
end
| _repl_slurp_eval($query.rewrite)
| _repl($opts)
)
end;
# just gives error, call appearing last will be renamed to _repl_slurp
def repl($_):
if options.repl then error("repl must be last")
else error("repl can only be used from interactive repl")
end;
def repl($_): error("repl must be last in pipeline. ex: ... | repl");
def repl: repl(null);
def _slurp($query):
if ($query.slurp_args | length != 1) then
_eval_error("compile"; "slurp requires one string argument. ex: ... | slurp(\"name\")")
else
# TODO: allow only one output?
( _repl_slurp_eval($query.slurp_args[0])[] as $name
| if ($name | _is_ident | not) then
_eval_error("compile"; "invalid slurp name \"\($name)\", must be a valid identifier. ex: ... | slurp(\"name\")")
else
( _repl_slurp_eval($query.rewrite) as $v
| _slurps(.[$name] |= $v)
| empty
)
end
)
end;
def slurp($_): error("slurp must be last in pipeline. ex: ... | slurp(\"name\")");
def slurp: slurp(null);
def spew($name):
( _slurps[$name]
| if . then .[]
else error("no such slurp: \($name)")
end
);
def spew:
_slurps;

4
pkg/interp/testdata/argvars.fqtest vendored
View File

@ -28,11 +28,11 @@ null> ^D
$ fq -n --raw-file filea /nonexisting
exitcode: 2
stderr:
error: open testdata/nonexisting: no such file or directory
error: /nonexisting: no such file or directory
$ fq -n --decode-file filea /nonexisting
exitcode: 2
stderr:
error: --decode-file filea: open testdata/nonexisting: no such file or directory
error: --decode-file filea: no such file or directory
$ fq -n -d mp4 --decode-file filea /test.mp3 '$filea'
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: /test.mp3 (mp4)
| | | error: mp4: error at position 0x8: no styp, ftyp, free or moov box found

4
pkg/interp/testdata/completion.fqtest vendored
View File

@ -9,11 +9,11 @@ hex
hexdump
null> _is_ide\t
_is_ident
null> {aa: 123} | var("test")
null> {aa: 123} | slurp("test")
null> $\t
$ENV
$test
null> $test.a\t
null> $test[].a\t
aa
null> {bb: 123} as $aa | $aa.b\t
bb

2
pkg/interp/testdata/exitcode.fqtest vendored
View File

@ -23,4 +23,4 @@ error: arg:1:2: unexpected token <EOF>
$ fq . non-existing
exitcode: 2
stderr:
error: open testdata/non-existing: no such file or directory
error: non-existing: no such file or directory

4
pkg/interp/testdata/exprfile.fqtest vendored
View File

@ -16,3 +16,7 @@ $ fq -nf /err.jq
exitcode: 3
stderr:
error: /err.jq:1:6: unexpected token ")"
$ fq -n -f missing
exitcode: 2
stderr:
error: missing: no such file or directory

43
pkg/interp/testdata/help.fqtest vendored Normal file
View File

@ -0,0 +1,43 @@
$ fq -ni
null> help
Type expression to evaluate
\t Completion
Up/Down History
^C Interrupt execution
... | repl Start a new REPL
^D Exit REPL
null> help | abc
error: expr: function not defined: abc/0
null> help | 1
error: help must be alone or last in pipeline. ex: help(length) or ... | help
null> abc | help
error: expr: function not defined: abc/0
null> "a"+1 | help
error: cannot add: string ("a") and number (1)
"value help"
[]
null> help(length)
length: Length of string, array, object, etc
- For string number of unicode codepoints
- For array number of elements in array
- For object number of key-value pairs
- For null zero
- For number the number itself
- For boolean is an error
Examples:
> [1,2,3] | length
3
> "abc" | length
3
> {"a":1,"b":2} | length
2
> null | length
0
> 123 | length
123
> true | length
error: length cannot be applied to: boolean (true)
null> help(1;2)
error: expr: help must be last in pipeline. ex: help(length) or ... | help
null> ^D

12
pkg/interp/testdata/inputs.fqtest vendored
View File

@ -18,7 +18,7 @@ $ fq -d raw '(.,input,input,input) | try todescription catch .' /a /b /c
"/c"
exitcode: 5
stderr:
error: break
error: /c: break
$ fq -d raw -n '(.,inputs) | try todescription catch .' /a /b /c
"expected decode value but got: null (null)"
"/a"
@ -40,7 +40,7 @@ $ fq -d raw -n '(input,input,input,input) | todescription' /a /b /c
"/c"
exitcode: 5
stderr:
error: break
error: /c: break
$ fq -d raw input_filename
"<stdin>"
stdin:
@ -54,7 +54,7 @@ $ fq -d raw input_filename /a /non-existing /c
"/c"
exitcode: 2
stderr:
error: open testdata/non-existing: no such file or directory
error: /non-existing: no such file or directory
$ fq -d raw '(' /a /b /c
exitcode: 3
stderr:
@ -66,9 +66,9 @@ error: arg: function not defined: bla/0
$ fq -d raw '1+"a"' /a /b /c
exitcode: 5
stderr:
error: cannot add: number (1) and string ("a")
error: cannot add: number (1) and string ("a")
error: cannot add: number (1) and string ("a")
error: /a: cannot add: number (1) and string ("a")
error: /b: cannot add: number (1) and string ("a")
error: /c: cannot add: number (1) and string ("a")
$ fq -s -d raw '[.[] | todescription]' /a /b /c
[
"/a",

2
pkg/interp/testdata/options.fqtest vendored
View File

@ -79,7 +79,7 @@ $ fq -n -o expr=123
$ fq -o expr_file=test.jq -n options.expr_file
exitcode: 2
stderr:
error: open testdata/test.jq: no such file or directory
error: test.jq: no such file or directory
$ fq -o 'filenames=["/test.mp3"]' format
"mp3"
$ fq -o 'force=true' -n options.force

36
pkg/interp/testdata/repl.fqtest vendored
View File

@ -5,11 +5,20 @@ null
null> 1+1
2
null> (
error: expr:1:2: unexpected token <EOF>
^ unexpected token <EOF>
null> )
^ unexpected token ")"
null> abc
error: expr: function not defined: abc/0
null> 1+"a"
error: cannot add: number (1) and string ("a")
null> abc | repl
error: expr: function not defined: abc/0
null> "a"+1 | repl
error: cannot add: string ("a") and number (1)
> empty> ^D
null> repl | 1
error: repl must be last in pipeline. ex: ... | repl
null> 1 | repl
> number> .+1
2
@ -24,6 +33,11 @@ null> 1,2,3 | repl
2
3
> number, ...[0:3][]> ^D
null> 1,error("err"),3 | repl
error: err
> number> .
1
> number> ^D
null> (1 | raw | .unknown0), 1 | repl
> .unknown0 string, ...[0:2][]> ^D
null> def f: 1; f,f | repl
@ -45,6 +59,16 @@ null> [1] | repl
1
]
> [number]> ^D
null> 1,2,error("err"),3 | repl
error: err
> number, ...[0:2][]> .
1
2
> number, ...[0:2][]> ^D
null> repl(123)
error: expr: options must be an object
null> repl(123; 123)
error: expr: repl requires none or one options argument. ex: ... | repl or ... | repl({compact: true})
null> [] | repl
> []> ^D
null> ^D
@ -56,6 +80,12 @@ number, ...[0:3][]> .*2
2
4
6
number, ...[0:3][]> .*2 | repl
> number, ...[0:3][]> .
2
4
6
> number, ...[0:3][]> ^D
number, ...[0:3][]> ^D
$ fq -i '[1,2,3]'
[number, ...][0:3]> repl({compact: true})
@ -74,6 +104,6 @@ json!> ^D
$ fq -i -n '"[]" | json'
json> ^D
$ fq -n repl
exitcode: 5
exitcode: 3
stderr:
error: repl can only be used from interactive repl
error: arg: repl can only be used from interactive repl

106
pkg/interp/testdata/slurp.fqtest vendored Normal file
View File

@ -0,0 +1,106 @@
$ fq -ni
null> abc | slurp
error: expr: slurp requires one string argument. ex: ... | slurp("name")
null> slurp | 1
error: slurp must be last in pipeline. ex: ... | slurp("name")
null> abc | slurp("a")
error: expr: function not defined: abc/0
null> "a"+1 | slurp("a")
error: cannot add: string ("a") and number (1)
null> 123 | slurp("a")
null> 123, "bb" | slurp("bb")
null> 123, 456, error("err"), "bb" | slurp("err")
error: err
null> spew
{
"a": [
123
],
"bb": [
123,
"bb"
],
"err": [
123,
456
]
}
null> spew("bb")
123
"bb"
null> $a
[
123
]
null> "aa" | slurp("a")
null> spew
{
"a": [
"aa"
],
"bb": [
123,
"bb"
],
"err": [
123,
456
]
}
null> $a
[
"aa"
]
null> . | repl
> null> $bb
[
123,
"bb"
]
> null> ^D
null> 123 | slurp("a b")
error: expr: invalid slurp name "a b", must be a valid identifier. ex: ... | slurp("name")
null> 123 | slurp(null)
error: expr: invalid slurp name "null", must be a valid identifier. ex: ... | slurp("name")
null> ^D
$ fq -i
null> 1,2,3 | repl
> number, ...[0:3][]> ., .*2 | slurp("b")
> number, ...[0:3][]> if . == 2 then error("err") end | slurp("c")
error: err
> number, ...[0:3][]> ^D
null> spew
{
"b": [
1,
2,
2,
4,
3,
6
],
"c": [
1,
3
]
}
null> ^D
$ fq -d mp3 -i . /test.mp3
mp3> .frames[0] | slurp("f")
mp3> $f[]
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.frames[0]{}: (mp3_frame)
0x20| ff fb 40| ..@| header{}:
0x30|c0 |. |
0x30| 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00| ...............| side_info{}:
0x40|00 00 |.. |
0x40| 49 6e 66 6f 00 00 00 0f 00 00 00 02 00 00| Info..........| xing{}: (xing)
0x50|02 57 00 a6 a6 a6 a6 a6 a6 a6 a6 a6 a6 a6 a6 a6|.W..............|
* |until 0xdd.7 (156) | |
0xd0| 00 00| ..| padding: raw bits
0xe0|00 00 00 |... |
| | | crc_calculated: "827a" (raw bits)
mp3> ^D
$ fq -n slurp
exitcode: 3
stderr:
error: arg: slurp can only be used from interactive repl

4
pkg/interp/testdata/string_input.fqtest vendored
View File

@ -57,9 +57,9 @@ c
$ fq -R . missing
exitcode: 2
stderr:
error: open testdata/missing: no such file or directory
error: missing: no such file or directory
$ fq -Rs . missing
""
exitcode: 2
stderr:
error: open testdata/missing: no such file or directory
error: missing: no such file or directory

52
pkg/interp/testdata/var.fqtest vendored
View File

@ -1,52 +0,0 @@
$ fq -ni
null> 123 | var("a")
null> "bb" | var("bb")
null> var
{
"a": 123,
"bb": "bb"
}
null> $a
123
null> "aa" | var("a")
null> var
{
"a": "aa",
"bb": "bb"
}
null> $a
"aa"
null> var("a"; empty)
null> $a
error: expr: variable not defined: $a
null> var
{
"bb": "bb"
}
null> . | repl
> null> $bb
"bb"
> null> ^D
null> var("bb"; empty)
null> var
{}
null> 123 | var("a b")
error: invalid variable name: a b
null> 123 | var(null)
error: invalid variable name: null
null> ^D
$ fq -d mp3 -i . /test.mp3
mp3> .frames[0] | var("f")
mp3> $f
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.frames[0]{}: (mp3_frame)
0x20| ff fb 40| ..@| header{}:
0x30|c0 |. |
0x30| 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00| ...............| side_info{}:
0x40|00 00 |.. |
0x40| 49 6e 66 6f 00 00 00 0f 00 00 00 02 00 00| Info..........| xing{}: (xing)
0x50|02 57 00 a6 a6 a6 a6 a6 a6 a6 a6 a6 a6 a6 a6 a6|.W..............|
* |until 0xdd.7 (156) | |
0xd0| 00 00| ..| padding: raw bits
0xe0|00 00 00 |... |
| | | crc_calculated: "827a" (raw bits)
mp3> ^D