1
1
mirror of https://github.com/wader/fq.git synced 2024-12-23 21:31:33 +03:00

interp: Reorganize, move out repl and options, more functions to funcs.jq

This commit is contained in:
Mattias Wadman 2021-09-22 21:08:36 +02:00
parent b75da3001a
commit 0cce5ec61f
6 changed files with 418 additions and 419 deletions

View File

@ -1,5 +1,36 @@
# TODO: figure out a saner way to force int
def _to_int: (. % (. + 1));
def print: stdout;
def println: ., "\n" | stdout;
def debug:
( ((["DEBUG", .] | tojson), "\n" | stderr)
, .
);
def debug(f): . as $c | f | debug | $c;
# TODO: introspect and show doc, reflection somehow?
def help:
( "Type jq expression to evaluate"
, "\\t Auto completion"
, "Up/Down History"
, "^C Interrupt execution"
, "... | repl Start a new REPL"
, "^D Exit REPL"
) | println;
def display($opts): _display($opts);
def display: _display({});
def d($opts): _display($opts);
def d: _display({});
def full($opts): _display({array_truncate: 0} + $opts);
def full: full({});
def f($opts): full($opts);
def f: full;
def verbose($opts): _display({verbose: true, array_truncate: 0} + $opts);
def verbose: verbose({});
def v($opts): verbose($opts);
def v: verbose;
def formats:
_registry.formats;
# integer division
# inspried by https://github.com/itchyny/gojq/issues/63#issuecomment-765066351
@ -9,6 +40,12 @@ def intdiv($a; $b):
| ($a - ($a % $b)) / $b
);
def _esc: "\u001b";
def _ansi:
{
clear_line: "\(_esc)[2K",
};
# valid jq identifier, start with alpha or underscore then zero or more alpha, num or underscore
def _is_ident: type == "string" and test("^[a-zA-Z_][a-zA-Z_0-9]*$");
# escape " and \

View File

@ -1,11 +1,3 @@
def print: stdout;
def println: ., "\n" | stdout;
def debug:
( ((["DEBUG", .] | tojson), "\n" | stderr)
, .
);
def debug(f): . as $c | f | debug | $c;
# eval f and finally eval fin even on empty or error
def _finally(f; fin):
( try f // (fin | empty)
@ -14,6 +6,9 @@ def _finally(f; fin):
| .
);
# TODO: figure out a saner way to force int
def _to_int: (. % (. + 1));
def _repeat_break(f):
try repeat(f)
catch
@ -21,6 +16,18 @@ def _repeat_break(f):
else error
end;
# TODO: better way? what about nested eval errors?
def _eval_is_compile_error: type == "object" and .error != null and .what != null;
def _eval_compile_error_tostring:
"\(.filename // "src"):\(.line):\(.column): \(.error)";
def _eval($expr; $filename; f; on_error; on_compile_error):
( try eval($expr; $filename) | f
catch
if _eval_is_compile_error then on_compile_error
else on_error
end
);
def _error_str: "error: \(.)";
def _errorln: ., "\n" | stderr;

View File

@ -33,9 +33,11 @@ import (
//go:embed interp.jq
//go:embed internal.jq
//go:embed options.jq
//go:embed funcs.jq
//go:embed args.jq
//go:embed query.jq
//go:embed repl.jq
//go:embed formats.jq
var builtinFS embed.FS

View File

@ -1,7 +1,9 @@
include "internal";
include "funcs";
include "options";
include "args";
include "query";
include "repl";
# generated decode functions per format and format helpers
include "formats";
# optional user init
@ -23,415 +25,6 @@ def _exit_code_input_decode_error: 4;
def _exit_code_expr_error: 5;
def _obj_to_csv_kv:
[to_entries[] | [.key, .value] | join("=")] | join(",");
def _build_default_fixed_options:
( (null | stdout) as $stdout
| {
addrbase: 16,
arg: [],
argjson: [],
array_truncate: 50,
bits_format: "snippet",
byte_colors: "0-0xff=brightwhite,0=brightblack,32-126:9-13=white",
color: ($stdout.is_terminal and (env.NO_COLOR | . == null or . == "")),
colors: (
{
null: "brightblack",
false: "yellow",
true: "yellow",
number: "cyan",
string: "green",
objectkey: "brightblue",
array: "white",
object: "white",
index: "white",
value: "white",
error: "brightred",
dumpheader: "yellow+underline",
dumpaddr: "yellow"
} | _obj_to_csv_kv
),
compact: false,
decode_file: [],
decode_format: "probe",
decode_progress: (env.NO_DECODE_PROGRESS == null),
depth: 0,
expr: ".",
expr_file: null,
expr_eval_path: "arg",
filenames: ["-"],
include_path: null,
join_string: "\n",
null_input: false,
rawfile: [],
raw_output: ($stdout.is_terminal | not),
raw_string: false,
repl: false,
sizebase: 10,
show_formats: false,
show_help: false,
show_options: false,
slurp: false,
string_input: false,
unicode: ($stdout.is_terminal and env.CLIUNICODE != null),
verbose: false,
}
);
def _build_default_dynamic_options:
( (null | stdout) as $stdout
| {
# TODO: intdiv 2 * 2 to get even number, nice or maybe not needed?
display_bytes: (if $stdout.is_terminal then [intdiv(intdiv($stdout.width; 8); 2) * 2, 4] | max else 16 end),
line_bytes: (if $stdout.is_terminal then [intdiv(intdiv($stdout.width; 8); 2) * 2, 4] | max else 16 end),
}
);
def _toboolean:
try
if . == "true" then true
elif . == "false" then false
else tonumber != 0
end
catch
null;
def _tonumber:
try tonumber catch null;
def _tostring:
if . != null then
( "\"\(.)\""
| try
( fromjson
| if type != "string" then error end
)
catch null
)
end;
def _toarray(f):
try
( fromjson
| if type == "array" and (all(f) | not) then null end
)
catch null;
def _is_string_pair:
type == "array" and length == 2 and all(type == "string");
def _to_options:
( {
addrbase: (.addrbase | _tonumber),
arg: (.arg | _toarray(_is_string_pair)),
argjson: (.argjson | _toarray(_is_string_pair)),
array_truncate: (.array_truncate | _tonumber),
bits_format: (.bits_format | _tostring),
byte_colors: (.byte_colors | _tostring),
color: (.color | _toboolean),
colors: (.colors | _tostring),
compact: (.compact | _toboolean),
decode_file: (.decode_file | _toarray(type == "string")),
decode_format: (.decode_format | _tostring),
decode_progress: (.decode_progress | _toboolean),
depth: (.depth | _tonumber),
display_bytes: (.display_bytes | _tonumber),
expr: (.expr | _tostring),
expr_file: (.expr_file | _tostring),
filename: (.filenames | _toarray(type == "string")),
include_path: (.include_path | _tostring),
join_string: (.join_string | _tostring),
line_bytes: (.line_bytes | _tonumber),
null_input: (.null_input | _toboolean),
rawfile: (.rawfile| _toarray(_is_string_pair)),
raw_output: (.raw_output | _toboolean),
raw_string: (.raw_string | _toboolean),
repl: (.repl | _toboolean),
sizebase: (.sizebase | _tonumber),
show_formats: (.show_formats | _toboolean),
show_help: (.show_help | _toboolean),
show_options: (.show_options | _toboolean),
slurp: (.slurp | _toboolean),
string_input: (.string_input | _toboolean),
unicode: (.unicode | _toboolean),
verbose: (.verbose | _toboolean),
}
| with_entries(select(.value != null))
);
# . will have additional array of options taking priority
# NOTE: is called from go *interp.Interp Options()
def options($opts):
[_build_default_dynamic_options] + _options_stack + $opts | add;
def options: options([{}]);
# TODO: currently only make sense to allow keywords start start a term or directive
def _complete_keywords:
[
"and",
#"as",
#"break",
#"catch",
"def",
#"elif",
#"else",
#"end",
"false",
"foreach",
"if",
"import",
"include",
"label",
"module",
"null",
"or",
"reduce",
#"then",
"true",
"try"
];
def _complete_scope:
[scope[], _complete_keywords[]];
# TODO: handle variables via ast walk?
# TODO: refactor this
# TODO: completionMode
# TODO: return escaped identifier, not sure current readline implementation supports
# modifying "previous" characters if quoting is needed
# completions that needs to change previous input, ex: .a\t -> ."a \" b" etc
def _complete($line; $cursor_pos):
# TODO: reverse this? word or non-ident char?
def _is_separator: . as $c | " .;[]()|=" | contains($c);
def _is_internal: startswith("_") or startswith("$_");
def _query_index_or_key($q):
( ([.[] | eval($q) | type]) as $n
| if ($n | all(. == "object")) then "."
elif ($n | all(. == "array")) then "[]"
else null
end
);
# only complete if at end or there is a whitespace for now
if ($line[$cursor_pos] | . == "" or _is_separator) then
( . as $c
| $line[0:$cursor_pos]
| . as $line_query
# expr -> map(partial-expr | . | f?) | add
# TODO: move map/add logic to here?
| _query_completion(
if .type | . == "func" or . == "var" then "_complete_scope"
elif .type == "index" then
if (.prefix | startswith("_")) then "_extkeys"
else "keys"
end
else error("unreachable")
end
) as {$type, $query, $prefix}
| {
prefix: $prefix,
names: (
if $type == "none" then
( $c
| _query_index_or_key($line_query)
| if . then [.] else [] end
)
else
( $c
| eval($query)
| ($prefix | _is_internal) as $prefix_is_internal
| map(
select(
strings and
# TODO: var type really needed? just func?
(_is_ident or $type == "var") and
((_is_internal | not) or $prefix_is_internal or $type == "index") and
startswith($prefix)
)
)
| unique
| sort
| if length == 1 and .[0] == $prefix then
( $c
| _query_index_or_key($line_query)
| if . then [$prefix+.] else [$prefix] end
)
end
)
end
)
}
)
else
{prefix: "", names: []}
end;
def _complete($line): _complete($line; $line | length);
def _prompt:
def _type_name_error:
( . as $c
| try
( _display_name
, if ._error then "!" else empty end
)
catch ($c | type)
);
def _path_prefix:
(._path? // []) | if . == [] then "" else path_to_expr + " " end;
def _preview:
if format != null or type != "array" then
_type_name_error
else
( "["
, if length > 0 then (.[0] | _type_name_error) else empty end
, if length > 1 then ", ..." else empty end
, "]"
, if length > 1 then "[\(length)]" else empty end
)
end;
( [ (_options_stack | length | if . > 2 then ((.-2) * ">") + " " else empty end)
, if length == 0 then
"empty"
else
( .[0]
| _path_prefix
, _preview
)
end
, if length > 1 then ", [\(length)]" else empty end
, "> "
]
) | join("");
# TODO: better way? what about nested eval errors?
def _eval_is_compile_error: type == "object" and .error != null and .what != null;
def _eval_compile_error_tostring:
"\(.filename // "src"):\(.line):\(.column): \(.error)";
def _eval($expr; $filename; f; on_error; on_compile_error):
( try eval($expr; $filename) | f
catch
if _eval_is_compile_error then on_compile_error
else on_error
end
);
def _repl_display: _display({depth: 1});
def _repl_on_error:
( if _eval_is_compile_error then _eval_compile_error_tostring end
| (_error_str | println)
);
def _repl_on_compile_error: _repl_on_error;
def _repl_eval($expr): _eval($expr; "repl"; _repl_display; _repl_on_error; _repl_on_compile_error);
# run read-eval-print-loop
def _repl($opts): #:: a|(Opts) => @
def _read_expr:
# both _prompt and _complete want arrays
( . as $c
| _readline(_prompt; {complete: "_complete", timeout: 0.5})
| if trim == "" then
$c | _read_expr
end
);
def _repl_loop:
( . as $c
| try
( _read_expr
| . as $expr
| try _query_fromstring
# TODO: nicer way to set filename for error message
catch (. | .filename = "repl")
| if _query_pipe_last | _query_is_func("repl") then
( _query_slurp_wrap(_query_func_rename("_repl_slurp"))
| _query_tostring as $wrap_expr
| $c
| _repl_eval($wrap_expr)
)
else
( $c
| .[]
| _repl_eval($expr)
)
end
)
catch
if . == "interrupt" then empty
elif . == "eof" then error("break")
elif _eval_is_compile_error then _repl_on_error
else error
end
);
( _options_stack(. + [$opts]) as $_
| _finally(
_repeat_break(_repl_loop);
_options_stack(.[:-1])
)
);
def _repl_slurp($opts): _repl($opts);
def _repl_slurp: _repl({});
# just gives error, call appearing last will be renamed to _repl_slurp
def repl($_):
if options.repl then error("repl must be last")
else error("repl can only be used from interactive repl")
end;
def repl: repl(null);
def _cli_expr_on_error:
( . as $err
| _cli_last_expr_error($err) as $_
| (_error_str | _errorln)
);
def _cli_expr_on_compile_error:
( _eval_compile_error_tostring
| halt_error(_exit_code_compile_error)
);
# _cli_expr_eval halts on compile errors
def _cli_expr_eval($expr; $filename; f):
_eval($expr; $filename; f; _cli_expr_on_error; _cli_expr_on_compile_error);
def _cli_expr_eval($expr; $filename):
_eval($expr; $filename; .; _cli_expr_on_error; _cli_expr_on_compile_error);
# TODO: introspect and show doc, reflection somehow?
def help:
( "Type jq expression to evaluate"
, "\\t Auto completion"
, "Up/Down History"
, "^C Interrupt execution"
, "... | repl Start a new REPL"
, "^D Exit REPL"
) | println;
def display($opts): _display($opts);
def display: _display({});
def d($opts): _display($opts);
def d: _display({});
def full($opts): _display({array_truncate: 0} + $opts);
def full: full({});
def f($opts): full($opts);
def f: full;
def verbose($opts): _display({verbose: true, array_truncate: 0} + $opts);
def verbose: verbose({});
def v($opts): verbose($opts);
def v: verbose;
def formats:
_registry.formats;
def _esc: "\u001b";
def _ansi:
{
clear_line: "\(_esc)[2K",
};
# null input means done, otherwise {approx_read_bytes: 123, total_size: 123}
# TODO: decode provide even more detailed progress, post-process sort etc?
def _decode_progress:
@ -565,6 +158,22 @@ def var($k; f):
def var($k): . as $c | var($k; $c);
def _cli_expr_on_error:
( . as $err
| _cli_last_expr_error($err) as $_
| (_error_str | _errorln)
);
def _cli_expr_on_compile_error:
( _eval_compile_error_tostring
| halt_error(_exit_code_compile_error)
);
# _cli_expr_eval halts on compile errors
def _cli_expr_eval($expr; $filename; f):
_eval($expr; $filename; f; _cli_expr_on_error; _cli_expr_on_compile_error);
def _cli_expr_eval($expr; $filename):
_eval($expr; $filename; .; _cli_expr_on_error; _cli_expr_on_compile_error);
def _main:
def _formats_list:
[ ( formats

144
pkg/interp/options.jq Normal file
View File

@ -0,0 +1,144 @@
def _obj_to_csv_kv:
[to_entries[] | [.key, .value] | join("=")] | join(",");
def _build_default_fixed_options:
( (null | stdout) as $stdout
| {
addrbase: 16,
arg: [],
argjson: [],
array_truncate: 50,
bits_format: "snippet",
byte_colors: "0-0xff=brightwhite,0=brightblack,32-126:9-13=white",
color: ($stdout.is_terminal and (env.NO_COLOR | . == null or . == "")),
colors: (
{
null: "brightblack",
false: "yellow",
true: "yellow",
number: "cyan",
string: "green",
objectkey: "brightblue",
array: "white",
object: "white",
index: "white",
value: "white",
error: "brightred",
dumpheader: "yellow+underline",
dumpaddr: "yellow"
} | _obj_to_csv_kv
),
compact: false,
decode_file: [],
decode_format: "probe",
decode_progress: (env.NO_DECODE_PROGRESS == null),
depth: 0,
expr: ".",
expr_file: null,
expr_eval_path: "arg",
filenames: ["-"],
include_path: null,
join_string: "\n",
null_input: false,
rawfile: [],
raw_output: ($stdout.is_terminal | not),
raw_string: false,
repl: false,
sizebase: 10,
show_formats: false,
show_help: false,
show_options: false,
slurp: false,
string_input: false,
unicode: ($stdout.is_terminal and env.CLIUNICODE != null),
verbose: false,
}
);
def _build_default_dynamic_options:
( (null | stdout) as $stdout
| {
# TODO: intdiv 2 * 2 to get even number, nice or maybe not needed?
display_bytes: (if $stdout.is_terminal then [intdiv(intdiv($stdout.width; 8); 2) * 2, 4] | max else 16 end),
line_bytes: (if $stdout.is_terminal then [intdiv(intdiv($stdout.width; 8); 2) * 2, 4] | max else 16 end),
}
);
# these _to* function do a bit for fuzzy string to type conversions
def _toboolean:
try
if . == "true" then true
elif . == "false" then false
else tonumber != 0
end
catch
null;
def _tonumber:
try tonumber catch null;
def _tostring:
if . != null then
( "\"\(.)\""
| try
( fromjson
| if type != "string" then error end
)
catch null
)
end;
def _toarray(f):
try
( fromjson
| if type == "array" and (all(f) | not) then null end
)
catch null;
def _is_string_pair:
type == "array" and length == 2 and all(type == "string");
def _to_options:
( {
addrbase: (.addrbase | _tonumber),
arg: (.arg | _toarray(_is_string_pair)),
argjson: (.argjson | _toarray(_is_string_pair)),
array_truncate: (.array_truncate | _tonumber),
bits_format: (.bits_format | _tostring),
byte_colors: (.byte_colors | _tostring),
color: (.color | _toboolean),
colors: (.colors | _tostring),
compact: (.compact | _toboolean),
decode_file: (.decode_file | _toarray(type == "string")),
decode_format: (.decode_format | _tostring),
decode_progress: (.decode_progress | _toboolean),
depth: (.depth | _tonumber),
display_bytes: (.display_bytes | _tonumber),
expr: (.expr | _tostring),
expr_file: (.expr_file | _tostring),
filename: (.filenames | _toarray(type == "string")),
include_path: (.include_path | _tostring),
join_string: (.join_string | _tostring),
line_bytes: (.line_bytes | _tonumber),
null_input: (.null_input | _toboolean),
rawfile: (.rawfile| _toarray(_is_string_pair)),
raw_output: (.raw_output | _toboolean),
raw_string: (.raw_string | _toboolean),
repl: (.repl | _toboolean),
sizebase: (.sizebase | _tonumber),
show_formats: (.show_formats | _toboolean),
show_help: (.show_help | _toboolean),
show_options: (.show_options | _toboolean),
slurp: (.slurp | _toboolean),
string_input: (.string_input | _toboolean),
unicode: (.unicode | _toboolean),
verbose: (.verbose | _toboolean),
}
| with_entries(select(.value != null))
);
# . will have additional array of options taking priority
# NOTE: is called from go *interp.Interp Options()
def options($opts):
[_build_default_dynamic_options] + _options_stack + $opts | add;
def options: options([{}]);

200
pkg/interp/repl.jq Normal file
View File

@ -0,0 +1,200 @@
# TODO: currently only make sense to allow keywords start start a term or directive
def _complete_keywords:
[
"and",
#"as",
#"break",
#"catch",
"def",
#"elif",
#"else",
#"end",
"false",
"foreach",
"if",
"import",
"include",
"label",
"module",
"null",
"or",
"reduce",
#"then",
"true",
"try"
];
def _complete_scope:
[scope[], _complete_keywords[]];
# TODO: handle variables via ast walk?
# TODO: refactor this
# TODO: completionMode
# TODO: return escaped identifier, not sure current readline implementation supports
# modifying "previous" characters if quoting is needed
# completions that needs to change previous input, ex: .a\t -> ."a \" b" etc
def _complete($line; $cursor_pos):
# TODO: reverse this? word or non-ident char?
def _is_separator: . as $c | " .;[]()|=" | contains($c);
def _is_internal: startswith("_") or startswith("$_");
def _query_index_or_key($q):
( ([.[] | eval($q) | type]) as $n
| if ($n | all(. == "object")) then "."
elif ($n | all(. == "array")) then "[]"
else null
end
);
# only complete if at end or there is a whitespace for now
if ($line[$cursor_pos] | . == "" or _is_separator) then
( . as $c
| $line[0:$cursor_pos]
| . as $line_query
# expr -> map(partial-expr | . | f?) | add
# TODO: move map/add logic to here?
| _query_completion(
if .type | . == "func" or . == "var" then "_complete_scope"
elif .type == "index" then
if (.prefix | startswith("_")) then "_extkeys"
else "keys"
end
else error("unreachable")
end
) as {$type, $query, $prefix}
| {
prefix: $prefix,
names: (
if $type == "none" then
( $c
| _query_index_or_key($line_query)
| if . then [.] else [] end
)
else
( $c
| eval($query)
| ($prefix | _is_internal) as $prefix_is_internal
| map(
select(
strings and
# TODO: var type really needed? just func?
(_is_ident or $type == "var") and
((_is_internal | not) or $prefix_is_internal or $type == "index") and
startswith($prefix)
)
)
| unique
| sort
| if length == 1 and .[0] == $prefix then
( $c
| _query_index_or_key($line_query)
| if . then [$prefix+.] else [$prefix] end
)
end
)
end
)
}
)
else
{prefix: "", names: []}
end;
def _complete($line): _complete($line; $line | length);
def _prompt:
def _type_name_error:
( . as $c
| try
( _display_name
, if ._error then "!" else empty end
)
catch ($c | type)
);
def _path_prefix:
(._path? // []) | if . == [] then "" else path_to_expr + " " end;
def _preview:
if format != null or type != "array" then
_type_name_error
else
( "["
, if length > 0 then (.[0] | _type_name_error) else empty end
, if length > 1 then ", ..." else empty end
, "]"
, if length > 1 then "[\(length)]" else empty end
)
end;
( [ (_options_stack | length | if . > 2 then ((.-2) * ">") + " " else empty end)
, if length == 0 then
"empty"
else
( .[0]
| _path_prefix
, _preview
)
end
, if length > 1 then ", [\(length)]" else empty end
, "> "
]
) | join("");
def _repl_display: _display({depth: 1});
def _repl_on_error:
( if _eval_is_compile_error then _eval_compile_error_tostring end
| (_error_str | println)
);
def _repl_on_compile_error: _repl_on_error;
def _repl_eval($expr): _eval($expr; "repl"; _repl_display; _repl_on_error; _repl_on_compile_error);
# run read-eval-print-loop
def _repl($opts): #:: a|(Opts) => @
def _read_expr:
# both _prompt and _complete want arrays
( . as $c
| _readline(_prompt; {complete: "_complete", timeout: 0.5})
| if trim == "" then
$c | _read_expr
end
);
def _repl_loop:
( . as $c
| try
( _read_expr
| . as $expr
| try _query_fromstring
# TODO: nicer way to set filename for error message
catch (. | .filename = "repl")
| if _query_pipe_last | _query_is_func("repl") then
( _query_slurp_wrap(_query_func_rename("_repl_slurp"))
| _query_tostring as $wrap_expr
| $c
| _repl_eval($wrap_expr)
)
else
( $c
| .[]
| _repl_eval($expr)
)
end
)
catch
if . == "interrupt" then empty
elif . == "eof" then error("break")
elif _eval_is_compile_error then _repl_on_error
else error
end
);
( _options_stack(. + [$opts]) as $_
| _finally(
_repeat_break(_repl_loop);
_options_stack(.[:-1])
)
);
def _repl_slurp($opts): _repl($opts);
def _repl_slurp: _repl({});
# just gives error, call appearing last will be renamed to _repl_slurp
def repl($_):
if options.repl then error("repl must be last")
else error("repl can only be used from interactive repl")
end;
def repl: repl(null);