wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-09-11 12:05:39 +03:00
Code Issues Projects Releases Wiki Activity

cli: Make --raw-string work with input/0 and inputs/0

Browse Source 
Also lots of refactor
This commit is contained in:
Mattias Wadman 2021-09-01 19:53:12 +02:00
parent 845bc6b917
commit ac8cfca500
10 changed files with 242 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.vscode/settings.json vendored
View File

@ -119,6 +119,7 @@
"tfhd",
"tfra",
"tmpl",
"toarray",
"toboolean",
"tobytes",
"toimage",

BIN
doc/file.mp3
View File

Binary file not shown.

BIN
doc/file.mp4
View File

Binary file not shown.

4
doc/usage.md
View File

@ -8,7 +8,7 @@ For more information see https://github.com/wader/fq
Usage: fq [OPTIONS] [--] [EXPR] [FILE...]
--compact,-c Compact output
--decode,-d NAME Decode format (probe)
--decode,-d NAME Decode format (default probe)
--file,-f PATH Read EXPR from file
--formats Show supported formats
--help,-h Show help
@ -18,10 +18,10 @@ Usage: fq [OPTIONS] [--] [EXPR] [FILE...]
--null-output,-0 Null byte between outputs
--option,-o KEY=VALUE Set option, eg: color=true
--options Show all options
--raw-input,-R Read raw input strings (don't decode)
--raw-output,-r Raw string output (without quotes)
--repl,-i Interactive REPL
--slurp,-s Read (slurp) all inputs into an array
--raw-input,-R Read raw input strings (don't decode)
--version,-v Show version (dev)
</pre>

4
pkg/interp/args.jq
View File

@ -107,7 +107,9 @@ def args_help_text($opts):
end;
( _maxoptlen as $l
| $opts
| to_entries[]
| to_entries
| sort_by(.value.long)
| .[]
| (.value | .help_default // .default) as $default
| [ "\(.value | _opthelp | rpad(" "; $l)) \(.value.description)"
, if $default then

9
pkg/interp/internal.jq
View File

@ -29,9 +29,6 @@ def _default_options(f): _global_var("default_options"; f);
def _options_stack: _global_var("options_stack");
def _options_stack(f): _global_var("options_stack"; f);
def _parsed_args: _global_var("parsed_args");
def _parsed_args(f): _global_var("parsed_args"; f);
def _cli_last_expr_error: _global_var("cli_last_expr_error");
def _cli_last_expr_error(f): _global_var("cli_last_expr_error"; f);
@ -41,6 +38,12 @@ def _input_filename(f): _global_var("input_filename"; f);
def _input_filenames: _global_var("input_filenames");
def _input_filenames(f): _global_var("input_filenames"; f);
def _input_strings: _global_var("input_strings");
def _input_strings(f): _global_var("input_strings"; f);
def _input_strings_lines: _global_var("input_strings_lines");
def _input_strings_lines(f): _global_var("input_strings_lines"; f);
def _input_io_errors: _global_var("input_io_errors");
def _input_io_errors(f): _global_var("input_io_errors"; f);

300
pkg/interp/interp.jq
View File

@ -71,11 +71,15 @@ def _build_default_options:
} | _obj_to_csv_kv
),
compact: false,
decode_format: "probe",
decode_progress: (env.NODECODEPROGRESS == null),
depth: 0,
# TODO: intdiv 2 * 2 to get even number, nice or maybe not needed?
displaybytes: (if $stdout.is_terminal then [intdiv(intdiv($stdout.width; 8); 2) * 2, 4] | max else 16 end),
expr: ".",
expr_file: null,
expr_eval_path: "arg",
filenames: ["-"],
include_path: null,
join_string: "\n",
linebytes: (if $stdout.is_terminal then [intdiv(intdiv($stdout.width; 8); 2) * 2, 4] | max else 16 end),
@ -84,6 +88,9 @@ def _build_default_options:
raw_string: false,
repl: false,
sizebase: 10,
show_formats: false,
show_help: false,
show_options: false,
slurp: false,
string_input: false,
unicode: ($stdout.is_terminal and env.CLIUNICODE != null),
@ -106,6 +113,13 @@ def _tonumber:
def _tostring:
if . != null then "\"\(.)\"" | fromjson end;
def _toarray:
if . != null then
( fromjson
| if type != "array" then null end
)
end;
def _to_options:
( {
addrbase: (.addrbase | _tonumber),
@ -115,10 +129,13 @@ def _to_options:
color: (.color | _toboolean),
colors: (.colors | _tostring),
compact: (.compact | _toboolean),
decode_format: (.decode_format | _tostring),
decode_progress: (.decode_progress | _toboolean),
depth: (.depth | _tonumber),
displaybytes: (.displaybytes | _tonumber),
expr: (.expr | _tostring),
expr_file: (.expr_file | _tostring),
filename: (.filenames | _toarray),
include_path: (.include_path | _tostring),
join_string: (.join_string | _tostring),
linebytes: (.linebytes | _tonumber),
@ -127,6 +144,9 @@ def _to_options:
raw_string: (.raw_string | _toboolean),
repl: (.repl | _toboolean),
sizebase: (.sizebase | _tonumber),
show_formats: (.show_formats | _toboolean),
show_help: (.show_help | _toboolean),
show_options: (.show_options | _toboolean),
slurp: (.slurp | _toboolean),
string_input: (.string_input | _toboolean),
unicode: (.unicode | _toboolean),
@ -299,44 +319,101 @@ def _cli_expr_on_compile_error:
def _cli_expr_eval($e; $filename; f): _eval($e; $filename; f; _cli_expr_on_error; _cli_expr_on_compile_error);
def _cli_expr_eval($e; $filename): _eval($e; $filename; .; _cli_expr_on_error; _cli_expr_on_compile_error);
def _repeat_break(f):
try repeat(f)
catch if . == "break" then empty else error end;
# next valid input
def input:
( _input_filenames
| if length == 0 then error("break") end
| [.[0], .[1:]] as [$h, $t]
| _input_filenames($t)
| _input_filename(null) as $_
| $h
| try
( open
| _input_filename($h) as $_
| .
)
catch
( . as $err
| _input_io_errors(. += {($h): $err}) as $_
| $err
| (_error_str | _errorln)
, input
)
| try
if _parsed_args.string_input then (tobytes | tostring)
else decode(_parsed_args.decode_format)
end
catch
( . as $err
| _input_decode_errors(. += {($h): $err}) as $_
| "\($h): failed to decode (\(_parsed_args.decode_format)), try -d FORMAT to force"
| (_error_str | _errorln)
, input
def _input($opts; f):
( _input_filenames
| if length == 0 then error("break") end
| [.[0], .[1:]] as [$h, $t]
| _input_filenames($t)
| _input_filename(null) as $_
| $h
| try
( open
| _input_filename($h) as $_
| .
)
catch
( . as $err
| _input_io_errors(. += {($h): $err}) as $_
| $err
| (_error_str | _errorln)
, _input($opts; f)
)
| try f
catch
( . as $err
| _input_decode_errors(. += {($h): $err}) as $_
| "\($h): failed to decode (\($opts.decode_format)), try -d FORMAT to force"
| (_error_str | _errorln)
, _input($opts; f)
)
);
# TODO: don't rebuild options each time
( options as $opts
| if $opts.string_input then
( _input_strings_lines
| if . then
if length == 0 then error("break")
else
( [.[0], .[1:]] as [$h, $t]
| _input_strings_lines($t)
| $h
)
end
else
( [_repeat_break(_input($opts; tobytes | tostring))]
| join("") as $all
| if $opts.slurp then
# jq --raw-input combined with --slurp reads all inputs into a string
# make next input break
( _input_strings_lines([]) as $_
| $all
)
else
# TODO: different line endings?
# jq strips last newline, "a\nb" and "a\nb\n" behaves the same
# also jq -R . <(echo -ne 'a\nb') <(echo c) produces "a" and "bc"
( _input_strings_lines(
( $all
| rtrimstr("\n")
| split("\n")
)
) as $_
| input
)
end
)
end
)
else _input($opts; decode($opts.decode_format))
end
);
# iterate all valid inputs
def inputs:
try repeat(input)
catch if . == "break" then empty else error end;
def inputs: _repeat_break(input);
def _inputs:
( options as $opts
| if $opts.null_input then null
elif $opts.string_input then inputs
# ( [inputs]
# | join("")
# | if $opts.slurp then .
# else
# ( rtrimstr("\n")
# | split("\n")[]
# )
# end
# )
elif $opts.slurp then [inputs]
else inputs
end
);
def input_filename: _input_filename;
@ -384,8 +461,7 @@ def _main:
"decode_format": {
short: "-d",
long: "--decode",
description: "Decode format",
default: "probe",
description: "Decode format (probe)",
string: "NAME"
},
"expr_file": {
@ -394,12 +470,12 @@ def _main:
description: "Read EXPR from file",
string: "PATH"
},
"formats": {
"show_formats": {
long: "--formats",
description: "Show supported formats",
bool: true
},
"help": {
"show_help": {
short: "-h",
long: "--help",
description: "Show help",
@ -436,7 +512,7 @@ def _main:
description: "Set option, eg: color=true",
object: "KEY=VALUE",
},
"options": {
"show_options": {
long: "--options",
description: "Show all options",
bool: true
@ -467,7 +543,7 @@ def _main:
description: "Read (slurp) all inputs into an array",
bool: true
},
"version": {
"show_version": {
short: "-v",
long: "--version",
description: "Show version (\($version))",
@ -488,48 +564,70 @@ def _main:
| ( try args_parse($args[1:]; _opts($version))
catch halt_error(_exit_code_args_error)
) as {parsed: $parsed_args, $rest}
# store parsed arguments, .decode_format is used by input/0
| _parsed_args($parsed_args) as $_
| _default_options(_build_default_options) as $_
# combine --args and -o key=value args
| ( ($parsed_args.option | _to_options)
+ $parsed_args
) as $args_opts
| _options_stack(
[ ($parsed_args.option | _to_options)
+ ({
compact: $parsed_args.compact,
expr_file: $parsed_args.expr_file,
join_string: (
if $parsed_args.join_output then ""
elif $parsed_args.null_output then "\u0000"
else null
end
),
include_path: $parsed_args.include_path,
null_input: $parsed_args.null_input,
repl: $parsed_args.repl,
raw_string: (
if $parsed_args.raw_string
or $parsed_args.join_output
or $parsed_args.null_output
then true
else null
end
),
slurp: $parsed_args.slurp,
string_input: $parsed_args.string_input
} | with_entries(select(.value != null)))
[ $args_opts
+ ( {
expr: (
( $args_opts.expr_file
| if . then
try (open | tobytes | tostring)
catch halt_error(_exit_code_args_error)
else $rest[0] // null
end
)
),
filenames: (
( if $args_opts.expr_file then $rest
else $rest[1:]
end
| if . == [] then null end
)
),
join_string: (
if $args_opts.join_output then ""
elif $args_opts.null_output then "\u0000"
else null
end
),
null_input: (
( if $args_opts.expr_file then $rest
else $rest[1:]
end
| if . == [] and $args_opts.repl then true
else null
end
)
),
raw_string: (
if $args_opts.raw_string
or $args_opts.join_output
or $args_opts.null_output
then true
else null
end
)
}
| with_entries(select(.value != null))
)
]
) as $_
| options as $opts
| if $parsed_args.help then
| if $opts.show_help then
( _banner
, ""
, _usage($arg0; $version)
, args_help_text(_opts($version))
) | println
elif $parsed_args.version then
elif $opts.show_version then
$version | println
elif $parsed_args.formats then
elif $opts.show_formats then
_formats_list | println
elif $parsed_args.options then
elif $opts.show_options then
$opts | display
elif
( ($rest | length) == 0 and
@ -543,64 +641,24 @@ def _main:
else
# use finally as display etc prints and results in empty
finally(
( { null_input: ($opts.null_input == true) }
| if $opts.expr_file then
( .expr = ($opts.expr_file | open | tobytes | tostring)
| .expr_filename = $opts.expr_file
| .filenames = $rest
)
else
( .expr = ($rest[0] // ".")
| .expr_filename = "arg"
| .filenames = $rest[1:]
)
end
| if $opts.repl and .filenames == [] then
.null_input = true
elif .filenames == [] then
.filenames = ["-"]
end
| . as {$expr, $expr_filename, $filenames, $null_input}
| _include_paths([
( _include_paths([
$opts.include_path // empty
]) as $_
| _input_filenames($filenames) as $_ # store inputs
| ( def _inputs:
if $null_input then null
elif $opts.string_input then
( [inputs]
| join("")
| if $opts.slurp then
# jq --raw-input combined with --slurp reads all inputs into a string
.
else
# TODO: different line endings?
# jq strips last newline, "a\nb" and "a\nb\n" behaves the same
# also jq -R . <(echo -ne 'a\nb') <(echo c) produces "a" and "bc"
( rtrimstr("\n")
| split("\n")[]
)
end
)
elif $opts.slurp then [inputs]
else inputs
end;
if $opts.repl then
( [_inputs]
| ( [.[] | _cli_expr_eval($expr; $expr_filename)]
| repl({}; .[])
)
| _input_filenames($opts.filenames) as $_ # store inputs
| if $opts.repl then
( [_inputs]
| ( [.[] | _cli_expr_eval($opts.expr; $opts.expr_eval_path)]
| repl({}; .[])
)
else
( _inputs
# iterate all inputs
| ( _cli_last_expr_error(null) as $_
| _cli_expr_eval($expr; $expr_filename; _repl_display)
)
)
else
( _inputs
# iterate all inputs
| ( _cli_last_expr_error(null) as $_
| _cli_expr_eval($opts.expr; $opts.expr_eval_path; _repl_display)
)
end
)
)
end
)
; # finally
( if _input_io_errors then

11
pkg/interp/testdata/args.fqtest vendored
View File

@ -20,10 +20,10 @@ Usage: fq [OPTIONS] [--] [EXPR] [FILE...]
--null-output,-0 Null byte between outputs
--option,-o KEY=VALUE Set option, eg: color=true
--options Show all options
--raw-input,-R Read raw input strings (don't decode)
--raw-output,-r Raw string output (without quotes)
--repl,-i Interactive REPL
--slurp,-s Read (slurp) all inputs into an array
--raw-input,-R Read raw input strings (don't decode)
--version,-v Show version (dev)
$ fq -i
null> ^D
@ -120,10 +120,16 @@ $ fq --options
"color": false,
"colors": "array=white,dumpaddr=yellow,dumpheader=yellow+underline,error=brightred,false=yellow,index=white,null=brightblack,number=cyan,object=white,objectkey=brightblue,string=green,true=yellow,value=white",
"compact": false,
"decode_format": "probe",
"decode_progress": false,
"depth": 0,
"displaybytes": 16,
"expr": ".",
"expr_eval_path": "arg",
"expr_file": null,
"filenames": [
"-"
],
"include_path": null,
"join_string": "\n",
"linebytes": 16,
@ -131,6 +137,9 @@ $ fq --options
"raw_output": false,
"raw_string": false,
"repl": false,
"show_formats": false,
"show_help": false,
"show_options": true,
"sizebase": 10,
"slurp": false,
"string_input": false,

11
pkg/interp/testdata/options.fqtest vendored
View File

@ -7,10 +7,16 @@ $ fq -n options
"color": false,
"colors": "array=white,dumpaddr=yellow,dumpheader=yellow+underline,error=brightred,false=yellow,index=white,null=brightblack,number=cyan,object=white,objectkey=brightblue,string=green,true=yellow,value=white",
"compact": false,
"decode_format": "probe",
"decode_progress": false,
"depth": 0,
"displaybytes": 16,
"expr": "options",
"expr_eval_path": "arg",
"expr_file": null,
"filenames": [
"-"
],
"include_path": null,
"join_string": "\n",
"linebytes": 16,
@ -18,6 +24,9 @@ $ fq -n options
"raw_output": false,
"raw_string": false,
"repl": false,
"show_formats": false,
"show_help": false,
"show_options": false,
"sizebase": 10,
"slurp": false,
"string_input": false,
@ -59,7 +68,7 @@ $ fq -o displaybytes=10 -n options.displaybytes
$ fq -o displaybytes=true -n options.displaybytes
16
$ fq -o expr_file=test.jq -n options.expr_file
exitcode: 5
exitcode: 2
stderr:
error: test.jq: file not found
$ fq -o include_path=path -n options.include_path

31
pkg/interp/testdata/string_input.fqtest vendored
View File

@ -1,3 +1,4 @@
# TODO: test for jq -R . <(echo -ne 'a\nb') <(echo c) produces "a" and "bc"
/a:
a
b
@ -9,6 +10,10 @@ $ fq -R . /a /c
"c"
$ fq -Rs . /a /c
"a\nb\nc\n"
$ fq -nRs input /a /c
"a\nb\nc\n"
$ fq -nRs inputs /a /c
"a\nb\nc\n"
$ fq -R
"a"
"b"
@ -17,9 +22,35 @@ stdin:
a
b
c
$ fq -nR input
"a"
stdin:
a
b
c
$ fq -nR inputs
"a"
"b"
"c"
stdin:
a
b
c
$ fq -Rs
"a\nb\nc\n"
stdin:
a
b
c
$ fq -nRs input
"a\nb\nc\n"
stdin:
a
b
c
$ fq -nRs inputs
"a\nb\nc\n"
stdin:
a
b
c