1
1
mirror of https://github.com/wader/fq.git synced 2024-11-23 00:57:15 +03:00

cli: Add --raw-string

This commit is contained in:
Mattias Wadman 2021-09-01 15:01:13 +02:00
parent 4242bf6013
commit 6356a84f15
9 changed files with 249 additions and 140 deletions

View File

@ -37,11 +37,13 @@ Usage: fq [OPTIONS] [--] [EXPR] [FILE...]
repl=false
sizebase=10
slurp=false
string_input=false
unicode=false
verbose=false
--raw-output,-r Raw string output (without quotes)
--repl,-i Interactive REPL
--slurp,-s Read (slurp) all inputs into an array
--raw-input,-R Read raw input strings (don't decode)
--version,-v Show version (dev)
</pre>

View File

@ -33,13 +33,13 @@ func (a autoCompleterFn) Do(line []rune, pos int) (newLine [][]rune, length int)
return a(line, pos)
}
type standardOS struct {
type stdOS struct {
rl *readline.Instance
interruptSignalChan chan os.Signal
interruptChan chan struct{}
}
func newStandardOS() *standardOS {
func newStandardOS() *stdOS {
interruptChan := make(chan struct{}, 1)
interruptSignalChan := make(chan os.Signal, 1)
signal.Notify(interruptSignalChan, os.Interrupt)
@ -53,53 +53,74 @@ func newStandardOS() *standardOS {
}
}()
return &standardOS{
return &stdOS{
interruptSignalChan: interruptSignalChan,
interruptChan: interruptChan,
}
}
func (o *standardOS) Stdin() fs.File {
return interp.FileReader{
R: os.Stdin,
FileInfo: interp.FixedFileInfo{
FName: "stdin",
FMode: fs.ModeIrregular,
type fdTerminal uintptr
func (fd fdTerminal) Size() (int, int) {
w, h, _ := readline.GetSize(int(fd))
return w, h
}
func (fd fdTerminal) IsTerminal() bool {
return readline.IsTerminal(int(fd))
}
type stdinInput struct {
fdTerminal
fs.File
}
func (o *stdOS) Stdin() interp.Input {
return stdinInput{
fdTerminal: fdTerminal(os.Stdin.Fd()),
File: interp.FileReader{
R: os.Stdin,
FileInfo: interp.FixedFileInfo{
FName: "stdin",
FMode: fs.ModeIrregular,
},
},
}
}
type standardOsOutput struct {
os *standardOS
type stdoutOutput struct {
fdTerminal
os *stdOS
}
func (o standardOsOutput) Write(p []byte) (n int, err error) {
func (o stdoutOutput) Write(p []byte) (n int, err error) {
// Let write go thru readline if it has been used. This to have ansi color emulation
// on windows thru readlins:s stdout rewriter
// TODO: check if tty instead? else only color when repl
if o.os.rl != nil {
return o.os.rl.Write(p)
}
return os.Stdout.Write(p)
}
func (o standardOsOutput) Size() (int, int) {
w, h, _ := readline.GetSize(int(os.Stdout.Fd()))
return w, h
func (o *stdOS) Stdout() interp.Output {
return stdoutOutput{fdTerminal: fdTerminal(os.Stdout.Fd()), os: o}
}
func (o standardOsOutput) IsTerminal() bool {
return readline.IsTerminal(int(os.Stdout.Fd()))
type stderrOutput struct {
fdTerminal
}
func (o *standardOS) Stdout() interp.Output { return standardOsOutput{os: o} }
func (o stderrOutput) Write(p []byte) (n int, err error) { return os.Stderr.Write(p) }
func (o *standardOS) Stderr() io.Writer { return os.Stderr }
func (o *stdOS) Stderr() interp.Output { return stderrOutput{fdTerminal: fdTerminal(os.Stderr.Fd())} }
func (o *standardOS) Interrupt() chan struct{} { return o.interruptChan }
func (o *stdOS) Interrupt() chan struct{} { return o.interruptChan }
func (*standardOS) Args() []string { return os.Args }
func (*stdOS) Args() []string { return os.Args }
func (*standardOS) Environ() []string { return os.Environ() }
func (*stdOS) Environ() []string { return os.Environ() }
func (*standardOS) ConfigDir() (string, error) {
func (*stdOS) ConfigDir() (string, error) {
p, err := os.UserConfigDir()
if err != nil {
return "", err
@ -107,13 +128,13 @@ func (*standardOS) ConfigDir() (string, error) {
return filepath.Join(p, "fq"), nil
}
type standardOSFS struct{}
type stdOSFS struct{}
func (standardOSFS) Open(name string) (fs.File, error) { return os.Open(name) }
func (stdOSFS) Open(name string) (fs.File, error) { return os.Open(name) }
func (*standardOS) FS() fs.FS { return standardOSFS{} }
func (*stdOS) FS() fs.FS { return stdOSFS{} }
func (o *standardOS) Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error) {
func (o *stdOS) Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error) {
if o.rl == nil {
var err error
@ -159,7 +180,7 @@ func (o *standardOS) Readline(prompt string, complete func(line string, pos int)
return line, nil
}
func (o *standardOS) History() ([]string, error) {
func (o *stdOS) History() ([]string, error) {
// TODO: refactor history handling to use internal fs?
r, err := os.Open(o.rl.Config.HistoryFile)
if err != nil {
@ -177,7 +198,7 @@ func (o *standardOS) History() ([]string, error) {
return hs, nil
}
func (o *standardOS) Close() error {
func (o *stdOS) Close() error {
// only close if is terminal otherwise ansi reset will write
// to stdout and mess up raw output
if o.rl != nil {

View File

@ -29,12 +29,20 @@ type testCaseReadline struct {
expectedStdout string
}
type testCaseRunInput struct {
interp.FileReader
isTerminal bool
}
func (testCaseRunInput) Size() (int, int) { return 130, 25 }
func (i testCaseRunInput) IsTerminal() bool { return i.isTerminal }
type testCaseRunOutput struct {
io.Writer
}
func (o testCaseRunOutput) Size() (int, int) { return 130, 25 }
func (o testCaseRunOutput) IsTerminal() bool { return true }
func (testCaseRunOutput) Size() (int, int) { return 130, 25 }
func (testCaseRunOutput) IsTerminal() bool { return true }
type testCaseRun struct {
lineNr int
@ -53,13 +61,18 @@ type testCaseRun struct {
func (tcr *testCaseRun) Line() int { return tcr.lineNr }
func (tcr *testCaseRun) Stdin() fs.File {
return interp.FileReader{R: bytes.NewBufferString(tcr.stdin)}
func (tcr *testCaseRun) Stdin() interp.Input {
return testCaseRunInput{
FileReader: interp.FileReader{
R: bytes.NewBufferString(tcr.stdin),
},
isTerminal: tcr.stdin == "",
}
}
func (tcr *testCaseRun) Stdout() interp.Output { return testCaseRunOutput{tcr.actualStdoutBuf} }
func (tcr *testCaseRun) Stderr() io.Writer { return tcr.actualStderrBuf }
func (tcr *testCaseRun) Stderr() interp.Output { return testCaseRunOutput{tcr.actualStderrBuf} }
func (tcr *testCaseRun) Interrupt() chan struct{} { return nil }

View File

@ -23,6 +23,7 @@ import (
"github.com/wader/fq/format/registry"
"github.com/wader/fq/internal/aheadreadseeker"
"github.com/wader/fq/internal/ctxreadseeker"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/ioextra"
"github.com/wader/fq/internal/progressreadseeker"
"github.com/wader/fq/pkg/bitio"
@ -34,12 +35,11 @@ import (
// TODO: make it nicer somehow? generate generators? remove from struct?
func (i *Interp) makeFunctions(registry *registry.Registry) []Function {
fs := []Function{
{[]string{"tty"}, 0, 0, i.tty, nil},
{[]string{"readline"}, 0, 2, i.readline, nil},
{[]string{"eval"}, 1, 2, nil, i.eval},
{[]string{"stdout"}, 0, 0, nil, i.stdout},
{[]string{"stderr"}, 0, 0, nil, i.stderr},
{[]string{"stdin"}, 0, 0, nil, i.makeStdioFn(i.os.Stdin())},
{[]string{"stdout"}, 0, 0, nil, i.makeStdioFn(i.os.Stdout())},
{[]string{"stderr"}, 0, 0, nil, i.makeStdioFn(i.os.Stderr())},
{[]string{"_complete_query"}, 0, 0, i._completeQuery, nil},
{[]string{"_display_name"}, 0, 0, i._displayName, nil},
@ -103,15 +103,6 @@ func (i *Interp) makeFunctions(registry *registry.Registry) []Function {
return fs
}
func (i *Interp) tty(c interface{}, a []interface{}) interface{} {
w, h := i.evalContext.stdout.Size()
return map[string]interface{}{
"is_terminal": i.evalContext.stdout.IsTerminal(),
"width": w,
"height": h,
}
}
// transform byte string <-> buffer using fn:s
func makeStringBitBufTransformFn(
decodeFn func(r io.Reader) (io.Reader, error),
@ -259,7 +250,7 @@ func (i *Interp) eval(c interface{}, a []interface{}) gojq.Iter {
}
}
iter, err := i.Eval(i.evalContext.ctx, ScriptMode, c, src, filenameHint, i.evalContext.stdout)
iter, err := i.Eval(i.evalContext.ctx, ScriptMode, c, src, filenameHint, i.evalContext.output)
if err != nil {
return gojq.NewIter(err)
}
@ -267,18 +258,26 @@ func (i *Interp) eval(c interface{}, a []interface{}) gojq.Iter {
return iter
}
func (i *Interp) stdout(c interface{}, a []interface{}) gojq.Iter {
if _, err := fmt.Fprint(i.os.Stdout(), c); err != nil {
return gojq.NewIter(err)
}
return gojq.NewIter()
}
func (i *Interp) makeStdioFn(t Terminal) func(c interface{}, a []interface{}) gojq.Iter {
return func(c interface{}, a []interface{}) gojq.Iter {
if c == nil {
w, h := t.Size()
return gojq.NewIter(map[string]interface{}{
"is_terminal": t.IsTerminal(),
"width": w,
"height": h,
})
}
func (i *Interp) stderr(c interface{}, a []interface{}) gojq.Iter {
if _, err := fmt.Fprint(i.os.Stderr(), c); err != nil {
return gojq.NewIter(err)
if w, ok := t.(io.Writer); ok {
if _, err := fmt.Fprint(w, c); err != nil {
return gojq.NewIter(err)
}
return gojq.NewIter()
}
return gojq.NewIter(fmt.Errorf("%v: it not writeable", c))
}
return gojq.NewIter()
}
func (i *Interp) _completeQuery(c interface{}, a []interface{}) interface{} {
@ -386,6 +385,8 @@ func (i *Interp) history(c interface{}, a []interface{}) interface{} {
}
type bitBufFile struct {
gojq.JQValue
bb *bitio.Buffer
filename string
@ -395,7 +396,7 @@ type bitBufFile struct {
var _ ToBuffer = (*bitBufFile)(nil)
func (bbf *bitBufFile) Display(w io.Writer, opts Options) error {
_, err := fmt.Fprintf(w, "<%s>\n", bbf.filename)
_, err := fmt.Fprintln(w, bbf.JQValue.JQValueToString())
return err
}
@ -484,6 +485,7 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
}
return &bitBufFile{
JQValue: gojqextra.String(fmt.Sprintf("<bitBufFile %s>", path)),
bb: bb,
filename: path,
decodeDoneFn: decodeDoneFn,
@ -563,23 +565,23 @@ func (i *Interp) makeDisplayFn(fnOpts map[string]interface{}) func(c interface{}
switch v := c.(type) {
case Display:
if err := v.Display(i.evalContext.stdout, opts); err != nil {
if err := v.Display(i.evalContext.output, opts); err != nil {
return gojq.NewIter(err)
}
return gojq.NewIter()
case nil, bool, float64, int, string, *big.Int, map[string]interface{}, []interface{}, gojq.JQValue:
if s, ok := v.(string); ok && opts.RawString {
fmt.Fprint(i.evalContext.stdout, s)
fmt.Fprint(i.evalContext.output, s)
} else {
cj, err := i.NewColorJSON(opts)
if err != nil {
return gojq.NewIter(err)
}
if err := cj.Marshal(v, i.evalContext.stdout); err != nil {
if err := cj.Marshal(v, i.evalContext.output); err != nil {
return gojq.NewIter(err)
}
}
fmt.Fprint(i.evalContext.stdout, opts.JoinString)
fmt.Fprint(i.evalContext.output, opts.JoinString)
return gojq.NewIter()
case error:
@ -599,7 +601,7 @@ func (i *Interp) preview(c interface{}, a []interface{}) gojq.Iter {
switch v := c.(type) {
case Preview:
if err := v.Preview(i.evalContext.stdout, opts); err != nil {
if err := v.Preview(i.evalContext.output, opts); err != nil {
return gojq.NewIter(err)
}
return gojq.NewIter()
@ -619,7 +621,7 @@ func (i *Interp) hexdump(c interface{}, a []interface{}) gojq.Iter {
return gojq.NewIter(err)
}
if err := hexdumpRange(bbr, i.evalContext.stdout, opts); err != nil {
if err := hexdumpRange(bbr, i.evalContext.output, opts); err != nil {
return gojq.NewIter(err)
}

View File

@ -82,16 +82,25 @@ type IsEmptyErrorer interface {
IsEmptyError() bool
}
type Output interface {
io.Writer
type Terminal interface {
Size() (int, int)
IsTerminal() bool
}
type Input interface {
fs.File
Terminal
}
type Output interface {
io.Writer
Terminal
}
type OS interface {
Stdin() fs.File
Stdin() Input
Stdout() Output
Stderr() io.Writer
Stderr() Output
Interrupt() chan struct{}
Args() []string
Environ() []string
@ -143,7 +152,7 @@ func (o DiscardOutput) Write(p []byte) (n int, err error) {
}
type CtxOutput struct {
Output
io.Writer
Ctx context.Context
}
@ -153,7 +162,7 @@ func (o CtxOutput) Write(p []byte) (n int, err error) {
return 0, err
}
}
return o.Output.Write(p)
return o.Writer.Write(p)
}
type InterpValue interface {
@ -440,7 +449,7 @@ const (
type evalContext struct {
// structcheck has problems with embedding https://gitlab.com/opennota/check#known-limitations
ctx context.Context
stdout Output
output io.Writer
mode RunMode
}
@ -532,7 +541,7 @@ func (i *Interp) Main(ctx context.Context, stdout Output, version string) error
return nil
}
func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src string, filename string, stdout Output) (gojq.Iter, error) {
func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src string, filename string, output io.Writer) (gojq.Iter, error) {
gq, err := gojq.Parse(src)
if err != nil {
p := queryErrorPosition(src, err)
@ -728,7 +737,7 @@ func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src stri
runCtx, runCtxCancelFn := i.interruptStack.Push(ctx)
ni.evalContext.ctx = runCtx
ni.evalContext.stdout = CtxOutput{Output: stdout, Ctx: runCtx}
ni.evalContext.output = CtxOutput{Writer: output, Ctx: runCtx}
iter := gc.RunWithContext(runCtx, c, variableValues...)
iterWrapper := iterFn(func() (interface{}, bool) {
@ -743,7 +752,7 @@ func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src stri
return iterWrapper, nil
}
func (i *Interp) EvalFunc(ctx context.Context, mode RunMode, c interface{}, name string, args []interface{}, stdout Output) (gojq.Iter, error) {
func (i *Interp) EvalFunc(ctx context.Context, mode RunMode, c interface{}, name string, args []interface{}, output Output) (gojq.Iter, error) {
var argsExpr []string
for i := range args {
argsExpr = append(argsExpr, fmt.Sprintf("$_args[%d]", i))
@ -760,15 +769,15 @@ func (i *Interp) EvalFunc(ctx context.Context, mode RunMode, c interface{}, name
/// _args to mark variable as internal and hide it from completion
// {input: ..., args: [...]} | .args as {args: $_args} | .input | name[($_args[0]; ...)]
trampolineExpr := fmt.Sprintf(". as {args: $_args} | .input | %s%s", name, argExpr)
iter, err := i.Eval(ctx, mode, trampolineInput, trampolineExpr, "", stdout)
iter, err := i.Eval(ctx, mode, trampolineInput, trampolineExpr, "", output)
if err != nil {
return nil, err
}
return iter, nil
}
func (i *Interp) EvalFuncValues(ctx context.Context, mode RunMode, c interface{}, name string, args []interface{}, stdout Output) ([]interface{}, error) {
iter, err := i.EvalFunc(ctx, mode, c, name, args, stdout)
func (i *Interp) EvalFuncValues(ctx context.Context, mode RunMode, c interface{}, name string, args []interface{}, output Output) ([]interface{}, error) {
iter, err := i.EvalFunc(ctx, mode, c, name, args, output)
if err != nil {
return nil, err
}

View File

@ -46,47 +46,50 @@ def _obj_to_csv_kv:
[to_entries[] | [.key, .value] | join("=")] | join(",");
def _build_default_options:
{
addrbase: 16,
arraytruncate: 50,
bitsformat: "snippet",
bytecolors: "0-0xff=brightwhite,0=brightblack,32-126:9-13=white",
color: (tty.is_terminal and env.CLICOLOR != null),
colors: (
{
null: "brightblack",
false: "yellow",
true: "yellow",
number: "cyan",
string: "green",
objectkey: "brightblue",
array: "white",
object: "white",
index: "white",
value: "white",
error: "brightred",
dumpheader: "yellow+underline",
dumpaddr: "yellow"
} | _obj_to_csv_kv
),
compact: false,
decode_progress: (env.NODECODEPROGRESS == null),
depth: 0,
# TODO: intdiv 2 * 2 to get even number, nice or maybe not needed?
displaybytes: (if tty.is_terminal then [intdiv(intdiv(tty.width; 8); 2) * 2, 4] | max else 16 end),
expr_file: null,
include_path: null,
join_string: "\n",
linebytes: (if tty.is_terminal then [intdiv(intdiv(tty.width; 8); 2) * 2, 4] | max else 16 end),
null_input: false,
raw_output: (tty.is_terminal | not),
raw_string: false,
repl: false,
sizebase: 10,
slurp: false,
unicode: (tty.is_terminal and env.CLIUNICODE != null),
verbose: false,
};
( (null | stdout) as $stdout
| {
addrbase: 16,
arraytruncate: 50,
bitsformat: "snippet",
bytecolors: "0-0xff=brightwhite,0=brightblack,32-126:9-13=white",
color: ($stdout.is_terminal and env.CLICOLOR != null),
colors: (
{
null: "brightblack",
false: "yellow",
true: "yellow",
number: "cyan",
string: "green",
objectkey: "brightblue",
array: "white",
object: "white",
index: "white",
value: "white",
error: "brightred",
dumpheader: "yellow+underline",
dumpaddr: "yellow"
} | _obj_to_csv_kv
),
compact: false,
decode_progress: (env.NODECODEPROGRESS == null),
depth: 0,
# TODO: intdiv 2 * 2 to get even number, nice or maybe not needed?
displaybytes: (if $stdout.is_terminal then [intdiv(intdiv($stdout.width; 8); 2) * 2, 4] | max else 16 end),
expr_file: null,
include_path: null,
join_string: "\n",
linebytes: (if $stdout.is_terminal then [intdiv(intdiv($stdout.width; 8); 2) * 2, 4] | max else 16 end),
null_input: false,
raw_output: ($stdout.is_terminal | not),
raw_string: false,
repl: false,
sizebase: 10,
slurp: false,
string_input: false,
unicode: ($stdout.is_terminal and env.CLIUNICODE != null),
verbose: false,
}
);
def _toboolean:
try
@ -125,6 +128,7 @@ def _to_options:
repl: (.repl | _toboolean),
sizebase: (.sizebase | _tonumber),
slurp: (.slurp | _toboolean),
string_input: (.string_input | _toboolean),
unicode: (.unicode | _toboolean),
verbose: (.verbose | _toboolean),
}
@ -317,7 +321,9 @@ def input:
, input
)
| try
decode(_parsed_args.decode_format)
if _parsed_args.string_input then (tobytes | tostring)
else decode(_parsed_args.decode_format)
end
catch
( . as $err
| _input_decode_errors(. += {($h): $err}) as $_
@ -432,10 +438,16 @@ def _main:
default: {},
help_default: _build_default_options
},
"string_input": {
short: "-R",
long: "--raw-input",
description: "Read raw input strings (don't decode)",
bool: true
},
"raw_string": {
short: "-r",
# for jq compat, is called raw string internally, raw output is
# if we can output raw bytes or not
# for jq compat, is called raw string internally, "raw output" is if
# we can output raw bytes or not
long: "--raw-output",
description: "Raw string output (without quotes)",
bool: true
@ -467,6 +479,7 @@ def _main:
def _usage($arg0; $version):
"Usage: \($arg0) [OPTIONS] [--] [EXPR] [FILE...]";
( . as {$version, $args, args: [$arg0]}
| (null | [stdin, stdout]) as [$stdin, $stdout]
# make sure we don't unintentionally use . to make things clearer
| null
| ( try args_parse($args[1:]; _opts($version))
@ -497,7 +510,8 @@ def _main:
else null
end
),
slurp: $parsed_args.slurp
slurp: $parsed_args.slurp,
string_input: $parsed_args.string_input
} | with_entries(select(.value != null)))
]
) as $_
@ -512,7 +526,12 @@ def _main:
$version | println
elif $parsed_args.formats then
_formats_list | println
elif ($rest | length) == 0 and (($opts.repl | not) and ($opts.expr_file | not)) then
elif
( ($rest | length) == 0 and
($opts.repl | not) and
($opts.expr_file | not) and
$stdin.is_terminal and $stdout.is_terminal
) then
( (( _usage($arg0; $version), "\n") | stderr)
, null | halt_error(_exit_code_args_error)
)
@ -541,27 +560,42 @@ def _main:
$opts.include_path // empty
]) as $_
| _input_filenames($filenames) as $_ # store inputs
| if $opts.repl then
( [ if $null_input then null
elif $opts.slurp then [inputs]
else inputs
end
]
| ( [.[] | _cli_expr_eval($expr; $expr_filename)]
| repl({}; .[])
)
)
else
( if $null_input then null
| ( def _inputs:
if $null_input then null
elif $opts.string_input then
( [inputs]
| join("")
| if $opts.slurp then
# jq --raw-input combined with --slurp reads all inputs into a string
.
else
# TODO: different line endings?
# jq strips last newline, "a\nb" and "a\nb\n" behaves the same
# also jq -R . <(echo -ne 'a\nb') <(echo c) produces "a" and "bc"
( rtrimstr("\n")
| split("\n")[]
)
end
)
elif $opts.slurp then [inputs]
else inputs
end
# iterate inputs
end;
if $opts.repl then
( [_inputs]
| ( [.[] | _cli_expr_eval($expr; $expr_filename)]
| repl({}; .[])
)
)
else
( _inputs
# iterate all inputs
| ( _cli_last_expr_error(null) as $_
| _cli_expr_eval($expr; $expr_filename; _repl_display)
)
)
end
)
end
)
)
; # finally
( if _input_io_errors then

View File

@ -39,11 +39,13 @@ Usage: fq [OPTIONS] [--] [EXPR] [FILE...]
repl=false
sizebase=10
slurp=false
string_input=false
unicode=false
verbose=false
--raw-output,-r Raw string output (without quotes)
--repl,-i Interactive REPL
--slurp,-s Read (slurp) all inputs into an array
--raw-input,-R Read raw input strings (don't decode)
--version,-v Show version (dev)
$ fq -i
null> ^D

View File

@ -20,6 +20,7 @@ $ fq -n options
"repl": false,
"sizebase": 10,
"slurp": false,
"string_input": false,
"unicode": false,
"verbose": false
}

25
pkg/interp/testdata/string_input.fqtest vendored Normal file
View File

@ -0,0 +1,25 @@
/a:
a
b
/c:
c
$ fq -R . /a /c
"a"
"b"
"c"
$ fq -Rs . /a /c
"a\nb\nc\n"
$ fq -R
"a"
"b"
"c"
stdin:
a
b
c
$ fq -Rs
"a\nb\nc\n"
stdin:
a
b
c