diff --git a/doc/TODO.md b/doc/TODO.md index 41d070e8..f272d760 100644 --- a/doc/TODO.md +++ b/doc/TODO.md @@ -13,6 +13,7 @@ - Rework cli/repl user interrupt (context cancel via ctrl-c), see comment in Interp.Main - Optimize `Interp.Options` calls, now called per display. Cache per eval? needs to handle nested evals. - `[{start: ...: end: ...}]` syntax a bit broken. +- REPL completion might have side effcts. Make interp.Function type know and wrap somehow? input, inputs, open, ... ### TODO and ideas diff --git a/doc/usage.md b/doc/usage.md index d02e8c7d..99cd58f9 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -418,6 +418,8 @@ you currently have to do `fq -d raw 'mp3({force: true})' file`. - `p`/`preview` show preview of field tree - `hd`/`hexdump` hexdump value - `repl` nested REPL, must be last in a pipeline. `1 | repl`, can "slurp" outputs `1, 2, 3 | repl`. +- `paste` read string from stdin until ^D. Useful for pasting text. + - Ex: `paste | frompem | asn1_ber | repl` read from stdin then decode and start a new sub-REPL with result. ## Color and unicode output diff --git a/internal/script/script.go b/internal/script/script.go index 6c69af5f..f638d075 100644 --- a/internal/script/script.go +++ b/internal/script/script.go @@ -149,8 +149,8 @@ func (cr *CaseRun) ConfigDir() (string, error) { return "/config", nil } func (cr *CaseRun) FS() fs.FS { return cr.Case } -func (cr *CaseRun) Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error) { - cr.ActualStdoutBuf.WriteString(prompt) +func (cr *CaseRun) Readline(opts interp.ReadlineOpts) (string, error) { + cr.ActualStdoutBuf.WriteString(opts.Prompt) if cr.ReadlinesPos >= len(cr.Readlines) { return "", io.EOF } @@ -165,7 +165,7 @@ func (cr *CaseRun) Readline(prompt string, complete func(line string, pos int) ( cr.ActualStdoutBuf.WriteString(lineRaw + "\n") l := len(line) - 1 - newLine, shared := complete(line[0:l], l) + newLine, shared := opts.CompleteFn(line[0:l], l) // TODO: shared _ = shared for _, nl := range newLine { diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 3b091556..2b276118 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -158,7 +158,7 @@ func (stdOSFS) Open(name string) (fs.File, error) { return os.Open(name) } func (*stdOS) FS() fs.FS { return stdOSFS{} } -func (o *stdOS) Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error) { +func (o *stdOS) Readline(opts interp.ReadlineOpts) (string, error) { if o.rl == nil { var err error @@ -179,9 +179,9 @@ func (o *stdOS) Readline(prompt string, complete func(line string, pos int) (new } } - if complete != nil { + if opts.CompleteFn != nil { o.rl.Config.AutoComplete = autoCompleterFn(func(line []rune, pos int) (newLine [][]rune, length int) { - names, shared := complete(string(line), pos) + names, shared := opts.CompleteFn(string(line), pos) var runeNames [][]rune for _, name := range names { runeNames = append(runeNames, []rune(name[shared:])) @@ -191,7 +191,7 @@ func (o *stdOS) Readline(prompt string, complete func(line string, pos int) (new }) } - o.rl.SetPrompt(prompt) + o.rl.SetPrompt(opts.Prompt) line, err := o.rl.Readline() if errors.Is(err, readline.ErrInterrupt) { return "", interp.ErrInterrupt diff --git a/pkg/interp/binary.go b/pkg/interp/binary.go index 25adad8e..c399f203 100644 --- a/pkg/interp/binary.go +++ b/pkg/interp/binary.go @@ -16,13 +16,14 @@ import ( "github.com/wader/fq/internal/progressreadseeker" "github.com/wader/fq/pkg/bitio" "github.com/wader/fq/pkg/ranges" + "github.com/wader/gojq" ) func init() { functionRegisterFns = append(functionRegisterFns, func(i *Interp) []Function { return []Function{ {"_tobits", 3, 3, i._toBits, nil}, - {"open", 0, 0, i._open, nil}, + {"open", 0, 0, nil, i._open}, } }) } @@ -175,7 +176,11 @@ func (of *openFile) ToBinary() (Binary, error) { // def open: #:: string| => binary // opens a file for reading from filesystem // TODO: when to close? when br loses all refs? need to use finalizer somehow? -func (i *Interp) _open(c interface{}, a []interface{}) interface{} { +func (i *Interp) _open(c interface{}, a []interface{}) gojq.Iter { + if i.evalContext.isCompleting { + return gojq.NewIter() + } + var err error var f fs.File var path string @@ -187,11 +192,11 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} { default: path, err = toString(c) if err != nil { - return fmt.Errorf("%s: %w", path, err) + return gojq.NewIter(fmt.Errorf("%s: %w", path, err)) } f, err = i.os.FS().Open(path) if err != nil { - return err + return gojq.NewIter(err) } } @@ -201,7 +206,7 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} { fFI, err := f.Stat() if err != nil { f.Close() - return err + return gojq.NewIter(err) } // ctxreadseeker is used to make sure any io calls can be canceled @@ -219,7 +224,7 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} { buf, err := ioutil.ReadAll(ctxreadseeker.New(i.evalContext.ctx, &ioextra.ReadErrSeeker{Reader: f})) if err != nil { f.Close() - return err + return gojq.NewIter(err) } fRS = bytes.NewReader(buf) bEnd = int64(len(buf)) @@ -246,10 +251,10 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} { bbf.br = bitio.NewIOBitReadSeeker(aheadRs) if err != nil { - return err + return gojq.NewIter(err) } - return bbf + return gojq.NewIter(bbf) } var _ Value = Binary{} diff --git a/pkg/interp/decode.go b/pkg/interp/decode.go index 6d0e022d..88451ee1 100644 --- a/pkg/interp/decode.go +++ b/pkg/interp/decode.go @@ -162,7 +162,7 @@ func (i *Interp) _decode(c interface{}, a []interface{}) interface{} { c, opts.Progress, nil, - ioextra.DiscardCtxWriter{Ctx: i.evalContext.ctx}, + EvalOpts{output: ioextra.DiscardCtxWriter{Ctx: i.evalContext.ctx}}, ) } lastProgress := time.Now() diff --git a/pkg/interp/funcs.jq b/pkg/interp/funcs.jq index 0ea830ef..ac2cb8d7 100644 --- a/pkg/interp/funcs.jq +++ b/pkg/interp/funcs.jq @@ -305,3 +305,14 @@ def topem($label): | join("\n") ); def topem: topem(""); + +def paste: + if _is_completing | not then + ( [ _repeat_break( + try _stdin(64*1024) + catch if . == "eof" then error("break") end + ) + ] + | join("") + ) + end; diff --git a/pkg/interp/interp.go b/pkg/interp/interp.go index f1950854..f38aee4e 100644 --- a/pkg/interp/interp.go +++ b/pkg/interp/interp.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "io/fs" + "io/ioutil" "math/big" "path" "strconv" @@ -22,6 +23,7 @@ import ( "github.com/wader/fq/internal/bitioextra" "github.com/wader/fq/internal/colorjson" "github.com/wader/fq/internal/ctxstack" + "github.com/wader/fq/internal/gojqextra" "github.com/wader/fq/internal/ioextra" "github.com/wader/fq/internal/mathextra" "github.com/wader/fq/internal/pos" @@ -53,11 +55,11 @@ var functionRegisterFns []func(i *Interp) []Function func init() { functionRegisterFns = append(functionRegisterFns, func(i *Interp) []Function { return []Function{ - {"_readline", 0, 2, i._readline, nil}, + {"_readline", 0, 1, nil, i._readline}, {"eval", 1, 2, nil, i.eval}, - {"_stdin", 0, 0, nil, i.makeStdioFn(i.os.Stdin())}, - {"_stdout", 0, 0, nil, i.makeStdioFn(i.os.Stdout())}, - {"_stderr", 0, 0, nil, i.makeStdioFn(i.os.Stderr())}, + {"_stdin", 0, 1, nil, i.makeStdioFn("stdin", i.os.Stdin())}, + {"_stdout", 0, 0, nil, i.makeStdioFn("stdout", i.os.Stdout())}, + {"_stderr", 0, 0, nil, i.makeStdioFn("stderr", i.os.Stderr())}, {"_extkeys", 0, 0, i._extKeys, nil}, {"_exttype", 0, 0, i._extType, nil}, {"_global_state", 0, 1, i.makeStateFn(i.state), nil}, @@ -65,6 +67,7 @@ func init() { {"_display", 1, 1, nil, i._display}, {"_can_display", 0, 0, i._canDisplay, nil}, {"_print_color_json", 0, 1, nil, i._printColorJSON}, + {"_is_completing", 0, 1, i._isCompleting, nil}, } }) } @@ -95,7 +98,7 @@ func (ce compileError) Value() interface{} { func (ce compileError) Error() string { filename := ce.filename if filename == "" { - filename = "src" + filename = "expr" } return fmt.Sprintf("%s:%d:%d: %s: %s", filename, ce.pos.Line, ce.pos.Column, ce.what, ce.err.Error()) } @@ -133,6 +136,11 @@ type Platform struct { Arch string } +type ReadlineOpts struct { + Prompt string + CompleteFn func(line string, pos int) (newLine []string, shared int) +} + type OS interface { Platform() Platform Stdin() Input @@ -144,7 +152,7 @@ type OS interface { ConfigDir() (string, error) // FS.File returned by FS().Open() can optionally implement io.Seeker FS() fs.FS - Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error) + Readline(opts ReadlineOpts) (string, error) History() ([]string, error) } @@ -283,14 +291,14 @@ func toBytes(v interface{}) ([]byte, error) { } } -func queryErrorPosition(src string, v error) pos.Pos { +func queryErrorPosition(expr string, v error) pos.Pos { var offset int if tokIf, ok := v.(interface{ Token() (string, int) }); ok { //nolint:errorlint _, offset = tokIf.Token() } if offset >= 0 { - return pos.NewFromOffset(src, offset) + return pos.NewFromOffset(expr, offset) } return pos.Pos{} } @@ -317,8 +325,9 @@ const ( ) type evalContext struct { - ctx context.Context - output io.Writer + ctx context.Context + output io.Writer + isCompleting bool } type Interp struct { @@ -380,7 +389,7 @@ func (i *Interp) Main(ctx context.Context, output Output, versionStr string) err "arch": platform.Arch, } - iter, err := i.EvalFunc(ctx, input, "_main", nil, output) + iter, err := i.EvalFunc(ctx, input, "_main", nil, EvalOpts{output: output}) if err != nil { fmt.Fprintln(i.os.Stderr(), err) return err @@ -414,28 +423,24 @@ func (i *Interp) Main(ctx context.Context, output Output, versionStr string) err return nil } -func (i *Interp) _readline(c interface{}, a []interface{}) interface{} { +func (i *Interp) _readline(c interface{}, a []interface{}) gojq.Iter { + if i.evalContext.isCompleting { + return gojq.NewIter() + } + var opts struct { + Promopt string `mapstructure:"prompt"` Complete string `mapstructure:"complete"` Timeout float64 `mapstructure:"timeout"` } - var err error - prompt := "" - if len(a) > 0 { - prompt, err = toString(a[0]) - if err != nil { - return fmt.Errorf("prompt: %w", err) - } - } - if len(a) > 1 { - _ = mapstructure.Decode(a[1], &opts) + _ = mapstructure.Decode(a[0], &opts) } - src, err := i.os.Readline( - prompt, - func(line string, pos int) (newLine []string, shared int) { + expr, err := i.os.Readline(ReadlineOpts{ + Prompt: opts.Promopt, + CompleteFn: func(line string, pos int) (newLine []string, shared int) { completeCtx := i.evalContext.ctx if opts.Timeout > 0 { var completeCtxCancelFn context.CancelFunc @@ -450,7 +455,10 @@ func (i *Interp) _readline(c interface{}, a []interface{}) interface{} { c, opts.Complete, []interface{}{line, pos}, - ioextra.DiscardCtxWriter{Ctx: completeCtx}, + EvalOpts{ + output: ioextra.DiscardCtxWriter{Ctx: completeCtx}, + isCompleting: true, + }, ) if err != nil { return nil, pos, err @@ -485,24 +493,24 @@ func (i *Interp) _readline(c interface{}, a []interface{}) interface{} { return names, shared }, - ) + }) if errors.Is(err, ErrInterrupt) { - return valueError{"interrupt"} + return gojq.NewIter(valueError{"interrupt"}) } else if errors.Is(err, ErrEOF) { - return valueError{"eof"} + return gojq.NewIter(valueError{"eof"}) } else if err != nil { - return err + return gojq.NewIter(err) } - return src + return gojq.NewIter(expr) } func (i *Interp) eval(c interface{}, a []interface{}) gojq.Iter { var err error - src, err := toString(a[0]) + expr, err := toString(a[0]) if err != nil { - return gojq.NewIter(fmt.Errorf("src: %w", err)) + return gojq.NewIter(fmt.Errorf("expr: %w", err)) } var filenameHint string if len(a) >= 2 { @@ -512,7 +520,10 @@ func (i *Interp) eval(c interface{}, a []interface{}) gojq.Iter { } } - iter, err := i.Eval(i.evalContext.ctx, c, src, filenameHint, i.evalContext.output) + iter, err := i.Eval(i.evalContext.ctx, c, expr, EvalOpts{ + filename: filenameHint, + output: i.evalContext.output, + }) if err != nil { return gojq.NewIter(err) } @@ -547,25 +558,53 @@ func (i *Interp) makeStateFn(state *interface{}) func(c interface{}, a []interfa } } -func (i *Interp) makeStdioFn(t Terminal) func(c interface{}, a []interface{}) gojq.Iter { +func (i *Interp) makeStdioFn(name string, t Terminal) func(c interface{}, a []interface{}) gojq.Iter { return func(c interface{}, a []interface{}) gojq.Iter { - if c == nil { + if i.evalContext.isCompleting { + return gojq.NewIter("") + } + + switch { + case len(a) == 1: + r, ok := t.(io.Reader) + if !ok { + return gojq.NewIter(fmt.Errorf("%s is not readable", name)) + } + l, ok := gojqextra.ToInt(a[0]) + if !ok { + return gojq.NewIter(gojqextra.FuncTypeError{Name: name, V: a[0]}) + } + + buf := make([]byte, l) + n, err := io.ReadFull(r, buf) + s := string(buf[0:n]) + + vs := []interface{}{s} + switch { + case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF): + vs = append(vs, valueError{"eof"}) + default: + vs = append(vs, err) + } + + return gojq.NewIter(vs...) + case c == nil: w, h := t.Size() return gojq.NewIter(map[string]interface{}{ "is_terminal": t.IsTerminal(), "width": w, "height": h, }) - } - - if w, ok := t.(io.Writer); ok { + default: + w, ok := t.(io.Writer) + if !ok { + return gojq.NewIter(fmt.Errorf("%v: it not writeable", c)) + } if _, err := fmt.Fprint(w, c); err != nil { return gojq.NewIter(err) } return gojq.NewIter() } - - return gojq.NewIter(fmt.Errorf("%v: it not writeable", c)) } } @@ -595,6 +634,11 @@ func (i *Interp) _display(c interface{}, a []interface{}) gojq.Iter { } } +func (i *Interp) _canDisplay(c interface{}, a []interface{}) interface{} { + _, ok := c.(Display) + return ok +} + func (i *Interp) _printColorJSON(c interface{}, a []interface{}) gojq.Iter { opts := i.Options(a[0]) @@ -609,9 +653,8 @@ func (i *Interp) _printColorJSON(c interface{}, a []interface{}) gojq.Iter { return gojq.NewIter() } -func (i *Interp) _canDisplay(c interface{}, a []interface{}) interface{} { - _, ok := c.(Display) - return ok +func (i *Interp) _isCompleting(c interface{}, a []interface{}) interface{} { + return i.evalContext.isCompleting } type pathResolver struct { @@ -667,14 +710,20 @@ func (i *Interp) lookupPathResolver(filename string) (pathResolver, error) { return pathResolver{}, fmt.Errorf("could not resolve path: %s", filename) } -func (i *Interp) Eval(ctx context.Context, c interface{}, src string, srcFilename string, output io.Writer) (gojq.Iter, error) { - gq, err := gojq.Parse(src) +type EvalOpts struct { + filename string + output io.Writer + isCompleting bool +} + +func (i *Interp) Eval(ctx context.Context, c interface{}, expr string, opts EvalOpts) (gojq.Iter, error) { + gq, err := gojq.Parse(expr) if err != nil { - p := queryErrorPosition(src, err) + p := queryErrorPosition(expr, err) return nil, compileError{ err: err, what: "parse", - filename: srcFilename, + filename: opts.filename, pos: p, } } @@ -827,18 +876,25 @@ func (i *Interp) Eval(ctx context.Context, c interface{}, src string, srcFilenam gc, err := gojq.Compile(gq, compilerOpts...) if err != nil { - p := queryErrorPosition(src, err) + p := queryErrorPosition(expr, err) return nil, compileError{ err: err, what: "compile", - filename: srcFilename, + filename: opts.filename, pos: p, } } + output := opts.output + if opts.output == nil { + output = ioutil.Discard + } + runCtx, runCtxCancelFn := i.interruptStack.Push(ctx) ni.evalContext.ctx = runCtx ni.evalContext.output = ioextra.CtxWriter{Writer: output, Ctx: runCtx} + // inherit or set + ni.evalContext.isCompleting = i.evalContext.isCompleting || opts.isCompleting iter := gc.RunWithContext(runCtx, c, variableValues...) iterWrapper := iterFn(func() (interface{}, bool) { @@ -853,7 +909,7 @@ func (i *Interp) Eval(ctx context.Context, c interface{}, src string, srcFilenam return iterWrapper, nil } -func (i *Interp) EvalFunc(ctx context.Context, c interface{}, name string, args []interface{}, output io.Writer) (gojq.Iter, error) { +func (i *Interp) EvalFunc(ctx context.Context, c interface{}, name string, args []interface{}, opts EvalOpts) (gojq.Iter, error) { var argsExpr []string for i := range args { argsExpr = append(argsExpr, fmt.Sprintf("$_args[%d]", i)) @@ -870,15 +926,15 @@ func (i *Interp) EvalFunc(ctx context.Context, c interface{}, name string, args // _args to mark variable as internal and hide it from completion // {input: ..., args: [...]} | .args as {args: $_args} | .input | name[($_args[0]; ...)] trampolineExpr := fmt.Sprintf(". as {args: $_args} | .input | %s%s", name, argExpr) - iter, err := i.Eval(ctx, trampolineInput, trampolineExpr, "", output) + iter, err := i.Eval(ctx, trampolineInput, trampolineExpr, opts) if err != nil { return nil, err } return iter, nil } -func (i *Interp) EvalFuncValues(ctx context.Context, c interface{}, name string, args []interface{}, output io.Writer) ([]interface{}, error) { - iter, err := i.EvalFunc(ctx, c, name, args, output) +func (i *Interp) EvalFuncValues(ctx context.Context, c interface{}, name string, args []interface{}, opts EvalOpts) ([]interface{}, error) { + iter, err := i.EvalFunc(ctx, c, name, args, opts) if err != nil { return nil, err } diff --git a/pkg/interp/repl.jq b/pkg/interp/repl.jq index 5e5a67f5..531715f6 100644 --- a/pkg/interp/repl.jq +++ b/pkg/interp/repl.jq @@ -182,7 +182,7 @@ def _repl($opts): #:: a|(Opts) => @ def _read_expr: _repeat_break( # both _prompt and _complete want input arrays - ( _readline(_prompt; {complete: "_complete", timeout: 1}) + ( _readline({prompt: _prompt, complete: "_complete", timeout: 1}) | if trim == "" then empty else (., error("break")) end @@ -216,12 +216,15 @@ def _repl($opts): #:: a|(Opts) => @ else error end ); - ( _options_stack(. + [$opts]) as $_ - | _finally( - _repeat_break(_repl_loop); - _options_stack(.[:-1]) + if _is_completing | not then + ( _options_stack(. + [$opts]) as $_ + | _finally( + _repeat_break(_repl_loop); + _options_stack(.[:-1]) + ) ) - ); + else empty + end; def _repl_slurp($opts): _repl($opts); def _repl_slurp: _repl({}); @@ -229,7 +232,7 @@ def _repl_slurp: _repl({}); # TODO: introspect and show doc, reflection somehow? def help: ( "Type expression to evaluate" - , "\\t Auto completion" + , "\\t Completion" , "Up/Down History" , "^C Interrupt execution" , "... | repl Start a new REPL" diff --git a/pkg/interp/testdata/paste.fqtest b/pkg/interp/testdata/paste.fqtest new file mode 100644 index 00000000..3a5842b1 --- /dev/null +++ b/pkg/interp/testdata/paste.fqtest @@ -0,0 +1,6 @@ +$ fq -i +null> paste +"test\n" +null> ^D +stdin: +test