Merge pull request #143 from wader/paste

interp: Add paste function to allow pasting text into REPL etc

doc/TODO.md

@@ -13,6 +13,7 @@
 - Rework cli/repl user interrupt (context cancel via ctrl-c), see comment in Interp.Main
 - Optimize `Interp.Options` calls, now called per display. Cache per eval? needs to handle nested evals.
 - `<array decode value>[{start: ...: end: ...}]` syntax a bit broken.
+- REPL completion might have side effcts. Make interp.Function type know and wrap somehow? input, inputs, open, ...
 
 ### TODO and ideas
 

doc/usage.md

@@ -418,6 +418,8 @@ you currently have to do `fq -d raw 'mp3({force: true})' file`.
 - `p`/`preview` show preview of field tree
 - `hd`/`hexdump` hexdump value
 - `repl` nested REPL, must be last in a pipeline. `1 | repl`, can "slurp" outputs `1, 2, 3 | repl`.
+- `paste` read string from stdin until ^D. Useful for pasting text.
+    - Ex: `paste | frompem | asn1_ber | repl` read from stdin then decode and start a new sub-REPL with result.
 
 ## Color and unicode output
 

internal/script/script.go

@@ -149,8 +149,8 @@ func (cr *CaseRun) ConfigDir() (string, error) { return "/config", nil }
 
 func (cr *CaseRun) FS() fs.FS { return cr.Case }
 
-func (cr *CaseRun) Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error) {
+func (cr *CaseRun) Readline(opts interp.ReadlineOpts) (string, error) {
-	cr.ActualStdoutBuf.WriteString(prompt)
+	cr.ActualStdoutBuf.WriteString(opts.Prompt)
 	if cr.ReadlinesPos >= len(cr.Readlines) {
 		return "", io.EOF
 	}
@@ -165,7 +165,7 @@ func (cr *CaseRun) Readline(prompt string, complete func(line string, pos int) (
 		cr.ActualStdoutBuf.WriteString(lineRaw + "\n")
 
 		l := len(line) - 1
-		newLine, shared := complete(line[0:l], l)
+		newLine, shared := opts.CompleteFn(line[0:l], l)
 		// TODO: shared
 		_ = shared
 		for _, nl := range newLine {

pkg/cli/cli.go

@@ -158,7 +158,7 @@ func (stdOSFS) Open(name string) (fs.File, error) { return os.Open(name) }
 
 func (*stdOS) FS() fs.FS { return stdOSFS{} }
 
-func (o *stdOS) Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error) {
+func (o *stdOS) Readline(opts interp.ReadlineOpts) (string, error) {
 	if o.rl == nil {
 		var err error
 
@@ -179,9 +179,9 @@ func (o *stdOS) Readline(prompt string, complete func(line string, pos int) (new
 		}
 	}
 
-	if complete != nil {
+	if opts.CompleteFn != nil {
 		o.rl.Config.AutoComplete = autoCompleterFn(func(line []rune, pos int) (newLine [][]rune, length int) {
-			names, shared := complete(string(line), pos)
+			names, shared := opts.CompleteFn(string(line), pos)
 			var runeNames [][]rune
 			for _, name := range names {
 				runeNames = append(runeNames, []rune(name[shared:]))
@@ -191,7 +191,7 @@ func (o *stdOS) Readline(prompt string, complete func(line string, pos int) (new
 		})
 	}
 
-	o.rl.SetPrompt(prompt)
+	o.rl.SetPrompt(opts.Prompt)
 	line, err := o.rl.Readline()
 	if errors.Is(err, readline.ErrInterrupt) {
 		return "", interp.ErrInterrupt

pkg/interp/binary.go

@@ -16,13 +16,14 @@ import (
 	"github.com/wader/fq/internal/progressreadseeker"
 	"github.com/wader/fq/pkg/bitio"
 	"github.com/wader/fq/pkg/ranges"
+	"github.com/wader/gojq"
 )
 
 func init() {
 	functionRegisterFns = append(functionRegisterFns, func(i *Interp) []Function {
 		return []Function{
 			{"_tobits", 3, 3, i._toBits, nil},
-			{"open", 0, 0, i._open, nil},
+			{"open", 0, 0, nil, i._open},
 		}
 	})
 }
@@ -175,7 +176,11 @@ func (of *openFile) ToBinary() (Binary, error) {
 // def open: #:: string| => binary
 // opens a file for reading from filesystem
 // TODO: when to close? when br loses all refs? need to use finalizer somehow?
-func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
+func (i *Interp) _open(c interface{}, a []interface{}) gojq.Iter {
+	if i.evalContext.isCompleting {
+		return gojq.NewIter()
+	}
+
 	var err error
 	var f fs.File
 	var path string
@@ -187,11 +192,11 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
 	default:
 		path, err = toString(c)
 		if err != nil {
-			return fmt.Errorf("%s: %w", path, err)
+			return gojq.NewIter(fmt.Errorf("%s: %w", path, err))
 		}
 		f, err = i.os.FS().Open(path)
 		if err != nil {
-			return err
+			return gojq.NewIter(err)
 		}
 	}
 
@@ -201,7 +206,7 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
 	fFI, err := f.Stat()
 	if err != nil {
 		f.Close()
-		return err
+		return gojq.NewIter(err)
 	}
 
 	// ctxreadseeker is used to make sure any io calls can be canceled
@@ -219,7 +224,7 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
 		buf, err := ioutil.ReadAll(ctxreadseeker.New(i.evalContext.ctx, &ioextra.ReadErrSeeker{Reader: f}))
 		if err != nil {
 			f.Close()
-			return err
+			return gojq.NewIter(err)
 		}
 		fRS = bytes.NewReader(buf)
 		bEnd = int64(len(buf))
@@ -246,10 +251,10 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
 
 	bbf.br = bitio.NewIOBitReadSeeker(aheadRs)
 	if err != nil {
-		return err
+		return gojq.NewIter(err)
 	}
 
-	return bbf
+	return gojq.NewIter(bbf)
 }
 
 var _ Value = Binary{}

pkg/interp/decode.go

@@ -162,7 +162,7 @@ func (i *Interp) _decode(c interface{}, a []interface{}) interface{} {
 					c,
 					opts.Progress,
 					nil,
-					ioextra.DiscardCtxWriter{Ctx: i.evalContext.ctx},
+					EvalOpts{output: ioextra.DiscardCtxWriter{Ctx: i.evalContext.ctx}},
 				)
 			}
 			lastProgress := time.Now()

pkg/interp/funcs.jq

@@ -305,3 +305,14 @@ def topem($label):
   | join("\n")
   );
 def topem: topem("");
+
+def paste:
+  if _is_completing | not then
+    ( [ _repeat_break(
+          try _stdin(64*1024)
+          catch if . == "eof" then error("break") end
+        )
+      ]
+    | join("")
+    )
+  end;

pkg/interp/interp.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"io/ioutil"
 	"math/big"
 	"path"
 	"strconv"
@@ -22,6 +23,7 @@ import (
 	"github.com/wader/fq/internal/bitioextra"
 	"github.com/wader/fq/internal/colorjson"
 	"github.com/wader/fq/internal/ctxstack"
+	"github.com/wader/fq/internal/gojqextra"
 	"github.com/wader/fq/internal/ioextra"
 	"github.com/wader/fq/internal/mathextra"
 	"github.com/wader/fq/internal/pos"
@@ -53,11 +55,11 @@ var functionRegisterFns []func(i *Interp) []Function
 func init() {
 	functionRegisterFns = append(functionRegisterFns, func(i *Interp) []Function {
 		return []Function{
-			{"_readline", 0, 2, i._readline, nil},
+			{"_readline", 0, 1, nil, i._readline},
 			{"eval", 1, 2, nil, i.eval},
-			{"_stdin", 0, 0, nil, i.makeStdioFn(i.os.Stdin())},
+			{"_stdin", 0, 1, nil, i.makeStdioFn("stdin", i.os.Stdin())},
-			{"_stdout", 0, 0, nil, i.makeStdioFn(i.os.Stdout())},
+			{"_stdout", 0, 0, nil, i.makeStdioFn("stdout", i.os.Stdout())},
-			{"_stderr", 0, 0, nil, i.makeStdioFn(i.os.Stderr())},
+			{"_stderr", 0, 0, nil, i.makeStdioFn("stderr", i.os.Stderr())},
 			{"_extkeys", 0, 0, i._extKeys, nil},
 			{"_exttype", 0, 0, i._extType, nil},
 			{"_global_state", 0, 1, i.makeStateFn(i.state), nil},
@@ -65,6 +67,7 @@ func init() {
 			{"_display", 1, 1, nil, i._display},
 			{"_can_display", 0, 0, i._canDisplay, nil},
 			{"_print_color_json", 0, 1, nil, i._printColorJSON},
+			{"_is_completing", 0, 1, i._isCompleting, nil},
 		}
 	})
 }
@@ -95,7 +98,7 @@ func (ce compileError) Value() interface{} {
 func (ce compileError) Error() string {
 	filename := ce.filename
 	if filename == "" {
-		filename = "src"
+		filename = "expr"
 	}
 	return fmt.Sprintf("%s:%d:%d: %s: %s", filename, ce.pos.Line, ce.pos.Column, ce.what, ce.err.Error())
 }
@@ -133,6 +136,11 @@ type Platform struct {
 	Arch string
 }
 
+type ReadlineOpts struct {
+	Prompt     string
+	CompleteFn func(line string, pos int) (newLine []string, shared int)
+}
+
 type OS interface {
 	Platform() Platform
 	Stdin() Input
@@ -144,7 +152,7 @@ type OS interface {
 	ConfigDir() (string, error)
 	// FS.File returned by FS().Open() can optionally implement io.Seeker
 	FS() fs.FS
-	Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error)
+	Readline(opts ReadlineOpts) (string, error)
 	History() ([]string, error)
 }
 
@@ -283,14 +291,14 @@ func toBytes(v interface{}) ([]byte, error) {
 	}
 }
 
-func queryErrorPosition(src string, v error) pos.Pos {
+func queryErrorPosition(expr string, v error) pos.Pos {
 	var offset int
 
 	if tokIf, ok := v.(interface{ Token() (string, int) }); ok { //nolint:errorlint
 		_, offset = tokIf.Token()
 	}
 	if offset >= 0 {
-		return pos.NewFromOffset(src, offset)
+		return pos.NewFromOffset(expr, offset)
 	}
 	return pos.Pos{}
 }
@@ -317,8 +325,9 @@ const (
 )
 
 type evalContext struct {
-	ctx    context.Context
+	ctx          context.Context
-	output io.Writer
+	output       io.Writer
+	isCompleting bool
 }
 
 type Interp struct {
@@ -380,7 +389,7 @@ func (i *Interp) Main(ctx context.Context, output Output, versionStr string) err
 		"arch":    platform.Arch,
 	}
 
-	iter, err := i.EvalFunc(ctx, input, "_main", nil, output)
+	iter, err := i.EvalFunc(ctx, input, "_main", nil, EvalOpts{output: output})
 	if err != nil {
 		fmt.Fprintln(i.os.Stderr(), err)
 		return err
@@ -414,28 +423,24 @@ func (i *Interp) Main(ctx context.Context, output Output, versionStr string) err
 	return nil
 }
 
-func (i *Interp) _readline(c interface{}, a []interface{}) interface{} {
+func (i *Interp) _readline(c interface{}, a []interface{}) gojq.Iter {
+	if i.evalContext.isCompleting {
+		return gojq.NewIter()
+	}
+
 	var opts struct {
+		Promopt  string  `mapstructure:"prompt"`
 		Complete string  `mapstructure:"complete"`
 		Timeout  float64 `mapstructure:"timeout"`
 	}
 
-	var err error
-	prompt := ""
-
 	if len(a) > 0 {
-		prompt, err = toString(a[0])
+		_ = mapstructure.Decode(a[0], &opts)
-		if err != nil {
-			return fmt.Errorf("prompt: %w", err)
-		}
-	}
-	if len(a) > 1 {
-		_ = mapstructure.Decode(a[1], &opts)
 	}
 
-	src, err := i.os.Readline(
+	expr, err := i.os.Readline(ReadlineOpts{
-		prompt,
+		Prompt: opts.Promopt,
-		func(line string, pos int) (newLine []string, shared int) {
+		CompleteFn: func(line string, pos int) (newLine []string, shared int) {
 			completeCtx := i.evalContext.ctx
 			if opts.Timeout > 0 {
 				var completeCtxCancelFn context.CancelFunc
@@ -450,7 +455,10 @@ func (i *Interp) _readline(c interface{}, a []interface{}) interface{} {
 					c,
 					opts.Complete,
 					[]interface{}{line, pos},
-					ioextra.DiscardCtxWriter{Ctx: completeCtx},
+					EvalOpts{
+						output:       ioextra.DiscardCtxWriter{Ctx: completeCtx},
+						isCompleting: true,
+					},
 				)
 				if err != nil {
 					return nil, pos, err
@@ -485,24 +493,24 @@ func (i *Interp) _readline(c interface{}, a []interface{}) interface{} {
 
 			return names, shared
 		},
-	)
+	})
 
 	if errors.Is(err, ErrInterrupt) {
-		return valueError{"interrupt"}
+		return gojq.NewIter(valueError{"interrupt"})
 	} else if errors.Is(err, ErrEOF) {
-		return valueError{"eof"}
+		return gojq.NewIter(valueError{"eof"})
 	} else if err != nil {
-		return err
+		return gojq.NewIter(err)
 	}
 
-	return src
+	return gojq.NewIter(expr)
 }
 
 func (i *Interp) eval(c interface{}, a []interface{}) gojq.Iter {
 	var err error
-	src, err := toString(a[0])
+	expr, err := toString(a[0])
 	if err != nil {
-		return gojq.NewIter(fmt.Errorf("src: %w", err))
+		return gojq.NewIter(fmt.Errorf("expr: %w", err))
 	}
 	var filenameHint string
 	if len(a) >= 2 {
@@ -512,7 +520,10 @@ func (i *Interp) eval(c interface{}, a []interface{}) gojq.Iter {
 		}
 	}
 
-	iter, err := i.Eval(i.evalContext.ctx, c, src, filenameHint, i.evalContext.output)
+	iter, err := i.Eval(i.evalContext.ctx, c, expr, EvalOpts{
+		filename: filenameHint,
+		output:   i.evalContext.output,
+	})
 	if err != nil {
 		return gojq.NewIter(err)
 	}
@@ -547,25 +558,53 @@ func (i *Interp) makeStateFn(state *interface{}) func(c interface{}, a []interfa
 	}
 }
 
-func (i *Interp) makeStdioFn(t Terminal) func(c interface{}, a []interface{}) gojq.Iter {
+func (i *Interp) makeStdioFn(name string, t Terminal) func(c interface{}, a []interface{}) gojq.Iter {
 	return func(c interface{}, a []interface{}) gojq.Iter {
-		if c == nil {
+		if i.evalContext.isCompleting {
+			return gojq.NewIter("")
+		}
+
+		switch {
+		case len(a) == 1:
+			r, ok := t.(io.Reader)
+			if !ok {
+				return gojq.NewIter(fmt.Errorf("%s is not readable", name))
+			}
+			l, ok := gojqextra.ToInt(a[0])
+			if !ok {
+				return gojq.NewIter(gojqextra.FuncTypeError{Name: name, V: a[0]})
+			}
+
+			buf := make([]byte, l)
+			n, err := io.ReadFull(r, buf)
+			s := string(buf[0:n])
+
+			vs := []interface{}{s}
+			switch {
+			case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF):
+				vs = append(vs, valueError{"eof"})
+			default:
+				vs = append(vs, err)
+			}
+
+			return gojq.NewIter(vs...)
+		case c == nil:
 			w, h := t.Size()
 			return gojq.NewIter(map[string]interface{}{
 				"is_terminal": t.IsTerminal(),
 				"width":       w,
 				"height":      h,
 			})
-		}
+		default:
-
+			w, ok := t.(io.Writer)
-		if w, ok := t.(io.Writer); ok {
+			if !ok {
+				return gojq.NewIter(fmt.Errorf("%v: it not writeable", c))
+			}
 			if _, err := fmt.Fprint(w, c); err != nil {
 				return gojq.NewIter(err)
 			}
 			return gojq.NewIter()
 		}
-
-		return gojq.NewIter(fmt.Errorf("%v: it not writeable", c))
 	}
 }
 
@@ -595,6 +634,11 @@ func (i *Interp) _display(c interface{}, a []interface{}) gojq.Iter {
 	}
 }
 
+func (i *Interp) _canDisplay(c interface{}, a []interface{}) interface{} {
+	_, ok := c.(Display)
+	return ok
+}
+
 func (i *Interp) _printColorJSON(c interface{}, a []interface{}) gojq.Iter {
 	opts := i.Options(a[0])
 
@@ -609,9 +653,8 @@ func (i *Interp) _printColorJSON(c interface{}, a []interface{}) gojq.Iter {
 	return gojq.NewIter()
 }
 
-func (i *Interp) _canDisplay(c interface{}, a []interface{}) interface{} {
+func (i *Interp) _isCompleting(c interface{}, a []interface{}) interface{} {
-	_, ok := c.(Display)
+	return i.evalContext.isCompleting
-	return ok
 }
 
 type pathResolver struct {
@@ -667,14 +710,20 @@ func (i *Interp) lookupPathResolver(filename string) (pathResolver, error) {
 	return pathResolver{}, fmt.Errorf("could not resolve path: %s", filename)
 }
 
-func (i *Interp) Eval(ctx context.Context, c interface{}, src string, srcFilename string, output io.Writer) (gojq.Iter, error) {
+type EvalOpts struct {
-	gq, err := gojq.Parse(src)
+	filename     string
+	output       io.Writer
+	isCompleting bool
+}
+
+func (i *Interp) Eval(ctx context.Context, c interface{}, expr string, opts EvalOpts) (gojq.Iter, error) {
+	gq, err := gojq.Parse(expr)
 	if err != nil {
-		p := queryErrorPosition(src, err)
+		p := queryErrorPosition(expr, err)
 		return nil, compileError{
 			err:      err,
 			what:     "parse",
-			filename: srcFilename,
+			filename: opts.filename,
 			pos:      p,
 		}
 	}
@@ -827,18 +876,25 @@ func (i *Interp) Eval(ctx context.Context, c interface{}, src string, srcFilenam
 
 	gc, err := gojq.Compile(gq, compilerOpts...)
 	if err != nil {
-		p := queryErrorPosition(src, err)
+		p := queryErrorPosition(expr, err)
 		return nil, compileError{
 			err:      err,
 			what:     "compile",
-			filename: srcFilename,
+			filename: opts.filename,
 			pos:      p,
 		}
 	}
 
+	output := opts.output
+	if opts.output == nil {
+		output = ioutil.Discard
+	}
+
 	runCtx, runCtxCancelFn := i.interruptStack.Push(ctx)
 	ni.evalContext.ctx = runCtx
 	ni.evalContext.output = ioextra.CtxWriter{Writer: output, Ctx: runCtx}
+	// inherit or set
+	ni.evalContext.isCompleting = i.evalContext.isCompleting || opts.isCompleting
 	iter := gc.RunWithContext(runCtx, c, variableValues...)
 
 	iterWrapper := iterFn(func() (interface{}, bool) {
@@ -853,7 +909,7 @@ func (i *Interp) Eval(ctx context.Context, c interface{}, src string, srcFilenam
 	return iterWrapper, nil
 }
 
-func (i *Interp) EvalFunc(ctx context.Context, c interface{}, name string, args []interface{}, output io.Writer) (gojq.Iter, error) {
+func (i *Interp) EvalFunc(ctx context.Context, c interface{}, name string, args []interface{}, opts EvalOpts) (gojq.Iter, error) {
 	var argsExpr []string
 	for i := range args {
 		argsExpr = append(argsExpr, fmt.Sprintf("$_args[%d]", i))
@@ -870,15 +926,15 @@ func (i *Interp) EvalFunc(ctx context.Context, c interface{}, name string, args
 	// _args to mark variable as internal and hide it from completion
 	// {input: ..., args: [...]} | .args as {args: $_args} | .input | name[($_args[0]; ...)]
 	trampolineExpr := fmt.Sprintf(". as {args: $_args} | .input | %s%s", name, argExpr)
-	iter, err := i.Eval(ctx, trampolineInput, trampolineExpr, "", output)
+	iter, err := i.Eval(ctx, trampolineInput, trampolineExpr, opts)
 	if err != nil {
 		return nil, err
 	}
 	return iter, nil
 }
 
-func (i *Interp) EvalFuncValues(ctx context.Context, c interface{}, name string, args []interface{}, output io.Writer) ([]interface{}, error) {
+func (i *Interp) EvalFuncValues(ctx context.Context, c interface{}, name string, args []interface{}, opts EvalOpts) ([]interface{}, error) {
-	iter, err := i.EvalFunc(ctx, c, name, args, output)
+	iter, err := i.EvalFunc(ctx, c, name, args, opts)
 	if err != nil {
 		return nil, err
 	}

pkg/interp/repl.jq

@@ -182,7 +182,7 @@ def _repl($opts): #:: a|(Opts) => @
   def _read_expr:
     _repeat_break(
       # both _prompt and _complete want input arrays
-      ( _readline(_prompt; {complete: "_complete", timeout: 1})
+      ( _readline({prompt: _prompt, complete: "_complete", timeout: 1})
       | if trim == "" then empty
         else (., error("break"))
         end
@@ -216,12 +216,15 @@ def _repl($opts): #:: a|(Opts) => @
         else error
         end
     );
-  ( _options_stack(. + [$opts]) as $_
+  if _is_completing | not then
-  | _finally(
+    ( _options_stack(. + [$opts]) as $_
-      _repeat_break(_repl_loop);
+    | _finally(
-      _options_stack(.[:-1])
+        _repeat_break(_repl_loop);
+        _options_stack(.[:-1])
+      )
     )
-  );
+  else empty
+  end;
 
 def _repl_slurp($opts): _repl($opts);
 def _repl_slurp: _repl({});
@@ -229,7 +232,7 @@ def _repl_slurp: _repl({});
 # TODO: introspect and show doc, reflection somehow?
 def help:
   ( "Type expression to evaluate"
-  , "\\t          Auto completion"
+  , "\\t          Completion"
   , "Up/Down     History"
   , "^C          Interrupt execution"
   , "... | repl  Start a new REPL"

pkg/interp/testdata/paste.fqtest

@@ -0,0 +1,6 @@
+$ fq -i
+null> paste
+"test\n"
+null> ^D
+stdin:
+test