wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-12-23 05:13:30 +03:00
Code Issues Projects Releases Wiki Activity

interp: Add paste function to allow pasting text into REPL etc

Browse Source 
Also refactor readline and eval args into option struct and partinally start
addressing some side effects during completion.
This commit is contained in:
Mattias Wadman 2022-02-10 18:46:23 +01:00
parent 97a6bf694b
commit 48a19cb82c
10 changed files with 161 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/TODO.md
View File

@ -13,6 +13,7 @@
- Rework cli/repl user interrupt (context cancel via ctrl-c), see comment in Interp.Main
- Optimize `Interp.Options` calls, now called per display. Cache per eval? needs to handle nested evals.
- `<array decode value>[{start: ...: end: ...}]` syntax a bit broken.
- REPL completion might have side effcts. Make interp.Function type know and wrap somehow? input, inputs, open, ...
### TODO and ideas

2
doc/usage.md
View File

@ -418,6 +418,8 @@ you currently have to do `fq -d raw 'mp3({force: true})' file`.
- `p`/`preview` show preview of field tree
- `hd`/`hexdump` hexdump value
- `repl` nested REPL, must be last in a pipeline. `1 | repl`, can "slurp" outputs `1, 2, 3 | repl`.
- `paste` read string from stdin until ^D. Useful for pasting text.
- Ex: `paste | frompem | asn1_ber | repl` read from stdin then decode and start a new sub-REPL with result.
## Color and unicode output

6
internal/script/script.go
View File

@ -149,8 +149,8 @@ func (cr *CaseRun) ConfigDir() (string, error) { return "/config", nil }
func (cr *CaseRun) FS() fs.FS { return cr.Case }
func (cr *CaseRun) Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error) {
cr.ActualStdoutBuf.WriteString(prompt)
func (cr *CaseRun) Readline(opts interp.ReadlineOpts) (string, error) {
cr.ActualStdoutBuf.WriteString(opts.Prompt)
if cr.ReadlinesPos >= len(cr.Readlines) {
return "", io.EOF
}
@ -165,7 +165,7 @@ func (cr *CaseRun) Readline(prompt string, complete func(line string, pos int) (
cr.ActualStdoutBuf.WriteString(lineRaw + "\n")
l := len(line) - 1
newLine, shared := complete(line[0:l], l)
newLine, shared := opts.CompleteFn(line[0:l], l)
// TODO: shared
_ = shared
for _, nl := range newLine {

8
pkg/cli/cli.go
View File

@ -158,7 +158,7 @@ func (stdOSFS) Open(name string) (fs.File, error) { return os.Open(name) }
func (*stdOS) FS() fs.FS { return stdOSFS{} }
func (o *stdOS) Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error) {
func (o *stdOS) Readline(opts interp.ReadlineOpts) (string, error) {
if o.rl == nil {
var err error
@ -179,9 +179,9 @@ func (o *stdOS) Readline(prompt string, complete func(line string, pos int) (new
}
}
if complete != nil {
if opts.CompleteFn != nil {
o.rl.Config.AutoComplete = autoCompleterFn(func(line []rune, pos int) (newLine [][]rune, length int) {
names, shared := complete(string(line), pos)
names, shared := opts.CompleteFn(string(line), pos)
var runeNames [][]rune
for _, name := range names {
runeNames = append(runeNames, []rune(name[shared:]))
@ -191,7 +191,7 @@ func (o *stdOS) Readline(prompt string, complete func(line string, pos int) (new
})
}
o.rl.SetPrompt(prompt)
o.rl.SetPrompt(opts.Prompt)
line, err := o.rl.Readline()
if errors.Is(err, readline.ErrInterrupt) {
return "", interp.ErrInterrupt

21
pkg/interp/binary.go
View File

@ -16,13 +16,14 @@ import (
"github.com/wader/fq/internal/progressreadseeker"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/ranges"
"github.com/wader/gojq"
)
func init() {
functionRegisterFns = append(functionRegisterFns, func(i *Interp) []Function {
return []Function{
{"_tobits", 3, 3, i._toBits, nil},
{"open", 0, 0, i._open, nil},
{"open", 0, 0, nil, i._open},
}
})
}
@ -175,7 +176,11 @@ func (of *openFile) ToBinary() (Binary, error) {
// def open: #:: string| => binary
// opens a file for reading from filesystem
// TODO: when to close? when br loses all refs? need to use finalizer somehow?
func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
func (i *Interp) _open(c interface{}, a []interface{}) gojq.Iter {
if i.evalContext.isCompleting {
return gojq.NewIter()
}
var err error
var f fs.File
var path string
@ -187,11 +192,11 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
default:
path, err = toString(c)
if err != nil {
return fmt.Errorf("%s: %w", path, err)
return gojq.NewIter(fmt.Errorf("%s: %w", path, err))
}
f, err = i.os.FS().Open(path)
if err != nil {
return err
return gojq.NewIter(err)
}
}
@ -201,7 +206,7 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
fFI, err := f.Stat()
if err != nil {
f.Close()
return err
return gojq.NewIter(err)
}
// ctxreadseeker is used to make sure any io calls can be canceled
@ -219,7 +224,7 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
buf, err := ioutil.ReadAll(ctxreadseeker.New(i.evalContext.ctx, &ioextra.ReadErrSeeker{Reader: f}))
if err != nil {
f.Close()
return err
return gojq.NewIter(err)
}
fRS = bytes.NewReader(buf)
bEnd = int64(len(buf))
@ -246,10 +251,10 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
bbf.br = bitio.NewIOBitReadSeeker(aheadRs)
if err != nil {
return err
return gojq.NewIter(err)
}
return bbf
return gojq.NewIter(bbf)
}
var _ Value = Binary{}

2
pkg/interp/decode.go
View File

@ -162,7 +162,7 @@ func (i *Interp) _decode(c interface{}, a []interface{}) interface{} {
c,
opts.Progress,
nil,
ioextra.DiscardCtxWriter{Ctx: i.evalContext.ctx},
EvalOpts{output: ioextra.DiscardCtxWriter{Ctx: i.evalContext.ctx}},
)
}
lastProgress := time.Now()

11
pkg/interp/funcs.jq
View File

@ -305,3 +305,14 @@ def topem($label):
| join("\n")
);
def topem: topem("");
def paste:
if _is_completing | not then
( [ _repeat_break(
try _stdin(64*1024)
catch if . == "eof" then error("break") end
)
]
| join("")
)
end;

164
pkg/interp/interp.go
View File

@ -11,6 +11,7 @@ import (
"fmt"
"io"
"io/fs"
"io/ioutil"
"math/big"
"path"
"strconv"
@ -22,6 +23,7 @@ import (
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/internal/colorjson"
"github.com/wader/fq/internal/ctxstack"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/ioextra"
"github.com/wader/fq/internal/mathextra"
"github.com/wader/fq/internal/pos"
@ -53,11 +55,11 @@ var functionRegisterFns []func(i *Interp) []Function
func init() {
functionRegisterFns = append(functionRegisterFns, func(i *Interp) []Function {
return []Function{
{"_readline", 0, 2, i._readline, nil},
{"_readline", 0, 1, nil, i._readline},
{"eval", 1, 2, nil, i.eval},
{"_stdin", 0, 0, nil, i.makeStdioFn(i.os.Stdin())},
{"_stdout", 0, 0, nil, i.makeStdioFn(i.os.Stdout())},
{"_stderr", 0, 0, nil, i.makeStdioFn(i.os.Stderr())},
{"_stdin", 0, 1, nil, i.makeStdioFn("stdin", i.os.Stdin())},
{"_stdout", 0, 0, nil, i.makeStdioFn("stdout", i.os.Stdout())},
{"_stderr", 0, 0, nil, i.makeStdioFn("stderr", i.os.Stderr())},
{"_extkeys", 0, 0, i._extKeys, nil},
{"_exttype", 0, 0, i._extType, nil},
{"_global_state", 0, 1, i.makeStateFn(i.state), nil},
@ -65,6 +67,7 @@ func init() {
{"_display", 1, 1, nil, i._display},
{"_can_display", 0, 0, i._canDisplay, nil},
{"_print_color_json", 0, 1, nil, i._printColorJSON},
{"_is_completing", 0, 1, i._isCompleting, nil},
}
})
}
@ -95,7 +98,7 @@ func (ce compileError) Value() interface{} {
func (ce compileError) Error() string {
filename := ce.filename
if filename == "" {
filename = "src"
filename = "expr"
}
return fmt.Sprintf("%s:%d:%d: %s: %s", filename, ce.pos.Line, ce.pos.Column, ce.what, ce.err.Error())
}
@ -133,6 +136,11 @@ type Platform struct {
Arch string
}
type ReadlineOpts struct {
Prompt string
CompleteFn func(line string, pos int) (newLine []string, shared int)
}
type OS interface {
Platform() Platform
Stdin() Input
@ -144,7 +152,7 @@ type OS interface {
ConfigDir() (string, error)
// FS.File returned by FS().Open() can optionally implement io.Seeker
FS() fs.FS
Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error)
Readline(opts ReadlineOpts) (string, error)
History() ([]string, error)
}
@ -283,14 +291,14 @@ func toBytes(v interface{}) ([]byte, error) {
}
}
func queryErrorPosition(src string, v error) pos.Pos {
func queryErrorPosition(expr string, v error) pos.Pos {
var offset int
if tokIf, ok := v.(interface{ Token() (string, int) }); ok { //nolint:errorlint
_, offset = tokIf.Token()
}
if offset >= 0 {
return pos.NewFromOffset(src, offset)
return pos.NewFromOffset(expr, offset)
}
return pos.Pos{}
}
@ -317,8 +325,9 @@ const (
)
type evalContext struct {
ctx context.Context
output io.Writer
ctx context.Context
output io.Writer
isCompleting bool
}
type Interp struct {
@ -380,7 +389,7 @@ func (i *Interp) Main(ctx context.Context, output Output, versionStr string) err
"arch": platform.Arch,
}
iter, err := i.EvalFunc(ctx, input, "_main", nil, output)
iter, err := i.EvalFunc(ctx, input, "_main", nil, EvalOpts{output: output})
if err != nil {
fmt.Fprintln(i.os.Stderr(), err)
return err
@ -414,28 +423,24 @@ func (i *Interp) Main(ctx context.Context, output Output, versionStr string) err
return nil
}
func (i *Interp) _readline(c interface{}, a []interface{}) interface{} {
func (i *Interp) _readline(c interface{}, a []interface{}) gojq.Iter {
if i.evalContext.isCompleting {
return gojq.NewIter()
}
var opts struct {
Promopt string `mapstructure:"prompt"`
Complete string `mapstructure:"complete"`
Timeout float64 `mapstructure:"timeout"`
}
var err error
prompt := ""
if len(a) > 0 {
prompt, err = toString(a[0])
if err != nil {
return fmt.Errorf("prompt: %w", err)
}
}
if len(a) > 1 {
_ = mapstructure.Decode(a[1], &opts)
_ = mapstructure.Decode(a[0], &opts)
}
src, err := i.os.Readline(
prompt,
func(line string, pos int) (newLine []string, shared int) {
expr, err := i.os.Readline(ReadlineOpts{
Prompt: opts.Promopt,
CompleteFn: func(line string, pos int) (newLine []string, shared int) {
completeCtx := i.evalContext.ctx
if opts.Timeout > 0 {
var completeCtxCancelFn context.CancelFunc
@ -450,7 +455,10 @@ func (i *Interp) _readline(c interface{}, a []interface{}) interface{} {
c,
opts.Complete,
[]interface{}{line, pos},
ioextra.DiscardCtxWriter{Ctx: completeCtx},
EvalOpts{
output: ioextra.DiscardCtxWriter{Ctx: completeCtx},
isCompleting: true,
},
)
if err != nil {
return nil, pos, err
@ -485,24 +493,24 @@ func (i *Interp) _readline(c interface{}, a []interface{}) interface{} {
return names, shared
},
)
})
if errors.Is(err, ErrInterrupt) {
return valueError{"interrupt"}
return gojq.NewIter(valueError{"interrupt"})
} else if errors.Is(err, ErrEOF) {
return valueError{"eof"}
return gojq.NewIter(valueError{"eof"})
} else if err != nil {
return err
return gojq.NewIter(err)
}
return src
return gojq.NewIter(expr)
}
func (i *Interp) eval(c interface{}, a []interface{}) gojq.Iter {
var err error
src, err := toString(a[0])
expr, err := toString(a[0])
if err != nil {
return gojq.NewIter(fmt.Errorf("src: %w", err))
return gojq.NewIter(fmt.Errorf("expr: %w", err))
}
var filenameHint string
if len(a) >= 2 {
@ -512,7 +520,10 @@ func (i *Interp) eval(c interface{}, a []interface{}) gojq.Iter {
}
}
iter, err := i.Eval(i.evalContext.ctx, c, src, filenameHint, i.evalContext.output)
iter, err := i.Eval(i.evalContext.ctx, c, expr, EvalOpts{
filename: filenameHint,
output: i.evalContext.output,
})
if err != nil {
return gojq.NewIter(err)
}
@ -547,25 +558,53 @@ func (i *Interp) makeStateFn(state *interface{}) func(c interface{}, a []interfa
}
}
func (i *Interp) makeStdioFn(t Terminal) func(c interface{}, a []interface{}) gojq.Iter {
func (i *Interp) makeStdioFn(name string, t Terminal) func(c interface{}, a []interface{}) gojq.Iter {
return func(c interface{}, a []interface{}) gojq.Iter {
if c == nil {
if i.evalContext.isCompleting {
return gojq.NewIter("")
}
switch {
case len(a) == 1:
r, ok := t.(io.Reader)
if !ok {
return gojq.NewIter(fmt.Errorf("%s is not readable", name))
}
l, ok := gojqextra.ToInt(a[0])
if !ok {
return gojq.NewIter(gojqextra.FuncTypeError{Name: name, V: a[0]})
}
buf := make([]byte, l)
n, err := io.ReadFull(r, buf)
s := string(buf[0:n])
vs := []interface{}{s}
switch {
case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF):
vs = append(vs, valueError{"eof"})
default:
vs = append(vs, err)
}
return gojq.NewIter(vs...)
case c == nil:
w, h := t.Size()
return gojq.NewIter(map[string]interface{}{
"is_terminal": t.IsTerminal(),
"width": w,
"height": h,
})
}
if w, ok := t.(io.Writer); ok {
default:
w, ok := t.(io.Writer)
if !ok {
return gojq.NewIter(fmt.Errorf("%v: it not writeable", c))
}
if _, err := fmt.Fprint(w, c); err != nil {
return gojq.NewIter(err)
}
return gojq.NewIter()
}
return gojq.NewIter(fmt.Errorf("%v: it not writeable", c))
}
}
@ -595,6 +634,11 @@ func (i *Interp) _display(c interface{}, a []interface{}) gojq.Iter {
}
}
func (i *Interp) _canDisplay(c interface{}, a []interface{}) interface{} {
_, ok := c.(Display)
return ok
}
func (i *Interp) _printColorJSON(c interface{}, a []interface{}) gojq.Iter {
opts := i.Options(a[0])
@ -609,9 +653,8 @@ func (i *Interp) _printColorJSON(c interface{}, a []interface{}) gojq.Iter {
return gojq.NewIter()
}
func (i *Interp) _canDisplay(c interface{}, a []interface{}) interface{} {
_, ok := c.(Display)
return ok
func (i *Interp) _isCompleting(c interface{}, a []interface{}) interface{} {
return i.evalContext.isCompleting
}
type pathResolver struct {
@ -667,14 +710,20 @@ func (i *Interp) lookupPathResolver(filename string) (pathResolver, error) {
return pathResolver{}, fmt.Errorf("could not resolve path: %s", filename)
}
func (i *Interp) Eval(ctx context.Context, c interface{}, src string, srcFilename string, output io.Writer) (gojq.Iter, error) {
gq, err := gojq.Parse(src)
type EvalOpts struct {
filename string
output io.Writer
isCompleting bool
}
func (i *Interp) Eval(ctx context.Context, c interface{}, expr string, opts EvalOpts) (gojq.Iter, error) {
gq, err := gojq.Parse(expr)
if err != nil {
p := queryErrorPosition(src, err)
p := queryErrorPosition(expr, err)
return nil, compileError{
err: err,
what: "parse",
filename: srcFilename,
filename: opts.filename,
pos: p,
}
}
@ -827,18 +876,25 @@ func (i *Interp) Eval(ctx context.Context, c interface{}, src string, srcFilenam
gc, err := gojq.Compile(gq, compilerOpts...)
if err != nil {
p := queryErrorPosition(src, err)
p := queryErrorPosition(expr, err)
return nil, compileError{
err: err,
what: "compile",
filename: srcFilename,
filename: opts.filename,
pos: p,
}
}
output := opts.output
if opts.output == nil {
output = ioutil.Discard
}
runCtx, runCtxCancelFn := i.interruptStack.Push(ctx)
ni.evalContext.ctx = runCtx
ni.evalContext.output = ioextra.CtxWriter{Writer: output, Ctx: runCtx}
// inherit or set
ni.evalContext.isCompleting = i.evalContext.isCompleting || opts.isCompleting
iter := gc.RunWithContext(runCtx, c, variableValues...)
iterWrapper := iterFn(func() (interface{}, bool) {
@ -853,7 +909,7 @@ func (i *Interp) Eval(ctx context.Context, c interface{}, src string, srcFilenam
return iterWrapper, nil
}
func (i *Interp) EvalFunc(ctx context.Context, c interface{}, name string, args []interface{}, output io.Writer) (gojq.Iter, error) {
func (i *Interp) EvalFunc(ctx context.Context, c interface{}, name string, args []interface{}, opts EvalOpts) (gojq.Iter, error) {
var argsExpr []string
for i := range args {
argsExpr = append(argsExpr, fmt.Sprintf("$_args[%d]", i))
@ -870,15 +926,15 @@ func (i *Interp) EvalFunc(ctx context.Context, c interface{}, name string, args
// _args to mark variable as internal and hide it from completion
// {input: ..., args: [...]} | .args as {args: $_args} | .input | name[($_args[0]; ...)]
trampolineExpr := fmt.Sprintf(". as {args: $_args} | .input | %s%s", name, argExpr)
iter, err := i.Eval(ctx, trampolineInput, trampolineExpr, "", output)
iter, err := i.Eval(ctx, trampolineInput, trampolineExpr, opts)
if err != nil {
return nil, err
}
return iter, nil
}
func (i *Interp) EvalFuncValues(ctx context.Context, c interface{}, name string, args []interface{}, output io.Writer) ([]interface{}, error) {
iter, err := i.EvalFunc(ctx, c, name, args, output)
func (i *Interp) EvalFuncValues(ctx context.Context, c interface{}, name string, args []interface{}, opts EvalOpts) ([]interface{}, error) {
iter, err := i.EvalFunc(ctx, c, name, args, opts)
if err != nil {
return nil, err
}

17
pkg/interp/repl.jq
View File

@ -182,7 +182,7 @@ def _repl($opts): #:: a|(Opts) => @
def _read_expr:
_repeat_break(
# both _prompt and _complete want input arrays
( _readline(_prompt; {complete: "_complete", timeout: 1})
( _readline({prompt: _prompt, complete: "_complete", timeout: 1})
| if trim == "" then empty
else (., error("break"))
end
@ -216,12 +216,15 @@ def _repl($opts): #:: a|(Opts) => @
else error
end
);
( _options_stack(. + [$opts]) as $_
| _finally(
_repeat_break(_repl_loop);
_options_stack(.[:-1])
if _is_completing | not then
( _options_stack(. + [$opts]) as $_
| _finally(
_repeat_break(_repl_loop);
_options_stack(.[:-1])
)
)
);
else empty
end;
def _repl_slurp($opts): _repl($opts);
def _repl_slurp: _repl({});
@ -229,7 +232,7 @@ def _repl_slurp: _repl({});
# TODO: introspect and show doc, reflection somehow?
def help:
( "Type expression to evaluate"
, "\\t Auto completion"
, "\\t Completion"
, "Up/Down History"
, "^C Interrupt execution"
, "... | repl Start a new REPL"

6
pkg/interp/testdata/paste.fqtest vendored Normal file
View File

@ -0,0 +1,6 @@
$ fq -i
null> paste
"test\n"
null> ^D
stdin:
test