wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-12-29 00:22:38 +03:00
Code Issues Projects Releases Wiki Activity

Merge pull request #143 from wader/paste

Browse Source 
interp: Add paste function to allow pasting text into REPL etc
This commit is contained in:
Mattias Wadman 2022-02-11 18:31:50 +01:00 committed by GitHub
commit ca234bc1c9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 161 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/TODO.md
View File

@ -13,6 +13,7 @@
- Rework cli/repl user interrupt (context cancel via ctrl-c), see comment in Interp.Main
- Optimize `Interp.Options` calls, now called per display. Cache per eval? needs to handle nested evals.
- `<array decode value>[{start: ...: end: ...}]` syntax a bit broken.
- REPL completion might have side effcts. Make interp.Function type know and wrap somehow? input, inputs, open, ...
### TODO and ideas

2
doc/usage.md
View File

@ -418,6 +418,8 @@ you currently have to do `fq -d raw 'mp3({force: true})' file`.
- `p`/`preview` show preview of field tree
- `hd`/`hexdump` hexdump value
- `repl` nested REPL, must be last in a pipeline. `1 | repl`, can "slurp" outputs `1, 2, 3 | repl`.
- `paste` read string from stdin until ^D. Useful for pasting text.
- Ex: `paste | frompem | asn1_ber | repl` read from stdin then decode and start a new sub-REPL with result.
## Color and unicode output

6
internal/script/script.go
View File

@ -149,8 +149,8 @@ func (cr *CaseRun) ConfigDir() (string, error) { return "/config", nil }
func (cr *CaseRun) FS() fs.FS { return cr.Case }
func (cr *CaseRun) Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error) {
cr.ActualStdoutBuf.WriteString(prompt)
func (cr *CaseRun) Readline(opts interp.ReadlineOpts) (string, error) {
cr.ActualStdoutBuf.WriteString(opts.Prompt)
if cr.ReadlinesPos >= len(cr.Readlines) {
return "", io.EOF
}
@ -165,7 +165,7 @@ func (cr *CaseRun) Readline(prompt string, complete func(line string, pos int) (
cr.ActualStdoutBuf.WriteString(lineRaw + "\n")
l := len(line) - 1
newLine, shared := complete(line[0:l], l)
newLine, shared := opts.CompleteFn(line[0:l], l)
// TODO: shared
_ = shared
for _, nl := range newLine {

8
pkg/cli/cli.go
View File

@ -158,7 +158,7 @@ func (stdOSFS) Open(name string) (fs.File, error) { return os.Open(name) }
func (*stdOS) FS() fs.FS { return stdOSFS{} }
func (o *stdOS) Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error) {
func (o *stdOS) Readline(opts interp.ReadlineOpts) (string, error) {
if o.rl == nil {
var err error
@ -179,9 +179,9 @@ func (o *stdOS) Readline(prompt string, complete func(line string, pos int) (new
}
}
if complete != nil {
if opts.CompleteFn != nil {
o.rl.Config.AutoComplete = autoCompleterFn(func(line []rune, pos int) (newLine [][]rune, length int) {
names, shared := complete(string(line), pos)
names, shared := opts.CompleteFn(string(line), pos)
var runeNames [][]rune
for _, name := range names {
runeNames = append(runeNames, []rune(name[shared:]))
@ -191,7 +191,7 @@ func (o *stdOS) Readline(prompt string, complete func(line string, pos int) (new
})
}
o.rl.SetPrompt(prompt)
o.rl.SetPrompt(opts.Prompt)
line, err := o.rl.Readline()
if errors.Is(err, readline.ErrInterrupt) {
return "", interp.ErrInterrupt

21
pkg/interp/binary.go
View File

@ -16,13 +16,14 @@ import (
"github.com/wader/fq/internal/progressreadseeker"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/ranges"
"github.com/wader/gojq"
)
func init() {
functionRegisterFns = append(functionRegisterFns, func(i *Interp) []Function {
return []Function{
{"_tobits", 3, 3, i._toBits, nil},
{"open", 0, 0, i._open, nil},
{"open", 0, 0, nil, i._open},
}
})
}
@ -175,7 +176,11 @@ func (of *openFile) ToBinary() (Binary, error) {
// def open: #:: string| => binary
// opens a file for reading from filesystem
// TODO: when to close? when br loses all refs? need to use finalizer somehow?
func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
func (i *Interp) _open(c interface{}, a []interface{}) gojq.Iter {
if i.evalContext.isCompleting {
return gojq.NewIter()
}
var err error
var f fs.File
var path string
@ -187,11 +192,11 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
default:
path, err = toString(c)
if err != nil {
return fmt.Errorf("%s: %w", path, err)
return gojq.NewIter(fmt.Errorf("%s: %w", path, err))
}
f, err = i.os.FS().Open(path)
if err != nil {
return err
return gojq.NewIter(err)
}
}
@ -201,7 +206,7 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
fFI, err := f.Stat()
if err != nil {
f.Close()
return err
return gojq.NewIter(err)
}
// ctxreadseeker is used to make sure any io calls can be canceled
@ -219,7 +224,7 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
buf, err := ioutil.ReadAll(ctxreadseeker.New(i.evalContext.ctx, &ioextra.ReadErrSeeker{Reader: f}))
if err != nil {
f.Close()
return err
return gojq.NewIter(err)
}
fRS = bytes.NewReader(buf)
bEnd = int64(len(buf))
@ -246,10 +251,10 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
bbf.br = bitio.NewIOBitReadSeeker(aheadRs)
if err != nil {
return err
return gojq.NewIter(err)
}
return bbf
return gojq.NewIter(bbf)
}
var _ Value = Binary{}

2
pkg/interp/decode.go
View File

@ -162,7 +162,7 @@ func (i *Interp) _decode(c interface{}, a []interface{}) interface{} {
c,
opts.Progress,
nil,
ioextra.DiscardCtxWriter{Ctx: i.evalContext.ctx},
EvalOpts{output: ioextra.DiscardCtxWriter{Ctx: i.evalContext.ctx}},
)
}
lastProgress := time.Now()

11
pkg/interp/funcs.jq
View File

@ -305,3 +305,14 @@ def topem($label):
| join("\n")
);
def topem: topem("");
def paste:
if _is_completing | not then
( [ _repeat_break(
try _stdin(64*1024)
catch if . == "eof" then error("break") end
)
]
| join("")
)
end;

164
pkg/interp/interp.go
View File

@ -11,6 +11,7 @@ import (
"fmt"
"io"
"io/fs"
"io/ioutil"
"math/big"
"path"
"strconv"
@ -22,6 +23,7 @@ import (
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/internal/colorjson"
"github.com/wader/fq/internal/ctxstack"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/ioextra"
"github.com/wader/fq/internal/mathextra"
"github.com/wader/fq/internal/pos"
@ -53,11 +55,11 @@ var functionRegisterFns []func(i *Interp) []Function
func init() {
functionRegisterFns = append(functionRegisterFns, func(i *Interp) []Function {
return []Function{
{"_readline", 0, 2, i._readline, nil},
{"_readline", 0, 1, nil, i._readline},
{"eval", 1, 2, nil, i.eval},
{"_stdin", 0, 0, nil, i.makeStdioFn(i.os.Stdin())},
{"_stdout", 0, 0, nil, i.makeStdioFn(i.os.Stdout())},
{"_stderr", 0, 0, nil, i.makeStdioFn(i.os.Stderr())},
{"_stdin", 0, 1, nil, i.makeStdioFn("stdin", i.os.Stdin())},
{"_stdout", 0, 0, nil, i.makeStdioFn("stdout", i.os.Stdout())},
{"_stderr", 0, 0, nil, i.makeStdioFn("stderr", i.os.Stderr())},
{"_extkeys", 0, 0, i._extKeys, nil},
{"_exttype", 0, 0, i._extType, nil},
{"_global_state", 0, 1, i.makeStateFn(i.state), nil},
@ -65,6 +67,7 @@ func init() {
{"_display", 1, 1, nil, i._display},
{"_can_display", 0, 0, i._canDisplay, nil},
{"_print_color_json", 0, 1, nil, i._printColorJSON},
{"_is_completing", 0, 1, i._isCompleting, nil},
}
})
}
@ -95,7 +98,7 @@ func (ce compileError) Value() interface{} {
func (ce compileError) Error() string {
filename := ce.filename
if filename == "" {
filename = "src"
filename = "expr"
}
return fmt.Sprintf("%s:%d:%d: %s: %s", filename, ce.pos.Line, ce.pos.Column, ce.what, ce.err.Error())
}
@ -133,6 +136,11 @@ type Platform struct {
Arch string
}
type ReadlineOpts struct {
Prompt string
CompleteFn func(line string, pos int) (newLine []string, shared int)
}
type OS interface {
Platform() Platform
Stdin() Input
@ -144,7 +152,7 @@ type OS interface {
ConfigDir() (string, error)
// FS.File returned by FS().Open() can optionally implement io.Seeker
FS() fs.FS
Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error)
Readline(opts ReadlineOpts) (string, error)
History() ([]string, error)
}
@ -283,14 +291,14 @@ func toBytes(v interface{}) ([]byte, error) {
}
}
func queryErrorPosition(src string, v error) pos.Pos {
func queryErrorPosition(expr string, v error) pos.Pos {
var offset int
if tokIf, ok := v.(interface{ Token() (string, int) }); ok { //nolint:errorlint
_, offset = tokIf.Token()
}
if offset >= 0 {
return pos.NewFromOffset(src, offset)
return pos.NewFromOffset(expr, offset)
}
return pos.Pos{}
}
@ -317,8 +325,9 @@ const (
)
type evalContext struct {
ctx context.Context
output io.Writer
ctx context.Context
output io.Writer
isCompleting bool
}
type Interp struct {
@ -380,7 +389,7 @@ func (i *Interp) Main(ctx context.Context, output Output, versionStr string) err
"arch": platform.Arch,
}
iter, err := i.EvalFunc(ctx, input, "_main", nil, output)
iter, err := i.EvalFunc(ctx, input, "_main", nil, EvalOpts{output: output})
if err != nil {
fmt.Fprintln(i.os.Stderr(), err)
return err
@ -414,28 +423,24 @@ func (i *Interp) Main(ctx context.Context, output Output, versionStr string) err
return nil
}
func (i *Interp) _readline(c interface{}, a []interface{}) interface{} {
func (i *Interp) _readline(c interface{}, a []interface{}) gojq.Iter {
if i.evalContext.isCompleting {
return gojq.NewIter()
}
var opts struct {
Promopt string `mapstructure:"prompt"`
Complete string `mapstructure:"complete"`
Timeout float64 `mapstructure:"timeout"`
}
var err error
prompt := ""
if len(a) > 0 {
prompt, err = toString(a[0])
if err != nil {
return fmt.Errorf("prompt: %w", err)
}
}
if len(a) > 1 {
_ = mapstructure.Decode(a[1], &opts)
_ = mapstructure.Decode(a[0], &opts)
}
src, err := i.os.Readline(
prompt,
func(line string, pos int) (newLine []string, shared int) {
expr, err := i.os.Readline(ReadlineOpts{
Prompt: opts.Promopt,
CompleteFn: func(line string, pos int) (newLine []string, shared int) {
completeCtx := i.evalContext.ctx
if opts.Timeout > 0 {
var completeCtxCancelFn context.CancelFunc
@ -450,7 +455,10 @@ func (i *Interp) _readline(c interface{}, a []interface{}) interface{} {
c,
opts.Complete,
[]interface{}{line, pos},
ioextra.DiscardCtxWriter{Ctx: completeCtx},
EvalOpts{
output: ioextra.DiscardCtxWriter{Ctx: completeCtx},
isCompleting: true,
},
)
if err != nil {
return nil, pos, err
@ -485,24 +493,24 @@ func (i *Interp) _readline(c interface{}, a []interface{}) interface{} {
return names, shared
},
)
})
if errors.Is(err, ErrInterrupt) {
return valueError{"interrupt"}
return gojq.NewIter(valueError{"interrupt"})
} else if errors.Is(err, ErrEOF) {
return valueError{"eof"}
return gojq.NewIter(valueError{"eof"})
} else if err != nil {
return err
return gojq.NewIter(err)
}
return src
return gojq.NewIter(expr)
}
func (i *Interp) eval(c interface{}, a []interface{}) gojq.Iter {
var err error
src, err := toString(a[0])
expr, err := toString(a[0])
if err != nil {
return gojq.NewIter(fmt.Errorf("src: %w", err))
return gojq.NewIter(fmt.Errorf("expr: %w", err))
}
var filenameHint string
if len(a) >= 2 {
@ -512,7 +520,10 @@ func (i *Interp) eval(c interface{}, a []interface{}) gojq.Iter {
}
}
iter, err := i.Eval(i.evalContext.ctx, c, src, filenameHint, i.evalContext.output)
iter, err := i.Eval(i.evalContext.ctx, c, expr, EvalOpts{
filename: filenameHint,
output: i.evalContext.output,
})
if err != nil {
return gojq.NewIter(err)
}
@ -547,25 +558,53 @@ func (i *Interp) makeStateFn(state *interface{}) func(c interface{}, a []interfa
}
}
func (i *Interp) makeStdioFn(t Terminal) func(c interface{}, a []interface{}) gojq.Iter {
func (i *Interp) makeStdioFn(name string, t Terminal) func(c interface{}, a []interface{}) gojq.Iter {
return func(c interface{}, a []interface{}) gojq.Iter {
if c == nil {
if i.evalContext.isCompleting {
return gojq.NewIter("")
}
switch {
case len(a) == 1:
r, ok := t.(io.Reader)
if !ok {
return gojq.NewIter(fmt.Errorf("%s is not readable", name))
}
l, ok := gojqextra.ToInt(a[0])
if !ok {
return gojq.NewIter(gojqextra.FuncTypeError{Name: name, V: a[0]})
}
buf := make([]byte, l)
n, err := io.ReadFull(r, buf)
s := string(buf[0:n])
vs := []interface{}{s}
switch {
case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF):
vs = append(vs, valueError{"eof"})
default:
vs = append(vs, err)
}
return gojq.NewIter(vs...)
case c == nil:
w, h := t.Size()
return gojq.NewIter(map[string]interface{}{
"is_terminal": t.IsTerminal(),
"width": w,
"height": h,
})
}
if w, ok := t.(io.Writer); ok {
default:
w, ok := t.(io.Writer)
if !ok {
return gojq.NewIter(fmt.Errorf("%v: it not writeable", c))
}
if _, err := fmt.Fprint(w, c); err != nil {
return gojq.NewIter(err)
}
return gojq.NewIter()
}
return gojq.NewIter(fmt.Errorf("%v: it not writeable", c))
}
}
@ -595,6 +634,11 @@ func (i *Interp) _display(c interface{}, a []interface{}) gojq.Iter {
}
}
func (i *Interp) _canDisplay(c interface{}, a []interface{}) interface{} {
_, ok := c.(Display)
return ok
}
func (i *Interp) _printColorJSON(c interface{}, a []interface{}) gojq.Iter {
opts := i.Options(a[0])
@ -609,9 +653,8 @@ func (i *Interp) _printColorJSON(c interface{}, a []interface{}) gojq.Iter {
return gojq.NewIter()
}
func (i *Interp) _canDisplay(c interface{}, a []interface{}) interface{} {
_, ok := c.(Display)
return ok
func (i *Interp) _isCompleting(c interface{}, a []interface{}) interface{} {
return i.evalContext.isCompleting
}
type pathResolver struct {
@ -667,14 +710,20 @@ func (i *Interp) lookupPathResolver(filename string) (pathResolver, error) {
return pathResolver{}, fmt.Errorf("could not resolve path: %s", filename)
}
func (i *Interp) Eval(ctx context.Context, c interface{}, src string, srcFilename string, output io.Writer) (gojq.Iter, error) {
gq, err := gojq.Parse(src)
type EvalOpts struct {
filename string
output io.Writer
isCompleting bool
}
func (i *Interp) Eval(ctx context.Context, c interface{}, expr string, opts EvalOpts) (gojq.Iter, error) {
gq, err := gojq.Parse(expr)
if err != nil {
p := queryErrorPosition(src, err)
p := queryErrorPosition(expr, err)
return nil, compileError{
err: err,
what: "parse",
filename: srcFilename,
filename: opts.filename,
pos: p,
}
}
@ -827,18 +876,25 @@ func (i *Interp) Eval(ctx context.Context, c interface{}, src string, srcFilenam
gc, err := gojq.Compile(gq, compilerOpts...)
if err != nil {
p := queryErrorPosition(src, err)
p := queryErrorPosition(expr, err)
return nil, compileError{
err: err,
what: "compile",
filename: srcFilename,
filename: opts.filename,
pos: p,
}
}
output := opts.output
if opts.output == nil {
output = ioutil.Discard
}
runCtx, runCtxCancelFn := i.interruptStack.Push(ctx)
ni.evalContext.ctx = runCtx
ni.evalContext.output = ioextra.CtxWriter{Writer: output, Ctx: runCtx}
// inherit or set
ni.evalContext.isCompleting = i.evalContext.isCompleting || opts.isCompleting
iter := gc.RunWithContext(runCtx, c, variableValues...)
iterWrapper := iterFn(func() (interface{}, bool) {
@ -853,7 +909,7 @@ func (i *Interp) Eval(ctx context.Context, c interface{}, src string, srcFilenam
return iterWrapper, nil
}
func (i *Interp) EvalFunc(ctx context.Context, c interface{}, name string, args []interface{}, output io.Writer) (gojq.Iter, error) {
func (i *Interp) EvalFunc(ctx context.Context, c interface{}, name string, args []interface{}, opts EvalOpts) (gojq.Iter, error) {
var argsExpr []string
for i := range args {
argsExpr = append(argsExpr, fmt.Sprintf("$_args[%d]", i))
@ -870,15 +926,15 @@ func (i *Interp) EvalFunc(ctx context.Context, c interface{}, name string, args
// _args to mark variable as internal and hide it from completion
// {input: ..., args: [...]} | .args as {args: $_args} | .input | name[($_args[0]; ...)]
trampolineExpr := fmt.Sprintf(". as {args: $_args} | .input | %s%s", name, argExpr)
iter, err := i.Eval(ctx, trampolineInput, trampolineExpr, "", output)
iter, err := i.Eval(ctx, trampolineInput, trampolineExpr, opts)
if err != nil {
return nil, err
}
return iter, nil
}
func (i *Interp) EvalFuncValues(ctx context.Context, c interface{}, name string, args []interface{}, output io.Writer) ([]interface{}, error) {
iter, err := i.EvalFunc(ctx, c, name, args, output)
func (i *Interp) EvalFuncValues(ctx context.Context, c interface{}, name string, args []interface{}, opts EvalOpts) ([]interface{}, error) {
iter, err := i.EvalFunc(ctx, c, name, args, opts)
if err != nil {
return nil, err
}

17
pkg/interp/repl.jq
View File

@ -182,7 +182,7 @@ def _repl($opts): #:: a|(Opts) => @
def _read_expr:
_repeat_break(
# both _prompt and _complete want input arrays
( _readline(_prompt; {complete: "_complete", timeout: 1})
( _readline({prompt: _prompt, complete: "_complete", timeout: 1})
| if trim == "" then empty
else (., error("break"))
end
@ -216,12 +216,15 @@ def _repl($opts): #:: a|(Opts) => @
else error
end
);
( _options_stack(. + [$opts]) as $_
| _finally(
_repeat_break(_repl_loop);
_options_stack(.[:-1])
if _is_completing | not then
( _options_stack(. + [$opts]) as $_
| _finally(
_repeat_break(_repl_loop);
_options_stack(.[:-1])
)
)
);
else empty
end;
def _repl_slurp($opts): _repl($opts);
def _repl_slurp: _repl({});
@ -229,7 +232,7 @@ def _repl_slurp: _repl({});
# TODO: introspect and show doc, reflection somehow?
def help:
( "Type expression to evaluate"
, "\\t Auto completion"
, "\\t Completion"
, "Up/Down History"
, "^C Interrupt execution"
, "... | repl Start a new REPL"

6
pkg/interp/testdata/paste.fqtest vendored Normal file
View File

@ -0,0 +1,6 @@
$ fq -i
null> paste
"test\n"
null> ^D
stdin:
test