2020-06-08 03:29:51 +03:00
|
|
|
package interp
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
|
|
|
"context"
|
2021-08-12 02:35:40 +03:00
|
|
|
"crypto/md5"
|
2020-06-08 03:29:51 +03:00
|
|
|
"embed"
|
2021-08-12 02:35:40 +03:00
|
|
|
"encoding/base64"
|
|
|
|
"encoding/hex"
|
2020-06-08 03:29:51 +03:00
|
|
|
"errors"
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"io/fs"
|
|
|
|
"math/big"
|
|
|
|
"path/filepath"
|
|
|
|
"strconv"
|
|
|
|
"strings"
|
|
|
|
"time"
|
|
|
|
|
2021-09-22 02:35:27 +03:00
|
|
|
"github.com/mitchellh/mapstructure"
|
2021-08-17 13:06:32 +03:00
|
|
|
"github.com/wader/fq/format/registry"
|
|
|
|
"github.com/wader/fq/internal/ansi"
|
|
|
|
"github.com/wader/fq/internal/colorjson"
|
|
|
|
"github.com/wader/fq/internal/ctxstack"
|
|
|
|
"github.com/wader/fq/internal/num"
|
|
|
|
"github.com/wader/fq/internal/pos"
|
|
|
|
"github.com/wader/fq/pkg/bitio"
|
|
|
|
"github.com/wader/fq/pkg/decode"
|
|
|
|
|
|
|
|
"github.com/wader/gojq"
|
2020-06-08 03:29:51 +03:00
|
|
|
)
|
|
|
|
|
2021-08-14 21:00:11 +03:00
|
|
|
//go:embed interp.jq
|
2020-06-08 03:29:51 +03:00
|
|
|
//go:embed internal.jq
|
2021-09-22 22:08:36 +03:00
|
|
|
//go:embed options.jq
|
2020-06-08 03:29:51 +03:00
|
|
|
//go:embed funcs.jq
|
|
|
|
//go:embed args.jq
|
2021-08-19 19:11:37 +03:00
|
|
|
//go:embed query.jq
|
2021-09-22 22:08:36 +03:00
|
|
|
//go:embed repl.jq
|
2021-09-21 17:42:35 +03:00
|
|
|
//go:embed formats.jq
|
2020-06-08 03:29:51 +03:00
|
|
|
var builtinFS embed.FS
|
|
|
|
|
2021-08-14 21:00:11 +03:00
|
|
|
var initSource = `include "@builtin/interp";`
|
2020-06-08 03:29:51 +03:00
|
|
|
|
|
|
|
type valueError struct {
|
|
|
|
v interface{}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (v valueError) Error() string { return fmt.Sprintf("error: %v", v.v) }
|
|
|
|
func (v valueError) Value() interface{} { return v.v }
|
|
|
|
|
|
|
|
type compileError struct {
|
|
|
|
err error
|
|
|
|
what string
|
|
|
|
filename string
|
|
|
|
pos pos.Pos
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ce compileError) Value() interface{} {
|
|
|
|
return map[string]interface{}{
|
2021-08-18 19:47:13 +03:00
|
|
|
"error": ce.err.Error(),
|
|
|
|
"what": ce.what,
|
|
|
|
"filename": ce.filename,
|
|
|
|
"line": ce.pos.Line,
|
|
|
|
"column": ce.pos.Column,
|
2020-06-08 03:29:51 +03:00
|
|
|
}
|
|
|
|
}
|
2021-09-27 12:01:14 +03:00
|
|
|
func (ce compileError) Error() string {
|
|
|
|
filename := ce.filename
|
2020-06-08 03:29:51 +03:00
|
|
|
if filename == "" {
|
|
|
|
filename = "src"
|
|
|
|
}
|
2021-09-27 12:01:14 +03:00
|
|
|
return fmt.Sprintf("%s:%d:%d: %s: %s", filename, ce.pos.Line, ce.pos.Column, ce.what, ce.err.Error())
|
2020-06-08 03:29:51 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
var ErrEOF = io.EOF
|
|
|
|
var ErrInterrupt = errors.New("Interrupt")
|
|
|
|
|
|
|
|
// gojq errors can implement this to signal exit code
|
|
|
|
type Exiter interface {
|
|
|
|
ExitCode() int
|
|
|
|
}
|
|
|
|
|
|
|
|
// gojq halt_error uses this
|
|
|
|
type IsEmptyErrorer interface {
|
|
|
|
IsEmptyError() bool
|
|
|
|
}
|
|
|
|
|
2021-09-01 16:01:13 +03:00
|
|
|
type Terminal interface {
|
2020-06-08 03:29:51 +03:00
|
|
|
Size() (int, int)
|
|
|
|
IsTerminal() bool
|
|
|
|
}
|
|
|
|
|
2021-09-01 16:01:13 +03:00
|
|
|
type Input interface {
|
|
|
|
fs.File
|
|
|
|
Terminal
|
|
|
|
}
|
|
|
|
|
|
|
|
type Output interface {
|
|
|
|
io.Writer
|
|
|
|
Terminal
|
|
|
|
}
|
|
|
|
|
2020-06-08 03:29:51 +03:00
|
|
|
type OS interface {
|
2021-09-01 16:01:13 +03:00
|
|
|
Stdin() Input
|
2020-06-08 03:29:51 +03:00
|
|
|
Stdout() Output
|
2021-09-01 16:01:13 +03:00
|
|
|
Stderr() Output
|
2020-06-08 03:29:51 +03:00
|
|
|
Interrupt() chan struct{}
|
|
|
|
Args() []string
|
|
|
|
Environ() []string
|
|
|
|
ConfigDir() (string, error)
|
|
|
|
// returned Open() io.ReadSeeker can optionally implement io.Closer
|
|
|
|
FS() fs.FS
|
|
|
|
Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error)
|
|
|
|
History() ([]string, error)
|
|
|
|
}
|
|
|
|
|
|
|
|
type FixedFileInfo struct {
|
|
|
|
FName string
|
|
|
|
FSize int64
|
|
|
|
FMode fs.FileMode
|
|
|
|
FModTime time.Time
|
|
|
|
FIsDir bool
|
|
|
|
FSys interface{}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ffi FixedFileInfo) Name() string { return ffi.FName }
|
|
|
|
func (ffi FixedFileInfo) Size() int64 { return ffi.FSize }
|
|
|
|
func (ffi FixedFileInfo) Mode() fs.FileMode { return ffi.FMode }
|
|
|
|
func (ffi FixedFileInfo) ModTime() time.Time { return ffi.FModTime }
|
|
|
|
func (ffi FixedFileInfo) IsDir() bool { return ffi.FIsDir }
|
|
|
|
func (ffi FixedFileInfo) Sys() interface{} { return ffi.FSys }
|
|
|
|
|
|
|
|
type FileReader struct {
|
|
|
|
R io.Reader
|
|
|
|
FileInfo FixedFileInfo
|
|
|
|
}
|
|
|
|
|
|
|
|
func (rf FileReader) Stat() (fs.FileInfo, error) { return rf.FileInfo, nil }
|
|
|
|
func (rf FileReader) Read(p []byte) (int, error) { return rf.R.Read(p) }
|
|
|
|
func (FileReader) Close() error { return nil }
|
|
|
|
|
|
|
|
// TODO: move
|
2021-09-11 01:51:50 +03:00
|
|
|
type DiscardCtxWriter struct {
|
2020-06-08 03:29:51 +03:00
|
|
|
Ctx context.Context
|
|
|
|
}
|
|
|
|
|
2021-09-11 01:51:50 +03:00
|
|
|
func (o DiscardCtxWriter) Write(p []byte) (n int, err error) {
|
2020-06-08 03:29:51 +03:00
|
|
|
if o.Ctx != nil {
|
|
|
|
if err := o.Ctx.Err(); err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return n, nil
|
|
|
|
}
|
|
|
|
|
2021-09-11 01:51:50 +03:00
|
|
|
type CtxWriter struct {
|
2021-09-01 16:01:13 +03:00
|
|
|
io.Writer
|
2020-06-08 03:29:51 +03:00
|
|
|
Ctx context.Context
|
|
|
|
}
|
|
|
|
|
2021-09-11 01:51:50 +03:00
|
|
|
func (o CtxWriter) Write(p []byte) (n int, err error) {
|
2020-06-08 03:29:51 +03:00
|
|
|
if o.Ctx != nil {
|
|
|
|
if err := o.Ctx.Err(); err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
}
|
2021-09-01 16:01:13 +03:00
|
|
|
return o.Writer.Write(p)
|
2020-06-08 03:29:51 +03:00
|
|
|
}
|
|
|
|
|
2021-09-27 12:01:14 +03:00
|
|
|
type Value interface {
|
2020-06-08 03:29:51 +03:00
|
|
|
gojq.JQValue
|
|
|
|
|
|
|
|
DisplayName() string
|
|
|
|
ExtKeys() []string
|
|
|
|
}
|
|
|
|
|
|
|
|
type Display interface {
|
|
|
|
Display(w io.Writer, opts Options) error
|
|
|
|
}
|
|
|
|
|
|
|
|
type ToBuffer interface {
|
|
|
|
ToBuffer() (*bitio.Buffer, error)
|
|
|
|
}
|
|
|
|
|
2021-09-28 00:14:08 +03:00
|
|
|
type ToBufferView interface {
|
|
|
|
ToBufferView() (BufferView, error)
|
2020-06-08 03:29:51 +03:00
|
|
|
}
|
|
|
|
|
2021-08-12 00:51:00 +03:00
|
|
|
type JQValueEx interface {
|
2021-08-12 02:35:40 +03:00
|
|
|
JQValueToGoJQEx(opts Options) interface{}
|
2021-08-12 00:51:00 +03:00
|
|
|
}
|
|
|
|
|
2020-06-08 03:29:51 +03:00
|
|
|
func valuePath(v *decode.Value) []interface{} {
|
|
|
|
var parts []interface{}
|
|
|
|
|
|
|
|
for v.Parent != nil {
|
|
|
|
switch v.Parent.V.(type) {
|
|
|
|
case decode.Struct:
|
|
|
|
parts = append([]interface{}{v.Name}, parts...)
|
|
|
|
case decode.Array:
|
|
|
|
parts = append([]interface{}{v.Index}, parts...)
|
|
|
|
}
|
|
|
|
v = v.Parent
|
|
|
|
}
|
|
|
|
|
|
|
|
return parts
|
|
|
|
}
|
|
|
|
|
|
|
|
func valuePathDecorated(v *decode.Value, d Decorator) string {
|
|
|
|
var parts []string
|
|
|
|
|
|
|
|
for _, p := range valuePath(v) {
|
|
|
|
switch p := p.(type) {
|
|
|
|
case string:
|
|
|
|
parts = append(parts, ".", d.ObjectKey.Wrap(p))
|
|
|
|
case int:
|
|
|
|
indexStr := strconv.Itoa(p)
|
|
|
|
parts = append(parts, fmt.Sprintf("%s%s%s", d.Index.F("["), d.Number.F(indexStr), d.Index.F("]")))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(parts) == 0 {
|
|
|
|
return "."
|
|
|
|
}
|
|
|
|
|
|
|
|
return strings.Join(parts, "")
|
|
|
|
}
|
|
|
|
|
|
|
|
type iterFn func() (interface{}, bool)
|
|
|
|
|
|
|
|
func (i iterFn) Next() (interface{}, bool) { return i() }
|
|
|
|
|
|
|
|
type loadModule struct {
|
|
|
|
init func() ([]*gojq.Query, error)
|
|
|
|
load func(name string) (*gojq.Query, error)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (l loadModule) LoadInitModules() ([]*gojq.Query, error) { return l.init() }
|
|
|
|
func (l loadModule) LoadModule(name string) (*gojq.Query, error) { return l.load(name) }
|
|
|
|
|
|
|
|
func toString(v interface{}) (string, error) {
|
|
|
|
switch v := v.(type) {
|
|
|
|
case string:
|
|
|
|
return v, nil
|
2021-08-16 19:39:37 +03:00
|
|
|
case gojq.JQValue:
|
|
|
|
return toString(v.JQValueToGoJQ())
|
2020-06-08 03:29:51 +03:00
|
|
|
default:
|
|
|
|
b, err := toBytes(v)
|
|
|
|
if err != nil {
|
|
|
|
return "", fmt.Errorf("value can't be a string")
|
|
|
|
}
|
|
|
|
|
|
|
|
return string(b), nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func toBigInt(v interface{}) (*big.Int, error) {
|
|
|
|
switch v := v.(type) {
|
|
|
|
case int:
|
|
|
|
return new(big.Int).SetInt64(int64(v)), nil
|
|
|
|
case float64:
|
|
|
|
return new(big.Int).SetInt64(int64(v)), nil
|
|
|
|
case *big.Int:
|
|
|
|
return v, nil
|
|
|
|
default:
|
|
|
|
return nil, fmt.Errorf("value is not a number")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func toBytes(v interface{}) ([]byte, error) {
|
|
|
|
switch v := v.(type) {
|
|
|
|
case []byte:
|
|
|
|
return v, nil
|
|
|
|
default:
|
|
|
|
bb, err := toBuffer(v)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("value is not bytes")
|
|
|
|
}
|
|
|
|
buf := &bytes.Buffer{}
|
|
|
|
if _, err := io.Copy(buf, bb); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return buf.Bytes(), nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func toBuffer(v interface{}) (*bitio.Buffer, error) {
|
|
|
|
return toBufferEx(v, false)
|
|
|
|
}
|
|
|
|
|
|
|
|
// TODO: refactor to return struct?
|
|
|
|
func toBufferEx(v interface{}, inArray bool) (*bitio.Buffer, error) {
|
|
|
|
switch vv := v.(type) {
|
|
|
|
case ToBuffer:
|
|
|
|
return vv.ToBuffer()
|
|
|
|
case string:
|
|
|
|
return bitio.NewBufferFromBytes([]byte(vv), -1), nil
|
|
|
|
case []byte:
|
|
|
|
return bitio.NewBufferFromBytes(vv, -1), nil
|
|
|
|
case int, float64, *big.Int:
|
|
|
|
bi, err := toBigInt(v)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
if inArray {
|
|
|
|
b := [1]byte{byte(bi.Uint64())}
|
|
|
|
return bitio.NewBufferFromBytes(b[:], -1), nil
|
|
|
|
}
|
2021-09-27 12:01:14 +03:00
|
|
|
|
|
|
|
padBefore := (8 - (bi.BitLen() % 8)) % 8
|
|
|
|
bb, err := bitio.NewBufferFromBytes(bi.Bytes(), -1).BitBufRange(int64(padBefore), int64(bi.BitLen()))
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return bb, nil
|
2020-06-08 03:29:51 +03:00
|
|
|
case []interface{}:
|
|
|
|
var rr []bitio.BitReadAtSeeker
|
|
|
|
// TODO: optimize byte array case
|
|
|
|
for _, e := range vv {
|
|
|
|
eBB, eErr := toBufferEx(e, true)
|
|
|
|
if eErr != nil {
|
|
|
|
return nil, eErr
|
|
|
|
}
|
|
|
|
rr = append(rr, eBB)
|
|
|
|
}
|
|
|
|
|
|
|
|
mb, err := bitio.NewMultiBitReader(rr)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
bb, err := bitio.NewBufferFromBitReadSeeker(mb)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return bb, nil
|
|
|
|
default:
|
|
|
|
return nil, fmt.Errorf("value can't be a buffer")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-08-12 02:35:40 +03:00
|
|
|
func toValue(opts Options, v interface{}) (interface{}, bool) {
|
2020-06-08 03:29:51 +03:00
|
|
|
switch v := v.(type) {
|
2021-08-12 00:51:00 +03:00
|
|
|
case JQValueEx:
|
2021-08-12 02:35:40 +03:00
|
|
|
return v.JQValueToGoJQEx(opts), true
|
2020-06-08 03:29:51 +03:00
|
|
|
case gojq.JQValue:
|
2021-08-12 00:51:00 +03:00
|
|
|
return v.JQValueToGoJQ(), true
|
2020-06-08 03:29:51 +03:00
|
|
|
case nil, bool, float64, int, string, *big.Int, map[string]interface{}, []interface{}:
|
2021-08-12 00:51:00 +03:00
|
|
|
return v, true
|
2020-06-08 03:29:51 +03:00
|
|
|
default:
|
2021-08-12 00:51:00 +03:00
|
|
|
return nil, false
|
2020-06-08 03:29:51 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-08-29 13:55:13 +03:00
|
|
|
func queryErrorPosition(src string, v error) pos.Pos {
|
2020-06-08 03:29:51 +03:00
|
|
|
var offset int
|
|
|
|
|
|
|
|
if tokIf, ok := v.(interface{ Token() (string, int) }); ok { //nolint:errorlint
|
|
|
|
_, offset = tokIf.Token()
|
|
|
|
}
|
|
|
|
if offset >= 0 {
|
2021-08-29 13:55:13 +03:00
|
|
|
return pos.NewFromOffset(src, offset)
|
2020-06-08 03:29:51 +03:00
|
|
|
}
|
|
|
|
return pos.Pos{}
|
|
|
|
}
|
|
|
|
|
|
|
|
type Variable struct {
|
|
|
|
Name string
|
|
|
|
Value interface{}
|
|
|
|
}
|
|
|
|
|
|
|
|
type Function struct {
|
|
|
|
Names []string
|
|
|
|
MinArity int
|
|
|
|
MaxArity int
|
|
|
|
Fn func(interface{}, []interface{}) interface{}
|
|
|
|
IterFn func(interface{}, []interface{}) gojq.Iter
|
|
|
|
}
|
|
|
|
|
|
|
|
type RunMode int
|
|
|
|
|
|
|
|
const (
|
|
|
|
ScriptMode RunMode = iota
|
|
|
|
REPLMode
|
|
|
|
CompletionMode
|
|
|
|
)
|
|
|
|
|
|
|
|
type evalContext struct {
|
|
|
|
// structcheck has problems with embedding https://gitlab.com/opennota/check#known-limitations
|
2021-08-14 20:50:17 +03:00
|
|
|
ctx context.Context
|
2021-09-01 16:01:13 +03:00
|
|
|
output io.Writer
|
2020-06-08 03:29:51 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
type Interp struct {
|
|
|
|
registry *registry.Registry
|
|
|
|
os OS
|
|
|
|
initFqQuery *gojq.Query
|
|
|
|
includeCache map[string]*gojq.Query
|
|
|
|
interruptStack *ctxstack.Stack
|
|
|
|
// global state, is ref as Interp i cloned per eval
|
|
|
|
state *interface{}
|
|
|
|
|
2021-08-30 02:56:28 +03:00
|
|
|
// new for each run, other values are copied by value
|
2020-06-08 03:29:51 +03:00
|
|
|
evalContext evalContext
|
|
|
|
}
|
|
|
|
|
|
|
|
func New(os OS, registry *registry.Registry) (*Interp, error) {
|
|
|
|
var err error
|
|
|
|
|
|
|
|
i := &Interp{
|
|
|
|
os: os,
|
|
|
|
registry: registry,
|
|
|
|
}
|
|
|
|
|
|
|
|
i.includeCache = map[string]*gojq.Query{}
|
2021-08-14 21:00:11 +03:00
|
|
|
i.initFqQuery, err = gojq.Parse(initSource)
|
2020-06-08 03:29:51 +03:00
|
|
|
if err != nil {
|
2021-08-29 13:55:13 +03:00
|
|
|
return nil, fmt.Errorf("init:%s: %w", queryErrorPosition(initSource, err), err)
|
2020-06-08 03:29:51 +03:00
|
|
|
}
|
|
|
|
// TODO: refactor ctxstack have a CancelTop and return c context to Stop?
|
|
|
|
i.interruptStack = ctxstack.New(func(stopCh chan struct{}) {
|
|
|
|
select {
|
|
|
|
case <-stopCh:
|
|
|
|
return
|
|
|
|
case <-os.Interrupt():
|
|
|
|
return
|
|
|
|
}
|
|
|
|
})
|
|
|
|
i.state = new(interface{})
|
|
|
|
|
|
|
|
return i, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (i *Interp) Stop() {
|
|
|
|
// TODO: cancel all run instances?
|
|
|
|
i.interruptStack.Stop()
|
|
|
|
}
|
|
|
|
|
2021-09-11 01:51:50 +03:00
|
|
|
func (i *Interp) Main(ctx context.Context, output Output, version string) error {
|
2020-06-08 03:29:51 +03:00
|
|
|
var args []interface{}
|
|
|
|
for _, a := range i.os.Args() {
|
|
|
|
args = append(args, a)
|
|
|
|
}
|
|
|
|
|
|
|
|
input := map[string]interface{}{
|
|
|
|
"args": args,
|
|
|
|
"version": version,
|
|
|
|
}
|
|
|
|
|
2021-09-22 02:35:27 +03:00
|
|
|
iter, err := i.EvalFunc(ctx, input, "_main", nil, output)
|
2020-06-08 03:29:51 +03:00
|
|
|
if err != nil {
|
|
|
|
fmt.Fprintln(i.os.Stderr(), err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
for {
|
|
|
|
v, ok := iter.Next()
|
|
|
|
if !ok {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
|
|
|
switch v := v.(type) {
|
|
|
|
case error:
|
|
|
|
if emptyErr, ok := v.(IsEmptyErrorer); ok && emptyErr.IsEmptyError() { //nolint:errorlint
|
|
|
|
// no output
|
|
|
|
} else {
|
|
|
|
fmt.Fprintln(i.os.Stderr(), v)
|
|
|
|
}
|
|
|
|
return v
|
|
|
|
case [2]interface{}:
|
|
|
|
fmt.Fprintln(i.os.Stderr(), v[:]...)
|
|
|
|
default:
|
|
|
|
// TODO: can this happen?
|
|
|
|
fmt.Fprintln(i.os.Stderr(), v)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2021-09-22 02:35:27 +03:00
|
|
|
func (i *Interp) Eval(ctx context.Context, c interface{}, src string, srcFilename string, output io.Writer) (gojq.Iter, error) {
|
2020-06-08 03:29:51 +03:00
|
|
|
gq, err := gojq.Parse(src)
|
|
|
|
if err != nil {
|
2021-08-29 13:55:13 +03:00
|
|
|
p := queryErrorPosition(src, err)
|
2020-06-08 03:29:51 +03:00
|
|
|
return nil, compileError{
|
2021-08-18 19:47:13 +03:00
|
|
|
err: err,
|
|
|
|
what: "parse",
|
2021-09-21 17:42:35 +03:00
|
|
|
filename: srcFilename,
|
2021-08-18 19:47:13 +03:00
|
|
|
pos: p,
|
2020-06-08 03:29:51 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// make copy of interp
|
|
|
|
ci := *i
|
|
|
|
ni := &ci
|
2021-09-22 02:35:27 +03:00
|
|
|
ni.evalContext = evalContext{}
|
2020-06-08 03:29:51 +03:00
|
|
|
|
2021-08-15 18:11:34 +03:00
|
|
|
var variableNames []string
|
|
|
|
var variableValues []interface{}
|
|
|
|
for k, v := range i.variables() {
|
|
|
|
variableNames = append(variableNames, "$"+k)
|
|
|
|
variableValues = append(variableValues, v)
|
|
|
|
}
|
2020-06-08 03:29:51 +03:00
|
|
|
|
2021-09-21 17:42:35 +03:00
|
|
|
var funcCompilerOpts []gojq.CompilerOption
|
2021-09-04 13:54:26 +03:00
|
|
|
for _, f := range ni.makeFunctions() {
|
2020-06-08 03:29:51 +03:00
|
|
|
for _, n := range f.Names {
|
|
|
|
if f.IterFn != nil {
|
2021-09-21 17:42:35 +03:00
|
|
|
funcCompilerOpts = append(funcCompilerOpts,
|
2020-06-08 03:29:51 +03:00
|
|
|
gojq.WithIterFunction(n, f.MinArity, f.MaxArity, f.IterFn))
|
|
|
|
} else {
|
2021-09-21 17:42:35 +03:00
|
|
|
funcCompilerOpts = append(funcCompilerOpts,
|
2020-06-08 03:29:51 +03:00
|
|
|
gojq.WithFunction(n, f.MinArity, f.MaxArity, f.Fn))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2021-09-21 17:42:35 +03:00
|
|
|
|
|
|
|
compilerOpts := append([]gojq.CompilerOption{}, funcCompilerOpts...)
|
2020-06-08 03:29:51 +03:00
|
|
|
compilerOpts = append(compilerOpts, gojq.WithEnvironLoader(ni.os.Environ))
|
2021-08-15 18:11:34 +03:00
|
|
|
compilerOpts = append(compilerOpts, gojq.WithVariables(variableNames))
|
2020-06-08 03:29:51 +03:00
|
|
|
compilerOpts = append(compilerOpts, gojq.WithModuleLoader(loadModule{
|
|
|
|
init: func() ([]*gojq.Query, error) {
|
|
|
|
return []*gojq.Query{i.initFqQuery}, nil
|
|
|
|
},
|
|
|
|
load: func(name string) (*gojq.Query, error) {
|
2021-09-21 17:42:35 +03:00
|
|
|
// log.Printf("name: %#+v\n", name)
|
2020-06-08 03:29:51 +03:00
|
|
|
if err := ctx.Err(); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
var filename string
|
|
|
|
// support include "nonexisting?" to ignore include error
|
|
|
|
var isTry bool
|
|
|
|
if strings.HasSuffix(name, "?") {
|
|
|
|
isTry = true
|
|
|
|
filename = name[0 : len(name)-1]
|
|
|
|
} else {
|
|
|
|
filename = name
|
|
|
|
}
|
|
|
|
filename = filename + ".jq"
|
|
|
|
|
|
|
|
pathPrefixes := []struct {
|
|
|
|
prefix string
|
|
|
|
cache bool
|
|
|
|
fn func(filename string) (io.Reader, error)
|
|
|
|
}{
|
|
|
|
{
|
|
|
|
"@builtin/", true, func(filename string) (io.Reader, error) {
|
|
|
|
return builtinFS.Open(filename)
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"@config/", false, func(filename string) (io.Reader, error) {
|
|
|
|
configDir, err := i.os.ConfigDir()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return i.os.FS().Open(filepath.Join(configDir, filename))
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"", false, func(filename string) (io.Reader, error) {
|
2021-08-14 20:50:17 +03:00
|
|
|
// TODO: jq $ORIGIN
|
|
|
|
for _, path := range append([]string{"./"}, i.includePaths()...) {
|
|
|
|
if f, err := i.os.FS().Open(filepath.Join(path, filename)); err == nil {
|
|
|
|
return f, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil, &fs.PathError{Op: "open", Path: filename, Err: fs.ErrNotExist}
|
2020-06-08 03:29:51 +03:00
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, p := range pathPrefixes {
|
|
|
|
if !strings.HasPrefix(filename, p.prefix) {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
if p.cache {
|
|
|
|
if q, ok := ni.includeCache[filename]; ok {
|
|
|
|
return q, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
filenamePart := strings.TrimPrefix(filename, p.prefix)
|
|
|
|
f, err := p.fn(filenamePart)
|
|
|
|
if err != nil {
|
|
|
|
if !isTry {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
f = &bytes.Buffer{}
|
|
|
|
}
|
|
|
|
|
|
|
|
b, err := io.ReadAll(f)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2021-08-29 13:55:13 +03:00
|
|
|
s := string(b)
|
|
|
|
q, err := gojq.Parse(s)
|
2020-06-08 03:29:51 +03:00
|
|
|
if err != nil {
|
2021-08-29 13:55:13 +03:00
|
|
|
p := queryErrorPosition(s, err)
|
2020-06-08 03:29:51 +03:00
|
|
|
return nil, compileError{
|
|
|
|
err: err,
|
|
|
|
what: "parse",
|
2021-08-18 19:47:13 +03:00
|
|
|
filename: filenamePart,
|
2020-06-08 03:29:51 +03:00
|
|
|
pos: p,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-09-21 17:42:35 +03:00
|
|
|
// not identity body means it returns something, threat as dynamic include
|
|
|
|
if q.Term.Type != gojq.TermTypeIdentity {
|
|
|
|
gc, err := gojq.Compile(q, funcCompilerOpts...)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
iter := gc.RunWithContext(context.Background(), nil)
|
|
|
|
var vs []interface{}
|
|
|
|
for {
|
|
|
|
v, ok := iter.Next()
|
|
|
|
if !ok {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
if err, ok := v.(error); ok {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
vs = append(vs, v)
|
|
|
|
}
|
|
|
|
if len(vs) != 1 {
|
|
|
|
return nil, fmt.Errorf("dynamic include: must output one string, got: %#v", vs)
|
|
|
|
}
|
|
|
|
s, sOk := vs[0].(string)
|
|
|
|
if !sOk {
|
|
|
|
return nil, fmt.Errorf("dynamic include: must be string, got %#v", s)
|
|
|
|
}
|
|
|
|
q, err = gojq.Parse(s)
|
|
|
|
if err != nil {
|
|
|
|
p := queryErrorPosition(s, err)
|
|
|
|
return nil, compileError{
|
|
|
|
err: err,
|
|
|
|
what: "parse",
|
|
|
|
filename: filenamePart,
|
|
|
|
pos: p,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-08-30 02:56:28 +03:00
|
|
|
// TODO: some better way of handling relative includes that
|
|
|
|
// works with @builtin etc
|
|
|
|
basePath := filepath.Dir(name)
|
|
|
|
for _, i := range q.Imports {
|
|
|
|
rewritePath := func(base, path string) string {
|
|
|
|
if strings.HasPrefix(i.IncludePath, "@") {
|
|
|
|
return path
|
|
|
|
}
|
|
|
|
if filepath.IsAbs(i.IncludePath) {
|
|
|
|
return path
|
|
|
|
}
|
|
|
|
return filepath.Join(base, path)
|
|
|
|
}
|
|
|
|
i.IncludePath = rewritePath(basePath, i.IncludePath)
|
|
|
|
i.ImportPath = rewritePath(basePath, i.ImportPath)
|
|
|
|
}
|
|
|
|
|
2020-06-08 03:29:51 +03:00
|
|
|
if p.cache {
|
|
|
|
i.includeCache[filename] = q
|
|
|
|
}
|
|
|
|
|
|
|
|
return q, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
panic("unreachable")
|
|
|
|
},
|
|
|
|
}))
|
|
|
|
|
|
|
|
gc, err := gojq.Compile(gq, compilerOpts...)
|
|
|
|
if err != nil {
|
2021-08-29 13:55:13 +03:00
|
|
|
p := queryErrorPosition(src, err)
|
2020-06-08 03:29:51 +03:00
|
|
|
return nil, compileError{
|
2021-08-18 19:47:13 +03:00
|
|
|
err: err,
|
|
|
|
what: "compile",
|
2021-09-21 17:42:35 +03:00
|
|
|
filename: srcFilename,
|
2021-08-18 19:47:13 +03:00
|
|
|
pos: p,
|
2020-06-08 03:29:51 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
runCtx, runCtxCancelFn := i.interruptStack.Push(ctx)
|
|
|
|
ni.evalContext.ctx = runCtx
|
2021-09-11 01:51:50 +03:00
|
|
|
ni.evalContext.output = CtxWriter{Writer: output, Ctx: runCtx}
|
2021-08-15 18:11:34 +03:00
|
|
|
iter := gc.RunWithContext(runCtx, c, variableValues...)
|
2020-06-08 03:29:51 +03:00
|
|
|
|
|
|
|
iterWrapper := iterFn(func() (interface{}, bool) {
|
|
|
|
v, ok := iter.Next()
|
|
|
|
// gojq ctx cancel will not return ok=false, just cancelled error
|
|
|
|
if !ok {
|
|
|
|
runCtxCancelFn()
|
|
|
|
}
|
|
|
|
return v, ok
|
|
|
|
})
|
|
|
|
|
|
|
|
return iterWrapper, nil
|
|
|
|
}
|
|
|
|
|
2021-09-22 02:35:27 +03:00
|
|
|
func (i *Interp) EvalFunc(ctx context.Context, c interface{}, name string, args []interface{}, output io.Writer) (gojq.Iter, error) {
|
2020-06-08 03:29:51 +03:00
|
|
|
var argsExpr []string
|
|
|
|
for i := range args {
|
2021-08-16 19:39:37 +03:00
|
|
|
argsExpr = append(argsExpr, fmt.Sprintf("$_args[%d]", i))
|
2020-06-08 03:29:51 +03:00
|
|
|
}
|
|
|
|
argExpr := ""
|
|
|
|
if len(argsExpr) > 0 {
|
|
|
|
argExpr = "(" + strings.Join(argsExpr, ";") + ")"
|
|
|
|
}
|
|
|
|
|
|
|
|
trampolineInput := map[string]interface{}{
|
|
|
|
"input": c,
|
|
|
|
"args": args,
|
|
|
|
}
|
2021-09-07 02:38:52 +03:00
|
|
|
// _args to mark variable as internal and hide it from completion
|
2021-08-16 19:39:37 +03:00
|
|
|
// {input: ..., args: [...]} | .args as {args: $_args} | .input | name[($_args[0]; ...)]
|
|
|
|
trampolineExpr := fmt.Sprintf(". as {args: $_args} | .input | %s%s", name, argExpr)
|
2021-09-22 02:35:27 +03:00
|
|
|
iter, err := i.Eval(ctx, trampolineInput, trampolineExpr, "", output)
|
2020-06-08 03:29:51 +03:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return iter, nil
|
|
|
|
}
|
|
|
|
|
2021-09-22 02:35:27 +03:00
|
|
|
func (i *Interp) EvalFuncValues(ctx context.Context, c interface{}, name string, args []interface{}, output io.Writer) ([]interface{}, error) {
|
|
|
|
iter, err := i.EvalFunc(ctx, c, name, args, output)
|
2020-06-08 03:29:51 +03:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
var vs []interface{}
|
|
|
|
for {
|
|
|
|
v, ok := iter.Next()
|
|
|
|
if !ok {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
vs = append(vs, v)
|
|
|
|
}
|
|
|
|
|
|
|
|
return vs, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
type Options struct {
|
2021-09-22 02:35:27 +03:00
|
|
|
Depth int `mapstructure:"depth"`
|
|
|
|
ArrayTruncate int `mapstructure:"array_truncate"`
|
|
|
|
Verbose bool `mapstructure:"verbose"`
|
|
|
|
DecodeProgress bool `mapstructure:"decode_progress"`
|
|
|
|
Color bool `mapstructure:"color"`
|
|
|
|
Colors string `mapstructure:"colors"`
|
|
|
|
ByteColors string `mapstructure:"byte_colors"`
|
|
|
|
Unicode bool `mapstructure:"unicode"`
|
|
|
|
RawOutput bool `mapstructure:"raw_output"`
|
|
|
|
REPL bool `mapstructure:"repl"`
|
|
|
|
RawString bool `mapstructure:"raw_string"`
|
|
|
|
JoinString string `mapstructure:"join_string"`
|
|
|
|
Compact bool `mapstructure:"compact"`
|
|
|
|
BitsFormat string `mapstructure:"bits_format"`
|
|
|
|
LineBytes int `mapstructure:"line_bytes"`
|
|
|
|
DisplayBytes int `mapstructure:"display_bytes"`
|
|
|
|
AddrBase int `mapstructure:"addrbase"`
|
|
|
|
SizeBase int `mapstructure:"sizebase"`
|
2021-08-12 02:35:40 +03:00
|
|
|
|
2021-08-14 21:00:11 +03:00
|
|
|
Decorator Decorator
|
2021-08-12 02:35:40 +03:00
|
|
|
BitsFormatFn func(bb *bitio.Buffer) (interface{}, error)
|
2020-06-08 03:29:51 +03:00
|
|
|
}
|
|
|
|
|
2021-08-12 02:35:40 +03:00
|
|
|
func bitsFormatFnFromOptions(opts Options) func(bb *bitio.Buffer) (interface{}, error) {
|
|
|
|
switch opts.BitsFormat {
|
|
|
|
case "md5":
|
|
|
|
return func(bb *bitio.Buffer) (interface{}, error) {
|
|
|
|
d := md5.New()
|
|
|
|
if _, err := io.Copy(d, bb); err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
return hex.EncodeToString(d.Sum(nil)), nil
|
|
|
|
}
|
|
|
|
case "base64":
|
|
|
|
return func(bb *bitio.Buffer) (interface{}, error) {
|
|
|
|
b := &bytes.Buffer{}
|
|
|
|
e := base64.NewEncoder(base64.StdEncoding, b)
|
|
|
|
if _, err := io.Copy(e, bb); err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
e.Close()
|
|
|
|
return b.String(), nil
|
|
|
|
}
|
|
|
|
case "truncate":
|
|
|
|
// TODO: configure
|
|
|
|
return func(bb *bitio.Buffer) (interface{}, error) {
|
|
|
|
b := &bytes.Buffer{}
|
|
|
|
if _, err := io.Copy(b, io.LimitReader(bb, 1024)); err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
return b.String(), nil
|
|
|
|
}
|
|
|
|
case "string":
|
|
|
|
return func(bb *bitio.Buffer) (interface{}, error) {
|
|
|
|
b := &bytes.Buffer{}
|
|
|
|
if _, err := io.Copy(b, bb); err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
return b.String(), nil
|
|
|
|
}
|
|
|
|
case "snippet":
|
|
|
|
fallthrough
|
|
|
|
default:
|
|
|
|
return func(bb *bitio.Buffer) (interface{}, error) {
|
|
|
|
b := &bytes.Buffer{}
|
|
|
|
e := base64.NewEncoder(base64.StdEncoding, b)
|
|
|
|
if _, err := io.Copy(e, io.LimitReader(bb, 256)); err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
e.Close()
|
|
|
|
return fmt.Sprintf("<%s>%s", num.Bits(bb.Len()).StringByteBits(opts.SizeBase), b.String()), nil
|
|
|
|
}
|
2020-06-08 03:29:51 +03:00
|
|
|
}
|
2021-08-14 20:50:17 +03:00
|
|
|
}
|
2021-08-12 02:35:40 +03:00
|
|
|
|
2021-08-14 20:50:17 +03:00
|
|
|
func (i *Interp) lookupState(key string) interface{} {
|
|
|
|
if i.state == nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
m, ok := (*i.state).(map[string]interface{})
|
|
|
|
if !ok {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
return m[key]
|
|
|
|
}
|
|
|
|
|
|
|
|
func (i *Interp) includePaths() []string {
|
|
|
|
pathsAny, _ := i.lookupState("include_paths").([]interface{})
|
|
|
|
var paths []string
|
|
|
|
for _, pathAny := range pathsAny {
|
|
|
|
paths = append(paths, pathAny.(string))
|
|
|
|
}
|
|
|
|
return paths
|
2020-06-08 03:29:51 +03:00
|
|
|
}
|
|
|
|
|
2021-08-15 18:11:34 +03:00
|
|
|
func (i *Interp) variables() map[string]interface{} {
|
|
|
|
variablesAny, _ := i.lookupState("variables").(map[string]interface{})
|
|
|
|
return variablesAny
|
|
|
|
}
|
|
|
|
|
2020-06-08 03:29:51 +03:00
|
|
|
func (i *Interp) Options(fnOptsV ...interface{}) (Options, error) {
|
2021-09-22 02:35:27 +03:00
|
|
|
vs, err := i.EvalFuncValues(i.evalContext.ctx, nil, "options", []interface{}{fnOptsV}, DiscardCtxWriter{Ctx: i.evalContext.ctx})
|
2020-06-08 03:29:51 +03:00
|
|
|
if err != nil {
|
|
|
|
return Options{}, err
|
|
|
|
}
|
|
|
|
if len(vs) < 1 {
|
|
|
|
return Options{}, fmt.Errorf("no options value")
|
|
|
|
}
|
|
|
|
v := vs[0]
|
|
|
|
if vErr, ok := v.(error); ok {
|
|
|
|
return Options{}, vErr
|
|
|
|
}
|
|
|
|
m, ok := v.(map[string]interface{})
|
|
|
|
if !ok {
|
|
|
|
return Options{}, fmt.Errorf("options value not a map: %v", m)
|
|
|
|
}
|
|
|
|
|
|
|
|
var opts Options
|
2021-09-22 02:35:27 +03:00
|
|
|
_ = mapstructure.Decode(m, &opts)
|
|
|
|
opts.Depth = num.MaxInt(0, opts.Depth)
|
|
|
|
opts.ArrayTruncate = num.MaxInt(0, opts.ArrayTruncate)
|
|
|
|
opts.AddrBase = num.ClampInt(2, 36, opts.AddrBase)
|
|
|
|
opts.SizeBase = num.ClampInt(2, 36, opts.SizeBase)
|
|
|
|
opts.LineBytes = num.MaxInt(0, opts.LineBytes)
|
|
|
|
opts.DisplayBytes = num.MaxInt(0, opts.DisplayBytes)
|
2020-06-08 03:29:51 +03:00
|
|
|
opts.Decorator = decoratorFromOptions(opts)
|
2021-08-12 02:35:40 +03:00
|
|
|
opts.BitsFormatFn = bitsFormatFnFromOptions(opts)
|
2020-06-08 03:29:51 +03:00
|
|
|
|
|
|
|
return opts, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (i *Interp) NewColorJSON(opts Options) (*colorjson.Encoder, error) {
|
|
|
|
indent := 2
|
|
|
|
if opts.Compact {
|
|
|
|
indent = 0
|
|
|
|
}
|
|
|
|
|
|
|
|
return colorjson.NewEncoder(
|
|
|
|
opts.Color,
|
|
|
|
false,
|
|
|
|
indent,
|
|
|
|
func(v interface{}) interface{} {
|
2021-08-12 02:35:40 +03:00
|
|
|
if v, ok := toValue(opts, v); ok {
|
2021-08-12 00:51:00 +03:00
|
|
|
return v
|
2020-06-08 03:29:51 +03:00
|
|
|
}
|
|
|
|
panic(fmt.Sprintf("toValue not a JQValue value: %#v", v))
|
|
|
|
},
|
|
|
|
colorjson.Colors{
|
|
|
|
Reset: []byte(ansi.Reset.SetString),
|
|
|
|
Null: []byte(opts.Decorator.Null.SetString),
|
|
|
|
False: []byte(opts.Decorator.False.SetString),
|
|
|
|
True: []byte(opts.Decorator.True.SetString),
|
|
|
|
Number: []byte(opts.Decorator.Number.SetString),
|
|
|
|
String: []byte(opts.Decorator.String.SetString),
|
|
|
|
ObjectKey: []byte(opts.Decorator.ObjectKey.SetString),
|
|
|
|
Array: []byte(opts.Decorator.Array.SetString),
|
|
|
|
Object: []byte(opts.Decorator.Object.SetString),
|
|
|
|
},
|
|
|
|
), nil
|
|
|
|
}
|