fq/pkg/interp/interp.go

package interp

import (
	"bytes"
	"context"
	"crypto/md5"
	"embed"
	"encoding/base64"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"math/big"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/wader/fq/internal/ansi"
	"github.com/wader/fq/internal/bitioex"
	"github.com/wader/fq/internal/colorjson"
	"github.com/wader/fq/internal/ctxstack"
	"github.com/wader/fq/internal/gojqex"
	"github.com/wader/fq/internal/ioex"
	"github.com/wader/fq/internal/mapstruct"
	"github.com/wader/fq/internal/mathex"
	"github.com/wader/fq/internal/pos"
	"github.com/wader/fq/pkg/bitio"
	"github.com/wader/fq/pkg/decode"
	"github.com/wader/fq/pkg/scalar"

	"github.com/wader/gojq"
)

//go:embed interp.jq
//go:embed internal.jq
//go:embed options.jq
//go:embed binary.jq
//go:embed decode.jq
//go:embed registry_include.jq
//go:embed format_decode.jq
//go:embed format_func.jq
//go:embed grep.jq
//go:embed args.jq
//go:embed eval.jq
//go:embed query.jq
//go:embed repl.jq
//go:embed help.jq
//go:embed funcs.jq
//go:embed ansi.jq
//go:embed init.jq
var builtinFS embed.FS

var initSource = `include "@builtin/init";`

func init() {
	RegisterIter1("_readline", (*Interp)._readline)
	RegisterIter2("_eval", (*Interp)._eval)

	RegisterIter2("_stdio_read", (*Interp)._stdioRead)
	RegisterIter1("_stdio_write", (*Interp)._stdioWrite)
	RegisterFunc1("_stdio_info", (*Interp)._stdioInfo)

	RegisterFunc0("_extkeys", (*Interp)._extKeys)
	RegisterFunc0("_exttype", (*Interp)._extType)

	RegisterFunc0("_global_state", func(i *Interp, c any) any { return *i.state })
	RegisterFunc1("_global_state", func(i *Interp, _ any, v any) any { *i.state = v; return v })

	RegisterFunc0("history", (*Interp).history)
	RegisterIter1("_display", (*Interp)._display)
	RegisterFunc0("_can_display", (*Interp)._canDisplay)
	RegisterIter1("_hexdump", (*Interp)._hexdump)
	RegisterIter1("_print_color_json", (*Interp)._printColorJSON)

	RegisterFunc0("_is_completing", (*Interp)._isCompleting)
}

type Scalarable interface {
	ScalarActual() any
	ScalarValue() any
	ScalarSym() any
	ScalarDescription() string
	ScalarGap() bool
	ScalarDisplayFormat() scalar.DisplayFormat
}

type valueError struct {
	v any
}

func (v valueError) Error() string { return fmt.Sprintf("error: %v", v.v) }
func (v valueError) Value() any    { return v.v }

type compileError struct {
	err      error
	what     string
	filename string
	pos      pos.Pos
}

func (ce compileError) Value() any {
	return map[string]any{
		"error":    ce.err.Error(),
		"what":     ce.what,
		"filename": ce.filename,
		"line":     ce.pos.Line,
		"column":   ce.pos.Column,
	}
}
func (ce compileError) Error() string {
	filename := ce.filename
	if filename == "" {
		filename = "expr"
	}
	return fmt.Sprintf("%s:%d:%d: %s: %s", filename, ce.pos.Line, ce.pos.Column, ce.what, ce.err.Error())
}

var ErrEOF = io.EOF
var ErrInterrupt = errors.New("Interrupt")

// gojq errors can implement this to signal exit code
type Exiter interface {
	ExitCode() int
}

// gojq halt_error uses this
type IsEmptyErrorer interface {
	IsEmptyError() bool
}

type Terminal interface {
	Size() (int, int)
	IsTerminal() bool
}

type Input interface {
	fs.File
	Terminal
}

type Output interface {
	io.Writer
	Terminal
}

type Platform struct {
	OS   string
	Arch string
}

type ReadlineOpts struct {
	Prompt     string
	CompleteFn func(line string, pos int) (newLine []string, shared int)
}

type OS interface {
	Platform() Platform
	Stdin() Input
	Stdout() Output
	Stderr() Output
	InterruptChan() chan struct{}
	Args() []string
	Environ() []string
	ConfigDir() (string, error)
	// FS.File returned by FS().Open() can optionally implement io.Seeker
	FS() fs.FS
	Readline(opts ReadlineOpts) (string, error)
	History() ([]string, error)
}

type FixedFileInfo struct {
	FName    string
	FSize    int64
	FMode    fs.FileMode
	FModTime time.Time
	FIsDir   bool
	FSys     any
}

func (ffi FixedFileInfo) Name() string       { return ffi.FName }
func (ffi FixedFileInfo) Size() int64        { return ffi.FSize }
func (ffi FixedFileInfo) Mode() fs.FileMode  { return ffi.FMode }
func (ffi FixedFileInfo) ModTime() time.Time { return ffi.FModTime }
func (ffi FixedFileInfo) IsDir() bool        { return ffi.FIsDir }
func (ffi FixedFileInfo) Sys() any           { return ffi.FSys }

type FileReader struct {
	R        io.Reader
	FileInfo FixedFileInfo
}

func (rf FileReader) Stat() (fs.FileInfo, error) { return rf.FileInfo, nil }
func (rf FileReader) Read(p []byte) (int, error) { return rf.R.Read(p) }
func (FileReader) Close() error                  { return nil }

type Value interface {
	gojq.JQValue

	ExtType() string
	ExtKeys() []string
}

type Display interface {
	Display(w io.Writer, opts Options) error
}

type JQValueEx interface {
	gojq.JQValue
	JQValueToGoJQEx(optsFn func() Options) any
}

func valuePath(v *decode.Value) []any {
	var parts []any

	for v.Parent != nil {
		switch vv := v.Parent.V.(type) {
		case *decode.Compound:
			if vv.IsArray {
				parts = append([]any{v.Index}, parts...)
			} else {
				parts = append([]any{v.Name}, parts...)
			}
		}
		v = v.Parent
	}

	return parts
}

func valuePathExprDecorated(v *decode.Value, d Decorator) string {
	parts := []string{"."}

	for i, p := range valuePath(v) {
		switch p := p.(type) {
		case string:
			if i > 0 {
				parts = append(parts, ".")
			}
			parts = append(parts, d.ObjectKey.Wrap(p))
		case int:
			indexStr := strconv.Itoa(p)
			parts = append(parts, fmt.Sprintf("%s%s%s", d.Index.F("["), d.Number.F(indexStr), d.Index.F("]")))
		}
	}

	return strings.Join(parts, "")
}

type iterFn func() (any, bool)

func (i iterFn) Next() (any, bool) { return i() }

type loadModule struct {
	init func() ([]*gojq.Query, error)
	load func(name string) (*gojq.Query, error)
}

func (l loadModule) LoadInitModules() ([]*gojq.Query, error)     { return l.init() }
func (l loadModule) LoadModule(name string) (*gojq.Query, error) { return l.load(name) }

func toString(v any) (string, error) {
	switch v := v.(type) {
	case string:
		return v, nil
	case gojq.JQValue:
		return toString(v.JQValueToGoJQ())
	default:
		b, err := toBytes(v)
		if err != nil {
			return "", fmt.Errorf("value can't be a string")
		}

		return string(b), nil
	}
}

func toBigInt(v any) (*big.Int, error) {
	switch v := v.(type) {
	case int:
		return new(big.Int).SetInt64(int64(v)), nil
	case float64:
		return new(big.Int).SetInt64(int64(v)), nil
	case *big.Int:
		return v, nil
	default:
		return nil, fmt.Errorf("value is not a number")
	}
}

func toBytes(v any) ([]byte, error) {
	switch v := v.(type) {
	default:
		br, err := ToBitReader(v)
		if err != nil {
			return nil, fmt.Errorf("value is not bytes")
		}
		buf := &bytes.Buffer{}
		if _, err := bitioex.CopyBits(buf, br); err != nil {
			return nil, err
		}

		return buf.Bytes(), nil
	}
}

func queryErrorPosition(expr string, v error) pos.Pos {
	var offset int

	if tokIf, ok := v.(interface{ Token() (string, int) }); ok { //nolint:errorlint
		_, offset = tokIf.Token()
	}
	if offset >= 0 {
		return pos.NewFromOffset(expr, offset)
	}
	return pos.Pos{}
}

type Variable struct {
	Name  string
	Value any
}

type RunMode int

const (
	ScriptMode RunMode = iota
	REPLMode
	CompletionMode
)

type EvalInstance struct {
	Ctx          context.Context
	Output       io.Writer
	IsCompleting bool

	includeSeen map[string]struct{}
}

type Interp struct {
	Registry *Registry
	OS       OS

	initQuery      *gojq.Query
	includeCache   map[string]*gojq.Query
	interruptStack *ctxstack.Stack
	// global state, is ref as Interp is cloned per eval
	state *any

	// new for each eval, other values are copied by value
	EvalInstance EvalInstance
}

func New(os OS, registry *Registry) (*Interp, error) {
	var err error

	i := &Interp{
		OS:       os,
		Registry: registry,
	}

	i.includeCache = map[string]*gojq.Query{}
	i.initQuery, err = gojq.Parse(initSource)
	if err != nil {
		return nil, fmt.Errorf("init:%s: %w", queryErrorPosition(initSource, err), err)
	}
	// TODO: refactor ctxstack have a CancelTop and return c context to Stop?
	i.interruptStack = ctxstack.New(func(stopCh chan struct{}) {
		select {
		case <-stopCh:
			return
		case <-os.InterruptChan():
			return
		}
	})
	i.state = new(any)

	return i, nil
}

func (i *Interp) Stop() {
	// TODO: cancel all run instances?
	i.interruptStack.Stop()
}

func (i *Interp) Main(ctx context.Context, output Output, versionStr string) error {
	var args []any
	for _, a := range i.OS.Args() {
		args = append(args, a)
	}

	platform := i.OS.Platform()
	input := map[string]any{
		"args":    args,
		"version": versionStr,
		"os":      platform.OS,
		"arch":    platform.Arch,
	}

	iter, err := i.EvalFunc(ctx, input, "_main", nil, EvalOpts{output: output})
	if err != nil {
		fmt.Fprintln(i.OS.Stderr(), err)
		return err
	}
	for {
		v, ok := iter.Next()
		if !ok {
			break
		}

		switch v := v.(type) {
		case error:
			if emptyErr, ok := v.(IsEmptyErrorer); ok && emptyErr.IsEmptyError() { //nolint:errorlint
				// no output
			} else if errors.Is(v, context.Canceled) {
				// ignore context cancel here for now, which means user somehow interrupted the interpreter
				// TODO: handle this inside interp.jq instead but then we probably have to do nested
				// eval and or also use different contexts for the interpreter and reading/decoding
			} else {
				fmt.Fprintln(i.OS.Stderr(), v)
			}
			return v
		case [2]any:
			fmt.Fprintln(i.OS.Stderr(), v[:]...)
		default:
			// TODO: can this happen?
			fmt.Fprintln(i.OS.Stderr(), v)
		}
	}

	return nil
}

type completionResult struct {
	Names  []string
	Prefix string
}

type readlineOpts struct {
	Prompt   string
	Complete string
	Timeout  float64
}

func (i *Interp) _readline(c any, opts readlineOpts) gojq.Iter {
	if i.EvalInstance.IsCompleting {
		return gojq.NewIter()
	}

	expr, err := i.OS.Readline(ReadlineOpts{
		Prompt: opts.Prompt,
		CompleteFn: func(line string, pos int) (newLine []string, shared int) {
			completeCtx := i.EvalInstance.Ctx
			if opts.Timeout > 0 {
				var completeCtxCancelFn context.CancelFunc
				completeCtx, completeCtxCancelFn = context.WithTimeout(i.EvalInstance.Ctx, time.Duration(opts.Timeout*float64(time.Second)))
				defer completeCtxCancelFn()
			}

			names, shared, err := func() (newLine []string, shared int, err error) {
				// c | opts.Complete(line; pos)
				vs, err := i.EvalFuncValues(
					completeCtx,
					c,
					opts.Complete,
					[]any{line, pos},
					EvalOpts{
						output:       ioex.DiscardCtxWriter{Ctx: completeCtx},
						isCompleting: true,
					},
				)
				if err != nil {
					return nil, pos, err
				}
				if len(vs) < 1 {
					return nil, pos, fmt.Errorf("no values")
				}
				v := vs[0]
				if vErr, ok := v.(error); ok {
					return nil, pos, vErr
				}

				// {abc: 123, abd: 123} | complete(".ab"; 3) will return {prefix: "ab", names: ["abc", "abd"]}
				r, ok := gojqex.CastFn[completionResult](v, mapstruct.ToStruct)
				if !ok {
					return nil, pos, fmt.Errorf("completion result not a map")
				}

				sharedLen := len(r.Prefix)

				return r.Names, sharedLen, nil
			}()

			// TODO: how to report err?
			_ = err

			return names, shared
		},
	})

	if errors.Is(err, ErrInterrupt) {
		return gojq.NewIter(valueError{"interrupt"})
	} else if errors.Is(err, ErrEOF) {
		return gojq.NewIter(valueError{"eof"})
	} else if err != nil {
		return gojq.NewIter(err)
	}

	return gojq.NewIter(expr)
}

type evalOpts struct {
	Filename string
}

func (i *Interp) _eval(c any, expr string, opts evalOpts) gojq.Iter {
	var err error

	iter, err := i.Eval(i.EvalInstance.Ctx, c, expr, EvalOpts{
		filename: opts.Filename,
		output:   i.EvalInstance.Output,
	})
	if err != nil {
		return gojq.NewIter(err)
	}

	return iter
}

func (i *Interp) _extKeys(c any) any {
	if v, ok := c.(Value); ok {
		var vs []any
		for _, s := range v.ExtKeys() {
			vs = append(vs, s)
		}
		return vs
	}
	return nil
}

func (i *Interp) _extType(c any) any {
	if v, ok := c.(Value); ok {
		return v.ExtType()
	}
	return gojq.TypeOf(c)
}

func (i *Interp) _stdioFdName(s string) (any, error) {
	switch s {
	case "stdin":
		return i.OS.Stdin(), nil
	case "stdout":
		return i.OS.Stdout(), nil
	case "stderr":
		return i.OS.Stderr(), nil
	default:
		return nil, fmt.Errorf("unknown fd %s", s)
	}
}

func (i *Interp) _stdioRead(c any, fdName string, l int) gojq.Iter {
	fd, err := i._stdioFdName(fdName)
	if err != nil {
		return gojq.NewIter(err)
	}
	r, ok := fd.(io.Reader)
	if !ok {
		return gojq.NewIter(fmt.Errorf("%s is not a writeable", fdName))
	}

	if i.EvalInstance.IsCompleting {
		return gojq.NewIter("")
	}

	buf := make([]byte, l)
	n, err := io.ReadFull(r, buf)
	s := string(buf[0:n])

	vs := []any{s}
	switch {
	case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF):
		vs = append(vs, valueError{"eof"})
	default:
		vs = append(vs, err)
	}

	return gojq.NewIter(vs...)
}

func (i *Interp) _stdioWrite(c any, fdName string) gojq.Iter {
	fd, err := i._stdioFdName(fdName)
	if err != nil {
		return gojq.NewIter(err)
	}
	w, ok := fd.(io.Writer)
	if !ok {
		return gojq.NewIter(fmt.Errorf("%s is not a writeable", fdName))
	}
	if i.EvalInstance.IsCompleting {
		return gojq.NewIter()
	}

	if _, err := fmt.Fprint(w, c); err != nil {
		return gojq.NewIter(err)
	}
	return gojq.NewIter()
}

func (i *Interp) _stdioInfo(c any, fdName string) any {
	fd, err := i._stdioFdName(fdName)
	if err != nil {
		return err
	}
	t, ok := fd.(Terminal)
	if !ok {
		return fmt.Errorf("%s is not a terminal", fdName)
	}

	w, h := t.Size()
	return map[string]any{
		"is_terminal": t.IsTerminal(),
		"width":       w,
		"height":      h,
	}
}

func (i *Interp) history(c any) any {
	hs, err := i.OS.History()
	if err != nil {
		return err
	}
	var vs []any
	for _, s := range hs {
		vs = append(vs, s)
	}
	return vs
}

func (i *Interp) _display(c any, v any) gojq.Iter {
	opts := OptionsFromValue(v)

	switch v := c.(type) {
	case Display:
		if err := v.Display(i.EvalInstance.Output, opts); err != nil {
			return gojq.NewIter(err)
		}
		return gojq.NewIter()
	default:
		return gojq.NewIter(fmt.Errorf("%+#v: not displayable", c))
	}
}

func (i *Interp) _canDisplay(c any) any {
	_, ok := c.(Display)
	return ok
}

func (i *Interp) _hexdump(c any, v any) gojq.Iter {
	opts := OptionsFromValue(v)
	bv, err := toBinary(c)
	if err != nil {
		return gojq.NewIter(err)
	}
	if err := hexdump(i.EvalInstance.Output, bv, opts); err != nil {
		return gojq.NewIter(err)
	}

	return gojq.NewIter()
}

func (i *Interp) _printColorJSON(c any, v any) gojq.Iter {
	opts := OptionsFromValue(v)
	indent := 2
	if opts.Compact {
		indent = 0
	}

	cj := colorjson.NewEncoder(colorjson.Options{
		Color:   opts.Color,
		Tab:     false,
		Indent:  indent,
		ValueFn: func(v any) any { return toValue(func() Options { return opts }, v) },
		Colors: colorjson.Colors{
			Reset:     []byte(ansi.Reset.SetString),
			Null:      []byte(opts.Decorator.Null.SetString),
			False:     []byte(opts.Decorator.False.SetString),
			True:      []byte(opts.Decorator.True.SetString),
			Number:    []byte(opts.Decorator.Number.SetString),
			String:    []byte(opts.Decorator.String.SetString),
			ObjectKey: []byte(opts.Decorator.ObjectKey.SetString),
			Array:     []byte(opts.Decorator.Array.SetString),
			Object:    []byte(opts.Decorator.Object.SetString),
		},
	})
	if err := cj.Marshal(c, i.EvalInstance.Output); err != nil {
		return gojq.NewIter(err)
	}

	return gojq.NewIter()
}

func (i *Interp) _isCompleting(c any) any {
	return i.EvalInstance.IsCompleting
}

type pathResolver struct {
	prefix string
	open   func(filename string) (io.ReadCloser, string, error)
}

func (i *Interp) lookupPathResolver(filename string) (pathResolver, error) {
	configDir, err := i.OS.ConfigDir()
	if err != nil {
		return pathResolver{}, err
	}

	resolvePaths := []pathResolver{
		{
			"@builtin/",
			func(filename string) (io.ReadCloser, string, error) {
				f, err := builtinFS.Open(filename)
				return f, "@builtin/" + filename, err
			},
		},
		{
			"@config/", func(filename string) (io.ReadCloser, string, error) {
				p := path.Join(configDir, filename)
				f, err := i.OS.FS().Open(p)
				return f, p, err
			},
		},
		{
			"", func(filename string) (io.ReadCloser, string, error) {
				if path.IsAbs(filename) {
					f, err := i.OS.FS().Open(filename)
					return f, filename, err
				}

				// TODO: jq $ORIGIN
				for _, includePath := range append([]string{"./"}, i.includePaths()...) {
					p := path.Join(includePath, filename)
					if f, err := i.OS.FS().Open(path.Join(includePath, filename)); err == nil {
						return f, p, nil
					}
				}

				return nil, "", &fs.PathError{Op: "open", Path: filename, Err: fs.ErrNotExist}
			},
		},
	}
	for _, p := range resolvePaths {
		if strings.HasPrefix(filename, p.prefix) {
			return p, nil
		}
	}
	return pathResolver{}, fmt.Errorf("could not resolve path: %s", filename)
}

type EvalOpts struct {
	filename     string
	output       io.Writer
	isCompleting bool
}

func (i *Interp) Eval(ctx context.Context, c any, expr string, opts EvalOpts) (gojq.Iter, error) {
	gq, err := gojq.Parse(expr)
	if err != nil {
		p := queryErrorPosition(expr, err)
		return nil, compileError{
			err:      err,
			what:     "parse",
			filename: opts.filename,
			pos:      p,
		}
	}

	// make copy of interp and give it its own eval context
	ci := *i
	ni := &ci
	ni.EvalInstance = EvalInstance{
		includeSeen: map[string]struct{}{},
	}

	var variableNames []string
	var variableValues []any
	for k, v := range i.slurps() {
		variableNames = append(variableNames, "$"+k)
		variableValues = append(variableValues, v)
	}

	var funcCompilerOpts []gojq.CompilerOption

	for _, fn := range i.Registry.EnvFuncFns {
		f := fn(ni)
		if f.IterFn != nil {
			funcCompilerOpts = append(funcCompilerOpts,
				gojq.WithIterFunction(f.Name, f.MinArity, f.MaxArity, f.IterFn))
		} else {
			funcCompilerOpts = append(funcCompilerOpts,
				gojq.WithFunction(f.Name, f.MinArity, f.MaxArity, f.FuncFn))
		}
	}

	compilerOpts := append([]gojq.CompilerOption{}, funcCompilerOpts...)
	compilerOpts = append(compilerOpts, gojq.WithEnvironLoader(ni.OS.Environ))
	compilerOpts = append(compilerOpts, gojq.WithVariables(variableNames))
	compilerOpts = append(compilerOpts, gojq.WithModuleLoader(loadModule{
		init: func() ([]*gojq.Query, error) {
			return []*gojq.Query{i.initQuery}, nil
		},
		load: func(name string) (*gojq.Query, error) {
			if err := ctx.Err(); err != nil {
				return nil, err
			}

			var filename string
			// support include "nonexisting?" to ignore include error
			var isTry bool
			if strings.HasSuffix(name, "?") {
				isTry = true
				filename = name[0 : len(name)-1]
			} else {
				filename = name
			}
			filename = filename + ".jq"

			pr, err := i.lookupPathResolver(filename)
			if err != nil {
				return nil, err
			}

			// skip if this eval instance has already included the file
			if _, ok := ni.EvalInstance.includeSeen[filename]; ok {
				return &gojq.Query{Term: &gojq.Term{Type: gojq.TermTypeIdentity}}, nil
			}
			ni.EvalInstance.includeSeen[filename] = struct{}{}

			// return cached version if file has already been parsed
			if q, ok := ni.includeCache[filename]; ok {
				return q, nil
			}

			filenamePart := strings.TrimPrefix(filename, pr.prefix)

			f, absPath, err := pr.open(filenamePart)
			if err != nil {
				if !isTry {
					return nil, err
				}
				f = io.NopCloser(&bytes.Buffer{})
			}
			defer f.Close()

			b, err := io.ReadAll(f)
			if err != nil {
				return nil, err
			}
			s := string(b)
			q, err := gojq.Parse(s)
			if err != nil {
				p := queryErrorPosition(s, err)
				return nil, compileError{
					err:      err,
					what:     "parse",
					filename: absPath,
					pos:      p,
				}
			}

			// not identity body means it returns something, threat as dynamic include
			if q.Term == nil || q.Term.Type != gojq.TermTypeIdentity {
				gc, err := gojq.Compile(q, funcCompilerOpts...)
				if err != nil {
					return nil, err
				}
				iter := gc.RunWithContext(context.Background(), nil)
				var vs []any
				for {
					v, ok := iter.Next()
					if !ok {
						break
					}
					if err, ok := v.(error); ok {
						return nil, err
					}
					vs = append(vs, v)
				}
				if len(vs) != 1 {
					return nil, fmt.Errorf("dynamic include: must output one string, got: %#v", vs)
				}
				s, sOk := vs[0].(string)
				if !sOk {
					return nil, fmt.Errorf("dynamic include: must be string, got %#v", s)
				}
				q, err = gojq.Parse(s)
				if err != nil {
					p := queryErrorPosition(s, err)
					return nil, compileError{
						err:      err,
						what:     "dynamic include parse",
						filename: filenamePart,
						pos:      p,
					}
				}
			}

			// TODO: some better way of handling relative includes that
			// works with @builtin etc
			basePath := path.Dir(name)
			for _, qi := range q.Imports {
				rewritePath := func(base, includePath string) string {
					if strings.HasPrefix(includePath, "@") || path.IsAbs(includePath) {
						return includePath
					}

					return path.Join(base, includePath)
				}
				if qi.IncludePath != "" {
					qi.IncludePath = rewritePath(basePath, qi.IncludePath)
				}
				if qi.ImportPath != "" {
					qi.ImportPath = rewritePath(basePath, qi.ImportPath)
				}
			}

			i.includeCache[filename] = q

			return q, nil
		},
	}))

	gc, err := gojq.Compile(gq, compilerOpts...)
	if err != nil {
		p := queryErrorPosition(expr, err)
		return nil, compileError{
			err:      err,
			what:     "compile",
			filename: opts.filename,
			pos:      p,
		}
	}

	output := opts.output
	if opts.output == nil {
		output = io.Discard
	}

	runCtx, runCtxCancelFn := i.interruptStack.Push(ctx)
	ni.EvalInstance.Ctx = runCtx
	ni.EvalInstance.Output = ioex.CtxWriter{Writer: output, Ctx: runCtx}
	// inherit or maybe set
	ni.EvalInstance.IsCompleting = i.EvalInstance.IsCompleting || opts.isCompleting
	iter := gc.RunWithContext(runCtx, c, variableValues...)

	iterWrapper := iterFn(func() (any, bool) {
		v, ok := iter.Next()
		// gojq ctx cancel will not return ok=false, just cancelled error
		if !ok {
			runCtxCancelFn()
		} else if _, ok := v.(error); ok {
			runCtxCancelFn()
		}
		return v, ok
	})

	return iterWrapper, nil
}

func (i *Interp) EvalFunc(ctx context.Context, c any, name string, args []any, opts EvalOpts) (gojq.Iter, error) {
	var argsExpr []string
	for i := range args {
		argsExpr = append(argsExpr, fmt.Sprintf("$_args[%d]", i))
	}
	argExpr := ""
	if len(argsExpr) > 0 {
		argExpr = "(" + strings.Join(argsExpr, ";") + ")"
	}

	trampolineInput := map[string]any{
		"input": c,
		"args":  args,
	}
	// _args to mark variable as internal and hide it from completion
	// {input: ..., args: [...]} | .args as {args: $_args} | .input | name[($_args[0]; ...)]
	trampolineExpr := fmt.Sprintf(". as {args: $_args} | .input | %s%s", name, argExpr)
	iter, err := i.Eval(ctx, trampolineInput, trampolineExpr, opts)
	if err != nil {
		return nil, err
	}
	return iter, nil
}

func (i *Interp) EvalFuncValues(ctx context.Context, c any, name string, args []any, opts EvalOpts) ([]any, error) {
	iter, err := i.EvalFunc(ctx, c, name, args, opts)
	if err != nil {
		return nil, err
	}

	var vs []any
	for {
		v, ok := iter.Next()
		if !ok {
			break
		}
		vs = append(vs, v)
	}

	return vs, nil
}

type Options struct {
	Depth          int
	ArrayTruncate  int
	Verbose        bool
	Width          int
	DecodeProgress bool
	Color          bool
	Colors         map[string]string
	ByteColors     []struct {
		Ranges [][2]int
		Value  string
	}
	Unicode      bool
	RawOutput    bool
	REPL         bool
	RawString    bool
	JoinString   string
	Compact      bool
	BitsFormat   string
	LineBytes    int
	DisplayBytes int
	Addrbase     int
	Sizebase     int

	Decorator    Decorator
	BitsFormatFn func(br bitio.ReaderAtSeeker) (any, error)
}

func OptionsFromValue(v any) Options {
	var opts Options
	_ = mapstruct.ToStruct(v, &opts)
	opts.ArrayTruncate = mathex.Max(0, opts.ArrayTruncate)
	opts.Depth = mathex.Max(0, opts.Depth)
	opts.Addrbase = mathex.Clamp(2, 36, opts.Addrbase)
	opts.Sizebase = mathex.Clamp(2, 36, opts.Sizebase)
	opts.LineBytes = mathex.Max(0, opts.LineBytes)
	opts.DisplayBytes = mathex.Max(0, opts.DisplayBytes)
	opts.Decorator = decoratorFromOptions(opts)
	opts.BitsFormatFn = bitsFormatFnFromOptions(opts)

	return opts
}

func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, error) {
	switch opts.BitsFormat {
	case "md5":
		return func(br bitio.ReaderAtSeeker) (any, error) {
			d := md5.New()
			if _, err := bitioex.CopyBits(d, br); err != nil {
				return "", err
			}
			return hex.EncodeToString(d.Sum(nil)), nil
		}
	case "base64":
		return func(br bitio.ReaderAtSeeker) (any, error) {
			b := &bytes.Buffer{}
			e := base64.NewEncoder(base64.StdEncoding, b)
			if _, err := bitioex.CopyBits(e, br); err != nil {
				return "", err
			}
			e.Close()
			return b.String(), nil
		}
	case "truncate":
		// TODO: configure
		return func(br bitio.ReaderAtSeeker) (any, error) {
			b := &bytes.Buffer{}
			if _, err := bitioex.CopyBits(b, bitio.NewLimitReader(br, 1024*8)); err != nil {
				return "", err
			}
			return b.String(), nil
		}
	case "string":
		return func(br bitio.ReaderAtSeeker) (any, error) {
			b := &bytes.Buffer{}
			if _, err := bitioex.CopyBits(b, br); err != nil {
				return "", err
			}
			return b.String(), nil
		}
	case "snippet":
		fallthrough
	default:
		return func(br bitio.ReaderAtSeeker) (any, error) {
			b := &bytes.Buffer{}
			e := base64.NewEncoder(base64.StdEncoding, b)
			if _, err := bitioex.CopyBits(e, bitio.NewLimitReader(br, 256*8)); err != nil {
				return "", err
			}
			e.Close()

			brLen, err := bitioex.Len(br)
			if err != nil {
				return nil, err
			}

			return fmt.Sprintf("<%s>%s", mathex.Bits(brLen).StringByteBits(opts.Sizebase), b.String()), nil
		}
	}
}

func (i *Interp) lookupState(key string) any {
	if i.state == nil {
		return nil
	}
	m, ok := (*i.state).(map[string]any)
	if !ok {
		return nil
	}
	return m[key]
}

func (i *Interp) includePaths() []string {
	pathsAny, _ := i.lookupState("include_paths").([]any)
	var paths []string
	for _, pathAny := range pathsAny {
		path, ok := pathAny.(string)
		if !ok {
			panic("path not string")
		}
		paths = append(paths, path)
	}
	return paths
}

func (i *Interp) slurps() map[string]any {
	slurpsAny, _ := i.lookupState("slurps").(map[string]any)
	return slurpsAny
}