cli: Include paths and some refactor

.vscode/settings.json

@@ -47,6 +47,7 @@
         "dumpaddr",
         "elif",
         "endians",
+        "errorln",
         "esds",
         "fpbits",
         "fqtest",
@@ -60,6 +61,7 @@
         "golangci",
         "gosec",
         "gosimple",
+        "gsub",
         "iinf",
         "iloc",
         "ilst",

Makefile

@@ -17,7 +17,7 @@ testwrite: test
 jqtest:
 	@for f in $$(find . -name "*_test.jq"); do \
 		echo $$f ; \
-		jq -L "$$(dirname $$f)" -f "$$f" -n -r ; \
+		go run main.go -L "$$(dirname $$f)" -f "$$f" -n -r ; \
 	done
 
 .PHONY: doc

doc/TODO.md

@@ -10,6 +10,8 @@
 - Configurable history file/name?
 - Reset color at prompt? context cancel
 - Auto complete $variables
+- Auto complete keys that need escaping, now just filtered out
+- Auto complete add "." just one and is object
 
 #### Language
 
@@ -41,7 +43,6 @@
 - empty file test
 - CLI tests, raw write, colors?
 - Interactive tests
-- Completion tests
 
 #### Documentation
 

pkg/fqtest/fqtest.go

@@ -290,7 +290,7 @@ func parseTestCases(s string) *testCase {
 	replDepth := 0
 
 	// TODO: better section splitter, too much heuristics now
-	for _, section := range SectionParser(regexp.MustCompile(`^\$ .*$|^stdin:$|^stderr:$|^exitcode:.*$|^#.*$|^/.*:|^(?:>* )?[a-z\d]+> .*$`), s) {
+	for _, section := range SectionParser(regexp.MustCompile(`^\$ .*$|^stdin:$|^stderr:$|^exitcode:.*$|^#.*$|^/.*:|^(?:>+ )?[a-z\d]+(?:, \.\.\.)?> .*$`), s) {
 		n, v := section.Name, section.Value
 
 		switch {

pkg/interp/assert.jq

@@ -1,9 +1,9 @@
-def assert($name; $a; $b):
-  ( if $a == $b then "PASS \($name)\n"
+def assert($name; $expected; $actual):
+  ( if $expected == $actual then
+      "PASS \($name)\n" | stderr
     else
-      ( "FAIL \($name) \($a) != \($b)\n"
+      ( "FAIL \($name): expected \($expected) got \($actual)\n" | stderr
       , (null | halt_error(1))
       )
     end
-  , empty
   );

pkg/interp/completion.go

@@ -105,7 +105,7 @@ func transformToCompletionQuery(q *gojq.Query) (*gojq.Query, CompletionType, str
 
 func completeTrampoline(ctx context.Context, completeFn string, c interface{}, i *Interp, line string, pos int) (newLine []string, shared int, err error) {
 	lineStr := line[0:pos]
-	vs, err := i.EvalFuncValues(ctx, CompletionMode, c, completeFn, []interface{}{lineStr}, DiscardOutput{Ctx: ctx}, "")
+	vs, err := i.EvalFuncValues(ctx, CompletionMode, c, completeFn, []interface{}{lineStr}, DiscardOutput{Ctx: ctx})
 	if err != nil {
 		return nil, pos, err
 	}

pkg/interp/fq.jq

@@ -36,6 +36,8 @@ def _exit_code_input_decode_error: 4;
 def _exit_code_expr_error: 5;
 
 # TODO: completionMode
+# TODO: return escaped identifier, not sure current readline implementation supports
+# completions that needs to change previous input, ex: .a\t -> ."a \" b" etc
 def _complete($e):
   ( ( $e | _complete_query) as {$type, $query, $prefix}
   | {
@@ -48,7 +50,7 @@ def _complete($e):
           else
             []
           end
-        | map(select(strings and startswith($prefix)))
+        | map(select(strings and _is_ident and startswith($prefix)))
         | unique
         | sort
         )
@@ -151,9 +153,6 @@ def _eval_is_compile_error: type == "object" and .error != null and .what != nul
 def _eval_compile_error_tostring:
   "\(.filename // "src"):\(.line):\(.column): \(.error)";
 
-def _eval_debug:
-  (["DEBUG", .] | tojson, "\n") | stderr;
-
 def _eval($e; f; on_error; on_compile_error):
   ( _default_options(_build_default_options) as $_
   | try eval($e; "_eval_debug") | f
@@ -166,7 +165,7 @@ def _eval($e; f; on_error; on_compile_error):
 def _repl_display: display({depth: 1});
 def _repl_on_error:
   ( if _eval_is_compile_error then _eval_compile_error_tostring end
-  | _print_error
+  | (_error_str | println)
   );
 def _repl_on_compile_error: _repl_on_error;
 def _repl_eval($e): _eval($e; _repl_display; _repl_on_error; _repl_on_compile_error);
@@ -209,7 +208,7 @@ def repl: repl({}; .); #:: a| => @
 def _cli_expr_on_error:
   ( . as $err
   | _cli_last_expr_error($err) as $_
-  | _stderr_error
+  | (_error_str | _errorln)
   );
 def _cli_expr_on_compile_error:
   ( _eval_compile_error_tostring
@@ -221,10 +220,10 @@ def _cli_expr_eval($e): _eval($e; .; _cli_expr_on_error; _cli_expr_on_compile_er
 
 # next valid input
 def input:
-  ( _inputs
+  ( _input_filenames
   | if length == 0 then error("break") end
   | [.[0], .[1:]] as [$h, $t]
-  | _inputs($t)
+  | _input_filenames($t)
   | _input_filename(null) as $_
   | $h
   | try
@@ -236,7 +235,7 @@ def input:
       ( . as $err
       | _input_io_errors(. += {($h): $err}) as $_
       | $err
-      | _stderr_error
+      | (_error_str | _errorln)
       , input
       )
   | try
@@ -245,7 +244,7 @@ def input:
       ( . as $err
       | _input_decode_errors(. += {($h): $err}) as $_
       | "\($h): failed to decode (\(_parsed_args.decode_format)), try -d FORMAT to force"
-      | _stderr_error
+      | (_error_str | _errorln)
       , input
       )
   );
@@ -254,7 +253,6 @@ def input:
 def inputs:
   try repeat(input)
   catch if . == "break" then empty else error end;
-def inputs($v): _inputs($v);
 
 def input_filename: _input_filename;
 
@@ -319,6 +317,13 @@ def _main:
         description: "No newline between outputs",
         bool: true
       },
+      # TODO: multiple paths
+      "include_path": {
+        short: "-L",
+        long: "--include-path",
+        description: "Include search path",
+        string: "PATH"
+      },
       "null_output": {
         short: "-0",
         long: "--null-output",
@@ -432,7 +437,10 @@ def _main:
             .filenames = ["-"]
           end
         | . as {$expr, $filenames, $null_input}
-        | inputs($filenames) as $_ # store inputs
+        | _include_paths([
+            $parsed_args.include_path // empty
+          ]) as $_
+        | _input_filenames($filenames) as $_ # store inputs
         | if $null_input then null
           elif $parsed_args.slurp then [inputs]
           else inputs

pkg/interp/funcs.go

@@ -8,7 +8,6 @@ import (
 	"crypto/md5"
 	"encoding/base64"
 	"encoding/hex"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"fq/format"
@@ -39,10 +38,8 @@ func (i *Interp) makeFunctions(registry *registry.Registry) []Function {
 
 		{[]string{"readline"}, 0, 2, i.readline, nil},
 		{[]string{"eval"}, 1, 2, nil, i.eval},
-		{[]string{"print"}, 0, 0, nil, i.print},
-		{[]string{"println"}, 0, 0, nil, i.println},
+		{[]string{"stdout"}, 0, 0, nil, i.stdout},
 		{[]string{"stderr"}, 0, 0, nil, i.stderr},
-		{[]string{"debug"}, 0, 0, i.debug, nil},
 
 		{[]string{"_complete_query"}, 0, 0, i._completeQuery, nil},
 		{[]string{"_display_name"}, 0, 0, i._displayName, nil},
@@ -252,15 +249,8 @@ func (i *Interp) eval(c interface{}, a []interface{}) gojq.Iter {
 	if !ok {
 		return gojq.NewIter(fmt.Errorf("%v: src is not a string", a[0]))
 	}
-	debugFn := ""
-	if len(a) >= 2 {
-		debugFn, ok = a[1].(string)
-		if !ok {
-			return gojq.NewIter(fmt.Errorf("%v: debugFn is not a string", a[1]))
-		}
-	}
 
-	iter, err := i.Eval(i.evalContext.ctx, ScriptMode, c, src, i.evalContext.stdout, debugFn)
+	iter, err := i.Eval(i.evalContext.ctx, ScriptMode, c, src, i.evalContext.stdout)
 	if err != nil {
 		return gojq.NewIter(err)
 	}
@@ -268,15 +258,8 @@ func (i *Interp) eval(c interface{}, a []interface{}) gojq.Iter {
 	return iter
 }
 
-func (i *Interp) print(c interface{}, a []interface{}) gojq.Iter {
-	if _, err := fmt.Fprint(i.evalContext.stdout, c); err != nil {
-		return gojq.NewIter(err)
-	}
-	return gojq.NewIter()
-}
-
-func (i *Interp) println(c interface{}, a []interface{}) gojq.Iter {
-	if _, err := fmt.Fprintln(i.evalContext.stdout, c); err != nil {
+func (i *Interp) stdout(c interface{}, a []interface{}) gojq.Iter {
+	if _, err := fmt.Fprint(i.os.Stdout(), c); err != nil {
 		return gojq.NewIter(err)
 	}
 	return gojq.NewIter()
@@ -289,32 +272,6 @@ func (i *Interp) stderr(c interface{}, a []interface{}) gojq.Iter {
 	return gojq.NewIter()
 }
 
-func (i *Interp) debug(c interface{}, a []interface{}) interface{} {
-	if i.evalContext.debugFn != "" {
-		di, err := i.EvalFunc(i.evalContext.ctx, ScriptMode, c, i.evalContext.debugFn, []interface{}{}, i.evalContext.stdout, "")
-		if err != nil {
-			return err
-		}
-		for {
-			v, ok := di.Next()
-			if !ok {
-				break
-			}
-			if err, ok := v.(error); ok {
-				// TODO: how to log?
-				fmt.Fprintf(i.os.Stderr(), "%v", err)
-			}
-		}
-	} else {
-		// TODO: how to log?
-		if err := json.NewEncoder(i.os.Stderr()).Encode([]interface{}{"DEBUG", c}); err != nil {
-			return err
-		}
-	}
-
-	return c
-}
-
 func (i *Interp) _completeQuery(c interface{}, a []interface{}) interface{} {
 	s, ok := c.(string)
 	if !ok {

pkg/interp/funcs.jq

@@ -5,13 +5,34 @@ def help:
   , "^D exit REPL"
   ) | println;
 
+# valid jq identifer, start with alpha or underscore then zero or more alpha, num or underscore
+def _is_ident: test("^[a-zA-Z_][a-zA-Z_0-9]*$");
+# escape " and \
+def _escape_ident: gsub("(?<g>[\\\\\"])"; "\\\(.g)");
+
 # TODO: escape for safe key names
 # path ["a", 1, "b"] -> "a[1].b"
 def path_to_expr:
-  if length == 0 then "."
-  else
-    map(if type == "number" then "[", ., "]" else ".", . end) | join("")
-  end;
+  ( if length == 0 or (.[0] | type) != "string" then
+      [""] + .
+    end
+  | map(
+      if type == "number" then "[", ., "]"
+      else
+        ( "."
+        , # empty (special case for leading index or empty path) or key
+          if . == "" or _is_ident then .
+          else
+            ( "\""
+            , _escape_ident
+            , "\""
+            )
+          end
+        )
+      end
+    )
+  | join("")
+  );
 
 # TODO: don't use eval? should support '.a.b[1]."c.c"' and escapes?
 def expr_to_path:

pkg/interp/funcs_test.jq

@@ -0,0 +1,14 @@
+include "assert";
+include "funcs";
+
+[
+  ".",
+  ".a",
+  ".a[0]",
+  ".a[123].bb",
+  ".[123].a",
+  ".[123][123].a",
+  ".\"b b\"",
+  ".\"a \\\\ b\"",
+  ".\"a \\\" b\""
+][] | assert("\(.) | expr_to_path | path_to_expr"; .; expr_to_path | path_to_expr)

pkg/interp/internal.jq

@@ -1,3 +1,10 @@
+def print: stdout;
+def println: ., "\n" | stdout;
+def debug:
+  ( ((["DEBUG", .] | tojson), "\n" | stderr)
+  , .
+  );
+
 # eval f and finally eval fin even on empty or error
 def finally(f; fin):
   ( try f // (fin | empty)
@@ -6,12 +13,15 @@ def finally(f; fin):
   | .
   );
 
-def _print_error: "error: \(.)" | println;
-def _stderr_error: "error: \(.)\n" | stderr;
+def _error_str: "error: \(.)";
+def _errorln: ., "\n" | stderr;
 
 def _global_var($k): _global_state[$k];
 def _global_var($k; f): _global_state(_global_state | .[$k] |= f);
 
+def _include_paths: _global_var("include_paths");
+def _include_paths(f): _global_var("include_paths"; f);
+
 def _default_options: _global_var("default_options");
 def _default_options(f): _global_var("default_options"; f);
 
@@ -24,12 +34,12 @@ def _parsed_args(f): _global_var("parsed_args"; f);
 def _cli_last_expr_error: _global_var("cli_last_expr_error");
 def _cli_last_expr_error(f): _global_var("cli_last_expr_error"; f);
 
-def _inputs: _global_var("inputs");
-def _inputs(f): _global_var("inputs"; f);
-
 def _input_filename: _global_var("input_filename");
 def _input_filename(f): _global_var("input_filename"; f);
 
+def _input_filenames: _global_var("input_filenames");
+def _input_filenames(f): _global_var("input_filenames"; f);
+
 def _input_io_errors: _global_var("input_io_errors");
 def _input_io_errors(f): _global_var("input_io_errors"; f);
 

pkg/interp/interp.go

@@ -443,10 +443,9 @@ const (
 
 type evalContext struct {
 	// structcheck has problems with embedding https://gitlab.com/opennota/check#known-limitations
-	ctx     context.Context
-	stdout  Output // TODO: rename?
-	mode    RunMode
-	debugFn string
+	ctx    context.Context
+	stdout Output // TODO: rename?
+	mode   RunMode
 }
 
 type Interp struct {
@@ -508,7 +507,7 @@ func (i *Interp) Main(ctx context.Context, stdout io.Writer, version string) err
 		"version": version,
 	}
 
-	iter, err := i.EvalFunc(ctx, runMode, input, "_main", nil, i.os.Stdout(), "")
+	iter, err := i.EvalFunc(ctx, runMode, input, "_main", nil, i.os.Stdout())
 	if err != nil {
 		fmt.Fprintln(i.os.Stderr(), err)
 		return err
@@ -538,7 +537,7 @@ func (i *Interp) Main(ctx context.Context, stdout io.Writer, version string) err
 	return nil
 }
 
-func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src string, stdout Output, debugFn string) (gojq.Iter, error) {
+func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src string, stdout Output) (gojq.Iter, error) {
 	var err error
 	// TODO: did not work
 	// nq := &(*q)
@@ -558,8 +557,7 @@ func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src stri
 	ni := &ci
 
 	ni.evalContext = evalContext{
-		mode:    mode,
-		debugFn: debugFn,
+		mode: mode,
 	}
 
 	// var variableNames []string
@@ -612,6 +610,7 @@ func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src stri
 					"@format/", true, func(filename string) (io.Reader, error) {
 						allFormats := i.registry.MustGroup("all")
 						if filename == "all.jq" {
+							// special case, a file that include all other format files
 							sb := &bytes.Buffer{}
 							for _, f := range allFormats {
 								if f.FS == nil {
@@ -649,7 +648,13 @@ func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src stri
 				},
 				{
 					"", false, func(filename string) (io.Reader, error) {
-						return i.os.FS().Open(filename)
+						// TODO: jq $ORIGIN
+						for _, path := range append([]string{"./"}, i.includePaths()...) {
+							if f, err := i.os.FS().Open(filepath.Join(path, filename)); err == nil {
+								return f, nil
+							}
+						}
+						return nil, &fs.PathError{Op: "open", Path: filename, Err: fs.ErrNotExist}
 					},
 				},
 			}
@@ -727,7 +732,7 @@ func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src stri
 	return iterWrapper, nil
 }
 
-func (i *Interp) EvalFunc(ctx context.Context, mode RunMode, c interface{}, name string, args []interface{}, stdout Output, debugFn string) (gojq.Iter, error) {
+func (i *Interp) EvalFunc(ctx context.Context, mode RunMode, c interface{}, name string, args []interface{}, stdout Output) (gojq.Iter, error) {
 	var argsExpr []string
 	for i := range args {
 		argsExpr = append(argsExpr, fmt.Sprintf("$args[%d]", i))
@@ -743,15 +748,15 @@ func (i *Interp) EvalFunc(ctx context.Context, mode RunMode, c interface{}, name
 	}
 	// {input: ..., args: [...]} | .args as $args | .input | name[($args[0]; ...)]
 	trampolineExpr := fmt.Sprintf(". as {$args} | .input | %s%s", name, argExpr)
-	iter, err := i.Eval(ctx, mode, trampolineInput, trampolineExpr, stdout, debugFn)
+	iter, err := i.Eval(ctx, mode, trampolineInput, trampolineExpr, stdout)
 	if err != nil {
 		return nil, err
 	}
 	return iter, nil
 }
 
-func (i *Interp) EvalFuncValues(ctx context.Context, mode RunMode, c interface{}, name string, args []interface{}, stdout Output, debugFn string) ([]interface{}, error) {
-	iter, err := i.EvalFunc(ctx, mode, c, name, args, stdout, debugFn)
+func (i *Interp) EvalFuncValues(ctx context.Context, mode RunMode, c interface{}, name string, args []interface{}, stdout Output) ([]interface{}, error) {
+	iter, err := i.EvalFunc(ctx, mode, c, name, args, stdout)
 	if err != nil {
 		return nil, err
 	}
@@ -897,11 +902,30 @@ func bitsFormatFnFromOptions(opts Options) func(bb *bitio.Buffer) (interface{},
 			return fmt.Sprintf("<%s>%s", num.Bits(bb.Len()).StringByteBits(opts.SizeBase), b.String()), nil
 		}
 	}
+}
 
+func (i *Interp) lookupState(key string) interface{} {
+	if i.state == nil {
+		return nil
+	}
+	m, ok := (*i.state).(map[string]interface{})
+	if !ok {
+		return nil
+	}
+	return m[key]
+}
+
+func (i *Interp) includePaths() []string {
+	pathsAny, _ := i.lookupState("include_paths").([]interface{})
+	var paths []string
+	for _, pathAny := range pathsAny {
+		paths = append(paths, pathAny.(string))
+	}
+	return paths
 }
 
 func (i *Interp) Options(fnOptsV ...interface{}) (Options, error) {
-	vs, err := i.EvalFuncValues(i.evalContext.ctx, ScriptMode, nil, "options", []interface{}{fnOptsV}, DiscardOutput{Ctx: i.evalContext.ctx}, "")
+	vs, err := i.EvalFuncValues(i.evalContext.ctx, ScriptMode, nil, "options", []interface{}{fnOptsV}, DiscardOutput{Ctx: i.evalContext.ctx})
 	if err != nil {
 		return Options{}, err
 	}

pkg/interp/testdata/args.fqtest

@@ -14,6 +14,7 @@ Usage: fq [OPTIONS] [--] [EXPR] [FILE...]
 --file,-f=PATH             Read EXPR from file
 --formats                  Show supported formats
 --help,-h                  Show help
+--include-path,-L=PATH     Include search path
 --join-output,-j           No newline between outputs
 --null-input,-n            Null input (can still use input/0 and inputs/0)
 --null-output,-0           Null byte between outputs

pkg/interp/testdata/incudepath.fqtest

@@ -0,0 +1,8 @@
+/library/a.jq:
+def a: "a";
+$ fq -L /library -n 'include "a"; a'
+"a"
+$ fq -L /wrong -n 'include "a"; a'
+exitcode: 3
+stderr:
+error: src:1:0: open a.jq: file does not exist

pkg/interp/testdata/repl.fqtest

@@ -0,0 +1,26 @@
+$ fq -i
+null> 1+1
+2
+null> (
+error: src:1:2: unexpected token <EOF>
+null> abc
+error: src:1:0: function not defined: abc/0
+null> 1+"a"
+error: cannot add: number (1) and string ("a")
+null> 1 | repl
+> number> .+1
+2
+> number> ^D
+null> [1,2,3] | repl({}; .[])
+> number, ...> .
+1
+2
+3
+> number, ...> ^D
+null> [] | repl({}; .[])
+> empty> 1
+> empty> ^D
+null> ^D
+$ fq -i 'empty'
+empty> 1
+empty> ^D