interp: Move formats func def to jq

format/matroska/matroska.go

@@ -77,7 +77,7 @@ func init() {
 			{Names: []string{format.VP9_CFM}, Formats: &vp9CFMFormat},
 			{Names: []string{format.VP9_FRAME}, Formats: &vp9FrameFormat},
 		},
-		FS: matroskaFS,
+		Files: matroskaFS,
 	})
 
 	codecToFormat = map[string]*[]*decode.Format{

format/mp4/mp4.go

@@ -78,7 +78,7 @@ func init() {
 			{Names: []string{format.VP9_FRAME}, Formats: &vp9FrameFormat},
 			{Names: []string{format.VPX_CCR}, Formats: &vpxCCRFormat},
 		},
-		FS: mp4FS,
+		Files: mp4FS,
 	})
 }
 

pkg/decode/format.go

@@ -16,7 +16,7 @@ type Format struct {
 	RootV        interface{}
 	RootName     string
 	Dependencies []Dependency
-	FS           fs.FS
+	Files        fs.ReadDirFS
 }
 
 func FormatFn(d func(d *D, in interface{}) interface{}) []*Format {

pkg/interp/formats.jq

@@ -0,0 +1,16 @@
+# note this is a "dynamic" include, outputted string will be used as source
+
+( [ ( _registry.groups
+    | to_entries[]
+    # TODO: nicer way to skip "all" which also would override builtin all/*
+    | select(.key != "all")
+    | "def \(.key)($opts): _decode(\(.key | tojson); $opts);"
+    , "def \(.key): _decode(\(.key | tojson); {});"
+    )
+  , ( _registry.formats[]
+    | select(.files)
+    | .files[]
+    )
+  ]
+  | join("\n")
+)

pkg/interp/funcs.go

@@ -47,7 +47,7 @@ func (i *Interp) makeFunctions() []Function {
 		{[]string{"_extkeys"}, 0, 0, i._extKeys, nil},
 		{[]string{"_global_state"}, 0, 1, i.makeStateFn(i.state), nil},
 
-		{[]string{"formats"}, 0, 0, i.formats, nil},
+		{[]string{"_registry"}, 0, 0, i._registry, nil},
 		{[]string{"history"}, 0, 0, i.history, nil},
 
 		{[]string{"open"}, 0, 0, i._open, nil},
@@ -397,24 +397,32 @@ func (i *Interp) makeStateFn(state *interface{}) func(c interface{}, a []interfa
 	}
 }
 
-func (i *Interp) formats(c interface{}, a []interface{}) interface{} {
-	allFormats := map[string]*decode.Format{}
+func (i *Interp) _registry(c interface{}, a []interface{}) interface{} {
+	uniqueFormats := map[string]*decode.Format{}
+
+	groups := map[string]interface{}{}
+	formats := map[string]interface{}{}
+
+	for fsName, fs := range i.registry.Groups {
+		var group []interface{}
 
-	for _, fs := range i.registry.Groups {
 		for _, f := range fs {
-			if _, ok := allFormats[f.Name]; ok {
+			group = append(group, f.Name)
+			if _, ok := uniqueFormats[f.Name]; ok {
 				continue
 			}
-			allFormats[f.Name] = f
+			uniqueFormats[f.Name] = f
 		}
+
+		groups[fsName] = group
 	}
 
-	vs := map[string]interface{}{}
-	for _, f := range allFormats {
+	for _, f := range uniqueFormats {
 		vf := map[string]interface{}{
 			"name":        f.Name,
 			"description": f.Description,
 			"probe_order": f.ProbeOrder,
+			"root_name":   f.RootName,
 		}
 
 		var dependenciesVs []interface{}
@@ -436,10 +444,36 @@ func (i *Interp) formats(c interface{}, a []interface{}) interface{} {
 			vf["groups"] = groupsVs
 		}
 
-		vs[f.Name] = vf
+		if f.Files != nil {
+			files := map[string]interface{}{}
+
+			entries, err := f.Files.ReadDir(".")
+			if err != nil {
+				return err
+			}
+
+			for _, e := range entries {
+				f, err := f.Files.Open(e.Name())
+				if err != nil {
+					return err
+				}
+				b, err := ioutil.ReadAll(f)
+				if err != nil {
+					return err
+				}
+				files[e.Name()] = string(b)
+			}
+
+			vf["files"] = files
+		}
+
+		formats[f.Name] = vf
 	}
 
-	return vs
+	return map[string]interface{}{
+		"groups":  groups,
+		"formats": formats,
+	}
 }
 
 func (i *Interp) history(c interface{}, a []interface{}) interface{} {

pkg/interp/interp.go

@@ -35,6 +35,7 @@ import (
 //go:embed funcs.jq
 //go:embed args.jq
 //go:embed query.jq
+//go:embed formats.jq
 var builtinFS embed.FS
 
 var initSource = `include "@builtin/interp";`
@@ -541,14 +542,14 @@ func (i *Interp) Main(ctx context.Context, output Output, version string) error
 	return nil
 }
 
-func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src string, filename string, output io.Writer) (gojq.Iter, error) {
+func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src string, srcFilename string, output io.Writer) (gojq.Iter, error) {
 	gq, err := gojq.Parse(src)
 	if err != nil {
 		p := queryErrorPosition(src, err)
 		return nil, compileError{
 			err:      err,
 			what:     "parse",
-			filename: filename,
+			filename: srcFilename,
 			pos:      p,
 		}
 	}
@@ -568,18 +569,20 @@ func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src stri
 		variableValues = append(variableValues, v)
 	}
 
-	var compilerOpts []gojq.CompilerOption
+	var funcCompilerOpts []gojq.CompilerOption
 	for _, f := range ni.makeFunctions() {
 		for _, n := range f.Names {
 			if f.IterFn != nil {
-				compilerOpts = append(compilerOpts,
+				funcCompilerOpts = append(funcCompilerOpts,
 					gojq.WithIterFunction(n, f.MinArity, f.MaxArity, f.IterFn))
 			} else {
-				compilerOpts = append(compilerOpts,
+				funcCompilerOpts = append(funcCompilerOpts,
 					gojq.WithFunction(n, f.MinArity, f.MaxArity, f.Fn))
 			}
 		}
 	}
+
+	compilerOpts := append([]gojq.CompilerOption{}, funcCompilerOpts...)
 	compilerOpts = append(compilerOpts, gojq.WithEnvironLoader(ni.os.Environ))
 	compilerOpts = append(compilerOpts, gojq.WithVariables(variableNames))
 	compilerOpts = append(compilerOpts, gojq.WithModuleLoader(loadModule{
@@ -587,6 +590,7 @@ func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src stri
 			return []*gojq.Query{i.initFqQuery}, nil
 		},
 		load: func(name string) (*gojq.Query, error) {
+			// log.Printf("name: %#+v\n", name)
 			if err := ctx.Err(); err != nil {
 				return nil, err
 			}
@@ -607,45 +611,6 @@ func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src stri
 				cache  bool
 				fn     func(filename string) (io.Reader, error)
 			}{
-				{
-					"@format/", true, func(filename string) (io.Reader, error) {
-						allFormats := i.registry.MustGroup("all")
-						if filename == "all.jq" {
-							// special case, a file that include all other format files
-							sb := &bytes.Buffer{}
-							for _, f := range allFormats {
-								if f.FS == nil {
-									continue
-								}
-								fmt.Fprintf(sb, "include \"@format/%s\";\n", f.Name)
-							}
-							return bytes.NewReader(sb.Bytes()), nil
-						} else if filename == "decode.jq" {
-							sb := &bytes.Buffer{}
-							for name := range i.registry.Groups {
-								// TODO: nicer way to skip all which also would override builtin all/*
-								if name == "all" {
-									continue
-								}
-								fmt.Fprintf(sb, ""+
-									"def %[1]s($opts): _decode(%[1]q; $opts);\n"+
-									"def %[1]s: _decode(%[1]q; {});\n",
-									name)
-							}
-							return bytes.NewReader(sb.Bytes()), nil
-						} else {
-							formatName := strings.TrimRight(filename, ".jq")
-							for _, f := range allFormats {
-								if f.Name != formatName {
-									continue
-								}
-								return f.FS.Open(filename)
-							}
-						}
-
-						return builtinFS.Open(filename)
-					},
-				},
 				{
 					"@builtin/", true, func(filename string) (io.Reader, error) {
 						return builtinFS.Open(filename)
@@ -709,6 +674,43 @@ func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src stri
 					}
 				}
 
+				// not identity body means it returns something, threat as dynamic include
+				if q.Term.Type != gojq.TermTypeIdentity {
+					gc, err := gojq.Compile(q, funcCompilerOpts...)
+					if err != nil {
+						return nil, err
+					}
+					iter := gc.RunWithContext(context.Background(), nil)
+					var vs []interface{}
+					for {
+						v, ok := iter.Next()
+						if !ok {
+							break
+						}
+						if err, ok := v.(error); ok {
+							return nil, err
+						}
+						vs = append(vs, v)
+					}
+					if len(vs) != 1 {
+						return nil, fmt.Errorf("dynamic include: must output one string, got: %#v", vs)
+					}
+					s, sOk := vs[0].(string)
+					if !sOk {
+						return nil, fmt.Errorf("dynamic include: must be string, got %#v", s)
+					}
+					q, err = gojq.Parse(s)
+					if err != nil {
+						p := queryErrorPosition(s, err)
+						return nil, compileError{
+							err:      err,
+							what:     "parse",
+							filename: filenamePart,
+							pos:      p,
+						}
+					}
+				}
+
 				// TODO: some better way of handling relative includes that
 				// works with @builtin etc
 				basePath := filepath.Dir(name)
@@ -743,7 +745,7 @@ func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src stri
 		return nil, compileError{
 			err:      err,
 			what:     "compile",
-			filename: filename,
+			filename: srcFilename,
 			pos:      p,
 		}
 	}

pkg/interp/interp.jq

@@ -2,11 +2,8 @@ include "internal";
 include "funcs";
 include "args";
 include "query";
-
-# generated decode functions per format
-include "@format/decode";
-# include per format specific functions
-include "@format/all";
+# generated decode functions per format and format helpers
+include "formats";
 # optional user init
 include "@config/init?";
 
@@ -421,6 +418,9 @@ def verbose: verbose({});
 def v($opts): verbose($opts);
 def v: verbose;
 
+def formats:
+  _registry.formats;
+
 # null input means done, otherwise {approx_read_bytes: 123, total_size: 123}
 # TODO: decode provide even more detailed progress, post-process sort etc?
 def _decode_progress: