1
1
mirror of https://github.com/wader/fq.git synced 2024-12-24 22:05:31 +03:00
fq/pkg/interp/decode.go

775 lines
17 KiB
Go
Raw Normal View History

2020-06-08 03:29:51 +03:00
package interp
import (
"bytes"
2020-06-08 03:29:51 +03:00
"errors"
2021-08-12 00:51:00 +03:00
"fmt"
2020-06-08 03:29:51 +03:00
"io"
"math/big"
"reflect"
2020-06-08 03:29:51 +03:00
"strings"
"time"
2020-06-08 03:29:51 +03:00
"github.com/mitchellh/copystructure"
"github.com/wader/fq/internal/bitioex"
"github.com/wader/fq/internal/gojqex"
"github.com/wader/fq/internal/ioex"
"github.com/wader/fq/internal/mapstruct"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/gojq"
2020-06-08 03:29:51 +03:00
)
func init() {
RegisterFunc0("_registry", (*Interp)._registry)
RegisterFunc1("_tovalue", (*Interp)._toValue)
RegisterFunc2("_decode", (*Interp)._decode)
}
2021-08-12 00:51:00 +03:00
type expectedExtkeyError struct {
Key string
}
func (err expectedExtkeyError) Error() string {
return "expected a extkey but got: " + err.Key
}
// TODO: redo/rename
// used by _isDecodeValue
2021-09-23 19:35:04 +03:00
type DecodeValue interface {
2021-09-27 12:01:14 +03:00
Value
ToBinary
2021-09-23 19:35:04 +03:00
DecodeValue() *decode.Value
2020-06-08 03:29:51 +03:00
}
func (i *Interp) _registry(c any) any {
uniqueFormats := map[string]decode.Format{}
groups := map[string]any{}
formats := map[string]any{}
for groupName := range i.Registry.FormatGroups {
var group []any
for _, f := range i.Registry.MustFormatGroup(groupName) {
group = append(group, f.Name)
if _, ok := uniqueFormats[f.Name]; ok {
continue
}
uniqueFormats[f.Name] = f
}
groups[groupName] = group
}
for _, f := range uniqueFormats {
vf := map[string]any{
"name": f.Name,
"description": f.Description,
"probe_order": f.ProbeOrder,
"root_name": f.RootName,
"root_array": f.RootArray,
"skip_decode_function": f.SkipDecodeFunction,
}
var dependenciesVs []any
for _, d := range f.Dependencies {
var dNamesVs []any
for _, n := range d.Names {
dNamesVs = append(dNamesVs, n)
}
dependenciesVs = append(dependenciesVs, dNamesVs)
}
if len(dependenciesVs) > 0 {
vf["dependencies"] = dependenciesVs
}
var groupsVs []any
for _, n := range f.Groups {
groupsVs = append(groupsVs, n)
}
if len(groupsVs) > 0 {
vf["groups"] = groupsVs
}
if f.DecodeInArg != nil {
doc := map[string]any{}
st := reflect.TypeOf(f.DecodeInArg)
for i := 0; i < st.NumField(); i++ {
f := st.Field(i)
if v, ok := f.Tag.Lookup("doc"); ok {
doc[mapstruct.CamelToSnake(f.Name)] = v
}
}
vf["decode_in_arg_doc"] = doc
args, err := mapstruct.ToMap(f.DecodeInArg)
if err != nil {
return err
}
// filter out internal field without documentation
for k := range args {
if _, ok := doc[k]; !ok {
delete(args, k)
}
}
vf["decode_in_arg"] = gojqex.Normalize(args)
}
if f.Functions != nil {
var ss []any
for _, f := range f.Functions {
ss = append(ss, f)
}
vf["functions"] = ss
}
formats[f.Name] = vf
}
var files []any
for _, fs := range i.Registry.FSs {
ventries := []any{}
entries, err := fs.ReadDir(".")
if err != nil {
return err
}
for _, e := range entries {
f, err := fs.Open(e.Name())
if err != nil {
return err
}
b, err := io.ReadAll(f)
if err != nil {
return err
}
ventries = append(ventries, map[string]any{
"name": e.Name(),
"data": string(b),
})
}
files = append(files, ventries)
}
return map[string]any{
"groups": groups,
"formats": formats,
"files": files,
}
}
func (i *Interp) _toValue(c any, opts map[string]any) any {
v, _ := toValue(
func() Options { return OptionsFromValue(opts) },
c,
)
return v
}
type decodeOpts struct {
Force bool
Progress string
Remain map[string]any `mapstruct:",remain"`
}
func (i *Interp) _decode(c any, format string, opts decodeOpts) any {
var filename string
// TODO: progress hack
// would be nice to move all progress code into decode but it might be
// tricky to keep track of absolute positions in the underlaying readers
// when it uses BitBuf slices, maybe only in Pos()?
if bbf, ok := c.(*openFile); ok {
filename = bbf.filename
if opts.Progress != "" {
evalProgress := func(c any) {
// {approx_read_bytes: 123, total_size: 123} | opts.Progress
_, _ = i.EvalFuncValues(
i.EvalInstance.Ctx,
c,
opts.Progress,
nil,
EvalOpts{output: ioex.DiscardCtxWriter{Ctx: i.EvalInstance.Ctx}},
)
}
lastProgress := time.Now()
bbf.progressFn = func(approxReadBytes, totalSize int64) {
// make sure to not call too often as it's quite expensive
n := time.Now()
if n.Sub(lastProgress) < 200*time.Millisecond {
return
}
lastProgress = n
evalProgress(
map[string]any{
"approx_read_bytes": approxReadBytes,
"total_size": totalSize,
},
)
}
// when done decoding, tell progress function were done and disable it
defer func() {
bbf.progressFn = nil
evalProgress(nil)
}()
}
}
bv, err := toBinary(c)
if err != nil {
return err
}
formatName, err := toString(format)
if err != nil {
return err
}
decodeFormat, err := i.Registry.FormatGroup(formatName)
if err != nil {
return err
}
dv, formatOut, err := decode.Decode(i.EvalInstance.Ctx, bv.br, decodeFormat,
decode.Options{
IsRoot: true,
FillGaps: true,
Force: opts.Force,
Range: bv.r,
Description: filename,
FormatInArgFn: func(f decode.Format) (any, error) {
inArg := f.DecodeInArg
if inArg == nil {
return nil, nil
}
var err error
inArg, err = copystructure.Copy(inArg)
if err != nil {
return f.DecodeInArg, err
}
if len(opts.Remain) > 0 {
if err := mapstruct.ToStruct(opts.Remain, &inArg); err != nil {
// TODO: currently ignores failed struct mappings
//nolint: nilerr
return f.DecodeInArg, nil
}
}
return inArg, nil
},
},
)
if dv == nil {
var decodeFormatsErr decode.FormatsError
if errors.As(err, &decodeFormatsErr) {
var vs []any
for _, fe := range decodeFormatsErr.Errs {
vs = append(vs, fe.Value())
}
return valueError{vs}
}
return valueError{err}
}
var formatOutMap any
if formatOut != nil {
formatOutMap, err = mapstruct.ToMap(formatOut)
if err != nil {
return err
}
}
return makeDecodeValueOut(dv, decodeValueValue, formatOutMap)
}
func valueKey(name string, a, b func(name string) any) any {
2021-08-12 00:51:00 +03:00
if strings.HasPrefix(name, "_") {
return a(name)
}
return b(name)
}
func valueHas(key any, a func(name string) any, b func(key any) any) any {
2021-09-28 00:06:46 +03:00
stringKey, ok := key.(string)
if ok && strings.HasPrefix(stringKey, "_") {
if err, ok := a(stringKey).(error); ok {
return err
}
return true
}
return b(key)
}
2021-08-12 00:51:00 +03:00
// optsFn is a function as toValue is used by tovalue/0 so needs to be fast
func toValue(optsFn func() Options, v any) (any, bool) {
switch v := v.(type) {
case JQValueEx:
if optsFn == nil {
return v.JQValueToGoJQ(), true
}
return v.JQValueToGoJQEx(optsFn), true
case gojq.JQValue:
return v.JQValueToGoJQ(), true
case nil, bool, float64, int, string, *big.Int, map[string]any, []any:
return v, true
default:
return nil, false
}
}
type decodeValueKind int
const (
decodeValueValue decodeValueKind = iota
decodeValueActual
decodeValueSym
)
func makeDecodeValue(dv *decode.Value, kind decodeValueKind) any {
return makeDecodeValueOut(dv, kind, nil)
}
func makeDecodeValueOut(dv *decode.Value, kind decodeValueKind, out any) any {
2020-06-08 03:29:51 +03:00
switch vv := dv.V.(type) {
case *decode.Compound:
if vv.IsArray {
return NewArrayDecodeValue(dv, out, vv)
2020-06-08 03:29:51 +03:00
}
return NewStructDecodeValue(dv, out, vv)
case Scalarable:
// TODO: rethink value/actual/sym handling
var vvv any
switch kind {
case decodeValueValue:
vvv = vv.ScalarValue()
case decodeValueActual:
vvv = vv.ScalarActual()
case decodeValueSym:
vvv = vv.ScalarSym()
}
switch vvv := vvv.(type) {
case bitio.ReaderAtSeeker:
// is lazy so that in situations where the decode value is only used to
// create another binary we don't have to read and create a string, ex:
// .unknown0 | tobytes[1:] | ...
return decodeValue{
JQValue: &gojqex.Lazy{
Type: "string",
IsScalar: true,
Fn: func() (gojq.JQValue, error) {
buf := &bytes.Buffer{}
vvvC, err := bitio.CloneReader(vvv)
if err != nil {
return nil, err
}
if _, err := bitioex.CopyBits(buf, vvvC); err != nil {
return nil, err
}
return gojqex.String([]rune(buf.String())), nil
},
},
decodeValueBase: decodeValueBase{dv: dv},
bitsFormat: true,
}
case bool:
return decodeValue{
JQValue: gojqex.Boolean(vvv),
decodeValueBase: decodeValueBase{dv: dv},
}
case int:
return decodeValue{
JQValue: gojqex.Number{V: vvv},
decodeValueBase: decodeValueBase{dv: dv},
}
case int64:
return decodeValue{
JQValue: gojqex.Number{V: big.NewInt(vvv)},
decodeValueBase: decodeValueBase{dv: dv},
}
case uint64:
return decodeValue{
JQValue: gojqex.Number{V: new(big.Int).SetUint64(vvv)},
decodeValueBase: decodeValueBase{dv: dv},
}
case float64:
return decodeValue{
JQValue: gojqex.Number{V: vvv},
decodeValueBase: decodeValueBase{dv: dv},
}
case string:
return decodeValue{
JQValue: gojqex.String(vvv),
decodeValueBase: decodeValueBase{dv: dv},
}
case []any:
return decodeValue{
JQValue: gojqex.Array(vvv),
decodeValueBase: decodeValueBase{dv: dv},
}
case map[string]any:
return decodeValue{
JQValue: gojqex.Object(vvv),
decodeValueBase: decodeValueBase{dv: dv},
}
case nil:
return decodeValue{
JQValue: gojqex.Null{},
decodeValueBase: decodeValueBase{dv: dv},
}
case *big.Int:
return decodeValue{
JQValue: gojqex.Number{V: vvv},
decodeValueBase: decodeValueBase{dv: dv},
}
case Binary:
// used by bytes and bits format
return vvv
default:
panic(fmt.Sprintf("unreachable vv %#+v", vvv))
2020-06-08 03:29:51 +03:00
}
default:
panic(fmt.Sprintf("unreachable dv %#+v", dv))
2020-06-08 03:29:51 +03:00
}
}
type decodeValueBase struct {
dv *decode.Value
out any
2020-06-08 03:29:51 +03:00
}
2021-09-23 19:35:04 +03:00
func (dvb decodeValueBase) DecodeValue() *decode.Value {
return dvb.dv
}
2020-06-08 03:29:51 +03:00
func (dvb decodeValueBase) Display(w io.Writer, opts Options) error { return dump(dvb.dv, w, opts) }
func (dvb decodeValueBase) ToBinary() (Binary, error) {
return Binary{br: dvb.dv.RootReader, r: dvb.dv.InnerRange(), unit: 8}, nil
2020-06-08 03:29:51 +03:00
}
func (decodeValueBase) ExtType() string { return "decode_value" }
2020-06-08 03:29:51 +03:00
func (dvb decodeValueBase) ExtKeys() []string {
kv := []string{
"_start",
"_stop",
"_len",
"_name",
"_root",
"_buffer_root",
"_format_root",
"_parent",
"_actual",
"_sym",
2020-06-08 03:29:51 +03:00
"_description",
"_path",
"_bits",
"_bytes",
"_gap",
"_index", // TODO: only if parent is array?
2020-06-08 03:29:51 +03:00
}
if _, ok := dvb.dv.V.(*decode.Compound); ok {
kv = append(kv,
"_error",
"_format",
"_out",
)
if dvb.dv.Index != -1 {
kv = append(kv, "_index")
}
2020-06-08 03:29:51 +03:00
}
return kv
}
func (dvb decodeValueBase) JQValueKey(name string) any {
2021-08-12 00:51:00 +03:00
dv := dvb.dv
switch name {
case "_start":
return big.NewInt(dv.Range.Start)
case "_stop":
return big.NewInt(dv.Range.Stop())
case "_len":
return big.NewInt(dv.Range.Len)
case "_name":
return dv.Name
case "_root":
return makeDecodeValue(dv.Root(), decodeValueValue)
case "_buffer_root":
// TODO: rename?
return makeDecodeValue(dv.BufferRoot(), decodeValueValue)
case "_format_root":
// TODO: rename?
return makeDecodeValue(dv.FormatRoot(), decodeValueValue)
case "_parent":
if dv.Parent == nil {
return nil
}
return makeDecodeValue(dv.Parent, decodeValueValue)
case "_actual":
switch dv.V.(type) {
case Scalarable:
return makeDecodeValue(dv, decodeValueActual)
default:
return nil
}
case "_sym":
switch dv.V.(type) {
case Scalarable:
return makeDecodeValue(dv, decodeValueSym)
default:
return nil
}
2021-08-12 00:51:00 +03:00
case "_description":
switch vv := dv.V.(type) {
case *decode.Compound:
if vv.Description == "" {
return nil
}
return vv.Description
case Scalarable:
desc := vv.ScalarDescription()
if desc == "" {
return nil
}
return desc
default:
return nil
}
2021-08-12 00:51:00 +03:00
case "_path":
return valuePath(dv)
case "_error":
var formatErr decode.FormatError
if errors.As(dv.Err, &formatErr) {
return formatErr.Value()
2020-06-08 03:29:51 +03:00
}
return nil
2021-08-12 00:51:00 +03:00
case "_bits":
return Binary{
br: dv.RootReader,
2021-10-05 23:26:05 +03:00
r: dv.Range,
unit: 1,
2021-08-12 00:51:00 +03:00
}
case "_bytes":
return Binary{
br: dv.RootReader,
2021-10-05 23:26:05 +03:00
r: dv.Range,
unit: 8,
2021-08-12 00:51:00 +03:00
}
case "_format":
if dv.Format != nil {
return dv.Format.Name
2021-08-12 00:51:00 +03:00
}
return nil
case "_out":
return dvb.out
case "_gap":
switch vv := dv.V.(type) {
case Scalarable:
return vv.ScalarGap()
default:
return false
}
case "_index":
if dv.Index != -1 {
return dv.Index
}
2020-06-08 03:29:51 +03:00
}
2021-08-12 00:51:00 +03:00
return expectedExtkeyError{Key: name}
2020-06-08 03:29:51 +03:00
}
2021-09-23 19:35:04 +03:00
var _ DecodeValue = decodeValue{}
2020-06-08 03:29:51 +03:00
type decodeValue struct {
2021-08-12 00:51:00 +03:00
gojq.JQValue
decodeValueBase
bitsFormat bool
2021-08-12 00:51:00 +03:00
}
func (v decodeValue) JQValueKey(name string) any {
2021-09-28 00:06:46 +03:00
return valueKey(name, v.decodeValueBase.JQValueKey, v.JQValue.JQValueKey)
}
func (v decodeValue) JQValueHas(key any) any {
2021-09-28 00:06:46 +03:00
return valueHas(key, v.decodeValueBase.JQValueKey, v.JQValue.JQValueHas)
2021-08-12 00:51:00 +03:00
}
func (v decodeValue) JQValueToGoJQEx(optsFn func() Options) any {
if !v.bitsFormat {
return v.JQValueToGoJQ()
2021-08-12 00:51:00 +03:00
}
bv, err := v.decodeValueBase.ToBinary()
if err != nil {
return err
2020-06-08 03:29:51 +03:00
}
br, err := bv.toReader()
if err != nil {
return err
}
brC, err := bitio.CloneReaderAtSeeker(br)
if err != nil {
return err
2020-06-08 03:29:51 +03:00
}
s, err := optsFn().BitsFormatFn(brC)
if err != nil {
return err
}
return s
2020-06-08 03:29:51 +03:00
}
// decode value array
2021-09-23 19:35:04 +03:00
var _ DecodeValue = ArrayDecodeValue{}
2020-06-08 03:29:51 +03:00
2021-09-27 12:01:14 +03:00
type ArrayDecodeValue struct {
gojqex.Base
2021-08-12 00:51:00 +03:00
decodeValueBase
*decode.Compound
2020-06-08 03:29:51 +03:00
}
func NewArrayDecodeValue(dv *decode.Value, out any, c *decode.Compound) ArrayDecodeValue {
2021-09-27 12:01:14 +03:00
return ArrayDecodeValue{
decodeValueBase: decodeValueBase{dv: dv, out: out},
Base: gojqex.Base{Typ: gojq.JQTypeArray},
Compound: c,
2021-08-12 00:51:00 +03:00
}
}
func (v ArrayDecodeValue) JQValueKey(name string) any {
2021-09-28 00:06:46 +03:00
return valueKey(name, v.decodeValueBase.JQValueKey, v.Base.JQValueKey)
2021-08-12 00:51:00 +03:00
}
func (v ArrayDecodeValue) JQValueSliceLen() any { return len(v.Compound.Children) }
func (v ArrayDecodeValue) JQValueLength() any { return len(v.Compound.Children) }
func (v ArrayDecodeValue) JQValueIndex(index int) any {
2020-06-08 03:29:51 +03:00
// -1 outside after string, -2 outside before string
if index < 0 {
return nil
}
return makeDecodeValue((v.Compound.Children)[index], decodeValueValue)
2020-06-08 03:29:51 +03:00
}
func (v ArrayDecodeValue) JQValueSlice(start int, end int) any {
vs := make([]any, end-start)
for i, e := range (v.Compound.Children)[start:end] {
vs[i] = makeDecodeValue(e, decodeValueValue)
2020-06-08 03:29:51 +03:00
}
return vs
}
func (v ArrayDecodeValue) JQValueEach() any {
props := make([]gojq.PathValue, len(v.Compound.Children))
for i, f := range v.Compound.Children {
props[i] = gojq.PathValue{Path: i, Value: makeDecodeValue(f, decodeValueValue)}
2020-06-08 03:29:51 +03:00
}
return props
}
func (v ArrayDecodeValue) JQValueKeys() any {
vs := make([]any, len(v.Compound.Children))
for i := range v.Compound.Children {
2020-06-08 03:29:51 +03:00
vs[i] = i
}
return vs
}
func (v ArrayDecodeValue) JQValueHas(key any) any {
2021-09-28 00:06:46 +03:00
return valueHas(
key,
v.decodeValueBase.JQValueKey,
func(key any) any {
2021-09-28 00:06:46 +03:00
intKey, ok := key.(int)
if !ok {
return gojqex.HasKeyTypeError{L: gojq.JQTypeArray, R: fmt.Sprintf("%v", key)}
2021-09-28 00:06:46 +03:00
}
return intKey >= 0 && intKey < len(v.Compound.Children)
2021-09-28 00:06:46 +03:00
})
2020-06-08 03:29:51 +03:00
}
func (v ArrayDecodeValue) JQValueToGoJQ() any {
vs := make([]any, len(v.Compound.Children))
for i, f := range v.Compound.Children {
vs[i] = makeDecodeValue(f, decodeValueValue)
2020-06-08 03:29:51 +03:00
}
return vs
}
// decode value struct
2021-09-23 19:35:04 +03:00
var _ DecodeValue = StructDecodeValue{}
2020-06-08 03:29:51 +03:00
2021-09-27 12:01:14 +03:00
type StructDecodeValue struct {
gojqex.Base
2021-08-12 00:51:00 +03:00
decodeValueBase
*decode.Compound
2020-06-08 03:29:51 +03:00
}
func NewStructDecodeValue(dv *decode.Value, out any, c *decode.Compound) StructDecodeValue {
2021-09-27 12:01:14 +03:00
return StructDecodeValue{
decodeValueBase: decodeValueBase{dv: dv, out: out},
Base: gojqex.Base{Typ: gojq.JQTypeObject},
Compound: c,
2021-08-12 00:51:00 +03:00
}
}
func (v StructDecodeValue) JQValueLength() any { return len(v.Compound.Children) }
func (v StructDecodeValue) JQValueSliceLen() any { return len(v.Compound.Children) }
func (v StructDecodeValue) JQValueKey(name string) any {
2021-08-12 00:51:00 +03:00
if strings.HasPrefix(name, "_") {
return v.decodeValueBase.JQValueKey(name)
}
if v.Compound.ByName != nil {
if f, ok := v.Compound.ByName[name]; ok {
return makeDecodeValue(f, decodeValueValue)
2020-06-08 03:29:51 +03:00
}
}
2020-06-08 03:29:51 +03:00
return nil
}
func (v StructDecodeValue) JQValueEach() any {
props := make([]gojq.PathValue, len(v.Compound.Children))
for i, f := range v.Compound.Children {
props[i] = gojq.PathValue{Path: f.Name, Value: makeDecodeValue(f, decodeValueValue)}
2020-06-08 03:29:51 +03:00
}
return props
}
func (v StructDecodeValue) JQValueKeys() any {
vs := make([]any, len(v.Compound.Children))
for i, f := range v.Compound.Children {
2020-06-08 03:29:51 +03:00
vs[i] = f.Name
}
return vs
}
func (v StructDecodeValue) JQValueHas(key any) any {
2021-09-28 00:06:46 +03:00
return valueHas(
key,
v.decodeValueBase.JQValueKey,
func(key any) any {
2021-09-28 00:06:46 +03:00
stringKey, ok := key.(string)
if !ok {
return gojqex.HasKeyTypeError{L: gojq.JQTypeObject, R: fmt.Sprintf("%v", key)}
2021-09-28 00:06:46 +03:00
}
for _, f := range v.Compound.Children {
2021-09-28 00:06:46 +03:00
if f.Name == stringKey {
return true
}
}
return false
},
)
2020-06-08 03:29:51 +03:00
}
func (v StructDecodeValue) JQValueToGoJQ() any {
vm := make(map[string]any, len(v.Compound.Children))
for _, f := range v.Compound.Children {
vm[f.Name] = makeDecodeValue(f, decodeValueValue)
2020-06-08 03:29:51 +03:00
}
return vm
}