Merge pull request #286 from wader/encoding-fix-incorrect-type

interp: Cast jq value to go value properly for encoding functions

internal/gojqextra/error.go

@@ -44,6 +44,16 @@ func (err FuncTypeError) Error() string {
 	return err.Name + " cannot be applied to: " + TypeErrorPreview(err.V)
 }
 
+type FuncArgTypeError struct {
+	Name    string
+	ArgName string
+	V       any
+}
+
+func (err FuncArgTypeError) Error() string {
+	return fmt.Sprintf("%s %s argument cannot be: %s", err.Name, err.ArgName, TypeErrorPreview(err.V))
+}
+
 type FuncTypeNameError struct {
 	Name string
 	Typ  string

internal/gojqextra/types.go

@@ -4,12 +4,157 @@ package gojqextra
 import (
 	"bytes"
 	"fmt"
+	"math"
+	"math/big"
+	"reflect"
+	"strconv"
 
 	"github.com/wader/fq/internal/colorjson"
 
 	"github.com/wader/gojq"
 )
 
+// Cast gojq value to go value
+//nolint: forcetypeassert, unconvert
+func CastFn[T any](v any, structFn func(input map[string]any, result any) error) (T, bool) {
+	var t T
+	switch any(t).(type) {
+	case bool:
+		switch v := v.(type) {
+		case bool:
+			return any(v).(T), true
+		case gojq.JQValue:
+			return CastFn[T](v.JQValueToGoJQ(), structFn)
+		default:
+			return t, false
+		}
+	case int:
+		switch v := v.(type) {
+		case int:
+			return any(v).(T), true
+		case *big.Int:
+			if !v.IsInt64() {
+				return t, false
+			}
+			ci := v.Int64()
+			if ci < math.MinInt || ci > math.MaxInt {
+				return t, false
+			}
+			return any(int(ci)).(T), true
+		case float64:
+			return any(int(v)).(T), true
+		case gojq.JQValue:
+			return CastFn[T](v.JQValueToGoJQ(), structFn)
+		default:
+			return t, false
+		}
+	case float64:
+		switch v := v.(type) {
+		case float64:
+			return any(v).(T), true
+		case int:
+			return any(float64(v)).(T), true
+		case *big.Int:
+			if v.IsInt64() {
+				return any(float64(v.Int64())).(T), true
+			}
+			// TODO: use *big.Float SetInt
+			if f, err := strconv.ParseFloat(v.String(), 64); err == nil {
+				return any(f).(T), true
+			}
+			return any(float64(math.Inf(v.Sign()))).(T), true
+		case gojq.JQValue:
+			return CastFn[T](v.JQValueToGoJQ(), structFn)
+		default:
+			return t, false
+		}
+	case *big.Int:
+		switch v := v.(type) {
+		case *big.Int:
+			return any(v).(T), true
+		case int:
+			return any(new(big.Int).SetInt64(int64(v))).(T), true
+		case float64:
+			return any(new(big.Int).SetInt64(int64(v))).(T), true
+		case gojq.JQValue:
+			return CastFn[T](v.JQValueToGoJQ(), structFn)
+		default:
+			return t, false
+		}
+	case string:
+		switch v := v.(type) {
+		case string:
+			return any(v).(T), true
+		case gojq.JQValue:
+			return CastFn[T](v.JQValueToGoJQ(), structFn)
+		default:
+			return t, false
+		}
+	case map[string]any:
+		switch v := v.(type) {
+		case map[string]any:
+			return any(v).(T), true
+		case nil:
+			// return empty instantiated map, not nil map
+			return any(map[string]any{}).(T), true
+		case gojq.JQValue:
+			return CastFn[T](v.JQValueToGoJQ(), structFn)
+		default:
+			return t, false
+		}
+	case []any:
+		switch v := v.(type) {
+		case []any:
+			return any(v).(T), true
+		case nil:
+			return t, true
+		case gojq.JQValue:
+			return CastFn[T](v.JQValueToGoJQ(), structFn)
+		default:
+			return t, false
+		}
+	default:
+		ft := reflect.TypeOf(&t)
+		if ft.Elem().Kind() == reflect.Struct {
+			m := map[string]any{}
+			switch v := v.(type) {
+			case map[string]any:
+				m = v
+			case nil:
+				// nop use instantiated map
+			case gojq.JQValue:
+				if jm, ok := Cast[map[string]any](v.JQValueToGoJQ()); ok {
+					m = jm
+				}
+			default:
+				return t, false
+			}
+
+			err := structFn(m, &t)
+			if err != nil {
+				return t, false
+			}
+
+			return t, true
+		} else if ft.Elem().Kind() == reflect.Interface {
+			// TODO: panic on non any interface?
+			// ignore failed type assert as v can be nil
+			cv, ok := any(v).(T)
+			if !ok && v != nil {
+				return cv, false
+			}
+
+			return cv, true
+		}
+
+		panic(fmt.Sprintf("unsupported type %s", ft.Elem().Kind()))
+	}
+}
+
+func Cast[T any](v any) (T, bool) {
+	return CastFn[T](v, nil)
+}
+
 // array
 
 var _ gojq.JQValue = Array{}

pkg/interp/encoding.go

@@ -35,6 +35,9 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+// TODO: Fn1, Fn2 etc?
+// TODO: struct arg, own reflect code? no need for refs etc
+
 // TODO: xml default indent?
 // TODO: query dup key
 // TODO: walk tostring tests
@@ -81,47 +84,6 @@ func norm(v any) any {
 	}
 }
 
-func addFuncOpts[TOpt any, Tc any](name string, fn func(c Tc, opts TOpt) any) {
-	if name[0] != '_' {
-		panic(fmt.Sprintf("invalid addFunc name %q", name))
-	}
-	functionRegisterFns = append(
-		functionRegisterFns,
-		func(i *Interp) []Function {
-			return []Function{{
-				name, 1, 1, func(c any, a []any) any {
-					var opts TOpt
-					if a[0] != nil {
-						optsM, ok := a[0].(map[string]any)
-						if !ok {
-							return fmt.Errorf("options %v not a object", a[0])
-						}
-						_ = mapToStruct(optsM, &opts)
-					}
-
-					cv, ok := gojqextra.ToGoJQValue(c)
-					if !ok {
-						return fmt.Errorf("invalid jq value %#v", cv)
-					}
-
-					var ct Tc
-					switch cv.(type) {
-					case nil:
-						// TODO: better way to check if Tc can be nil?
-					default:
-						ct, ok = cv.(Tc)
-						if !ok {
-							return gojqextra.FuncTypeError{Name: name[1:], V: c}
-						}
-					}
-
-					return fn(ct, opts)
-				},
-				nil,
-			}}
-		})
-}
-
 func addFunc[Tc any](name string, fn func(c Tc) any) {
 	if name[0] != '_' {
 		panic(fmt.Sprintf("invalid addFunc name %q", name))
@@ -131,22 +93,37 @@ func addFunc[Tc any](name string, fn func(c Tc) any) {
 		func(i *Interp) []Function {
 			return []Function{{
 				name, 0, 0, func(c any, a []any) any {
-					cv, ok := gojqextra.ToGoJQValue(c)
+					cv, ok := gojqextra.CastFn[Tc](c, mapToStruct)
 					if !ok {
-						return fmt.Errorf("invalid jq value %#v", cv)
+						return gojqextra.FuncTypeError{Name: name[1:], V: c}
-					}
-					var ct Tc
-					switch cv.(type) {
-					case nil:
-						// TODO: better way to check if Tc can be nil?
-					default:
-						ct, ok = cv.(Tc)
-						if !ok {
-							return gojqextra.FuncTypeError{Name: name[1:], V: c}
-						}
 					}
 
-					return fn(ct)
+					return fn(cv)
+				},
+				nil,
+			}}
+		})
+}
+
+func addFunc1[Tc any, Ta0 any](name string, fn func(c Tc, a0 Ta0) any) {
+	if name[0] != '_' {
+		panic(fmt.Sprintf("invalid addFunc name %q", name))
+	}
+	functionRegisterFns = append(
+		functionRegisterFns,
+		func(i *Interp) []Function {
+			return []Function{{
+				name, 1, 1, func(c any, a []any) any {
+					cv, ok := gojqextra.CastFn[Tc](c, mapToStruct)
+					if !ok {
+						return gojqextra.FuncTypeError{Name: name[1:], V: c}
+					}
+					a0, ok := gojqextra.CastFn[Ta0](a[0], mapToStruct)
+					if !ok {
+						return gojqextra.FuncArgTypeError{Name: name[1:], ArgName: "first", V: c}
+					}
+
+					return fn(cv, a0)
 				},
 				nil,
 			}}
@@ -197,7 +174,7 @@ func init() {
 	type ToJSONOpts struct {
 		Indent int
 	}
-	addFuncOpts("_tojson", func(c any, opts ToJSONOpts) any {
+	addFunc1("_tojson", func(c any, opts ToJSONOpts) any {
 		// TODO: share
 		cj := colorjson.NewEncoder(
 			false,
@@ -347,7 +324,7 @@ func init() {
 
 		return f(n, nil)
 	}
-	addFuncOpts("_fromxml", func(s string, opts FromXMLOpts) any {
+	addFunc1("_fromxml", func(s string, opts FromXMLOpts) any {
 		if opts.Array {
 			return fromXMLArray(s)
 		}
@@ -531,7 +508,7 @@ func init() {
 
 		return bb.String()
 	}
-	addFuncOpts("_toxml", func(c any, opts ToXMLOpts) any {
+	addFunc1("_toxml", func(c any, opts ToXMLOpts) any {
 		switch c := c.(type) {
 		case map[string]any:
 			return toXMLObject(c, opts)
@@ -701,7 +678,7 @@ func init() {
 
 		return f(doc.FirstChild)
 	}
-	addFuncOpts("_fromhtml", func(s string, opts FromHTMLOpts) any {
+	addFunc1("_fromhtml", func(s string, opts FromHTMLOpts) any {
 		if opts.Array {
 			return fromHTMLArray(s)
 		}
@@ -744,7 +721,7 @@ func init() {
 		Comma   string
 		Comment string
 	}
-	addFuncOpts("_fromcsv", func(s string, opts FromCSVOpts) any {
+	addFunc1("_fromcsv", func(s string, opts FromCSVOpts) any {
 		var rvs []any
 		r := csv.NewReader(strings.NewReader(s))
 		r.TrimLeadingSpace = true
@@ -772,7 +749,7 @@ func init() {
 	type ToCSVOpts struct {
 		Comma string
 	}
-	addFuncOpts("_tocsv", func(c []any, opts ToCSVOpts) any {
+	addFunc1("_tocsv", func(c []any, opts ToCSVOpts) any {
 		b := &bytes.Buffer{}
 		w := csv.NewWriter(b)
 		if opts.Comma != "" {
@@ -839,7 +816,7 @@ func init() {
 	type FromBase64Opts struct {
 		Encoding string
 	}
-	addFuncOpts("_frombase64", func(s string, opts FromBase64Opts) any {
+	addFunc1("_frombase64", func(s string, opts FromBase64Opts) any {
 		b, err := base64Encoding(opts.Encoding).DecodeString(s)
 		if err != nil {
 			return err
@@ -853,7 +830,7 @@ func init() {
 	type ToBase64Opts struct {
 		Encoding string
 	}
-	addFuncOpts("_tobase64", func(c any, opts ToBase64Opts) any {
+	addFunc1("_tobase64", func(c any, opts ToBase64Opts) any {
 		br, err := toBitReader(c)
 		if err != nil {
 			return err
@@ -972,7 +949,7 @@ func init() {
 		return m
 	})
 	addFunc("_tourl", func(c map[string]any) any {
-		str := func(v any) string { s, _ := gojqextra.ToString(v); return s }
+		str := func(v any) string { s, _ := gojqextra.Cast[string](v); return s }
 		u := url.URL{
 			Scheme:   str(c["scheme"]),
 			Host:     str(c["host"]),
@@ -980,18 +957,20 @@ func init() {
 			Fragment: str(c["fragment"]),
 		}
 
-		if um, ok := gojqextra.ToObject(c["user"]); ok {
+		if um, ok := gojqextra.Cast[map[string]any](c["user"]); ok {
 			username, password := str(um["username"]), str(um["password"])
-			if password == "" {
+			if username != "" {
-				u.User = url.User(username)
+				if password == "" {
-			} else {
+					u.User = url.User(username)
-				u.User = url.UserPassword(username, password)
+				} else {
+					u.User = url.UserPassword(username, password)
+				}
 			}
 		}
-		if s, ok := gojqextra.ToString(c["rawquery"]); ok {
+		if s, ok := gojqextra.Cast[string](c["rawquery"]); ok {
 			u.RawQuery = s
 		}
-		if qm, ok := gojqextra.ToObject(c["query"]); ok {
+		if qm, ok := gojqextra.Cast[map[string]any](c["query"]); ok {
 			u.RawQuery = toURLValues(qm).Encode()
 		}
 
@@ -1025,7 +1004,7 @@ func init() {
 	type ToHashOpts struct {
 		Name string
 	}
-	addFuncOpts("_tohash", func(c any, opts ToHashOpts) any {
+	addFunc1("_tohash", func(c any, opts ToHashOpts) any {
 		inBR, err := toBitReader(c)
 		if err != nil {
 			return err
@@ -1157,7 +1136,7 @@ func init() {
 	type ToStrEncodingOpts struct {
 		Encoding string
 	}
-	addFuncOpts("_tostrencoding", func(c string, opts ToStrEncodingOpts) any {
+	addFunc1("_tostrencoding", func(c string, opts ToStrEncodingOpts) any {
 		h := strEncodingFn(opts.Encoding)
 		if h == nil {
 			return fmt.Errorf("unknown string encoding %s", opts.Encoding)
@@ -1178,7 +1157,7 @@ func init() {
 	type FromStrEncodingOpts struct {
 		Encoding string
 	}
-	addFuncOpts("_fromstrencoding", func(c string, opts FromStrEncodingOpts) any {
+	addFunc1("_fromstrencoding", func(c any, opts FromStrEncodingOpts) any {
 		inBR, err := toBitReader(c)
 		if err != nil {
 			return err

pkg/interp/interp.go

@@ -1147,23 +1147,12 @@ func mapToStruct(m map[string]any, v any) error {
 			return camelToSnake(fieldName) == mapKey
 		},
 		DecodeHook: func(
-			f reflect.Type,
+			f reflect.Value,
-			t reflect.Type,
+			t reflect.Value) (any, error) {
-			data any) (any, error) {
 
-			if t.Kind() == reflect.Slice && t.Elem().Kind() == reflect.Uint8 {
+			// log.Printf("f: %#+v -> t: %#+v\n", f, t)
-				switch d := data.(type) {
-				case string:
-					return []byte(d), nil
-				}
-			} else {
-				switch d := data.(type) {
-				case *big.Int:
-					return d.Uint64(), nil
-				}
-			}
 
-			return data, nil
+			return f.Interface(), nil
 		},
 		Result: v,
 	})

pkg/interp/testdata/encoding/hash.fqtest

@@ -0,0 +1,19 @@
+$ fq -i
+null> "test" | tomd4, tomd5, tosha1, tosha256, tosha512, tosha3_224, tosha3_256, tosha3_384, tosha3_512 | tohex
+"db346d691d7acc4dc2625db19f9e3f52"
+"098f6bcd4621d373cade4e832627b4f6"
+"a94a8fe5ccb19ba61c4c0873d391e987982fbbd3"
+"9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
+"ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff"
+"3797bf0afbbfca4a7bbba7602a2b552746876517a7f9b7ce2db0ae7b"
+"36f028580bb02cc8272a9a020f4200e346e276ae664e45ee80745574e2f5ab80"
+"e516dabb23b6e30026863543282780a3ae0dccf05551cf0295178d7ff0f1b41eecb9db3ff219007c4e097260d58621bd"
+"9ece086e9bac491fac5c1d1046ca11d737b92a2b2ebd93f005d7b710110c0a678288166e7fbe796883a4f2e9b3ca9f484f521d0ce464345cc1aec96779149c14"
+null> 0xf08 | tobits | .[:4,5,6,7,8,9] | tomd5 | tohex
+"8c493a43d8c1ef798860bb02b62e8e79"
+"8c493a43d8c1ef798860bb02b62e8e79"
+"8c493a43d8c1ef798860bb02b62e8e79"
+"8c493a43d8c1ef798860bb02b62e8e79"
+"8c493a43d8c1ef798860bb02b62e8e79"
+"bdf26d2a670238e9a568e34ee02ca31c"
+null> ^D

pkg/interp/testdata/encoding/string.fqtest

@@ -1,26 +1,42 @@
-$ fq -i '"åäö"'
+$ fq -i
-string> toutf8 | ., fromutf8
+null> "åäö" | toiso8859_1 | ., fromiso8859_1
-   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|c3 a5 c3 a4 c3 b6|                             |......|         |.: raw bits 0x0-0x5.7 (6)
-"åäö"
-string> toiso8859_1 | ., fromiso8859_1
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
 0x0|e5 e4 f6|                                      |...|            |.: raw bits 0x0-0x2.7 (3)
 "åäö"
-string> toutf8 | ., fromutf8
+null> "åäö" | toutf8 | ., fromutf8
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
 0x0|c3 a5 c3 a4 c3 b6|                             |......|         |.: raw bits 0x0-0x5.7 (6)
 "åäö"
-string> toutf16 | ., fromutf16
+null> "åäö" | toutf16 | ., fromutf16
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
 0x0|ff fe e5 00 e4 00 f6 00|                       |........|       |.: raw bits 0x0-0x7.7 (8)
 "åäö"
-string> toutf16le | ., fromutf16le
+null> "åäö" | toutf16le | ., fromutf16le
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
 0x0|e5 00 e4 00 f6 00|                             |......|         |.: raw bits 0x0-0x5.7 (6)
 "åäö"
-string> toutf16be | ., fromutf16be
+null> "åäö" | toutf16be | ., fromutf16be
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
 0x0|00 e5 00 e4 00 f6|                             |......|         |.: raw bits 0x0-0x5.7 (6)
 "åäö"
-string> ^D
+null> [97,98,99] | fromiso8859_1 | ., toiso8859_1
+"abc"
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|61 62 63|                                      |abc|            |.: raw bits 0x0-0x2.7 (3)
+null> [97,98,99] | fromutf8 | ., toutf8
+"abc"
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|61 62 63|                                      |abc|            |.: raw bits 0x0-0x2.7 (3)
+null> [97,0,98,0,99,0] | fromutf16 | ., toutf16
+"abc"
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|ff fe 61 00 62 00 63 00|                       |..a.b.c.|       |.: raw bits 0x0-0x7.7 (8)
+null> [97,0,98,0,99,0] | fromutf16le | ., toutf16le
+"abc"
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|61 00 62 00 63 00|                             |a.b.c.|         |.: raw bits 0x0-0x5.7 (6)
+null> [0,97,0,98,0,99] | fromutf16be | ., toutf16be
+"abc"
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|00 61 00 62 00 63|                             |.a.b.c|         |.: raw bits 0x0-0x5.7 (6)
+null> ^D