interp: Cast jq value to go value properly for encoding functions

Some encoding fuctions accept binary used string as input type, should be any.
Add cast helper functions, hopefully can be useful in future for even nicer
function bindings api.

internal/gojqextra/error.go

@@ -44,6 +44,16 @@ func (err FuncTypeError) Error() string {
 	return err.Name + " cannot be applied to: " + TypeErrorPreview(err.V)
 }
 
+type FuncArgTypeError struct {
+	Name    string
+	ArgName string
+	V       any
+}
+
+func (err FuncArgTypeError) Error() string {
+	return fmt.Sprintf("%s %s argument cannot be: %s", err.Name, err.ArgName, TypeErrorPreview(err.V))
+}
+
 type FuncTypeNameError struct {
 	Name string
 	Typ  string

internal/gojqextra/types.go

@@ -4,12 +4,157 @@ package gojqextra
 import (
 	"bytes"
 	"fmt"
+	"math"
+	"math/big"
+	"reflect"
+	"strconv"
 
 	"github.com/wader/fq/internal/colorjson"
 
 	"github.com/wader/gojq"
 )
 
+// Cast gojq value to go value
+//nolint: forcetypeassert, unconvert
+func CastFn[T any](v any, structFn func(input map[string]any, result any) error) (T, bool) {
+	var t T
+	switch any(t).(type) {
+	case bool:
+		switch v := v.(type) {
+		case bool:
+			return any(v).(T), true
+		case gojq.JQValue:
+			return CastFn[T](v.JQValueToGoJQ(), structFn)
+		default:
+			return t, false
+		}
+	case int:
+		switch v := v.(type) {
+		case int:
+			return any(v).(T), true
+		case *big.Int:
+			if !v.IsInt64() {
+				return t, false
+			}
+			ci := v.Int64()
+			if ci < math.MinInt || ci > math.MaxInt {
+				return t, false
+			}
+			return any(int(ci)).(T), true
+		case float64:
+			return any(int(v)).(T), true
+		case gojq.JQValue:
+			return CastFn[T](v.JQValueToGoJQ(), structFn)
+		default:
+			return t, false
+		}
+	case float64:
+		switch v := v.(type) {
+		case float64:
+			return any(v).(T), true
+		case int:
+			return any(float64(v)).(T), true
+		case *big.Int:
+			if v.IsInt64() {
+				return any(float64(v.Int64())).(T), true
+			}
+			// TODO: use *big.Float SetInt
+			if f, err := strconv.ParseFloat(v.String(), 64); err == nil {
+				return any(f).(T), true
+			}
+			return any(float64(math.Inf(v.Sign()))).(T), true
+		case gojq.JQValue:
+			return CastFn[T](v.JQValueToGoJQ(), structFn)
+		default:
+			return t, false
+		}
+	case *big.Int:
+		switch v := v.(type) {
+		case *big.Int:
+			return any(v).(T), true
+		case int:
+			return any(new(big.Int).SetInt64(int64(v))).(T), true
+		case float64:
+			return any(new(big.Int).SetInt64(int64(v))).(T), true
+		case gojq.JQValue:
+			return CastFn[T](v.JQValueToGoJQ(), structFn)
+		default:
+			return t, false
+		}
+	case string:
+		switch v := v.(type) {
+		case string:
+			return any(v).(T), true
+		case gojq.JQValue:
+			return CastFn[T](v.JQValueToGoJQ(), structFn)
+		default:
+			return t, false
+		}
+	case map[string]any:
+		switch v := v.(type) {
+		case map[string]any:
+			return any(v).(T), true
+		case nil:
+			// return empty instantiated map, not nil map
+			return any(map[string]any{}).(T), true
+		case gojq.JQValue:
+			return CastFn[T](v.JQValueToGoJQ(), structFn)
+		default:
+			return t, false
+		}
+	case []any:
+		switch v := v.(type) {
+		case []any:
+			return any(v).(T), true
+		case nil:
+			return t, true
+		case gojq.JQValue:
+			return CastFn[T](v.JQValueToGoJQ(), structFn)
+		default:
+			return t, false
+		}
+	default:
+		ft := reflect.TypeOf(&t)
+		if ft.Elem().Kind() == reflect.Struct {
+			m := map[string]any{}
+			switch v := v.(type) {
+			case map[string]any:
+				m = v
+			case nil:
+				// nop use instantiated map
+			case gojq.JQValue:
+				if jm, ok := Cast[map[string]any](v.JQValueToGoJQ()); ok {
+					m = jm
+				}
+			default:
+				return t, false
+			}
+
+			err := structFn(m, &t)
+			if err != nil {
+				return t, false
+			}
+
+			return t, true
+		} else if ft.Elem().Kind() == reflect.Interface {
+			// TODO: panic on non any interface?
+			// ignore failed type assert as v can be nil
+			cv, ok := any(v).(T)
+			if !ok && v != nil {
+				return cv, false
+			}
+
+			return cv, true
+		}
+
+		panic(fmt.Sprintf("unsupported type %s", ft.Elem().Kind()))
+	}
+}
+
+func Cast[T any](v any) (T, bool) {
+	return CastFn[T](v, nil)
+}
+
 // array
 
 var _ gojq.JQValue = Array{}

pkg/interp/encoding.go

@@ -35,6 +35,9 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+// TODO: Fn1, Fn2 etc?
+// TODO: struct arg, own reflect code? no need for refs etc
+
 // TODO: xml default indent?
 // TODO: query dup key
 // TODO: walk tostring tests
@@ -81,47 +84,6 @@ func norm(v any) any {
 	}
 }
 
-func addFuncOpts[TOpt any, Tc any](name string, fn func(c Tc, opts TOpt) any) {
-	if name[0] != '_' {
-		panic(fmt.Sprintf("invalid addFunc name %q", name))
-	}
-	functionRegisterFns = append(
-		functionRegisterFns,
-		func(i *Interp) []Function {
-			return []Function{{
-				name, 1, 1, func(c any, a []any) any {
-					var opts TOpt
-					if a[0] != nil {
-						optsM, ok := a[0].(map[string]any)
-						if !ok {
-							return fmt.Errorf("options %v not a object", a[0])
-						}
-						_ = mapToStruct(optsM, &opts)
-					}
-
-					cv, ok := gojqextra.ToGoJQValue(c)
-					if !ok {
-						return fmt.Errorf("invalid jq value %#v", cv)
-					}
-
-					var ct Tc
-					switch cv.(type) {
-					case nil:
-						// TODO: better way to check if Tc can be nil?
-					default:
-						ct, ok = cv.(Tc)
-						if !ok {
-							return gojqextra.FuncTypeError{Name: name[1:], V: c}
-						}
-					}
-
-					return fn(ct, opts)
-				},
-				nil,
-			}}
-		})
-}
-
 func addFunc[Tc any](name string, fn func(c Tc) any) {
 	if name[0] != '_' {
 		panic(fmt.Sprintf("invalid addFunc name %q", name))
@@ -131,22 +93,37 @@ func addFunc[Tc any](name string, fn func(c Tc) any) {
 		func(i *Interp) []Function {
 			return []Function{{
 				name, 0, 0, func(c any, a []any) any {
-					cv, ok := gojqextra.ToGoJQValue(c)
+					cv, ok := gojqextra.CastFn[Tc](c, mapToStruct)
 					if !ok {
-						return fmt.Errorf("invalid jq value %#v", cv)
-					}
-					var ct Tc
-					switch cv.(type) {
-					case nil:
-						// TODO: better way to check if Tc can be nil?
-					default:
-						ct, ok = cv.(Tc)
-						if !ok {
-							return gojqextra.FuncTypeError{Name: name[1:], V: c}
-						}
+						return gojqextra.FuncTypeError{Name: name[1:], V: c}
 					}
 
-					return fn(ct)
+					return fn(cv)
+				},
+				nil,
+			}}
+		})
+}
+
+func addFunc1[Tc any, Ta0 any](name string, fn func(c Tc, a0 Ta0) any) {
+	if name[0] != '_' {
+		panic(fmt.Sprintf("invalid addFunc name %q", name))
+	}
+	functionRegisterFns = append(
+		functionRegisterFns,
+		func(i *Interp) []Function {
+			return []Function{{
+				name, 1, 1, func(c any, a []any) any {
+					cv, ok := gojqextra.CastFn[Tc](c, mapToStruct)
+					if !ok {
+						return gojqextra.FuncTypeError{Name: name[1:], V: c}
+					}
+					a0, ok := gojqextra.CastFn[Ta0](a[0], mapToStruct)
+					if !ok {
+						return gojqextra.FuncArgTypeError{Name: name[1:], ArgName: "first", V: c}
+					}
+
+					return fn(cv, a0)
 				},
 				nil,
 			}}
@@ -197,7 +174,7 @@ func init() {
 	type ToJSONOpts struct {
 		Indent int
 	}
-	addFuncOpts("_tojson", func(c any, opts ToJSONOpts) any {
+	addFunc1("_tojson", func(c any, opts ToJSONOpts) any {
 		// TODO: share
 		cj := colorjson.NewEncoder(
 			false,
@@ -347,7 +324,7 @@ func init() {
 
 		return f(n, nil)
 	}
-	addFuncOpts("_fromxml", func(s string, opts FromXMLOpts) any {
+	addFunc1("_fromxml", func(s string, opts FromXMLOpts) any {
 		if opts.Array {
 			return fromXMLArray(s)
 		}
@@ -531,7 +508,7 @@ func init() {
 
 		return bb.String()
 	}
-	addFuncOpts("_toxml", func(c any, opts ToXMLOpts) any {
+	addFunc1("_toxml", func(c any, opts ToXMLOpts) any {
 		switch c := c.(type) {
 		case map[string]any:
 			return toXMLObject(c, opts)
@@ -701,7 +678,7 @@ func init() {
 
 		return f(doc.FirstChild)
 	}
-	addFuncOpts("_fromhtml", func(s string, opts FromHTMLOpts) any {
+	addFunc1("_fromhtml", func(s string, opts FromHTMLOpts) any {
 		if opts.Array {
 			return fromHTMLArray(s)
 		}
@@ -744,7 +721,7 @@ func init() {
 		Comma   string
 		Comment string
 	}
-	addFuncOpts("_fromcsv", func(s string, opts FromCSVOpts) any {
+	addFunc1("_fromcsv", func(s string, opts FromCSVOpts) any {
 		var rvs []any
 		r := csv.NewReader(strings.NewReader(s))
 		r.TrimLeadingSpace = true
@@ -772,7 +749,7 @@ func init() {
 	type ToCSVOpts struct {
 		Comma string
 	}
-	addFuncOpts("_tocsv", func(c []any, opts ToCSVOpts) any {
+	addFunc1("_tocsv", func(c []any, opts ToCSVOpts) any {
 		b := &bytes.Buffer{}
 		w := csv.NewWriter(b)
 		if opts.Comma != "" {
@@ -839,7 +816,7 @@ func init() {
 	type FromBase64Opts struct {
 		Encoding string
 	}
-	addFuncOpts("_frombase64", func(s string, opts FromBase64Opts) any {
+	addFunc1("_frombase64", func(s string, opts FromBase64Opts) any {
 		b, err := base64Encoding(opts.Encoding).DecodeString(s)
 		if err != nil {
 			return err
@@ -853,7 +830,7 @@ func init() {
 	type ToBase64Opts struct {
 		Encoding string
 	}
-	addFuncOpts("_tobase64", func(c any, opts ToBase64Opts) any {
+	addFunc1("_tobase64", func(c any, opts ToBase64Opts) any {
 		br, err := toBitReader(c)
 		if err != nil {
 			return err
@@ -972,7 +949,7 @@ func init() {
 		return m
 	})
 	addFunc("_tourl", func(c map[string]any) any {
-		str := func(v any) string { s, _ := gojqextra.ToString(v); return s }
+		str := func(v any) string { s, _ := gojqextra.Cast[string](v); return s }
 		u := url.URL{
 			Scheme:   str(c["scheme"]),
 			Host:     str(c["host"]),
@@ -980,18 +957,20 @@ func init() {
 			Fragment: str(c["fragment"]),
 		}
 
-		if um, ok := gojqextra.ToObject(c["user"]); ok {
+		if um, ok := gojqextra.Cast[map[string]any](c["user"]); ok {
 			username, password := str(um["username"]), str(um["password"])
-			if password == "" {
-				u.User = url.User(username)
-			} else {
-				u.User = url.UserPassword(username, password)
+			if username != "" {
+				if password == "" {
+					u.User = url.User(username)
+				} else {
+					u.User = url.UserPassword(username, password)
+				}
 			}
 		}
-		if s, ok := gojqextra.ToString(c["rawquery"]); ok {
+		if s, ok := gojqextra.Cast[string](c["rawquery"]); ok {
 			u.RawQuery = s
 		}
-		if qm, ok := gojqextra.ToObject(c["query"]); ok {
+		if qm, ok := gojqextra.Cast[map[string]any](c["query"]); ok {
 			u.RawQuery = toURLValues(qm).Encode()
 		}
 
@@ -1025,7 +1004,7 @@ func init() {
 	type ToHashOpts struct {
 		Name string
 	}
-	addFuncOpts("_tohash", func(c any, opts ToHashOpts) any {
+	addFunc1("_tohash", func(c any, opts ToHashOpts) any {
 		inBR, err := toBitReader(c)
 		if err != nil {
 			return err
@@ -1157,7 +1136,7 @@ func init() {
 	type ToStrEncodingOpts struct {
 		Encoding string
 	}
-	addFuncOpts("_tostrencoding", func(c string, opts ToStrEncodingOpts) any {
+	addFunc1("_tostrencoding", func(c string, opts ToStrEncodingOpts) any {
 		h := strEncodingFn(opts.Encoding)
 		if h == nil {
 			return fmt.Errorf("unknown string encoding %s", opts.Encoding)
@@ -1178,7 +1157,7 @@ func init() {
 	type FromStrEncodingOpts struct {
 		Encoding string
 	}
-	addFuncOpts("_fromstrencoding", func(c string, opts FromStrEncodingOpts) any {
+	addFunc1("_fromstrencoding", func(c any, opts FromStrEncodingOpts) any {
 		inBR, err := toBitReader(c)
 		if err != nil {
 			return err

pkg/interp/interp.go

@@ -1147,23 +1147,12 @@ func mapToStruct(m map[string]any, v any) error {
 			return camelToSnake(fieldName) == mapKey
 		},
 		DecodeHook: func(
-			f reflect.Type,
-			t reflect.Type,
-			data any) (any, error) {
+			f reflect.Value,
+			t reflect.Value) (any, error) {
 
-			if t.Kind() == reflect.Slice && t.Elem().Kind() == reflect.Uint8 {
-				switch d := data.(type) {
-				case string:
-					return []byte(d), nil
-				}
-			} else {
-				switch d := data.(type) {
-				case *big.Int:
-					return d.Uint64(), nil
-				}
-			}
+			// log.Printf("f: %#+v -> t: %#+v\n", f, t)
 
-			return data, nil
+			return f.Interface(), nil
 		},
 		Result: v,
 	})

pkg/interp/testdata/encoding/hash.fqtest

@@ -0,0 +1,19 @@
+$ fq -i
+null> "test" | tomd4, tomd5, tosha1, tosha256, tosha512, tosha3_224, tosha3_256, tosha3_384, tosha3_512 | tohex
+"db346d691d7acc4dc2625db19f9e3f52"
+"098f6bcd4621d373cade4e832627b4f6"
+"a94a8fe5ccb19ba61c4c0873d391e987982fbbd3"
+"9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
+"ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff"
+"3797bf0afbbfca4a7bbba7602a2b552746876517a7f9b7ce2db0ae7b"
+"36f028580bb02cc8272a9a020f4200e346e276ae664e45ee80745574e2f5ab80"
+"e516dabb23b6e30026863543282780a3ae0dccf05551cf0295178d7ff0f1b41eecb9db3ff219007c4e097260d58621bd"
+"9ece086e9bac491fac5c1d1046ca11d737b92a2b2ebd93f005d7b710110c0a678288166e7fbe796883a4f2e9b3ca9f484f521d0ce464345cc1aec96779149c14"
+null> 0xf08 | tobits | .[:4,5,6,7,8,9] | tomd5 | tohex
+"8c493a43d8c1ef798860bb02b62e8e79"
+"8c493a43d8c1ef798860bb02b62e8e79"
+"8c493a43d8c1ef798860bb02b62e8e79"
+"8c493a43d8c1ef798860bb02b62e8e79"
+"8c493a43d8c1ef798860bb02b62e8e79"
+"bdf26d2a670238e9a568e34ee02ca31c"
+null> ^D

pkg/interp/testdata/encoding/string.fqtest

@@ -1,26 +1,42 @@
-$ fq -i '"åäö"'
-string> toutf8 | ., fromutf8
-   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|c3 a5 c3 a4 c3 b6|                             |......|         |.: raw bits 0x0-0x5.7 (6)
-"åäö"
-string> toiso8859_1 | ., fromiso8859_1
+$ fq -i
+null> "åäö" | toiso8859_1 | ., fromiso8859_1
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
 0x0|e5 e4 f6|                                      |...|            |.: raw bits 0x0-0x2.7 (3)
 "åäö"
-string> toutf8 | ., fromutf8
+null> "åäö" | toutf8 | ., fromutf8
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
 0x0|c3 a5 c3 a4 c3 b6|                             |......|         |.: raw bits 0x0-0x5.7 (6)
 "åäö"
-string> toutf16 | ., fromutf16
+null> "åäö" | toutf16 | ., fromutf16
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
 0x0|ff fe e5 00 e4 00 f6 00|                       |........|       |.: raw bits 0x0-0x7.7 (8)
 "åäö"
-string> toutf16le | ., fromutf16le
+null> "åäö" | toutf16le | ., fromutf16le
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
 0x0|e5 00 e4 00 f6 00|                             |......|         |.: raw bits 0x0-0x5.7 (6)
 "åäö"
-string> toutf16be | ., fromutf16be
+null> "åäö" | toutf16be | ., fromutf16be
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
 0x0|00 e5 00 e4 00 f6|                             |......|         |.: raw bits 0x0-0x5.7 (6)
 "åäö"
-string> ^D
+null> [97,98,99] | fromiso8859_1 | ., toiso8859_1
+"abc"
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|61 62 63|                                      |abc|            |.: raw bits 0x0-0x2.7 (3)
+null> [97,98,99] | fromutf8 | ., toutf8
+"abc"
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|61 62 63|                                      |abc|            |.: raw bits 0x0-0x2.7 (3)
+null> [97,0,98,0,99,0] | fromutf16 | ., toutf16
+"abc"
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|ff fe 61 00 62 00 63 00|                       |..a.b.c.|       |.: raw bits 0x0-0x7.7 (8)
+null> [97,0,98,0,99,0] | fromutf16le | ., toutf16le
+"abc"
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|61 00 62 00 63 00|                             |a.b.c.|         |.: raw bits 0x0-0x5.7 (6)
+null> [0,97,0,98,0,99] | fromutf16be | ., toutf16be
+"abc"
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|00 61 00 62 00 63|                             |.a.b.c|         |.: raw bits 0x0-0x5.7 (6)
+null> ^D