1
1
mirror of https://github.com/wader/fq.git synced 2024-12-23 21:31:33 +03:00

Merge pull request #286 from wader/encoding-fix-incorrect-type

interp: Cast jq value to go value properly for encoding functions
This commit is contained in:
Mattias Wadman 2022-05-30 18:53:30 +02:00 committed by GitHub
commit 783387ed8a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 257 additions and 99 deletions

View File

@ -44,6 +44,16 @@ func (err FuncTypeError) Error() string {
return err.Name + " cannot be applied to: " + TypeErrorPreview(err.V)
}
type FuncArgTypeError struct {
Name string
ArgName string
V any
}
func (err FuncArgTypeError) Error() string {
return fmt.Sprintf("%s %s argument cannot be: %s", err.Name, err.ArgName, TypeErrorPreview(err.V))
}
type FuncTypeNameError struct {
Name string
Typ string

View File

@ -4,12 +4,157 @@ package gojqextra
import (
"bytes"
"fmt"
"math"
"math/big"
"reflect"
"strconv"
"github.com/wader/fq/internal/colorjson"
"github.com/wader/gojq"
)
// Cast gojq value to go value
//nolint: forcetypeassert, unconvert
func CastFn[T any](v any, structFn func(input map[string]any, result any) error) (T, bool) {
var t T
switch any(t).(type) {
case bool:
switch v := v.(type) {
case bool:
return any(v).(T), true
case gojq.JQValue:
return CastFn[T](v.JQValueToGoJQ(), structFn)
default:
return t, false
}
case int:
switch v := v.(type) {
case int:
return any(v).(T), true
case *big.Int:
if !v.IsInt64() {
return t, false
}
ci := v.Int64()
if ci < math.MinInt || ci > math.MaxInt {
return t, false
}
return any(int(ci)).(T), true
case float64:
return any(int(v)).(T), true
case gojq.JQValue:
return CastFn[T](v.JQValueToGoJQ(), structFn)
default:
return t, false
}
case float64:
switch v := v.(type) {
case float64:
return any(v).(T), true
case int:
return any(float64(v)).(T), true
case *big.Int:
if v.IsInt64() {
return any(float64(v.Int64())).(T), true
}
// TODO: use *big.Float SetInt
if f, err := strconv.ParseFloat(v.String(), 64); err == nil {
return any(f).(T), true
}
return any(float64(math.Inf(v.Sign()))).(T), true
case gojq.JQValue:
return CastFn[T](v.JQValueToGoJQ(), structFn)
default:
return t, false
}
case *big.Int:
switch v := v.(type) {
case *big.Int:
return any(v).(T), true
case int:
return any(new(big.Int).SetInt64(int64(v))).(T), true
case float64:
return any(new(big.Int).SetInt64(int64(v))).(T), true
case gojq.JQValue:
return CastFn[T](v.JQValueToGoJQ(), structFn)
default:
return t, false
}
case string:
switch v := v.(type) {
case string:
return any(v).(T), true
case gojq.JQValue:
return CastFn[T](v.JQValueToGoJQ(), structFn)
default:
return t, false
}
case map[string]any:
switch v := v.(type) {
case map[string]any:
return any(v).(T), true
case nil:
// return empty instantiated map, not nil map
return any(map[string]any{}).(T), true
case gojq.JQValue:
return CastFn[T](v.JQValueToGoJQ(), structFn)
default:
return t, false
}
case []any:
switch v := v.(type) {
case []any:
return any(v).(T), true
case nil:
return t, true
case gojq.JQValue:
return CastFn[T](v.JQValueToGoJQ(), structFn)
default:
return t, false
}
default:
ft := reflect.TypeOf(&t)
if ft.Elem().Kind() == reflect.Struct {
m := map[string]any{}
switch v := v.(type) {
case map[string]any:
m = v
case nil:
// nop use instantiated map
case gojq.JQValue:
if jm, ok := Cast[map[string]any](v.JQValueToGoJQ()); ok {
m = jm
}
default:
return t, false
}
err := structFn(m, &t)
if err != nil {
return t, false
}
return t, true
} else if ft.Elem().Kind() == reflect.Interface {
// TODO: panic on non any interface?
// ignore failed type assert as v can be nil
cv, ok := any(v).(T)
if !ok && v != nil {
return cv, false
}
return cv, true
}
panic(fmt.Sprintf("unsupported type %s", ft.Elem().Kind()))
}
}
func Cast[T any](v any) (T, bool) {
return CastFn[T](v, nil)
}
// array
var _ gojq.JQValue = Array{}

View File

@ -35,6 +35,9 @@ import (
"gopkg.in/yaml.v3"
)
// TODO: Fn1, Fn2 etc?
// TODO: struct arg, own reflect code? no need for refs etc
// TODO: xml default indent?
// TODO: query dup key
// TODO: walk tostring tests
@ -81,47 +84,6 @@ func norm(v any) any {
}
}
func addFuncOpts[TOpt any, Tc any](name string, fn func(c Tc, opts TOpt) any) {
if name[0] != '_' {
panic(fmt.Sprintf("invalid addFunc name %q", name))
}
functionRegisterFns = append(
functionRegisterFns,
func(i *Interp) []Function {
return []Function{{
name, 1, 1, func(c any, a []any) any {
var opts TOpt
if a[0] != nil {
optsM, ok := a[0].(map[string]any)
if !ok {
return fmt.Errorf("options %v not a object", a[0])
}
_ = mapToStruct(optsM, &opts)
}
cv, ok := gojqextra.ToGoJQValue(c)
if !ok {
return fmt.Errorf("invalid jq value %#v", cv)
}
var ct Tc
switch cv.(type) {
case nil:
// TODO: better way to check if Tc can be nil?
default:
ct, ok = cv.(Tc)
if !ok {
return gojqextra.FuncTypeError{Name: name[1:], V: c}
}
}
return fn(ct, opts)
},
nil,
}}
})
}
func addFunc[Tc any](name string, fn func(c Tc) any) {
if name[0] != '_' {
panic(fmt.Sprintf("invalid addFunc name %q", name))
@ -131,22 +93,37 @@ func addFunc[Tc any](name string, fn func(c Tc) any) {
func(i *Interp) []Function {
return []Function{{
name, 0, 0, func(c any, a []any) any {
cv, ok := gojqextra.ToGoJQValue(c)
cv, ok := gojqextra.CastFn[Tc](c, mapToStruct)
if !ok {
return fmt.Errorf("invalid jq value %#v", cv)
}
var ct Tc
switch cv.(type) {
case nil:
// TODO: better way to check if Tc can be nil?
default:
ct, ok = cv.(Tc)
if !ok {
return gojqextra.FuncTypeError{Name: name[1:], V: c}
}
return gojqextra.FuncTypeError{Name: name[1:], V: c}
}
return fn(ct)
return fn(cv)
},
nil,
}}
})
}
func addFunc1[Tc any, Ta0 any](name string, fn func(c Tc, a0 Ta0) any) {
if name[0] != '_' {
panic(fmt.Sprintf("invalid addFunc name %q", name))
}
functionRegisterFns = append(
functionRegisterFns,
func(i *Interp) []Function {
return []Function{{
name, 1, 1, func(c any, a []any) any {
cv, ok := gojqextra.CastFn[Tc](c, mapToStruct)
if !ok {
return gojqextra.FuncTypeError{Name: name[1:], V: c}
}
a0, ok := gojqextra.CastFn[Ta0](a[0], mapToStruct)
if !ok {
return gojqextra.FuncArgTypeError{Name: name[1:], ArgName: "first", V: c}
}
return fn(cv, a0)
},
nil,
}}
@ -197,7 +174,7 @@ func init() {
type ToJSONOpts struct {
Indent int
}
addFuncOpts("_tojson", func(c any, opts ToJSONOpts) any {
addFunc1("_tojson", func(c any, opts ToJSONOpts) any {
// TODO: share
cj := colorjson.NewEncoder(
false,
@ -347,7 +324,7 @@ func init() {
return f(n, nil)
}
addFuncOpts("_fromxml", func(s string, opts FromXMLOpts) any {
addFunc1("_fromxml", func(s string, opts FromXMLOpts) any {
if opts.Array {
return fromXMLArray(s)
}
@ -531,7 +508,7 @@ func init() {
return bb.String()
}
addFuncOpts("_toxml", func(c any, opts ToXMLOpts) any {
addFunc1("_toxml", func(c any, opts ToXMLOpts) any {
switch c := c.(type) {
case map[string]any:
return toXMLObject(c, opts)
@ -701,7 +678,7 @@ func init() {
return f(doc.FirstChild)
}
addFuncOpts("_fromhtml", func(s string, opts FromHTMLOpts) any {
addFunc1("_fromhtml", func(s string, opts FromHTMLOpts) any {
if opts.Array {
return fromHTMLArray(s)
}
@ -744,7 +721,7 @@ func init() {
Comma string
Comment string
}
addFuncOpts("_fromcsv", func(s string, opts FromCSVOpts) any {
addFunc1("_fromcsv", func(s string, opts FromCSVOpts) any {
var rvs []any
r := csv.NewReader(strings.NewReader(s))
r.TrimLeadingSpace = true
@ -772,7 +749,7 @@ func init() {
type ToCSVOpts struct {
Comma string
}
addFuncOpts("_tocsv", func(c []any, opts ToCSVOpts) any {
addFunc1("_tocsv", func(c []any, opts ToCSVOpts) any {
b := &bytes.Buffer{}
w := csv.NewWriter(b)
if opts.Comma != "" {
@ -839,7 +816,7 @@ func init() {
type FromBase64Opts struct {
Encoding string
}
addFuncOpts("_frombase64", func(s string, opts FromBase64Opts) any {
addFunc1("_frombase64", func(s string, opts FromBase64Opts) any {
b, err := base64Encoding(opts.Encoding).DecodeString(s)
if err != nil {
return err
@ -853,7 +830,7 @@ func init() {
type ToBase64Opts struct {
Encoding string
}
addFuncOpts("_tobase64", func(c any, opts ToBase64Opts) any {
addFunc1("_tobase64", func(c any, opts ToBase64Opts) any {
br, err := toBitReader(c)
if err != nil {
return err
@ -972,7 +949,7 @@ func init() {
return m
})
addFunc("_tourl", func(c map[string]any) any {
str := func(v any) string { s, _ := gojqextra.ToString(v); return s }
str := func(v any) string { s, _ := gojqextra.Cast[string](v); return s }
u := url.URL{
Scheme: str(c["scheme"]),
Host: str(c["host"]),
@ -980,18 +957,20 @@ func init() {
Fragment: str(c["fragment"]),
}
if um, ok := gojqextra.ToObject(c["user"]); ok {
if um, ok := gojqextra.Cast[map[string]any](c["user"]); ok {
username, password := str(um["username"]), str(um["password"])
if password == "" {
u.User = url.User(username)
} else {
u.User = url.UserPassword(username, password)
if username != "" {
if password == "" {
u.User = url.User(username)
} else {
u.User = url.UserPassword(username, password)
}
}
}
if s, ok := gojqextra.ToString(c["rawquery"]); ok {
if s, ok := gojqextra.Cast[string](c["rawquery"]); ok {
u.RawQuery = s
}
if qm, ok := gojqextra.ToObject(c["query"]); ok {
if qm, ok := gojqextra.Cast[map[string]any](c["query"]); ok {
u.RawQuery = toURLValues(qm).Encode()
}
@ -1025,7 +1004,7 @@ func init() {
type ToHashOpts struct {
Name string
}
addFuncOpts("_tohash", func(c any, opts ToHashOpts) any {
addFunc1("_tohash", func(c any, opts ToHashOpts) any {
inBR, err := toBitReader(c)
if err != nil {
return err
@ -1157,7 +1136,7 @@ func init() {
type ToStrEncodingOpts struct {
Encoding string
}
addFuncOpts("_tostrencoding", func(c string, opts ToStrEncodingOpts) any {
addFunc1("_tostrencoding", func(c string, opts ToStrEncodingOpts) any {
h := strEncodingFn(opts.Encoding)
if h == nil {
return fmt.Errorf("unknown string encoding %s", opts.Encoding)
@ -1178,7 +1157,7 @@ func init() {
type FromStrEncodingOpts struct {
Encoding string
}
addFuncOpts("_fromstrencoding", func(c string, opts FromStrEncodingOpts) any {
addFunc1("_fromstrencoding", func(c any, opts FromStrEncodingOpts) any {
inBR, err := toBitReader(c)
if err != nil {
return err

View File

@ -1147,23 +1147,12 @@ func mapToStruct(m map[string]any, v any) error {
return camelToSnake(fieldName) == mapKey
},
DecodeHook: func(
f reflect.Type,
t reflect.Type,
data any) (any, error) {
f reflect.Value,
t reflect.Value) (any, error) {
if t.Kind() == reflect.Slice && t.Elem().Kind() == reflect.Uint8 {
switch d := data.(type) {
case string:
return []byte(d), nil
}
} else {
switch d := data.(type) {
case *big.Int:
return d.Uint64(), nil
}
}
// log.Printf("f: %#+v -> t: %#+v\n", f, t)
return data, nil
return f.Interface(), nil
},
Result: v,
})

View File

@ -0,0 +1,19 @@
$ fq -i
null> "test" | tomd4, tomd5, tosha1, tosha256, tosha512, tosha3_224, tosha3_256, tosha3_384, tosha3_512 | tohex
"db346d691d7acc4dc2625db19f9e3f52"
"098f6bcd4621d373cade4e832627b4f6"
"a94a8fe5ccb19ba61c4c0873d391e987982fbbd3"
"9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
"ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff"
"3797bf0afbbfca4a7bbba7602a2b552746876517a7f9b7ce2db0ae7b"
"36f028580bb02cc8272a9a020f4200e346e276ae664e45ee80745574e2f5ab80"
"e516dabb23b6e30026863543282780a3ae0dccf05551cf0295178d7ff0f1b41eecb9db3ff219007c4e097260d58621bd"
"9ece086e9bac491fac5c1d1046ca11d737b92a2b2ebd93f005d7b710110c0a678288166e7fbe796883a4f2e9b3ca9f484f521d0ce464345cc1aec96779149c14"
null> 0xf08 | tobits | .[:4,5,6,7,8,9] | tomd5 | tohex
"8c493a43d8c1ef798860bb02b62e8e79"
"8c493a43d8c1ef798860bb02b62e8e79"
"8c493a43d8c1ef798860bb02b62e8e79"
"8c493a43d8c1ef798860bb02b62e8e79"
"8c493a43d8c1ef798860bb02b62e8e79"
"bdf26d2a670238e9a568e34ee02ca31c"
null> ^D

View File

@ -1,26 +1,42 @@
$ fq -i '"åäö"'
string> toutf8 | ., fromutf8
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|c3 a5 c3 a4 c3 b6| |......| |.: raw bits 0x0-0x5.7 (6)
"åäö"
string> toiso8859_1 | ., fromiso8859_1
$ fq -i
null> "åäö" | toiso8859_1 | ., fromiso8859_1
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|e5 e4 f6| |...| |.: raw bits 0x0-0x2.7 (3)
"åäö"
string> toutf8 | ., fromutf8
null> "åäö" | toutf8 | ., fromutf8
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|c3 a5 c3 a4 c3 b6| |......| |.: raw bits 0x0-0x5.7 (6)
"åäö"
string> toutf16 | ., fromutf16
null> "åäö" | toutf16 | ., fromutf16
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|ff fe e5 00 e4 00 f6 00| |........| |.: raw bits 0x0-0x7.7 (8)
"åäö"
string> toutf16le | ., fromutf16le
null> "åäö" | toutf16le | ., fromutf16le
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|e5 00 e4 00 f6 00| |......| |.: raw bits 0x0-0x5.7 (6)
"åäö"
string> toutf16be | ., fromutf16be
null> "åäö" | toutf16be | ., fromutf16be
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|00 e5 00 e4 00 f6| |......| |.: raw bits 0x0-0x5.7 (6)
"åäö"
string> ^D
null> [97,98,99] | fromiso8859_1 | ., toiso8859_1
"abc"
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|61 62 63| |abc| |.: raw bits 0x0-0x2.7 (3)
null> [97,98,99] | fromutf8 | ., toutf8
"abc"
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|61 62 63| |abc| |.: raw bits 0x0-0x2.7 (3)
null> [97,0,98,0,99,0] | fromutf16 | ., toutf16
"abc"
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|ff fe 61 00 62 00 63 00| |..a.b.c.| |.: raw bits 0x0-0x7.7 (8)
null> [97,0,98,0,99,0] | fromutf16le | ., toutf16le
"abc"
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|61 00 62 00 63 00| |a.b.c.| |.: raw bits 0x0-0x5.7 (6)
null> [0,97,0,98,0,99] | fromutf16be | ., toutf16be
"abc"
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|00 61 00 62 00 63| |.a.b.c| |.: raw bits 0x0-0x5.7 (6)
null> ^D