From 149cb3f45a17223d172869787321237cdb3d814f Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Mon, 10 Jan 2022 18:25:36 +0100 Subject: [PATCH] interp: Add torepr/0 that converts decode value into what it reptresents Ex: fq -d msgpack torepr file.msgpack Willoutput the JSON representation of the msgpack Make per format *_torepr functions internal --- doc/usage.md | 2 ++ format/bencode/bencode.go | 1 + format/bencode/bencode.jq | 6 ++--- format/bencode/testdata/bbb.fqtest | 2 +- format/bson/bson.go | 1 + format/bson/bson.jq | 12 ++++----- format/bson/testdata/test.fqtest | 2 +- format/msgpack/msgpack.go | 1 + format/msgpack/msgpack.jq | 4 +-- format/msgpack/testdata/ints.fqtest | 2 +- format/msgpack/testdata/test.fqtest | 2 +- pkg/decode/format.go | 1 + pkg/interp/decode.go | 1 + pkg/interp/formats.jq | 41 ++++++++++++++++++++--------- pkg/interp/testdata/torepr.fqtest | 6 +++++ 15 files changed, 56 insertions(+), 28 deletions(-) create mode 100644 pkg/interp/testdata/torepr.fqtest diff --git a/doc/usage.md b/doc/usage.md index eff61087..1f2f138d 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -281,6 +281,8 @@ unary uses input and if more than one argument all as arguments ignoring the inp - `toactual/0` actual value (decoded etc) - `tosym/0` symbolic value (mapped etc) - `todescription/0` description of value + - `torepr/0` convert decode value into what it reptresents. For example convert msgpack decode value + into a value representing its JSON representation. - All regexp functions work with buffers as input and pattern argument with these differences from the string versions: - All offset and length will be in bytes. diff --git a/format/bencode/bencode.go b/format/bencode/bencode.go index 7863de77..c02047a4 100644 --- a/format/bencode/bencode.go +++ b/format/bencode/bencode.go @@ -19,6 +19,7 @@ func init() { Description: "BitTorrent bencoding", DecodeFn: decodeBencode, Files: bencodeFS, + ToRepr: "_bencode_torepr", }) } diff --git a/format/bencode/bencode.jq b/format/bencode/bencode.jq index 1c769b82..54d68ff0 100644 --- a/format/bencode/bencode.jq +++ b/format/bencode/bencode.jq @@ -1,7 +1,7 @@ -def bencode_torepr: +def _bencode_torepr: def _f: - if .type == "string" then .value - elif .type == "integer" then .value + if .type == "string" then .value | tovalue + elif .type == "integer" then .value | tovalue elif .type == "list" then .values | map(_f) elif .type == "dictionary" then ( .pairs diff --git a/format/bencode/testdata/bbb.fqtest b/format/bencode/testdata/bbb.fqtest index 23322960..e67c77af 100644 --- a/format/bencode/testdata/bbb.fqtest +++ b/format/bencode/testdata/bbb.fqtest @@ -275,7 +275,7 @@ $ fq -d bencode v bbb.torrent 0x3780|62 33 64 2e 72 65 6e 64 65 72 66 61 72 6d 69 6e|b3d.renderfarmin| 0x3790|67 2e 6e 65 74 |g.net | 0x3790| 65| | e| | end: "e" (valid) 0x3795-0x3795.7 (1) -$ fq -d bencode bencode_torepr bbb.torrent +$ fq -d bencode torepr bbb.torrent { "announce": "udp://tracker.openbittorrent.com:80/announce", "announce-list": [ diff --git a/format/bson/bson.go b/format/bson/bson.go index 737e3496..e6b75bef 100644 --- a/format/bson/bson.go +++ b/format/bson/bson.go @@ -21,6 +21,7 @@ func init() { Description: "Binary JSON", DecodeFn: decodeBSON, Files: bsonFS, + ToRepr: "_bson_torepr", }) } diff --git a/format/bson/bson.jq b/format/bson/bson.jq index 2ff5792e..7624c6bf 100644 --- a/format/bson/bson.jq +++ b/format/bson/bson.jq @@ -1,18 +1,18 @@ -def bson_torepr: - def _torepr: +def _bson_torepr: + def _f: ( if .type == null or .type == "array" then ( .value.elements - | map(_torepr) + | map(_f) ) elif .type == "document" then ( .value.elements - | map({key: .name, value: _torepr}) + | map({key: .name, value: _f}) | from_entries ) elif .type == "boolean" then .value != 0 - else .value + else .value | tovalue end ); ( {type: "document", value: .} - | _torepr + | _f ); diff --git a/format/bson/testdata/test.fqtest b/format/bson/testdata/test.fqtest index e9c30dd0..e90ec352 100644 --- a/format/bson/testdata/test.fqtest +++ b/format/bson/testdata/test.fqtest @@ -60,7 +60,7 @@ $ fq -d bson verbose /test.bson 0x60| 6e 75 6c| nul| name: "null" 0x6d-0x71.7 (5) 0x70|6c 00 |l. | 0x70| 00| | .| | terminator: 0 (valid) 0x72-0x72.7 (1) -$ fq -d bson bson_torepr /test.bson +$ fq -d bson torepr /test.bson { "array": [ 1, diff --git a/format/msgpack/msgpack.go b/format/msgpack/msgpack.go index 6b861af6..d81467ae 100644 --- a/format/msgpack/msgpack.go +++ b/format/msgpack/msgpack.go @@ -22,6 +22,7 @@ func init() { Description: "MessagePack", DecodeFn: decodeMsgPack, Files: msgPackFS, + ToRepr: "_msgpack_torepr", }) } diff --git a/format/msgpack/msgpack.jq b/format/msgpack/msgpack.jq index 18c8efc7..126456b5 100644 --- a/format/msgpack/msgpack.jq +++ b/format/msgpack/msgpack.jq @@ -1,4 +1,4 @@ -def msgpack_torepr: +def _msgpack_torepr: def _f: ( if .type | . == "fixmap" or . == "map16" or . == "map32" then ( .pairs @@ -7,7 +7,7 @@ def msgpack_torepr: ) elif .type | . == "fixarray" or . == "array16" or . == "array32" then .elements | map(_f) elif .type | . == "bin8" or . == "bin16" or . == "bin32" then .value | tostring - else .value + else .value | tovalue end ); _f; diff --git a/format/msgpack/testdata/ints.fqtest b/format/msgpack/testdata/ints.fqtest index b4ffdd4e..f558bd4d 100644 --- a/format/msgpack/testdata/ints.fqtest +++ b/format/msgpack/testdata/ints.fqtest @@ -83,7 +83,7 @@ $ fq -d msgpack v ints.msgpack 0x20| d2 | . | type: "int32" (0xd2) 0x2d-0x2d.7 (1) 0x20| 80 00| ..| value: -2147483647 0x2e-0x31.7 (4) 0x30|00 01| |..| | -$ fq -d msgpack msgpack_torepr ints.msgpack +$ fq -d msgpack torepr ints.msgpack [ 0, 1, diff --git a/format/msgpack/testdata/test.fqtest b/format/msgpack/testdata/test.fqtest index d69bddfb..0eecddca 100644 --- a/format/msgpack/testdata/test.fqtest +++ b/format/msgpack/testdata/test.fqtest @@ -84,7 +84,7 @@ $ fq -d msgpack v test.msgpack 0x40|6c 6c |ll | | | | value{}: 0x42-0x42.7 (1) 0x40| c0| | .| | type: "nil" (0xc0) 0x42-0x42.7 (1) -$ fq -d msgpack msgpack_torepr test.msgpack +$ fq -d msgpack torepr test.msgpack { "array": [ 1, diff --git a/pkg/decode/format.go b/pkg/decode/format.go index b29dbf93..fa006cad 100644 --- a/pkg/decode/format.go +++ b/pkg/decode/format.go @@ -19,6 +19,7 @@ type Format struct { RootName string Dependencies []Dependency Files fs.ReadDirFS + ToRepr string } func FormatFn(d func(d *D, in interface{}) interface{}) Group { diff --git a/pkg/interp/decode.go b/pkg/interp/decode.go index a965bc29..40fca157 100644 --- a/pkg/interp/decode.go +++ b/pkg/interp/decode.go @@ -75,6 +75,7 @@ func (i *Interp) _registry(c interface{}, a []interface{}) interface{} { "probe_order": f.ProbeOrder, "root_name": f.RootName, "root_array": f.RootArray, + "to_repr": f.ToRepr, } var dependenciesVs []interface{} diff --git a/pkg/interp/formats.jq b/pkg/interp/formats.jq index 2a574083..8e2e0b8c 100644 --- a/pkg/interp/formats.jq +++ b/pkg/interp/formats.jq @@ -1,16 +1,31 @@ # note this is a "dynamic" include, outputted string will be used as source -( [ ( _registry.groups - | to_entries[] - # TODO: nicer way to skip "all" which also would override builtin all/* - | select(.key != "all") - | "def \(.key)($opts): decode(\(.key | tojson); $opts);" - , "def \(.key): decode(\(.key | tojson); {});" - ) - , ( _registry.formats[] - | select(.files) - | .files[] - ) - ] +def _formats_source: + ( [ ( _registry.groups + | to_entries[] + # TODO: nicer way to skip "all" which also would override builtin all/* + | select(.key != "all") + | "def \(.key)($opts): decode(\(.key | tojson); $opts);" + , "def \(.key): decode(\(.key | tojson); {});" + ) + , ( _registry.formats[] + | select(.files) + | .files[] + ) + , ( "def torepr:" + , " ( format as $f" + , " | if $f == null then error(\"value is not a format root\") end" + , " | if false then error(\"unreachable\")" + , ( _registry.formats[] + | select(.to_repr != "") + | " elif $f == \(.name | tojson) then \(.to_repr)" + ) + , " else error(\"format has no torepr\")" + , " end" + , " );" + ) + ] | join("\n") -) + ); + +_formats_source diff --git a/pkg/interp/testdata/torepr.fqtest b/pkg/interp/testdata/torepr.fqtest new file mode 100644 index 00000000..db5c4c27 --- /dev/null +++ b/pkg/interp/testdata/torepr.fqtest @@ -0,0 +1,6 @@ +$ fq -i +null> torepr +error: value is not a format root +null> "{}" | json | torepr +error: format has no torepr +null> ^D