wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-12-26 06:51:35 +03:00
Code Issues Projects Releases Wiki Activity

Merge pull request #64 from wader/bencode

Browse Source 
bencode: Add decoder
This commit is contained in:
Mattias Wadman 2022-01-10 10:56:46 +01:00 committed by GitHub
commit ece2fd78ec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 443 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -104,7 +104,7 @@ go run fq.go
[./formats_list.jq]: sh-start
aac_frame, adts, adts_frame, apev2, ar, av1_ccr, av1_frame, av1_obu, avc_annexb, avc_au, avc_dcr, avc_nalu, avc_pps, avc_sei, avc_sps, bsd_loopback_frame, bson, bzip2, dns, dns_tcp, elf, ether8023_frame, exif, flac, flac_frame, flac_metadatablock, flac_metadatablocks, flac_picture, flac_streaminfo, gif, gzip, hevc_annexb, hevc_au, hevc_dcr, hevc_nalu, icc_profile, icmp, id3v1, id3v11, id3v2, ipv4_packet, jpeg, json, matroska, mp3, mp3_frame, mp4, mpeg_asc, mpeg_es, mpeg_pes, mpeg_pes_packet, mpeg_spu, mpeg_ts, ogg, ogg_page, opus_packet, pcap, pcapng, png, protobuf, protobuf_widevine, pssh_playready, raw, sll2_packet, sll_packet, tar, tcp_segment, tiff, udp_datagram, vorbis_comment, vorbis_packet, vp8_frame, vp9_cfm, vp9_frame, vpx_ccr, wav, webp, xing, zip
aac_frame, adts, adts_frame, apev2, ar, av1_ccr, av1_frame, av1_obu, avc_annexb, avc_au, avc_dcr, avc_nalu, avc_pps, avc_sei, avc_sps, bencode, bsd_loopback_frame, bson, bzip2, dns, dns_tcp, elf, ether8023_frame, exif, flac, flac_frame, flac_metadatablock, flac_metadatablocks, flac_picture, flac_streaminfo, gif, gzip, hevc_annexb, hevc_au, hevc_dcr, hevc_nalu, icc_profile, icmp, id3v1, id3v11, id3v2, ipv4_packet, jpeg, json, matroska, mp3, mp3_frame, mp4, mpeg_asc, mpeg_es, mpeg_pes, mpeg_pes_packet, mpeg_spu, mpeg_ts, ogg, ogg_page, opus_packet, pcap, pcapng, png, protobuf, protobuf_widevine, pssh_playready, raw, sll2_packet, sll_packet, tar, tcp_segment, tiff, udp_datagram, vorbis_comment, vorbis_packet, vp8_frame, vp9_cfm, vp9_frame, vpx_ccr, wav, webp, xing, zip
[#]: sh-end

1
doc/formats.md
View File

@ -19,6 +19,7 @@
|`avc_pps` |H.264/AVC&nbsp;Picture&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`avc_sei` |H.264/AVC&nbsp;Supplemental&nbsp;Enhancement&nbsp;Information |<sub></sub>|
|`avc_sps` |H.264/AVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`bencode` |BitTorrent&nbsp;bencoding |<sub></sub>|
|`bsd_loopback_frame` |BSD&nbsp;loopback&nbsp;frame |<sub>`ipv4_packet`</sub>|
|`bson` |Binary&nbsp;JSON |<sub></sub>|
|`bzip2` |bzip2&nbsp;compression |<sub>`probe`</sub>|

24
doc/formats.svg
View File

@ -1568,23 +1568,29 @@
<path fill="none" stroke="black" d="M2670,-1423C2637.45,-1423 2676.67,-1472.76 2652,-1494 2590.91,-1546.61 2368.57,-1508.04 2291,-1530 2290.26,-1530.21 2289.53,-1530.43 2288.79,-1530.67"/>
<polygon fill="black" stroke="black" points="2287.86,-1528.39 2282.12,-1533.09 2289.54,-1533 2287.86,-1528.39"/>
</g>
<!-- bson -->
<!-- bencode -->
<g id="node82" class="node">
<title>bencode</title>
<polygon fill="paleturquoise" stroke="transparent" points="2308,-1539 2308,-1558 2358,-1558 2358,-1539 2308,-1539"/>
<text text-anchor="start" x="2310" y="-1544.3" font-family="Times,serif" font-size="14.00">bencode</text>
</g>
<!-- bson -->
<g id="node83" class="node">
<title>bson</title>
<polygon fill="paleturquoise" stroke="transparent" points="2312,-1539 2312,-1558 2342,-1558 2342,-1539 2312,-1539"/>
<text text-anchor="start" x="2314" y="-1544.3" font-family="Times,serif" font-size="14.00">bson</text>
<polygon fill="paleturquoise" stroke="transparent" points="2396,-1539 2396,-1558 2426,-1558 2426,-1539 2396,-1539"/>
<text text-anchor="start" x="2398" y="-1544.3" font-family="Times,serif" font-size="14.00">bson</text>
</g>
<!-- dns_tcp -->
<g id="node83" class="node">
<g id="node84" class="node">
<title>dns_tcp</title>
<polygon fill="paleturquoise" stroke="transparent" points="2380.5,-1539 2380.5,-1558 2427.5,-1558 2427.5,-1539 2380.5,-1539"/>
<text text-anchor="start" x="2383" y="-1544.3" font-family="Times,serif" font-size="14.00">dns_tcp</text>
<polygon fill="paleturquoise" stroke="transparent" points="2464.5,-1539 2464.5,-1558 2511.5,-1558 2511.5,-1539 2464.5,-1539"/>
<text text-anchor="start" x="2467" y="-1544.3" font-family="Times,serif" font-size="14.00">dns_tcp</text>
</g>
<!-- raw -->
<g id="node84" class="node">
<g id="node85" class="node">
<title>raw</title>
<polygon fill="paleturquoise" stroke="transparent" points="2468.5,-1539 2468.5,-1558 2493.5,-1558 2493.5,-1539 2468.5,-1539"/>
<text text-anchor="start" x="2471" y="-1544.3" font-family="Times,serif" font-size="14.00">raw</text>
<polygon fill="paleturquoise" stroke="transparent" points="2552.5,-1539 2552.5,-1558 2577.5,-1558 2577.5,-1539 2552.5,-1539"/>
<text text-anchor="start" x="2555" y="-1544.3" font-family="Times,serif" font-size="14.00">raw</text>
</g>
</g>
</svg>

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 101 KiB

After

Width:  |  Height:  |  Size: 101 KiB

1
format/all/all.go
View File

@ -6,6 +6,7 @@ import (
_ "github.com/wader/fq/format/ape"
_ "github.com/wader/fq/format/ar"
_ "github.com/wader/fq/format/av1"
_ "github.com/wader/fq/format/bencode"
_ "github.com/wader/fq/format/bson"
_ "github.com/wader/fq/format/bzip2"
_ "github.com/wader/fq/format/dns"

93
format/bencode/bencode.go Normal file
View File

@ -0,0 +1,93 @@
package bencode
import (
"embed"
"strconv"
"github.com/wader/fq/format"
"github.com/wader/fq/format/registry"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/scalar"
)
//go:embed *.jq
var bencodeFS embed.FS
func init() {
registry.MustRegister(decode.Format{
Name: format.BENCODE,
Description: "BitTorrent bencoding",
DecodeFn: decodeBencode,
Files: bencodeFS,
})
}
var typeToNames = scalar.StrToSymStr{
"d": "dictionary",
"i": "integer",
"l": "list",
"0": "string",
"1": "string",
"2": "string",
"3": "string",
"4": "string",
"5": "string",
"6": "string",
"7": "string",
"8": "string",
"9": "string",
}
func decodeStrIntUntil(b byte) func(d *decode.D) int64 {
return func(d *decode.D) int64 {
// 21 is sign + longest 64 bit in base 10
i := d.PeekFindByte(b, 21)
if i == -1 {
d.Fatalf("decodeStrIntUntil: failed to find %v", b)
}
s := d.UTF8(int(i))
n, err := strconv.ParseInt(s, 10, 64)
if err != nil {
d.Fatalf("decodeStrIntUntil: %q: %s", s, err)
}
return n
}
}
func decodeBencodeValue(d *decode.D) {
typ := d.FieldUTF8("type", 1, typeToNames)
switch typ {
case "0", "1", "2", "3", "4", "5", "6", "7", "8", "9":
d.SeekRel(-8)
length := d.FieldSFn("length", decodeStrIntUntil(':'))
d.FieldUTF8("separator", 1, d.AssertStr(":"))
d.FieldUTF8("value", int(length))
case "i":
d.FieldSFn("value", decodeStrIntUntil('e'))
d.FieldUTF8("end", 1, d.AssertStr("e"))
case "l":
d.FieldArray("values", func(d *decode.D) {
for d.PeekBits(8) != 'e' {
d.FieldStruct("value", decodeBencodeValue)
}
})
d.FieldUTF8("end", 1, d.AssertStr("e"))
case "d":
d.FieldArray("pairs", func(d *decode.D) {
for d.PeekBits(8) != 'e' {
d.FieldStruct("pair", func(d *decode.D) {
d.FieldStruct("key", decodeBencodeValue)
d.FieldStruct("value", decodeBencodeValue)
})
}
})
d.FieldUTF8("end", 1, d.AssertStr("e"))
default:
d.Fatalf("unknown type %v", typ)
}
}
func decodeBencode(d *decode.D, in interface{}) interface{} {
decodeBencodeValue(d)
return nil
}

13
format/bencode/bencode.jq Normal file
View File

@ -0,0 +1,13 @@
def bencode_torepr:
def _f:
if .type == "string" then .value
elif .type == "integer" then .value
elif .type == "list" then .values | map(_f)
elif .type == "dictionary" then
( .pairs
| map({key: (.key | _f), value: (.value | _f)})
| from_entries
)
else error("unknown type \(.type)")
end;
_f;

317
format/bencode/testdata/bbb.fqtest vendored Normal file
View File

File diff suppressed because one or more lines are too long

BIN
format/bencode/testdata/bbb.torrent vendored Normal file
View File

Binary file not shown.

1
format/format.go
View File

@ -35,6 +35,7 @@ const (
AV1_CCR = "av1_ccr"
AV1_FRAME = "av1_frame"
AV1_OBU = "av1_obu"
BENCODE = "bencode"
BZIP2 = "bzip2"
EXIF = "exif"
FLAC = "flac"

1
pkg/interp/testdata/args.fqtest vendored
View File

@ -66,6 +66,7 @@ avc_nalu H.264/AVC Network Access Layer Unit
avc_pps H.264/AVC Picture Parameter Set
avc_sei H.264/AVC Supplemental Enhancement Information
avc_sps H.264/AVC Sequence Parameter Set
bencode BitTorrent bencoding
bsd_loopback_frame BSD loopback frame
bson Binary JSON
bzip2 bzip2 compression