From a7a101ca67cb8127e511c12e4cf16d8f8a8da07f Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Thu, 22 Sep 2022 18:32:13 +0200 Subject: [PATCH] doc,help: Nicer format help and move help tests into each format Also add authors to avro and macho decoders Generated with: for i in $(go run . -r -n '_registry | [([([.files[][].name | split(".")[0]] | unique[]), (.formats[].name)] | count[] | select(.[1] == 2)[0]), (.formats[] | select(.decode_in_arg).name)] | unique[]'); do DIR=$(dirname $(find format -name $i.go)); echo "$ fq -h $i" > "$DIR/testdata/help_$i.fqtest" ; done --- doc/formats.md | 35 +- doc/formats.svg | 452 ++--- format/all/all.fqtest | 108 ++ format/all/help.fqtest | 1475 ----------------- format/asn1/asn1_ber.md | 2 +- format/asn1/testdata/help_asn1_ber.fqtest | 39 + format/avro/avro_ocf.md | 5 + format/avro/testdata/help_avro_ocf.fqtest | 28 + format/bencode/testdata/help_bencode.fqtest | 20 + .../testdata/help_bitcoin_block.fqtest | 20 + format/bson/testdata/help_bson.fqtest | 25 + format/cbor/testdata/help_cbor.fqtest | 21 + format/csv/testdata/help_csv.fqtest | 31 + format/flac/testdata/help_flac_frame.fqtest | 20 + format/macho/macho.md | 5 + format/macho/testdata/help_macho.fqtest | 26 + format/markdown/markdown.jq | 8 +- format/matroska/matroska.md | 4 +- format/matroska/testdata/help_matroska.fqtest | 29 + format/mp3/testdata/help_mp3.fqtest | 21 + format/mp4/mp4.md | 7 +- format/mp4/testdata/help_mp4.fqtest | 54 + format/mpeg/testdata/help_aac_frame.fqtest | 20 + format/mpeg/testdata/help_avc_au.fqtest | 20 + format/mpeg/testdata/help_hevc_au.fqtest | 20 + format/msgpack/testdata/help_msgpack.fqtest | 20 + format/protobuf/testdata/help_protobuf.fqtest | 20 + format/rtmp/testdata/help_rtmp.fqtest | 18 + format/wasm/testdata/help_wasm.fqtest | 29 + format/xml/html.md | 6 +- format/xml/testdata/help_html.fqtest | 81 + format/xml/testdata/help_xml.fqtest | 124 ++ format/xml/xml.md | 6 +- format/zip/testdata/help_zip.fqtest | 27 + pkg/interp/help.jq | 9 +- pkg/interp/testdata/args.fqtest | 108 -- 36 files changed, 1098 insertions(+), 1845 deletions(-) delete mode 100644 format/all/help.fqtest create mode 100644 format/asn1/testdata/help_asn1_ber.fqtest create mode 100644 format/avro/testdata/help_avro_ocf.fqtest create mode 100644 format/bencode/testdata/help_bencode.fqtest create mode 100644 format/bitcoin/testdata/help_bitcoin_block.fqtest create mode 100644 format/bson/testdata/help_bson.fqtest create mode 100644 format/cbor/testdata/help_cbor.fqtest create mode 100644 format/csv/testdata/help_csv.fqtest create mode 100644 format/flac/testdata/help_flac_frame.fqtest create mode 100644 format/macho/testdata/help_macho.fqtest create mode 100644 format/matroska/testdata/help_matroska.fqtest create mode 100644 format/mp3/testdata/help_mp3.fqtest create mode 100644 format/mp4/testdata/help_mp4.fqtest create mode 100644 format/mpeg/testdata/help_aac_frame.fqtest create mode 100644 format/mpeg/testdata/help_avc_au.fqtest create mode 100644 format/mpeg/testdata/help_hevc_au.fqtest create mode 100644 format/msgpack/testdata/help_msgpack.fqtest create mode 100644 format/protobuf/testdata/help_protobuf.fqtest create mode 100644 format/rtmp/testdata/help_rtmp.fqtest create mode 100644 format/wasm/testdata/help_wasm.fqtest create mode 100644 format/xml/testdata/help_html.fqtest create mode 100644 format/xml/testdata/help_xml.fqtest create mode 100644 format/zip/testdata/help_zip.fqtest diff --git a/doc/formats.md b/doc/formats.md index 1d9e6c39..72422436 100644 --- a/doc/formats.md +++ b/doc/formats.md @@ -162,7 +162,7 @@ Supports decoding BER, CER and DER (X.690). - Does not support specifying a schema. - Supports `torepr` but without schema all sequences and sets will be arrays. -### `frompem` can be used to decode certificates etc +### Can be used to decode certificates etc ```sh $ fq -d raw 'frompem | asn1_ber | d' cert.pem @@ -220,6 +220,11 @@ Limitations: ### References - https://avro.apache.org/docs/current/spec.html#Object+Container+Files +### Authors +- Xentripetal +xentripetal@fastmail.com +[@xentripetal](https://github.com/xentripetal) + ## bencode ### Convert represented value to JSON @@ -428,14 +433,10 @@ $ 'text' | fq -d html -o array=true ] ] ] -``` -```sh -# Decode html files to a {file: "title", ...} object +# decode html files to a {file: "title", ...} object $ fq -n -d html '[inputs | {key: input_filename, value: .html.head.title?}] | from_entries' *.html -``` -```sh # href:s in file $ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href' file.html ``` @@ -453,15 +454,20 @@ $ fq '.load_commands[] | select(.cmd=="segment_64")' file ### References - https://github.com/aidansteele/osx-abi-macho-file-format-reference +### Authors +- Sıddık AÇIL +acils@itu.edu.tr +[@Akaame](https://github.com/Akaame) + ## matroska -### Lookup element path using `matroska_path` +### Lookup element using path ```sh $ fq 'matroska_path(".Segment.Tracks[0)")' file.mkv ``` -### Get element path using `matroska_path` +### Get path to element ```sh $ fq 'grep_by(.id == "Tracks") | matroska_path' file.mkv @@ -516,14 +522,14 @@ Decode value as mp4 ... | mp4({allow_truncated:false,decode_samples:true}) ``` -### `mp4_path($path)` - Lookup mp4 box using a mp4 box path. +### Lookup mp4 box using a mp4 box path. ```sh # | mp4_path($path) -> $ fq 'mp4_path(".moov.trak[1]")' file.mp4 ``` -### `mp4_path` - Return mp4 box path for a decode value box. +### Get mp4 box path for a decode value box. ```sh # | mp4_path -> string @@ -536,9 +542,10 @@ $ fq 'grep_by(.type == "trak") | mp4_path' file.mp4 $ fq -n '"AAAAHGVsc3QAAAAAAAAAAQAAADIAAAQAAAEAAA==" | frombase64 | mp4({force:true}) | d' ``` -### Don't decode samples and manually decode first sample for first track as a `aac_frame` +### Speed up decoding by not decoding samples ```sh +# manually decode first sample as a aac_frame $ fq -o decode_samples=false '.tracks[0].samples[0] | aac_frame | d' file.mp4 ``` @@ -671,9 +678,7 @@ $ echo 'bbbccc' | fq -d xml -o seq=true # access text of the element $ echo 'bbbccc' | fq '.a.c["#text"]' "ccc" -``` -```sh # decode to object and encode to xml $ echo 'bbbccc' | fq -r -d xml -o seq=true 'toxml({indent:2})' @@ -716,11 +721,9 @@ Elements are arrays of the shape `["#text": "body text", "attr_name", {key: "att ] ] ] -``` -```sh # decode to array and encode to xml -✗ echo 'bbbccc' | fq -r -d xml -o array=true -o seq=true 'toxml({indent:2})' +$ echo 'bbbccc' | fq -r -d xml -o array=true -o seq=true 'toxml({indent:2})' bbb diff --git a/doc/formats.svg b/doc/formats.svg index 3ddbbe1e..34af8ca0 100644 --- a/doc/formats.svg +++ b/doc/formats.svg @@ -8,21 +8,21 @@ viewBox="0.00 0.00 4576.31 2076.01" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"> formats - + adts - + adts - + adts_frame adts_frame - + adts_frame - + aac_frame @@ -34,7 +34,7 @@ aac_frame - + aac_frame @@ -46,9 +46,9 @@ apev2 - + apev2 - + image @@ -66,11 +66,11 @@ jpeg - + jpeg - + exif - + icc_profile @@ -82,53 +82,53 @@ mp4 - + mp4 - + aac_frame - + av1_ccr - + av1_frame - + avc_au - + avc_dcr - + flac_frame - + flac_metadatablocks - + hevc_au - + hevc_dcr - + icc_profile - + id3v2 - + image - + jpeg - + mp3_frame - + mpeg_es - + mpeg_pes_packet - + opus_packet - + prores_frame - + protobuf_widevine - + pssh_playready - + vorbis_packet - + vp9_frame - + vpx_ccr @@ -140,11 +140,11 @@ png - + png - + icc_profile - + exif @@ -156,9 +156,9 @@ tiff - + tiff - + icc_profile @@ -170,9 +170,9 @@ webp - + webp - + vp8_frame @@ -184,7 +184,7 @@ gif - + gif @@ -196,9 +196,9 @@ ar - + ar - + probe @@ -228,9 +228,9 @@ bitcoin_blkdat - + bitcoin_blkdat - + bitcoin_block @@ -242,9 +242,9 @@ bzip2 - + bzip2 - + probe @@ -256,11 +256,11 @@ flac - + flac - + flac_metadatablocks - + flac_frame @@ -272,9 +272,9 @@ gzip - + gzip - + probe @@ -292,9 +292,9 @@ macho_fat - + macho_fat - + macho @@ -306,7 +306,7 @@ macho - + macho @@ -318,45 +318,45 @@ matroska - + matroska - + aac_frame - + av1_ccr - + av1_frame - + avc_au - + avc_dcr - + flac_frame - + flac_metadatablocks - + hevc_au - + hevc_dcr - + image - + mp3_frame - + mpeg_asc - + mpeg_pes_packet - + mpeg_spu - + opus_packet - + vorbis_packet - + vp8_frame - + vp9_cfm - + vp9_frame @@ -368,17 +368,17 @@ mp3 - + mp3 - + id3v2 - + id3v1 - + id3v11 - + apev2 - + mp3_frame @@ -396,17 +396,17 @@ ogg - + ogg - + ogg_page - + vorbis_packet - + opus_packet - + flac_metadatablock - + flac_frame @@ -418,13 +418,13 @@ pcap - + pcap - + link_frame - + tcp_stream - + ipv4_packet @@ -436,13 +436,13 @@ pcapng - + pcapng - + link_frame - + tcp_stream - + ipv4_packet @@ -460,9 +460,9 @@ tar - + tar - + probe @@ -480,13 +480,13 @@ wav - + wav - + id3v2 - + id3v1 - + id3v11 @@ -504,9 +504,9 @@ zip - + zip - + probe @@ -518,7 +518,7 @@ avro_ocf - + avro_ocf @@ -530,7 +530,7 @@ elf - + elf @@ -548,7 +548,7 @@ json - + json @@ -560,7 +560,7 @@ jsonl - + jsonl @@ -572,7 +572,7 @@ mpeg_ts - + mpeg_ts @@ -584,7 +584,7 @@ toml - + toml @@ -596,7 +596,7 @@ wasm - + wasm @@ -608,7 +608,7 @@ xml - + xml @@ -620,7 +620,7 @@ yaml - + yaml @@ -632,15 +632,15 @@ av1_frame - + av1_frame - + av1_obu av1_obu - + av1_obu @@ -652,21 +652,21 @@ avc_annexb - + avc_annexb - + avc_nalu avc_nalu - + avc_nalu - + avc_sps - + avc_pps - + avc_sei @@ -678,7 +678,7 @@ avc_sps - + avc_sps @@ -690,7 +690,7 @@ avc_pps - + avc_pps @@ -702,7 +702,7 @@ avc_sei - + avc_sei @@ -714,9 +714,9 @@ avc_au - + avc_au - + avc_nalu @@ -728,9 +728,9 @@ avc_dcr - + avc_dcr - + avc_nalu @@ -742,9 +742,9 @@ bitcoin_block - + bitcoin_block - + bitcoin_transaction @@ -756,9 +756,9 @@ bitcoin_transaction - + bitcoin_transaction - + bitcoin_script @@ -770,7 +770,7 @@ bitcoin_script - + bitcoin_script @@ -782,9 +782,9 @@ bsd_loopback_frame - + bsd_loopback_frame - + inet_packet @@ -802,9 +802,9 @@ ipv4_packet - + ipv4_packet - + ip_packet @@ -816,9 +816,9 @@ ipv6_packet - + ipv6_packet - + ip_packet @@ -836,9 +836,9 @@ ether8023_frame - + ether8023_frame - + inet_packet @@ -850,9 +850,9 @@ flac_metadatablocks - + flac_metadatablocks - + flac_metadatablock @@ -864,7 +864,7 @@ flac_frame - + flac_frame @@ -876,13 +876,13 @@ flac_metadatablock - + flac_metadatablock - + flac_streaminfo - + flac_picture - + vorbis_comment @@ -894,7 +894,7 @@ flac_streaminfo - + flac_streaminfo @@ -906,9 +906,9 @@ flac_picture - + flac_picture - + image @@ -920,9 +920,9 @@ vorbis_comment - + vorbis_comment - + flac_picture @@ -952,21 +952,21 @@ hevc_annexb - + hevc_annexb - + hevc_nalu hevc_nalu - + hevc_nalu - + hevc_vps - + hevc_pps - + hevc_sps @@ -978,7 +978,7 @@ hevc_vps - + hevc_vps @@ -990,7 +990,7 @@ hevc_pps - + hevc_pps @@ -1002,7 +1002,7 @@ hevc_sps - + hevc_sps @@ -1014,9 +1014,9 @@ hevc_au - + hevc_au - + hevc_nalu @@ -1028,9 +1028,9 @@ hevc_dcr - + hevc_dcr - + hevc_nalu @@ -1042,9 +1042,9 @@ id3v2 - + id3v2 - + image @@ -1068,9 +1068,9 @@ udp_datagram - + udp_datagram - + udp_payload @@ -1082,7 +1082,7 @@ icmp - + icmp @@ -1094,7 +1094,7 @@ icmpv6 - + icmpv6 @@ -1106,7 +1106,7 @@ tcp_segment - + tcp_segment @@ -1124,7 +1124,7 @@ exif - + exif @@ -1136,7 +1136,7 @@ icc_profile - + icc_profile @@ -1208,7 +1208,7 @@ av1_ccr - + av1_ccr @@ -1220,9 +1220,9 @@ mp3_frame - + mp3_frame - + xing @@ -1234,7 +1234,7 @@ mpeg_asc - + mpeg_asc @@ -1246,7 +1246,7 @@ mpeg_pes_packet - + mpeg_pes_packet @@ -1258,7 +1258,7 @@ mpeg_spu - + mpeg_spu @@ -1270,9 +1270,9 @@ opus_packet - + opus_packet - + vorbis_comment @@ -1284,9 +1284,9 @@ vorbis_packet - + vorbis_packet - + vorbis_comment @@ -1298,7 +1298,7 @@ vp8_frame - + vp8_frame @@ -1310,7 +1310,7 @@ vp9_cfm - + vp9_cfm @@ -1322,7 +1322,7 @@ vp9_frame - + vp9_frame @@ -1334,7 +1334,7 @@ xing - + xing @@ -1376,7 +1376,7 @@ id3v1 - + id3v1 @@ -1388,7 +1388,7 @@ id3v11 - + id3v11 @@ -1508,11 +1508,11 @@ mpeg_es - + mpeg_es - + mpeg_asc - + vorbis_packet @@ -1524,7 +1524,7 @@ prores_frame - + prores_frame @@ -1536,9 +1536,9 @@ protobuf_widevine - + protobuf_widevine - + protobuf @@ -1550,7 +1550,7 @@ pssh_playready - + pssh_playready @@ -1562,7 +1562,7 @@ vpx_ccr - + vpx_ccr @@ -1586,7 +1586,7 @@ protobuf - + protobuf @@ -1598,11 +1598,11 @@ mpeg_pes - + mpeg_pes - + mpeg_pes_packet - + mpeg_spu @@ -1644,7 +1644,7 @@ ogg_page - + ogg_page @@ -1698,9 +1698,9 @@ sll2_packet - + sll2_packet - + inet_packet @@ -1712,9 +1712,9 @@ sll_packet - + sll_packet - + inet_packet @@ -1726,11 +1726,11 @@ rtmp - + rtmp - + amf0 - + mpeg_asc @@ -1742,7 +1742,7 @@ dns_tcp - + dns_tcp @@ -1790,7 +1790,7 @@ amf0 - + amf0 @@ -1838,7 +1838,7 @@ dns - + dns @@ -1880,61 +1880,61 @@ asn1_ber - + asn1_ber bencode - + bencode bson - + bson cbor - + cbor csv - + csv fairplay_spc - + fairplay_spc html - + html markdown - + markdown msgpack - + msgpack raw - + raw diff --git a/format/all/all.fqtest b/format/all/all.fqtest index 3e17c293..9ecaa2df 100644 --- a/format/all/all.fqtest +++ b/format/all/all.fqtest @@ -32,3 +32,111 @@ $ fq -n _registry.groups.probe "xml", "yaml" ] +$ fq --help formats +aac_frame Advanced Audio Coding frame +adts Audio Data Transport Stream +adts_frame Audio Data Transport Stream frame +amf0 Action Message Format 0 +apev2 APEv2 metadata tag +ar Unix archive +asn1_ber ASN1 BER (basic encoding rules, also CER and DER) +av1_ccr AV1 Codec Configuration Record +av1_frame AV1 frame +av1_obu AV1 Open Bitstream Unit +avc_annexb H.264/AVC Annex B +avc_au H.264/AVC Access Unit +avc_dcr H.264/AVC Decoder Configuration Record +avc_nalu H.264/AVC Network Access Layer Unit +avc_pps H.264/AVC Picture Parameter Set +avc_sei H.264/AVC Supplemental Enhancement Information +avc_sps H.264/AVC Sequence Parameter Set +avro_ocf Avro object container file +bencode BitTorrent bencoding +bitcoin_blkdat Bitcoin blk.dat +bitcoin_block Bitcoin block +bitcoin_script Bitcoin script +bitcoin_transaction Bitcoin transaction +bsd_loopback_frame BSD loopback frame +bson Binary JSON +bzip2 bzip2 compression +cbor Concise Binary Object Representation +csv Comma separated values +dns DNS packet +dns_tcp DNS packet (TCP) +elf Executable and Linkable Format +ether8023_frame Ethernet 802.3 frame +exif Exchangeable Image File Format +fairplay_spc FairPlay Server Playback Context +flac Free Lossless Audio Codec file +flac_frame FLAC frame +flac_metadatablock FLAC metadatablock +flac_metadatablocks FLAC metadatablocks +flac_picture FLAC metadatablock picture +flac_streaminfo FLAC streaminfo +gif Graphics Interchange Format +gzip gzip compression +hevc_annexb H.265/HEVC Annex B +hevc_au H.265/HEVC Access Unit +hevc_dcr H.265/HEVC Decoder Configuration Record +hevc_nalu H.265/HEVC Network Access Layer Unit +hevc_pps H.265/HEVC Picture Parameter Set +hevc_sps H.265/HEVC Sequence Parameter Set +hevc_vps H.265/HEVC Video Parameter Set +html HyperText Markup Language +icc_profile International Color Consortium profile +icmp Internet Control Message Protocol +icmpv6 Internet Control Message Protocol v6 +id3v1 ID3v1 metadata +id3v11 ID3v1.1 metadata +id3v2 ID3v2 metadata +ipv4_packet Internet protocol v4 packet +ipv6_packet Internet protocol v6 packet +jpeg Joint Photographic Experts Group file +json JavaScript Object Notation +jsonl JavaScript Object Notation Lines +macho Mach-O macOS executable +macho_fat Fat Mach-O macOS executable (multi-architecture) +markdown Markdown +matroska Matroska file +mp3 MP3 file +mp3_frame MPEG audio layer 3 frame +mp4 ISOBMFF, QuickTime and similar +mpeg_asc MPEG-4 Audio Specific Config +mpeg_es MPEG Elementary Stream +mpeg_pes MPEG Packetized elementary stream +mpeg_pes_packet MPEG Packetized elementary stream packet +mpeg_spu Sub Picture Unit (DVD subtitle) +mpeg_ts MPEG Transport Stream +msgpack MessagePack +ogg OGG file +ogg_page OGG page +opus_packet Opus packet +pcap PCAP packet capture +pcapng PCAPNG packet capture +png Portable Network Graphics file +prores_frame Apple ProRes frame +protobuf Protobuf +protobuf_widevine Widevine protobuf +pssh_playready PlayReady PSSH +raw Raw bits +rtmp Real-Time Messaging Protocol +sll2_packet Linux cooked capture encapsulation v2 +sll_packet Linux cooked capture encapsulation +tar Tar archive +tcp_segment Transmission control protocol segment +tiff Tag Image File Format +toml Tom's Obvious, Minimal Language +udp_datagram User datagram protocol +vorbis_comment Vorbis comment +vorbis_packet Vorbis packet +vp8_frame VP8 frame +vp9_cfm VP9 Codec Feature Metadata +vp9_frame VP9 frame +vpx_ccr VPX Codec Configuration Record +wasm WebAssembly Binary Format +wav WAV file +webp WebP image +xing Xing header +xml Extensible Markup Language +yaml YAML Ain't Markup Language +zip ZIP archive diff --git a/format/all/help.fqtest b/format/all/help.fqtest deleted file mode 100644 index 9b758dfb..00000000 --- a/format/all/help.fqtest +++ /dev/null @@ -1,1475 +0,0 @@ -$ fq -n '_registry.formats | keys[] | "help(\(.))", ([_help("fq"; .)] | join("\n") | split("\n")[] | ("out \(.)" | println))' -"help(aac_frame)" -out aac_frame: Advanced Audio Coding frame decoder -out # Options -out -out object_type=1 Audio object type -out -out # Decode examples -out -out # Decode file as aac_frame -out $ fq -d aac_frame . file -out # Decode value as aac_frame -out ... | aac_frame -out # Decode file using aac_frame options -out $ fq -d aac_frame -o object_type=1 . file -out # Decode value as aac_frame -out ... | aac_frame({object_type:1}) -out -"help(adts)" -out adts: Audio Data Transport Stream decoder -out -out # Decode examples -out -out # Decode file as adts -out $ fq -d adts . file -out # Decode value as adts -out ... | adts -out -"help(adts_frame)" -out adts_frame: Audio Data Transport Stream frame decoder -out -out # Decode examples -out -out # Decode file as adts_frame -out $ fq -d adts_frame . file -out # Decode value as adts_frame -out ... | adts_frame -out -"help(amf0)" -out amf0: Action Message Format 0 decoder -out -out # Decode examples -out -out # Decode file as amf0 -out $ fq -d amf0 . file -out # Decode value as amf0 -out ... | amf0 -out -"help(apev2)" -out apev2: APEv2 metadata tag decoder -out -out # Decode examples -out -out # Decode file as apev2 -out $ fq -d apev2 . file -out # Decode value as apev2 -out ... | apev2 -out -"help(ar)" -out ar: Unix archive decoder -out -out # Decode examples -out -out # Decode file as ar -out $ fq -d ar . file -out # Decode value as ar -out ... | ar -out -"help(asn1_ber)" -out asn1_ber: ASN1 BER (basic encoding rules, also CER and DER) decoder -out -out # Decode examples -out -out # Decode file as asn1_ber -out $ fq -d asn1_ber . file -out # Decode value as asn1_ber -out ... | asn1_ber -out -out Supports decoding BER, CER and DER (X.690). -out -out - Currently no extra validation is done for CER and DER. -out - Does not support specifying a schema. -out - Supports torepr but without schema all sequences and sets will be arrays. -out -out # frompem can be used to decode certificates etc -out -out $ fq -d raw 'frompem | asn1_ber | d' cert.pem -out -out # Can decode nested values -out -out $ fq -d asn1_ber '.constructed[1].value | asn1_ber' file.ber -out -out # Manual schema -out -out $ fq -d asn1_ber 'torepr as $r | ["version", "modulus", "private_exponent", "private_exponen", "prime1", "prime2", "exponent1", "exponent2", "coefficient"] | with_entries({key: .value, value: $r[.key]})' pkcs1.der -out -out # References -out - https://www.itu.int/ITU-T/studygroups/com10/languages/X.690_1297.pdf -out - https://en.wikipedia.org/wiki/X.690 -out - https://letsencrypt.org/docs/a-warm-welcome-to-asn1-and-der/ -out - https://lapo.it/asn1js/ -out -"help(av1_ccr)" -out av1_ccr: AV1 Codec Configuration Record decoder -out -out # Decode examples -out -out # Decode file as av1_ccr -out $ fq -d av1_ccr . file -out # Decode value as av1_ccr -out ... | av1_ccr -out -"help(av1_frame)" -out av1_frame: AV1 frame decoder -out -out # Decode examples -out -out # Decode file as av1_frame -out $ fq -d av1_frame . file -out # Decode value as av1_frame -out ... | av1_frame -out -"help(av1_obu)" -out av1_obu: AV1 Open Bitstream Unit decoder -out -out # Decode examples -out -out # Decode file as av1_obu -out $ fq -d av1_obu . file -out # Decode value as av1_obu -out ... | av1_obu -out -"help(avc_annexb)" -out avc_annexb: H.264/AVC Annex B decoder -out -out # Decode examples -out -out # Decode file as avc_annexb -out $ fq -d avc_annexb . file -out # Decode value as avc_annexb -out ... | avc_annexb -out -"help(avc_au)" -out avc_au: H.264/AVC Access Unit decoder -out # Options -out -out length_size=4 Length value size -out -out # Decode examples -out -out # Decode file as avc_au -out $ fq -d avc_au . file -out # Decode value as avc_au -out ... | avc_au -out # Decode file using avc_au options -out $ fq -d avc_au -o length_size=4 . file -out # Decode value as avc_au -out ... | avc_au({length_size:4}) -out -"help(avc_dcr)" -out avc_dcr: H.264/AVC Decoder Configuration Record decoder -out -out # Decode examples -out -out # Decode file as avc_dcr -out $ fq -d avc_dcr . file -out # Decode value as avc_dcr -out ... | avc_dcr -out -"help(avc_nalu)" -out avc_nalu: H.264/AVC Network Access Layer Unit decoder -out -out # Decode examples -out -out # Decode file as avc_nalu -out $ fq -d avc_nalu . file -out # Decode value as avc_nalu -out ... | avc_nalu -out -"help(avc_pps)" -out avc_pps: H.264/AVC Picture Parameter Set decoder -out -out # Decode examples -out -out # Decode file as avc_pps -out $ fq -d avc_pps . file -out # Decode value as avc_pps -out ... | avc_pps -out -"help(avc_sei)" -out avc_sei: H.264/AVC Supplemental Enhancement Information decoder -out -out # Decode examples -out -out # Decode file as avc_sei -out $ fq -d avc_sei . file -out # Decode value as avc_sei -out ... | avc_sei -out -"help(avc_sps)" -out avc_sps: H.264/AVC Sequence Parameter Set decoder -out -out # Decode examples -out -out # Decode file as avc_sps -out $ fq -d avc_sps . file -out # Decode value as avc_sps -out ... | avc_sps -out -"help(avro_ocf)" -out avro_ocf: Avro object container file decoder -out -out # Decode examples -out -out # Decode file as avro_ocf -out $ fq -d avro_ocf . file -out # Decode value as avro_ocf -out ... | avro_ocf -out -out Supports reading Avro Object Container Format (OCF) files based on the 1.11.0 specification. -out -out Capable of handling null, deflate, and snappy codecs for data compression. -out -out Limitations: -out -out - Schema does not support self-referential types, only built-in types. -out - Decimal logical types are not supported for decoding, will just be treated as their primitive type -out -out # References -out - https://avro.apache.org/docs/current/spec.html#Object+Container+Files -out -"help(bencode)" -out bencode: BitTorrent bencoding decoder -out -out # Decode examples -out -out # Decode file as bencode -out $ fq -d bencode . file -out # Decode value as bencode -out ... | bencode -out -out # Convert represented value to JSON -out -out $ fq -d bencode torepr file.torrent -out -out # References -out - https://wiki.theory.org/BitTorrentSpecification#Bencoding -out -"help(bitcoin_blkdat)" -out bitcoin_blkdat: Bitcoin blk.dat decoder -out -out # Decode examples -out -out # Decode file as bitcoin_blkdat -out $ fq -d bitcoin_blkdat . file -out # Decode value as bitcoin_blkdat -out ... | bitcoin_blkdat -out -"help(bitcoin_block)" -out bitcoin_block: Bitcoin block decoder -out # Options -out -out has_header=false Has blkdat header -out -out # Decode examples -out -out # Decode file as bitcoin_block -out $ fq -d bitcoin_block . file -out # Decode value as bitcoin_block -out ... | bitcoin_block -out # Decode file using bitcoin_block options -out $ fq -d bitcoin_block -o has_header=false . file -out # Decode value as bitcoin_block -out ... | bitcoin_block({has_header:false}) -out -"help(bitcoin_script)" -out bitcoin_script: Bitcoin script decoder -out -out # Decode examples -out -out # Decode file as bitcoin_script -out $ fq -d bitcoin_script . file -out # Decode value as bitcoin_script -out ... | bitcoin_script -out -"help(bitcoin_transaction)" -out bitcoin_transaction: Bitcoin transaction decoder -out -out # Decode examples -out -out # Decode file as bitcoin_transaction -out $ fq -d bitcoin_transaction . file -out # Decode value as bitcoin_transaction -out ... | bitcoin_transaction -out -"help(bsd_loopback_frame)" -out bsd_loopback_frame: BSD loopback frame decoder -out -out # Decode examples -out -out # Decode file as bsd_loopback_frame -out $ fq -d bsd_loopback_frame . file -out # Decode value as bsd_loopback_frame -out ... | bsd_loopback_frame -out -"help(bson)" -out bson: Binary JSON decoder -out -out # Decode examples -out -out # Decode file as bson -out $ fq -d bson . file -out # Decode value as bson -out ... | bson -out -out # Convert represented value to JSON -out -out $ fq -d bson torepr file.bson -out -out # Filter represented value -out -out $ fq -d bson 'torepr | select(.name=="bob")' file.bson -out -out # References -out - https://bsonspec.org/spec.html -out -"help(bzip2)" -out bzip2: bzip2 compression decoder -out -out # Decode examples -out -out # Decode file as bzip2 -out $ fq -d bzip2 . file -out # Decode value as bzip2 -out ... | bzip2 -out -"help(cbor)" -out cbor: Concise Binary Object Representation decoder -out -out # Decode examples -out -out # Decode file as cbor -out $ fq -d cbor . file -out # Decode value as cbor -out ... | cbor -out -out # Convert represented value to JSON -out -out $ fq -d cbor torepr file.cbor -out -out # References -out - https://en.wikipedia.org/wiki/CBOR -out - https://www.rfc-editor.org/rfc/rfc8949.html -out -"help(csv)" -out csv: Comma separated values decoder -out # Options -out -out comma=, Separator character -out comment=# Comment line character -out -out # Decode examples -out -out # Decode file as csv -out $ fq -d csv . file -out # Decode value as csv -out ... | csv -out # Decode file using csv options -out $ fq -d csv -o comma="," -o comment="#" . file -out # Decode value as csv -out ... | csv({comma:",",comment:"#"}) -out -out # TSV to CSV -out -out $ fq -d csv -o comma="\t" tocsv file.tsv -out -out # Convert rows to objects based on header row -out -out $ fq -d csv '.[0] as $t | .[1:] | map(with_entries(.key = $t[.key]))' file.csv -out -"help(dns)" -out dns: DNS packet decoder -out -out # Decode examples -out -out # Decode file as dns -out $ fq -d dns . file -out # Decode value as dns -out ... | dns -out -"help(dns_tcp)" -out dns_tcp: DNS packet (TCP) decoder -out -out # Decode examples -out -out # Decode file as dns_tcp -out $ fq -d dns_tcp . file -out # Decode value as dns_tcp -out ... | dns_tcp -out -"help(elf)" -out elf: Executable and Linkable Format decoder -out -out # Decode examples -out -out # Decode file as elf -out $ fq -d elf . file -out # Decode value as elf -out ... | elf -out -"help(ether8023_frame)" -out ether8023_frame: Ethernet 802.3 frame decoder -out -out # Decode examples -out -out # Decode file as ether8023_frame -out $ fq -d ether8023_frame . file -out # Decode value as ether8023_frame -out ... | ether8023_frame -out -"help(exif)" -out exif: Exchangeable Image File Format decoder -out -out # Decode examples -out -out # Decode file as exif -out $ fq -d exif . file -out # Decode value as exif -out ... | exif -out -"help(fairplay_spc)" -out fairplay_spc: FairPlay Server Playback Context decoder -out -out # Decode examples -out -out # Decode file as fairplay_spc -out $ fq -d fairplay_spc . file -out # Decode value as fairplay_spc -out ... | fairplay_spc -out -"help(flac)" -out flac: Free Lossless Audio Codec file decoder -out -out # Decode examples -out -out # Decode file as flac -out $ fq -d flac . file -out # Decode value as flac -out ... | flac -out -"help(flac_frame)" -out flac_frame: FLAC frame decoder -out # Options -out -out bits_per_sample=16 Bits per sample -out -out # Decode examples -out -out # Decode file as flac_frame -out $ fq -d flac_frame . file -out # Decode value as flac_frame -out ... | flac_frame -out # Decode file using flac_frame options -out $ fq -d flac_frame -o bits_per_sample=16 . file -out # Decode value as flac_frame -out ... | flac_frame({bits_per_sample:16}) -out -"help(flac_metadatablock)" -out flac_metadatablock: FLAC metadatablock decoder -out -out # Decode examples -out -out # Decode file as flac_metadatablock -out $ fq -d flac_metadatablock . file -out # Decode value as flac_metadatablock -out ... | flac_metadatablock -out -"help(flac_metadatablocks)" -out flac_metadatablocks: FLAC metadatablocks decoder -out -out # Decode examples -out -out # Decode file as flac_metadatablocks -out $ fq -d flac_metadatablocks . file -out # Decode value as flac_metadatablocks -out ... | flac_metadatablocks -out -"help(flac_picture)" -out flac_picture: FLAC metadatablock picture decoder -out -out # Decode examples -out -out # Decode file as flac_picture -out $ fq -d flac_picture . file -out # Decode value as flac_picture -out ... | flac_picture -out -"help(flac_streaminfo)" -out flac_streaminfo: FLAC streaminfo decoder -out -out # Decode examples -out -out # Decode file as flac_streaminfo -out $ fq -d flac_streaminfo . file -out # Decode value as flac_streaminfo -out ... | flac_streaminfo -out -"help(gif)" -out gif: Graphics Interchange Format decoder -out -out # Decode examples -out -out # Decode file as gif -out $ fq -d gif . file -out # Decode value as gif -out ... | gif -out -"help(gzip)" -out gzip: gzip compression decoder -out -out # Decode examples -out -out # Decode file as gzip -out $ fq -d gzip . file -out # Decode value as gzip -out ... | gzip -out -"help(hevc_annexb)" -out hevc_annexb: H.265/HEVC Annex B decoder -out -out # Decode examples -out -out # Decode file as hevc_annexb -out $ fq -d hevc_annexb . file -out # Decode value as hevc_annexb -out ... | hevc_annexb -out -"help(hevc_au)" -out hevc_au: H.265/HEVC Access Unit decoder -out # Options -out -out length_size=4 Length value size -out -out # Decode examples -out -out # Decode file as hevc_au -out $ fq -d hevc_au . file -out # Decode value as hevc_au -out ... | hevc_au -out # Decode file using hevc_au options -out $ fq -d hevc_au -o length_size=4 . file -out # Decode value as hevc_au -out ... | hevc_au({length_size:4}) -out -"help(hevc_dcr)" -out hevc_dcr: H.265/HEVC Decoder Configuration Record decoder -out -out # Decode examples -out -out # Decode file as hevc_dcr -out $ fq -d hevc_dcr . file -out # Decode value as hevc_dcr -out ... | hevc_dcr -out -"help(hevc_nalu)" -out hevc_nalu: H.265/HEVC Network Access Layer Unit decoder -out -out # Decode examples -out -out # Decode file as hevc_nalu -out $ fq -d hevc_nalu . file -out # Decode value as hevc_nalu -out ... | hevc_nalu -out -"help(hevc_pps)" -out hevc_pps: H.265/HEVC Picture Parameter Set decoder -out -out # Decode examples -out -out # Decode file as hevc_pps -out $ fq -d hevc_pps . file -out # Decode value as hevc_pps -out ... | hevc_pps -out -"help(hevc_sps)" -out hevc_sps: H.265/HEVC Sequence Parameter Set decoder -out -out # Decode examples -out -out # Decode file as hevc_sps -out $ fq -d hevc_sps . file -out # Decode value as hevc_sps -out ... | hevc_sps -out -"help(hevc_vps)" -out hevc_vps: H.265/HEVC Video Parameter Set decoder -out -out # Decode examples -out -out # Decode file as hevc_vps -out $ fq -d hevc_vps . file -out # Decode value as hevc_vps -out ... | hevc_vps -out -"help(html)" -out html: HyperText Markup Language decoder -out # Options -out -out array=false Decode as nested arrays -out attribute_prefix=@ Prefix for attribute keys -out seq=false Use seq attribute to preserve element order -out -out # Decode examples -out -out # Decode file as html -out $ fq -d html . file -out # Decode value as html -out ... | html -out # Decode file using html options -out $ fq -d html -o array=false -o attribute_prefix="@" -o seq=false . file -out # Decode value as html -out ... | html({array:false,attribute_prefix:"@",seq:false}) -out -out HTML is decoded in HTML5 mode and will always include , and element. -out -out See xml format for more examples and how to preserve element order and how to encode to xml. -out -out There is no tohtml function, see toxml instead. -out -out # Element as object -out -out # decode as object is the default -out $ echo 'text' | fq -d html -out { -out "html": { -out "body": { -out "a": { -out "#text": "text", -out "@href": "url" -out } -out }, -out "head": "" -out } -out } -out -out # Element as array -out -out $ 'text' | fq -d html -o array=true -out [ -out "html", -out null, -out [ -out [ -out "head", -out null, -out [] -out ], -out [ -out "body", -out null, -out [ -out [ -out "a", -out { -out "#text": "text", -out "href": "url" -out }, -out [] -out ] -out ] -out ] -out ] -out ] -out -out -out # Decode html files to a {file: "title", ...} object -out $ fq -n -d html '[inputs | {key: input_filename, value: .html.head.title?}] | from_entries' *.html -out -out -out # href:s in file -out $ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href' file.html -out -"help(icc_profile)" -out icc_profile: International Color Consortium profile decoder -out -out # Decode examples -out -out # Decode file as icc_profile -out $ fq -d icc_profile . file -out # Decode value as icc_profile -out ... | icc_profile -out -"help(icmp)" -out icmp: Internet Control Message Protocol decoder -out -out # Decode examples -out -out # Decode file as icmp -out $ fq -d icmp . file -out # Decode value as icmp -out ... | icmp -out -"help(icmpv6)" -out icmpv6: Internet Control Message Protocol v6 decoder -out -out # Decode examples -out -out # Decode file as icmpv6 -out $ fq -d icmpv6 . file -out # Decode value as icmpv6 -out ... | icmpv6 -out -"help(id3v1)" -out id3v1: ID3v1 metadata decoder -out -out # Decode examples -out -out # Decode file as id3v1 -out $ fq -d id3v1 . file -out # Decode value as id3v1 -out ... | id3v1 -out -"help(id3v11)" -out id3v11: ID3v1.1 metadata decoder -out -out # Decode examples -out -out # Decode file as id3v11 -out $ fq -d id3v11 . file -out # Decode value as id3v11 -out ... | id3v11 -out -"help(id3v2)" -out id3v2: ID3v2 metadata decoder -out -out # Decode examples -out -out # Decode file as id3v2 -out $ fq -d id3v2 . file -out # Decode value as id3v2 -out ... | id3v2 -out -"help(ipv4_packet)" -out ipv4_packet: Internet protocol v4 packet decoder -out -out # Decode examples -out -out # Decode file as ipv4_packet -out $ fq -d ipv4_packet . file -out # Decode value as ipv4_packet -out ... | ipv4_packet -out -"help(ipv6_packet)" -out ipv6_packet: Internet protocol v6 packet decoder -out -out # Decode examples -out -out # Decode file as ipv6_packet -out $ fq -d ipv6_packet . file -out # Decode value as ipv6_packet -out ... | ipv6_packet -out -"help(jpeg)" -out jpeg: Joint Photographic Experts Group file decoder -out -out # Decode examples -out -out # Decode file as jpeg -out $ fq -d jpeg . file -out # Decode value as jpeg -out ... | jpeg -out -"help(json)" -out json: JavaScript Object Notation decoder -out -out # Decode examples -out -out # Decode file as json -out $ fq -d json . file -out # Decode value as json -out ... | json -out -"help(jsonl)" -out jsonl: JavaScript Object Notation Lines decoder -out -out # Decode examples -out -out # Decode file as jsonl -out $ fq -d jsonl . file -out # Decode value as jsonl -out ... | jsonl -out -"help(macho)" -out macho: Mach-O macOS executable decoder -out -out # Decode examples -out -out # Decode file as macho -out $ fq -d macho . file -out # Decode value as macho -out ... | macho -out -out Supports decoding vanilla and FAT Mach-O binaries. -out -out # Select 64bit load segments -out -out $ fq '.load_commands[] | select(.cmd=="segment_64")' file -out -out # References -out - https://github.com/aidansteele/osx-abi-macho-file-format-reference -out -"help(macho_fat)" -out macho_fat: Fat Mach-O macOS executable (multi-architecture) decoder -out -out # Decode examples -out -out # Decode file as macho_fat -out $ fq -d macho_fat . file -out # Decode value as macho_fat -out ... | macho_fat -out -"help(markdown)" -out markdown: Markdown decoder -out -out # Decode examples -out -out # Decode file as markdown -out $ fq -d markdown . file -out # Decode value as markdown -out ... | markdown -out -"help(matroska)" -out matroska: Matroska file decoder -out -out # Decode examples -out -out # Decode file as matroska -out $ fq -d matroska . file -out # Decode value as matroska -out ... | matroska -out -out # Lookup element path using matroska_path -out -out $ fq 'matroska_path(".Segment.Tracks[0)")' file.mkv -out -out # Get element path using matroska_path -out -out $ fq 'grep_by(.id == "Tracks") | matroska_path' file.mkv -out -out # References -out - https://tools.ietf.org/html/draft-ietf-cellar-ebml-00 -out - https://matroska.org/technical/specs/index.html -out - https://www.matroska.org/technical/basics.html -out - https://www.matroska.org/technical/codec_specs.html -out - https://wiki.xiph.org/MatroskaOpus -out -"help(mp3)" -out mp3: MP3 file decoder -out # Options -out -out max_sync_seek=32768 Max byte distance to next sync -out max_unique_header_configs=5 Max number of unique frame header configs allowed -out -out # Decode examples -out -out # Decode file as mp3 -out $ fq -d mp3 . file -out # Decode value as mp3 -out ... | mp3 -out # Decode file using mp3 options -out $ fq -d mp3 -o max_sync_seek=32768 -o max_unique_header_configs=5 . file -out # Decode value as mp3 -out ... | mp3({max_sync_seek:32768,max_unique_header_configs:5}) -out -"help(mp3_frame)" -out mp3_frame: MPEG audio layer 3 frame decoder -out -out # Decode examples -out -out # Decode file as mp3_frame -out $ fq -d mp3_frame . file -out # Decode value as mp3_frame -out ... | mp3_frame -out -"help(mp4)" -out mp4: ISOBMFF, QuickTime and similar decoder -out # Options -out -out allow_truncated=false Allow box to be truncated -out decode_samples=true Decode supported media samples -out -out # Decode examples -out -out # Decode file as mp4 -out $ fq -d mp4 . file -out # Decode value as mp4 -out ... | mp4 -out # Decode file using mp4 options -out $ fq -d mp4 -o allow_truncated=false -o decode_samples=true . file -out # Decode value as mp4 -out ... | mp4({allow_truncated:false,decode_samples:true}) -out -out # mp4_path($path) - Lookup mp4 box using a mp4 box path. -out -out # | mp4_path($path) -> -out $ fq 'mp4_path(".moov.trak[1]")' file.mp4 -out -out # mp4_path - Return mp4 box path for a decode value box. -out -out # | mp4_path -> string -out $ fq 'grep_by(.type == "trak") | mp4_path' file.mp4 -out -out # Force decode a single box -out -out $ fq -n '"AAAAHGVsc3QAAAAAAAAAAQAAADIAAAQAAAEAAA==" | frombase64 | mp4({force:true}) | d' -out -out # Don't decode samples and manually decode first sample for first track as a aac_frame -out -out $ fq -o decode_samples=false '.tracks[0].samples[0] | aac_frame | d' file.mp4 -out -out # Entries for first edit list as values -out -out $ fq 'first(grep_by(.type=="elst").entries) | tovalue' file.mp4 -out -out # References -out - ISO/IEC base media file format (MPEG-4 Part 12) (https://en.wikipedia.org/wiki/ISO/IEC_base_media_file_format) -out - Quicktime file format (https://developer.apple.com/standards/qtff-2001.pdf) -out -"help(mpeg_asc)" -out mpeg_asc: MPEG-4 Audio Specific Config decoder -out -out # Decode examples -out -out # Decode file as mpeg_asc -out $ fq -d mpeg_asc . file -out # Decode value as mpeg_asc -out ... | mpeg_asc -out -"help(mpeg_es)" -out mpeg_es: MPEG Elementary Stream decoder -out -out # Decode examples -out -out # Decode file as mpeg_es -out $ fq -d mpeg_es . file -out # Decode value as mpeg_es -out ... | mpeg_es -out -"help(mpeg_pes)" -out mpeg_pes: MPEG Packetized elementary stream decoder -out -out # Decode examples -out -out # Decode file as mpeg_pes -out $ fq -d mpeg_pes . file -out # Decode value as mpeg_pes -out ... | mpeg_pes -out -"help(mpeg_pes_packet)" -out mpeg_pes_packet: MPEG Packetized elementary stream packet decoder -out -out # Decode examples -out -out # Decode file as mpeg_pes_packet -out $ fq -d mpeg_pes_packet . file -out # Decode value as mpeg_pes_packet -out ... | mpeg_pes_packet -out -"help(mpeg_spu)" -out mpeg_spu: Sub Picture Unit (DVD subtitle) decoder -out -out # Decode examples -out -out # Decode file as mpeg_spu -out $ fq -d mpeg_spu . file -out # Decode value as mpeg_spu -out ... | mpeg_spu -out -"help(mpeg_ts)" -out mpeg_ts: MPEG Transport Stream decoder -out -out # Decode examples -out -out # Decode file as mpeg_ts -out $ fq -d mpeg_ts . file -out # Decode value as mpeg_ts -out ... | mpeg_ts -out -"help(msgpack)" -out msgpack: MessagePack decoder -out -out # Decode examples -out -out # Decode file as msgpack -out $ fq -d msgpack . file -out # Decode value as msgpack -out ... | msgpack -out -out # Convert represented value to JSON -out -out $ fq -d msgpack torepr file.msgpack -out -out # References -out - https://github.com/msgpack/msgpack/blob/master/spec.md -out -"help(ogg)" -out ogg: OGG file decoder -out -out # Decode examples -out -out # Decode file as ogg -out $ fq -d ogg . file -out # Decode value as ogg -out ... | ogg -out -"help(ogg_page)" -out ogg_page: OGG page decoder -out -out # Decode examples -out -out # Decode file as ogg_page -out $ fq -d ogg_page . file -out # Decode value as ogg_page -out ... | ogg_page -out -"help(opus_packet)" -out opus_packet: Opus packet decoder -out -out # Decode examples -out -out # Decode file as opus_packet -out $ fq -d opus_packet . file -out # Decode value as opus_packet -out ... | opus_packet -out -"help(pcap)" -out pcap: PCAP packet capture decoder -out -out # Decode examples -out -out # Decode file as pcap -out $ fq -d pcap . file -out # Decode value as pcap -out ... | pcap -out -"help(pcapng)" -out pcapng: PCAPNG packet capture decoder -out -out # Decode examples -out -out # Decode file as pcapng -out $ fq -d pcapng . file -out # Decode value as pcapng -out ... | pcapng -out -"help(png)" -out png: Portable Network Graphics file decoder -out -out # Decode examples -out -out # Decode file as png -out $ fq -d png . file -out # Decode value as png -out ... | png -out -"help(prores_frame)" -out prores_frame: Apple ProRes frame decoder -out -out # Decode examples -out -out # Decode file as prores_frame -out $ fq -d prores_frame . file -out # Decode value as prores_frame -out ... | prores_frame -out -"help(protobuf)" -out protobuf: Protobuf decoder -out -out # Decode examples -out -out # Decode file as protobuf -out $ fq -d protobuf . file -out # Decode value as protobuf -out ... | protobuf -out -out # Can decode sub messages -out -out $ fq -d protobuf '.fields[6].wire_value | protobuf | d' file -out -out # References -out - https://developers.google.com/protocol-buffers/docs/encoding -out -"help(protobuf_widevine)" -out protobuf_widevine: Widevine protobuf decoder -out -out # Decode examples -out -out # Decode file as protobuf_widevine -out $ fq -d protobuf_widevine . file -out # Decode value as protobuf_widevine -out ... | protobuf_widevine -out -"help(pssh_playready)" -out pssh_playready: PlayReady PSSH decoder -out -out # Decode examples -out -out # Decode file as pssh_playready -out $ fq -d pssh_playready . file -out # Decode value as pssh_playready -out ... | pssh_playready -out -"help(raw)" -out raw: Raw bits decoder -out -out # Decode examples -out -out # Decode file as raw -out $ fq -d raw . file -out # Decode value as raw -out ... | raw -out -"help(rtmp)" -out rtmp: Real-Time Messaging Protocol decoder -out -out # Decode examples -out -out # Decode file as rtmp -out $ fq -d rtmp . file -out # Decode value as rtmp -out ... | rtmp -out -out Current only supports plain RTMP (not RTMPT or encrypted variants etc) with AMF0 (not AMF3). -out -out # References -out - https://rtmp.veriskope.com/docs/spec/ -out - https://rtmp.veriskope.com/pdf/video_file_format_spec_v10.pdf -out -"help(sll2_packet)" -out sll2_packet: Linux cooked capture encapsulation v2 decoder -out -out # Decode examples -out -out # Decode file as sll2_packet -out $ fq -d sll2_packet . file -out # Decode value as sll2_packet -out ... | sll2_packet -out -"help(sll_packet)" -out sll_packet: Linux cooked capture encapsulation decoder -out -out # Decode examples -out -out # Decode file as sll_packet -out $ fq -d sll_packet . file -out # Decode value as sll_packet -out ... | sll_packet -out -"help(tar)" -out tar: Tar archive decoder -out -out # Decode examples -out -out # Decode file as tar -out $ fq -d tar . file -out # Decode value as tar -out ... | tar -out -"help(tcp_segment)" -out tcp_segment: Transmission control protocol segment decoder -out -out # Decode examples -out -out # Decode file as tcp_segment -out $ fq -d tcp_segment . file -out # Decode value as tcp_segment -out ... | tcp_segment -out -"help(tiff)" -out tiff: Tag Image File Format decoder -out -out # Decode examples -out -out # Decode file as tiff -out $ fq -d tiff . file -out # Decode value as tiff -out ... | tiff -out -"help(toml)" -out toml: Tom's Obvious, Minimal Language decoder -out -out # Decode examples -out -out # Decode file as toml -out $ fq -d toml . file -out # Decode value as toml -out ... | toml -out -"help(udp_datagram)" -out udp_datagram: User datagram protocol decoder -out -out # Decode examples -out -out # Decode file as udp_datagram -out $ fq -d udp_datagram . file -out # Decode value as udp_datagram -out ... | udp_datagram -out -"help(vorbis_comment)" -out vorbis_comment: Vorbis comment decoder -out -out # Decode examples -out -out # Decode file as vorbis_comment -out $ fq -d vorbis_comment . file -out # Decode value as vorbis_comment -out ... | vorbis_comment -out -"help(vorbis_packet)" -out vorbis_packet: Vorbis packet decoder -out -out # Decode examples -out -out # Decode file as vorbis_packet -out $ fq -d vorbis_packet . file -out # Decode value as vorbis_packet -out ... | vorbis_packet -out -"help(vp8_frame)" -out vp8_frame: VP8 frame decoder -out -out # Decode examples -out -out # Decode file as vp8_frame -out $ fq -d vp8_frame . file -out # Decode value as vp8_frame -out ... | vp8_frame -out -"help(vp9_cfm)" -out vp9_cfm: VP9 Codec Feature Metadata decoder -out -out # Decode examples -out -out # Decode file as vp9_cfm -out $ fq -d vp9_cfm . file -out # Decode value as vp9_cfm -out ... | vp9_cfm -out -"help(vp9_frame)" -out vp9_frame: VP9 frame decoder -out -out # Decode examples -out -out # Decode file as vp9_frame -out $ fq -d vp9_frame . file -out # Decode value as vp9_frame -out ... | vp9_frame -out -"help(vpx_ccr)" -out vpx_ccr: VPX Codec Configuration Record decoder -out -out # Decode examples -out -out # Decode file as vpx_ccr -out $ fq -d vpx_ccr . file -out # Decode value as vpx_ccr -out ... | vpx_ccr -out -"help(wasm)" -out wasm: WebAssembly Binary Format decoder -out -out # Decode examples -out -out # Decode file as wasm -out $ fq -d wasm . file -out # Decode value as wasm -out ... | wasm -out -out # Count opcode usage -out -out $ fq '.sections[] | select(.id == "code_section") | [.. | .opcode? // empty] | count | map({key: .[0], value: .[1]}) | from_entries' file.wasm -out -out # List exports and imports -out -out $ fq '.sections | {import: map(select(.id == "import_section").content.im.x[].nm.b), export: map(select(.id == "export_section").content.ex.x[].nm.b)}' file.wasm -out -out # Authors -out - Takashi Oguma @bitbears-dev (https://github.com/bitbears-dev) @0xb17bea125 (https://twitter.com/0xb17bea125) -out -out # References -out - https://webassembly.github.io/spec/core/ -out -"help(wav)" -out wav: WAV file decoder -out -out # Decode examples -out -out # Decode file as wav -out $ fq -d wav . file -out # Decode value as wav -out ... | wav -out -"help(webp)" -out webp: WebP image decoder -out -out # Decode examples -out -out # Decode file as webp -out $ fq -d webp . file -out # Decode value as webp -out ... | webp -out -"help(xing)" -out xing: Xing header decoder -out -out # Decode examples -out -out # Decode file as xing -out $ fq -d xing . file -out # Decode value as xing -out ... | xing -out -"help(xml)" -out xml: Extensible Markup Language decoder -out # Options -out -out array=false Decode as nested arrays -out attribute_prefix=@ Prefix for attribute keys -out seq=false Use seq attribute to preserve element order -out -out # Decode examples -out -out # Decode file as xml -out $ fq -d xml . file -out # Decode value as xml -out ... | xml -out # Decode file using xml options -out $ fq -d xml -o array=false -o attribute_prefix="@" -o seq=false . file -out # Decode value as xml -out ... | xml({array:false,attribute_prefix:"@",seq:false}) -out -out XML can be decoded and encoded into jq values in two ways, elements as object or array. Which variant to use depends a bit what you want to do. The object variant -out might be easier to query for a specific value but array might be easier to use to generate xml or to query after all elements of some kind etc. -out -out Encoding is done using the toxml function and it will figure what variant that is used based on the input value. Is has two optional options indent and -out attribute_prefix. -out -out # Elements as object -out Element can have different shapes depending on body text, attributes and children: -out -out - text is {"a":{"#text":"text","@key":"value"}}, has text (#text) and attributes (@key) -out - text is {"a":"text"} -out - text is {"a":{"b":"text"}} one child with only text and no attributes -out - text is {"a":{"b":["","text"]}} two children with same name end up in an array -out - text is {"a":{"b":["",{"#text":"text","@key":"value"}]}} -out -out If there is #seq attribute it encodes the child element order. Use -o seq=true to include sequence number when decoding, otherwise order might be lost. -out -out -out # decode as object is the default -out $ echo 'bbbccc' | fq -d xml -o seq=true -out { -out "a": { -out "b": [ -out { -out "#seq": 0 -out }, -out { -out "#seq": 1, -out "#text": "bbb" -out } -out ], -out "c": { -out "#seq": 2, -out "#text": "ccc", -out "@attr": "value" -out } -out } -out } -out -out # access text of the element -out $ echo 'bbbccc' | fq '.a.c["#text"]' -out "ccc" -out -out -out # decode to object and encode to xml -out $ echo 'bbbccc' | fq -r -d xml -o seq=true 'toxml({indent:2})' -out -out -out bbb -out ccc -out -out -out # Elements as array -out Elements are arrays of the shape ["#text": "body text", "attr_name", {key: "attr value"}|null, [, ...]]. -out -out -out # decode as array -out ✗ echo 'bbbccc' | fq -d xml -o array=true -out [ -out "a", -out null, -out [ -out [ -out "b", -out null, -out [] -out ], -out [ -out "b", -out { -out "#text": "bbb" -out }, -out [] -out ], -out [ -out "c", -out { -out "#text": "ccc", -out "attr": "value" -out }, -out [] -out ] -out ] -out ] -out -out -out # decode to array and encode to xml -out ✗ echo 'bbbccc' | fq -r -d xml -o array=true -o seq=true 'toxml({indent:2})' -out -out -out bbb -out ccc -out -out -out # access text of the element, the object variant above is probably easier to use -out $ echo 'bbbccc' | fq -o array=true '.[2][2][1]["#text"]' -out "ccc" -out -out # References -out - xml.com's Converting Between XML and JSON (https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html) -out -"help(yaml)" -out yaml: YAML Ain't Markup Language decoder -out -out # Decode examples -out -out # Decode file as yaml -out $ fq -d yaml . file -out # Decode value as yaml -out ... | yaml -out -"help(zip)" -out zip: ZIP archive decoder -out # Options -out -out uncompress=true Uncompress and probe files -out -out # Decode examples -out -out # Decode file as zip -out $ fq -d zip . file -out # Decode value as zip -out ... | zip -out # Decode file using zip options -out $ fq -d zip -o uncompress=true . file -out # Decode value as zip -out ... | zip({uncompress:true}) -out -out Supports ZIP64. -out -out # References -out - https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT -out - https://opensource.apple.com/source/zip/zip-6/unzip/unzip/proginfo/extra.fld -out diff --git a/format/asn1/asn1_ber.md b/format/asn1/asn1_ber.md index 07813dba..aca678be 100644 --- a/format/asn1/asn1_ber.md +++ b/format/asn1/asn1_ber.md @@ -4,7 +4,7 @@ Supports decoding BER, CER and DER (X.690). - Does not support specifying a schema. - Supports `torepr` but without schema all sequences and sets will be arrays. -### `frompem` can be used to decode certificates etc +### Can be used to decode certificates etc ```sh $ fq -d raw 'frompem | asn1_ber | d' cert.pem diff --git a/format/asn1/testdata/help_asn1_ber.fqtest b/format/asn1/testdata/help_asn1_ber.fqtest new file mode 100644 index 00000000..aaac2765 --- /dev/null +++ b/format/asn1/testdata/help_asn1_ber.fqtest @@ -0,0 +1,39 @@ +$ fq -h asn1_ber +asn1_ber: ASN1 BER (basic encoding rules, also CER and DER) decoder + +Decode examples +=============== + + # Decode file as asn1_ber + $ fq -d asn1_ber . file + # Decode value as asn1_ber + ... | asn1_ber + +Supports decoding BER, CER and DER (X.690). + +- Currently no extra validation is done for CER and DER. +- Does not support specifying a schema. +- Supports torepr but without schema all sequences and sets will be arrays. + +Can be used to decode certificates etc +====================================== + + $ fq -d raw 'frompem | asn1_ber | d' cert.pem + +Can decode nested values +======================== + + $ fq -d asn1_ber '.constructed[1].value | asn1_ber' file.ber + +Manual schema +============= + + $ fq -d asn1_ber 'torepr as $r | ["version", "modulus", "private_exponent", "private_exponen", "prime1", "prime2", "exponent1", "exponent2", "coefficient"] | with_entries({key: .value, value: $r[.key]})' pkcs1.der + +References +========== +- https://www.itu.int/ITU-T/studygroups/com10/languages/X.690_1297.pdf +- https://en.wikipedia.org/wiki/X.690 +- https://letsencrypt.org/docs/a-warm-welcome-to-asn1-and-der/ +- https://lapo.it/asn1js/ + diff --git a/format/avro/avro_ocf.md b/format/avro/avro_ocf.md index 51d9056e..e61f2f72 100644 --- a/format/avro/avro_ocf.md +++ b/format/avro/avro_ocf.md @@ -9,3 +9,8 @@ Limitations: ### References - https://avro.apache.org/docs/current/spec.html#Object+Container+Files + +### Authors +- Xentripetal +xentripetal@fastmail.com +[@xentripetal](https://github.com/xentripetal) diff --git a/format/avro/testdata/help_avro_ocf.fqtest b/format/avro/testdata/help_avro_ocf.fqtest new file mode 100644 index 00000000..acb31777 --- /dev/null +++ b/format/avro/testdata/help_avro_ocf.fqtest @@ -0,0 +1,28 @@ +$ fq -h avro_ocf +avro_ocf: Avro object container file decoder + +Decode examples +=============== + + # Decode file as avro_ocf + $ fq -d avro_ocf . file + # Decode value as avro_ocf + ... | avro_ocf + +Supports reading Avro Object Container Format (OCF) files based on the 1.11.0 specification. + +Capable of handling null, deflate, and snappy codecs for data compression. + +Limitations: + +- Schema does not support self-referential types, only built-in types. +- Decimal logical types are not supported for decoding, will just be treated as their primitive type + +References +========== +- https://avro.apache.org/docs/current/spec.html#Object+Container+Files + +Authors +======= +- Xentripetal xentripetal@fastmail.com @xentripetal (https://github.com/xentripetal) + diff --git a/format/bencode/testdata/help_bencode.fqtest b/format/bencode/testdata/help_bencode.fqtest new file mode 100644 index 00000000..5af2c3d4 --- /dev/null +++ b/format/bencode/testdata/help_bencode.fqtest @@ -0,0 +1,20 @@ +$ fq -h bencode +bencode: BitTorrent bencoding decoder + +Decode examples +=============== + + # Decode file as bencode + $ fq -d bencode . file + # Decode value as bencode + ... | bencode + +Convert represented value to JSON +================================= + + $ fq -d bencode torepr file.torrent + +References +========== +- https://wiki.theory.org/BitTorrentSpecification#Bencoding + diff --git a/format/bitcoin/testdata/help_bitcoin_block.fqtest b/format/bitcoin/testdata/help_bitcoin_block.fqtest new file mode 100644 index 00000000..027d97de --- /dev/null +++ b/format/bitcoin/testdata/help_bitcoin_block.fqtest @@ -0,0 +1,20 @@ +$ fq -h bitcoin_block +bitcoin_block: Bitcoin block decoder + +Options +======= + + has_header=false Has blkdat header + +Decode examples +=============== + + # Decode file as bitcoin_block + $ fq -d bitcoin_block . file + # Decode value as bitcoin_block + ... | bitcoin_block + # Decode file using bitcoin_block options + $ fq -d bitcoin_block -o has_header=false . file + # Decode value as bitcoin_block + ... | bitcoin_block({has_header:false}) + diff --git a/format/bson/testdata/help_bson.fqtest b/format/bson/testdata/help_bson.fqtest new file mode 100644 index 00000000..814c8658 --- /dev/null +++ b/format/bson/testdata/help_bson.fqtest @@ -0,0 +1,25 @@ +$ fq -h bson +bson: Binary JSON decoder + +Decode examples +=============== + + # Decode file as bson + $ fq -d bson . file + # Decode value as bson + ... | bson + +Convert represented value to JSON +================================= + + $ fq -d bson torepr file.bson + +Filter represented value +======================== + + $ fq -d bson 'torepr | select(.name=="bob")' file.bson + +References +========== +- https://bsonspec.org/spec.html + diff --git a/format/cbor/testdata/help_cbor.fqtest b/format/cbor/testdata/help_cbor.fqtest new file mode 100644 index 00000000..a0175d07 --- /dev/null +++ b/format/cbor/testdata/help_cbor.fqtest @@ -0,0 +1,21 @@ +$ fq -h cbor +cbor: Concise Binary Object Representation decoder + +Decode examples +=============== + + # Decode file as cbor + $ fq -d cbor . file + # Decode value as cbor + ... | cbor + +Convert represented value to JSON +================================= + + $ fq -d cbor torepr file.cbor + +References +========== +- https://en.wikipedia.org/wiki/CBOR +- https://www.rfc-editor.org/rfc/rfc8949.html + diff --git a/format/csv/testdata/help_csv.fqtest b/format/csv/testdata/help_csv.fqtest new file mode 100644 index 00000000..f5b85907 --- /dev/null +++ b/format/csv/testdata/help_csv.fqtest @@ -0,0 +1,31 @@ +$ fq -h csv +csv: Comma separated values decoder + +Options +======= + + comma=, Separator character + comment=# Comment line character + +Decode examples +=============== + + # Decode file as csv + $ fq -d csv . file + # Decode value as csv + ... | csv + # Decode file using csv options + $ fq -d csv -o comma="," -o comment="#" . file + # Decode value as csv + ... | csv({comma:",",comment:"#"}) + +TSV to CSV +========== + + $ fq -d csv -o comma="\t" tocsv file.tsv + +Convert rows to objects based on header row +=========================================== + + $ fq -d csv '.[0] as $t | .[1:] | map(with_entries(.key = $t[.key]))' file.csv + diff --git a/format/flac/testdata/help_flac_frame.fqtest b/format/flac/testdata/help_flac_frame.fqtest new file mode 100644 index 00000000..566d1a5a --- /dev/null +++ b/format/flac/testdata/help_flac_frame.fqtest @@ -0,0 +1,20 @@ +$ fq -h flac_frame +flac_frame: FLAC frame decoder + +Options +======= + + bits_per_sample=16 Bits per sample + +Decode examples +=============== + + # Decode file as flac_frame + $ fq -d flac_frame . file + # Decode value as flac_frame + ... | flac_frame + # Decode file using flac_frame options + $ fq -d flac_frame -o bits_per_sample=16 . file + # Decode value as flac_frame + ... | flac_frame({bits_per_sample:16}) + diff --git a/format/macho/macho.md b/format/macho/macho.md index a11a5b70..346a0249 100644 --- a/format/macho/macho.md +++ b/format/macho/macho.md @@ -8,3 +8,8 @@ $ fq '.load_commands[] | select(.cmd=="segment_64")' file ### References - https://github.com/aidansteele/osx-abi-macho-file-format-reference + +### Authors +- Sıddık AÇIL +acils@itu.edu.tr +[@Akaame](https://github.com/Akaame) diff --git a/format/macho/testdata/help_macho.fqtest b/format/macho/testdata/help_macho.fqtest new file mode 100644 index 00000000..1987e260 --- /dev/null +++ b/format/macho/testdata/help_macho.fqtest @@ -0,0 +1,26 @@ +$ fq -h macho +macho: Mach-O macOS executable decoder + +Decode examples +=============== + + # Decode file as macho + $ fq -d macho . file + # Decode value as macho + ... | macho + +Supports decoding vanilla and FAT Mach-O binaries. + +Select 64bit load segments +========================== + + $ fq '.load_commands[] | select(.cmd=="segment_64")' file + +References +========== +- https://github.com/aidansteele/osx-abi-macho-file-format-reference + +Authors +======= +- Sıddık AÇIL acils@itu.edu.tr @Akaame (https://github.com/Akaame) + diff --git a/format/markdown/markdown.jq b/format/markdown/markdown.jq index 6c518005..907f5ce2 100644 --- a/format/markdown/markdown.jq +++ b/format/markdown/markdown.jq @@ -21,7 +21,13 @@ def _markdown_to_text($width; $header_depth): def _f($pln): if type == "string" then gsub("\n"; " ") elif .type == "document" then .children[] | _f("\n\n") - elif .type == "heading" then "#" * (.level+$header_depth), " ", (.children[] | _f("\n\n")), "\n" + elif .type == "heading" then + ( (.children[] | _f("\n\n")) as $title + | $title + , "\n" + , ("=" * ($title | length)) + , "\n" + ) elif .type == "paragraph" then ( ( [.children[] | _f("\n\n")] | join("") diff --git a/format/matroska/matroska.md b/format/matroska/matroska.md index 0ebe2f97..86c81fa1 100644 --- a/format/matroska/matroska.md +++ b/format/matroska/matroska.md @@ -1,10 +1,10 @@ -### Lookup element path using `matroska_path` +### Lookup element using path ```sh $ fq 'matroska_path(".Segment.Tracks[0)")' file.mkv ``` -### Get element path using `matroska_path` +### Get path to element ```sh $ fq 'grep_by(.id == "Tracks") | matroska_path' file.mkv diff --git a/format/matroska/testdata/help_matroska.fqtest b/format/matroska/testdata/help_matroska.fqtest new file mode 100644 index 00000000..ef4d880f --- /dev/null +++ b/format/matroska/testdata/help_matroska.fqtest @@ -0,0 +1,29 @@ +$ fq -h matroska +matroska: Matroska file decoder + +Decode examples +=============== + + # Decode file as matroska + $ fq -d matroska . file + # Decode value as matroska + ... | matroska + +Lookup element using path +========================= + + $ fq 'matroska_path(".Segment.Tracks[0)")' file.mkv + +Get path to element +=================== + + $ fq 'grep_by(.id == "Tracks") | matroska_path' file.mkv + +References +========== +- https://tools.ietf.org/html/draft-ietf-cellar-ebml-00 +- https://matroska.org/technical/specs/index.html +- https://www.matroska.org/technical/basics.html +- https://www.matroska.org/technical/codec_specs.html +- https://wiki.xiph.org/MatroskaOpus + diff --git a/format/mp3/testdata/help_mp3.fqtest b/format/mp3/testdata/help_mp3.fqtest new file mode 100644 index 00000000..a618b733 --- /dev/null +++ b/format/mp3/testdata/help_mp3.fqtest @@ -0,0 +1,21 @@ +$ fq -h mp3 +mp3: MP3 file decoder + +Options +======= + + max_sync_seek=32768 Max byte distance to next sync + max_unique_header_configs=5 Max number of unique frame header configs allowed + +Decode examples +=============== + + # Decode file as mp3 + $ fq -d mp3 . file + # Decode value as mp3 + ... | mp3 + # Decode file using mp3 options + $ fq -d mp3 -o max_sync_seek=32768 -o max_unique_header_configs=5 . file + # Decode value as mp3 + ... | mp3({max_sync_seek:32768,max_unique_header_configs:5}) + diff --git a/format/mp4/mp4.md b/format/mp4/mp4.md index fd32c951..8ab20e62 100644 --- a/format/mp4/mp4.md +++ b/format/mp4/mp4.md @@ -1,11 +1,11 @@ -### `mp4_path($path)` - Lookup mp4 box using a mp4 box path. +### Lookup mp4 box using a mp4 box path. ```sh # | mp4_path($path) -> $ fq 'mp4_path(".moov.trak[1]")' file.mp4 ``` -### `mp4_path` - Return mp4 box path for a decode value box. +### Get mp4 box path for a decode value box. ```sh # | mp4_path -> string @@ -18,9 +18,10 @@ $ fq 'grep_by(.type == "trak") | mp4_path' file.mp4 $ fq -n '"AAAAHGVsc3QAAAAAAAAAAQAAADIAAAQAAAEAAA==" | frombase64 | mp4({force:true}) | d' ``` -### Don't decode samples and manually decode first sample for first track as a `aac_frame` +### Speed up decoding by not decoding samples ```sh +# manually decode first sample as a aac_frame $ fq -o decode_samples=false '.tracks[0].samples[0] | aac_frame | d' file.mp4 ``` diff --git a/format/mp4/testdata/help_mp4.fqtest b/format/mp4/testdata/help_mp4.fqtest new file mode 100644 index 00000000..0bebd2b7 --- /dev/null +++ b/format/mp4/testdata/help_mp4.fqtest @@ -0,0 +1,54 @@ +$ fq -h mp4 +mp4: ISOBMFF, QuickTime and similar decoder + +Options +======= + + allow_truncated=false Allow box to be truncated + decode_samples=true Decode supported media samples + +Decode examples +=============== + + # Decode file as mp4 + $ fq -d mp4 . file + # Decode value as mp4 + ... | mp4 + # Decode file using mp4 options + $ fq -d mp4 -o allow_truncated=false -o decode_samples=true . file + # Decode value as mp4 + ... | mp4({allow_truncated:false,decode_samples:true}) + +Lookup mp4 box using a mp4 box path. +==================================== + + # | mp4_path($path) -> + $ fq 'mp4_path(".moov.trak[1]")' file.mp4 + +Get mp4 box path for a decode value box. +======================================== + + # | mp4_path -> string + $ fq 'grep_by(.type == "trak") | mp4_path' file.mp4 + +Force decode a single box +========================= + + $ fq -n '"AAAAHGVsc3QAAAAAAAAAAQAAADIAAAQAAAEAAA==" | frombase64 | mp4({force:true}) | d' + +Speed up decoding by not decoding samples +========================================= + + # manually decode first sample as a aac_frame + $ fq -o decode_samples=false '.tracks[0].samples[0] | aac_frame | d' file.mp4 + +Entries for first edit list as values +===================================== + + $ fq 'first(grep_by(.type=="elst").entries) | tovalue' file.mp4 + +References +========== +- ISO/IEC base media file format (MPEG-4 Part 12) (https://en.wikipedia.org/wiki/ISO/IEC_base_media_file_format) +- Quicktime file format (https://developer.apple.com/standards/qtff-2001.pdf) + diff --git a/format/mpeg/testdata/help_aac_frame.fqtest b/format/mpeg/testdata/help_aac_frame.fqtest new file mode 100644 index 00000000..53c11398 --- /dev/null +++ b/format/mpeg/testdata/help_aac_frame.fqtest @@ -0,0 +1,20 @@ +$ fq -h aac_frame +aac_frame: Advanced Audio Coding frame decoder + +Options +======= + + object_type=1 Audio object type + +Decode examples +=============== + + # Decode file as aac_frame + $ fq -d aac_frame . file + # Decode value as aac_frame + ... | aac_frame + # Decode file using aac_frame options + $ fq -d aac_frame -o object_type=1 . file + # Decode value as aac_frame + ... | aac_frame({object_type:1}) + diff --git a/format/mpeg/testdata/help_avc_au.fqtest b/format/mpeg/testdata/help_avc_au.fqtest new file mode 100644 index 00000000..3e986e05 --- /dev/null +++ b/format/mpeg/testdata/help_avc_au.fqtest @@ -0,0 +1,20 @@ +$ fq -h avc_au +avc_au: H.264/AVC Access Unit decoder + +Options +======= + + length_size=4 Length value size + +Decode examples +=============== + + # Decode file as avc_au + $ fq -d avc_au . file + # Decode value as avc_au + ... | avc_au + # Decode file using avc_au options + $ fq -d avc_au -o length_size=4 . file + # Decode value as avc_au + ... | avc_au({length_size:4}) + diff --git a/format/mpeg/testdata/help_hevc_au.fqtest b/format/mpeg/testdata/help_hevc_au.fqtest new file mode 100644 index 00000000..169a1d27 --- /dev/null +++ b/format/mpeg/testdata/help_hevc_au.fqtest @@ -0,0 +1,20 @@ +$ fq -h hevc_au +hevc_au: H.265/HEVC Access Unit decoder + +Options +======= + + length_size=4 Length value size + +Decode examples +=============== + + # Decode file as hevc_au + $ fq -d hevc_au . file + # Decode value as hevc_au + ... | hevc_au + # Decode file using hevc_au options + $ fq -d hevc_au -o length_size=4 . file + # Decode value as hevc_au + ... | hevc_au({length_size:4}) + diff --git a/format/msgpack/testdata/help_msgpack.fqtest b/format/msgpack/testdata/help_msgpack.fqtest new file mode 100644 index 00000000..78082b8b --- /dev/null +++ b/format/msgpack/testdata/help_msgpack.fqtest @@ -0,0 +1,20 @@ +$ fq -h msgpack +msgpack: MessagePack decoder + +Decode examples +=============== + + # Decode file as msgpack + $ fq -d msgpack . file + # Decode value as msgpack + ... | msgpack + +Convert represented value to JSON +================================= + + $ fq -d msgpack torepr file.msgpack + +References +========== +- https://github.com/msgpack/msgpack/blob/master/spec.md + diff --git a/format/protobuf/testdata/help_protobuf.fqtest b/format/protobuf/testdata/help_protobuf.fqtest new file mode 100644 index 00000000..95c9f31c --- /dev/null +++ b/format/protobuf/testdata/help_protobuf.fqtest @@ -0,0 +1,20 @@ +$ fq -h protobuf +protobuf: Protobuf decoder + +Decode examples +=============== + + # Decode file as protobuf + $ fq -d protobuf . file + # Decode value as protobuf + ... | protobuf + +Can decode sub messages +======================= + + $ fq -d protobuf '.fields[6].wire_value | protobuf | d' file + +References +========== +- https://developers.google.com/protocol-buffers/docs/encoding + diff --git a/format/rtmp/testdata/help_rtmp.fqtest b/format/rtmp/testdata/help_rtmp.fqtest new file mode 100644 index 00000000..564abccf --- /dev/null +++ b/format/rtmp/testdata/help_rtmp.fqtest @@ -0,0 +1,18 @@ +$ fq -h rtmp +rtmp: Real-Time Messaging Protocol decoder + +Decode examples +=============== + + # Decode file as rtmp + $ fq -d rtmp . file + # Decode value as rtmp + ... | rtmp + +Current only supports plain RTMP (not RTMPT or encrypted variants etc) with AMF0 (not AMF3). + +References +========== +- https://rtmp.veriskope.com/docs/spec/ +- https://rtmp.veriskope.com/pdf/video_file_format_spec_v10.pdf + diff --git a/format/wasm/testdata/help_wasm.fqtest b/format/wasm/testdata/help_wasm.fqtest new file mode 100644 index 00000000..29cf9f40 --- /dev/null +++ b/format/wasm/testdata/help_wasm.fqtest @@ -0,0 +1,29 @@ +$ fq -h wasm +wasm: WebAssembly Binary Format decoder + +Decode examples +=============== + + # Decode file as wasm + $ fq -d wasm . file + # Decode value as wasm + ... | wasm + +Count opcode usage +================== + + $ fq '.sections[] | select(.id == "code_section") | [.. | .opcode? // empty] | count | map({key: .[0], value: .[1]}) | from_entries' file.wasm + +List exports and imports +======================== + + $ fq '.sections | {import: map(select(.id == "import_section").content.im.x[].nm.b), export: map(select(.id == "export_section").content.ex.x[].nm.b)}' file.wasm + +Authors +======= +- Takashi Oguma @bitbears-dev (https://github.com/bitbears-dev) @0xb17bea125 (https://twitter.com/0xb17bea125) + +References +========== +- https://webassembly.github.io/spec/core/ + diff --git a/format/xml/html.md b/format/xml/html.md index 2d2b4ad3..67c27158 100644 --- a/format/xml/html.md +++ b/format/xml/html.md @@ -51,14 +51,10 @@ $ 'text' | fq -d html -o array=true ] ] ] -``` -```sh -# Decode html files to a {file: "title", ...} object +# decode html files to a {file: "title", ...} object $ fq -n -d html '[inputs | {key: input_filename, value: .html.head.title?}] | from_entries' *.html -``` -```sh # href:s in file $ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href' file.html ``` diff --git a/format/xml/testdata/help_html.fqtest b/format/xml/testdata/help_html.fqtest new file mode 100644 index 00000000..41639396 --- /dev/null +++ b/format/xml/testdata/help_html.fqtest @@ -0,0 +1,81 @@ +$ fq -h html +html: HyperText Markup Language decoder + +Options +======= + + array=false Decode as nested arrays + attribute_prefix=@ Prefix for attribute keys + seq=false Use seq attribute to preserve element order + +Decode examples +=============== + + # Decode file as html + $ fq -d html . file + # Decode value as html + ... | html + # Decode file using html options + $ fq -d html -o array=false -o attribute_prefix="@" -o seq=false . file + # Decode value as html + ... | html({array:false,attribute_prefix:"@",seq:false}) + +HTML is decoded in HTML5 mode and will always include , and element. + +See xml format for more examples and how to preserve element order and how to encode to xml. + +There is no tohtml function, see toxml instead. + +Element as object +================= + + # decode as object is the default + $ echo 'text' | fq -d html + { + "html": { + "body": { + "a": { + "#text": "text", + "@href": "url" + } + }, + "head": "" + } + } + +Element as array +================ + + $ 'text' | fq -d html -o array=true + [ + "html", + null, + [ + [ + "head", + null, + [] + ], + [ + "body", + null, + [ + [ + "a", + { + "#text": "text", + "href": "url" + }, + [] + ] + ] + ] + ] + ] + + # decode html files to a {file: "title", ...} object + $ fq -n -d html '[inputs | {key: input_filename, value: .html.head.title?}] | from_entries' *.html + + # href:s in file + $ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href' file.html + diff --git a/format/xml/testdata/help_xml.fqtest b/format/xml/testdata/help_xml.fqtest new file mode 100644 index 00000000..20232db6 --- /dev/null +++ b/format/xml/testdata/help_xml.fqtest @@ -0,0 +1,124 @@ +$ fq -h xml +xml: Extensible Markup Language decoder + +Options +======= + + array=false Decode as nested arrays + attribute_prefix=@ Prefix for attribute keys + seq=false Use seq attribute to preserve element order + +Decode examples +=============== + + # Decode file as xml + $ fq -d xml . file + # Decode value as xml + ... | xml + # Decode file using xml options + $ fq -d xml -o array=false -o attribute_prefix="@" -o seq=false . file + # Decode value as xml + ... | xml({array:false,attribute_prefix:"@",seq:false}) + +XML can be decoded and encoded into jq values in two ways, elements as object or array. Which variant to use depends a bit what you want to do. The object variant +might be easier to query for a specific value but array might be easier to use to generate xml or to query after all elements of some kind etc. + +Encoding is done using the toxml function and it will figure what variant that is used based on the input value. Is has two optional options indent and +attribute_prefix. + +Elements as object +================== +Element can have different shapes depending on body text, attributes and children: + +- text is {"a":{"#text":"text","@key":"value"}}, has text (#text) and attributes (@key) +- text is {"a":"text"} +- text is {"a":{"b":"text"}} one child with only text and no attributes +- text is {"a":{"b":["","text"]}} two children with same name end up in an array +- text is {"a":{"b":["",{"#text":"text","@key":"value"}]}} + +If there is #seq attribute it encodes the child element order. Use -o seq=true to include sequence number when decoding, otherwise order might be lost. + + + # decode as object is the default + $ echo 'bbbccc' | fq -d xml -o seq=true + { + "a": { + "b": [ + { + "#seq": 0 + }, + { + "#seq": 1, + "#text": "bbb" + } + ], + "c": { + "#seq": 2, + "#text": "ccc", + "@attr": "value" + } + } + } + + # access text of the element + $ echo 'bbbccc' | fq '.a.c["#text"]' + "ccc" + + # decode to object and encode to xml + $ echo 'bbbccc' | fq -r -d xml -o seq=true 'toxml({indent:2})' + + + bbb + ccc + + +Elements as array +================= +Elements are arrays of the shape ["#text": "body text", "attr_name", {key: "attr value"}|null, [, ...]]. + + + # decode as array + ✗ echo 'bbbccc' | fq -d xml -o array=true + [ + "a", + null, + [ + [ + "b", + null, + [] + ], + [ + "b", + { + "#text": "bbb" + }, + [] + ], + [ + "c", + { + "#text": "ccc", + "attr": "value" + }, + [] + ] + ] + ] + + # decode to array and encode to xml + $ echo 'bbbccc' | fq -r -d xml -o array=true -o seq=true 'toxml({indent:2})' + + + bbb + ccc + + + # access text of the element, the object variant above is probably easier to use + $ echo 'bbbccc' | fq -o array=true '.[2][2][1]["#text"]' + "ccc" + +References +========== +- xml.com's Converting Between XML and JSON (https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html) + diff --git a/format/xml/xml.md b/format/xml/xml.md index c483cdd0..d8112f61 100644 --- a/format/xml/xml.md +++ b/format/xml/xml.md @@ -44,9 +44,7 @@ $ echo 'bbbccc' | fq -d xml -o seq=true # access text of the element $ echo 'bbbccc' | fq '.a.c["#text"]' "ccc" -``` -```sh # decode to object and encode to xml $ echo 'bbbccc' | fq -r -d xml -o seq=true 'toxml({indent:2})' @@ -89,11 +87,9 @@ Elements are arrays of the shape `["#text": "body text", "attr_name", {key: "att ] ] ] -``` -```sh # decode to array and encode to xml -✗ echo 'bbbccc' | fq -r -d xml -o array=true -o seq=true 'toxml({indent:2})' +$ echo 'bbbccc' | fq -r -d xml -o array=true -o seq=true 'toxml({indent:2})' bbb diff --git a/format/zip/testdata/help_zip.fqtest b/format/zip/testdata/help_zip.fqtest new file mode 100644 index 00000000..7fff18a9 --- /dev/null +++ b/format/zip/testdata/help_zip.fqtest @@ -0,0 +1,27 @@ +$ fq -h zip +zip: ZIP archive decoder + +Options +======= + + uncompress=true Uncompress and probe files + +Decode examples +=============== + + # Decode file as zip + $ fq -d zip . file + # Decode value as zip + ... | zip + # Decode file using zip options + $ fq -d zip -o uncompress=true . file + # Decode value as zip + ... | zip({uncompress:true}) + +Supports ZIP64. + +References +========== +- https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT +- https://opensource.apple.com/source/zip/zip-6/unzip/unzip/proginfo/extra.fld + diff --git a/pkg/interp/help.jq b/pkg/interp/help.jq index 2a3208a1..4ce339b1 100644 --- a/pkg/interp/help.jq +++ b/pkg/interp/help.jq @@ -140,11 +140,13 @@ def _help($arg0; $topic): | (_format_func($f.name; "_help")? // {} | _help_format_enrich($arg0; $f; true)) as $fhelp | ((_registry.files[][] | select(.name=="\($topic).md").data) // false) as $doc | "\($f.name): \($f.description) decoder" + , "" , if $f.decode_in_arg then ( $f.decode_in_arg | to_entries | map([" \(.key)=\(.value) ", $f.decode_in_arg_doc[.key]]) - | "# Options" + | "Options" + , "=======" , "" , table( .; @@ -157,11 +159,12 @@ def _help($arg0; $topic): ) ) | join("") ) + , "" ) else empty end - , "" - , "# Decode examples" + , "Decode examples" + , "===============" , "" , ( $fhelp.examples[] | " # \(.comment)" diff --git a/pkg/interp/testdata/args.fqtest b/pkg/interp/testdata/args.fqtest index 9d71f2f8..b9f08748 100644 --- a/pkg/interp/testdata/args.fqtest +++ b/pkg/interp/testdata/args.fqtest @@ -98,114 +98,6 @@ string_input false unicode false verbose false width 135 -$ fq --help formats -aac_frame Advanced Audio Coding frame -adts Audio Data Transport Stream -adts_frame Audio Data Transport Stream frame -amf0 Action Message Format 0 -apev2 APEv2 metadata tag -ar Unix archive -asn1_ber ASN1 BER (basic encoding rules, also CER and DER) -av1_ccr AV1 Codec Configuration Record -av1_frame AV1 frame -av1_obu AV1 Open Bitstream Unit -avc_annexb H.264/AVC Annex B -avc_au H.264/AVC Access Unit -avc_dcr H.264/AVC Decoder Configuration Record -avc_nalu H.264/AVC Network Access Layer Unit -avc_pps H.264/AVC Picture Parameter Set -avc_sei H.264/AVC Supplemental Enhancement Information -avc_sps H.264/AVC Sequence Parameter Set -avro_ocf Avro object container file -bencode BitTorrent bencoding -bitcoin_blkdat Bitcoin blk.dat -bitcoin_block Bitcoin block -bitcoin_script Bitcoin script -bitcoin_transaction Bitcoin transaction -bsd_loopback_frame BSD loopback frame -bson Binary JSON -bzip2 bzip2 compression -cbor Concise Binary Object Representation -csv Comma separated values -dns DNS packet -dns_tcp DNS packet (TCP) -elf Executable and Linkable Format -ether8023_frame Ethernet 802.3 frame -exif Exchangeable Image File Format -fairplay_spc FairPlay Server Playback Context -flac Free Lossless Audio Codec file -flac_frame FLAC frame -flac_metadatablock FLAC metadatablock -flac_metadatablocks FLAC metadatablocks -flac_picture FLAC metadatablock picture -flac_streaminfo FLAC streaminfo -gif Graphics Interchange Format -gzip gzip compression -hevc_annexb H.265/HEVC Annex B -hevc_au H.265/HEVC Access Unit -hevc_dcr H.265/HEVC Decoder Configuration Record -hevc_nalu H.265/HEVC Network Access Layer Unit -hevc_pps H.265/HEVC Picture Parameter Set -hevc_sps H.265/HEVC Sequence Parameter Set -hevc_vps H.265/HEVC Video Parameter Set -html HyperText Markup Language -icc_profile International Color Consortium profile -icmp Internet Control Message Protocol -icmpv6 Internet Control Message Protocol v6 -id3v1 ID3v1 metadata -id3v11 ID3v1.1 metadata -id3v2 ID3v2 metadata -ipv4_packet Internet protocol v4 packet -ipv6_packet Internet protocol v6 packet -jpeg Joint Photographic Experts Group file -json JavaScript Object Notation -jsonl JavaScript Object Notation Lines -macho Mach-O macOS executable -macho_fat Fat Mach-O macOS executable (multi-architecture) -markdown Markdown -matroska Matroska file -mp3 MP3 file -mp3_frame MPEG audio layer 3 frame -mp4 ISOBMFF, QuickTime and similar -mpeg_asc MPEG-4 Audio Specific Config -mpeg_es MPEG Elementary Stream -mpeg_pes MPEG Packetized elementary stream -mpeg_pes_packet MPEG Packetized elementary stream packet -mpeg_spu Sub Picture Unit (DVD subtitle) -mpeg_ts MPEG Transport Stream -msgpack MessagePack -ogg OGG file -ogg_page OGG page -opus_packet Opus packet -pcap PCAP packet capture -pcapng PCAPNG packet capture -png Portable Network Graphics file -prores_frame Apple ProRes frame -protobuf Protobuf -protobuf_widevine Widevine protobuf -pssh_playready PlayReady PSSH -raw Raw bits -rtmp Real-Time Messaging Protocol -sll2_packet Linux cooked capture encapsulation v2 -sll_packet Linux cooked capture encapsulation -tar Tar archive -tcp_segment Transmission control protocol segment -tiff Tag Image File Format -toml Tom's Obvious, Minimal Language -udp_datagram User datagram protocol -vorbis_comment Vorbis comment -vorbis_packet Vorbis packet -vp8_frame VP8 frame -vp9_cfm VP9 Codec Feature Metadata -vp9_frame VP9 frame -vpx_ccr VPX Codec Configuration Record -wasm WebAssembly Binary Format -wav WAV file -webp WebP image -xing Xing header -xml Extensible Markup Language -yaml YAML Ain't Markup Language -zip ZIP archive $ fq -X exitcode: 2 stderr: