wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-11-21 13:33:22 +03:00
Code Issues Projects Releases Wiki Activity

help,doc: Use markdown for format documentation again

Browse Source 
Markdown is used as is in online documentation and in cli the markdown decoder
is used to decode and the some jq code massages it into something cli friendly.

Was just too much of a mess to have doc in jq.
This commit is contained in:
Mattias Wadman 2022-09-10 18:28:54 +02:00
parent 00a50662ea
commit 87b2c6c10c
50 changed files with 1079 additions and 635 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitattributes vendored
View File

@ -3,3 +3,4 @@
*.json eol=lf
*.jq eol=lf
*.xml eol=lf
*.md eol=lf

3
README.md
View File

@ -48,7 +48,7 @@ avc_sps,
[avro_ocf](doc/formats.md#avro_ocf),
[bencode](doc/formats.md#bencode),
bitcoin_blkdat,
bitcoin_block,
[bitcoin_block](doc/formats.md#bitcoin_block),
bitcoin_script,
bitcoin_transaction,
bsd_loopback_frame,
@ -91,6 +91,7 @@ json,
jsonl,
[macho](doc/formats.md#macho),
macho_fat,
markdown,
[matroska](doc/formats.md#matroska),
[mp3](doc/formats.md#mp3),
mp3_frame,

27
doc/formats.jq
View File

@ -2,14 +2,13 @@
def code: "`\(.)`";
def nbsp: gsub(" "; " ");
def has_section($f; $fhelp): $fhelp.notes or $fhelp.examples or $fhelp.links or $f.decode_in_arg;
def has_section($f; $fhelp): $fhelp.notes or $fhelp.examples or $f.decode_in_arg or ((_registry.files[][] | select(.name=="\($f.name).md").data) // false);
def formats_list:
[ formats
| to_entries[] as {$key, $value}
| (_format_func($key; "_help")? // {}) as $fhelp
| if has_section($value; $fhelp) then "[\($key)](doc/formats.md#\($key))"
else $key
[ formats[] as $f
| ({} | _help_format_enrich("fq"; $f; false)) as $fhelp
| if has_section($f; $fhelp) then "[\($f.name)](doc/formats.md#\($f.name))"
else $f.name
end
] | join(",\n");
@ -71,7 +70,8 @@ def formats_table:
def formats_sections:
( formats[] as $f
| (_format_func($f.name; "_help")? // {} | _help_format_enrich("fq"; $f; false)) as $fhelp
| ((_registry.files[][] | select(.name=="\($f.name).md").data) // false) as $doc
| ({} | _help_format_enrich("fq"; $f; false)) as $fhelp
| select(has_section($f; $fhelp))
| "### \($f.name)"
, ""
@ -134,16 +134,5 @@ def formats_sections:
)
else empty
end
, if $fhelp.links then
( "#### References and links"
, ""
, ( $fhelp.links[]
| if .title then "- [\(.title)](\(.url))"
else "- \(.url)"
end
)
, ""
)
else empty
end
, ($doc // empty)
);

433
doc/formats.md
View File

@ -2,121 +2,122 @@
[fq -rn -L . 'include "formats"; formats_table']: sh-start
|Name |Description |Dependencies|
|- |- |-|
|[`aac_frame`](#aac_frame) |Advanced&nbsp;Audio&nbsp;Coding&nbsp;frame |<sub></sub>|
|`adts` |Audio&nbsp;Data&nbsp;Transport&nbsp;Stream |<sub>`adts_frame`</sub>|
|`adts_frame` |Audio&nbsp;Data&nbsp;Transport&nbsp;Stream&nbsp;frame |<sub>`aac_frame`</sub>|
|`amf0` |Action&nbsp;Message&nbsp;Format&nbsp;0 |<sub></sub>|
|`apev2` |APEv2&nbsp;metadata&nbsp;tag |<sub>`image`</sub>|
|`ar` |Unix&nbsp;archive |<sub>`probe`</sub>|
|[`asn1_ber`](#asn1_ber) |ASN1&nbsp;BER&nbsp;(basic&nbsp;encoding&nbsp;rules,&nbsp;also&nbsp;CER&nbsp;and&nbsp;DER)|<sub></sub>|
|`av1_ccr` |AV1&nbsp;Codec&nbsp;Configuration&nbsp;Record |<sub></sub>|
|`av1_frame` |AV1&nbsp;frame |<sub>`av1_obu`</sub>|
|`av1_obu` |AV1&nbsp;Open&nbsp;Bitstream&nbsp;Unit |<sub></sub>|
|`avc_annexb` |H.264/AVC&nbsp;Annex&nbsp;B |<sub>`avc_nalu`</sub>|
|[`avc_au`](#avc_au) |H.264/AVC&nbsp;Access&nbsp;Unit |<sub>`avc_nalu`</sub>|
|`avc_dcr` |H.264/AVC&nbsp;Decoder&nbsp;Configuration&nbsp;Record |<sub>`avc_nalu`</sub>|
|`avc_nalu` |H.264/AVC&nbsp;Network&nbsp;Access&nbsp;Layer&nbsp;Unit |<sub>`avc_sps` `avc_pps` `avc_sei`</sub>|
|`avc_pps` |H.264/AVC&nbsp;Picture&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`avc_sei` |H.264/AVC&nbsp;Supplemental&nbsp;Enhancement&nbsp;Information |<sub></sub>|
|`avc_sps` |H.264/AVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set |<sub></sub>|
|[`avro_ocf`](#avro_ocf) |Avro&nbsp;object&nbsp;container&nbsp;file |<sub></sub>|
|[`bencode`](#bencode) |BitTorrent&nbsp;bencoding |<sub></sub>|
|`bitcoin_blkdat` |Bitcoin&nbsp;blk.dat |<sub>`bitcoin_block`</sub>|
|`bitcoin_block` |Bitcoin&nbsp;block |<sub>`bitcoin_transaction`</sub>|
|`bitcoin_script` |Bitcoin&nbsp;script |<sub></sub>|
|`bitcoin_transaction` |Bitcoin&nbsp;transaction |<sub>`bitcoin_script`</sub>|
|`bsd_loopback_frame` |BSD&nbsp;loopback&nbsp;frame |<sub>`inet_packet`</sub>|
|[`bson`](#bson) |Binary&nbsp;JSON |<sub></sub>|
|`bzip2` |bzip2&nbsp;compression |<sub>`probe`</sub>|
|[`cbor`](#cbor) |Concise&nbsp;Binary&nbsp;Object&nbsp;Representation |<sub></sub>|
|[`csv`](#csv) |Comma&nbsp;separated&nbsp;values |<sub></sub>|
|`dns` |DNS&nbsp;packet |<sub></sub>|
|`dns_tcp` |DNS&nbsp;packet&nbsp;(TCP) |<sub></sub>|
|`elf` |Executable&nbsp;and&nbsp;Linkable&nbsp;Format |<sub></sub>|
|`ether8023_frame` |Ethernet&nbsp;802.3&nbsp;frame |<sub>`inet_packet`</sub>|
|`exif` |Exchangeable&nbsp;Image&nbsp;File&nbsp;Format |<sub></sub>|
|`fairplay_spc` |FairPlay&nbsp;Server&nbsp;Playback&nbsp;Context |<sub></sub>|
|`flac` |Free&nbsp;Lossless&nbsp;Audio&nbsp;Codec&nbsp;file |<sub>`flac_metadatablocks` `flac_frame`</sub>|
|[`flac_frame`](#flac_frame) |FLAC&nbsp;frame |<sub></sub>|
|`flac_metadatablock` |FLAC&nbsp;metadatablock |<sub>`flac_streaminfo` `flac_picture` `vorbis_comment`</sub>|
|`flac_metadatablocks` |FLAC&nbsp;metadatablocks |<sub>`flac_metadatablock`</sub>|
|`flac_picture` |FLAC&nbsp;metadatablock&nbsp;picture |<sub>`image`</sub>|
|`flac_streaminfo` |FLAC&nbsp;streaminfo |<sub></sub>|
|`gif` |Graphics&nbsp;Interchange&nbsp;Format |<sub></sub>|
|`gzip` |gzip&nbsp;compression |<sub>`probe`</sub>|
|`hevc_annexb` |H.265/HEVC&nbsp;Annex&nbsp;B |<sub>`hevc_nalu`</sub>|
|[`hevc_au`](#hevc_au) |H.265/HEVC&nbsp;Access&nbsp;Unit |<sub>`hevc_nalu`</sub>|
|`hevc_dcr` |H.265/HEVC&nbsp;Decoder&nbsp;Configuration&nbsp;Record |<sub>`hevc_nalu`</sub>|
|`hevc_nalu` |H.265/HEVC&nbsp;Network&nbsp;Access&nbsp;Layer&nbsp;Unit |<sub>`hevc_vps` `hevc_pps` `hevc_sps`</sub>|
|`hevc_pps` |H.265/HEVC&nbsp;Picture&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`hevc_sps` |H.265/HEVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`hevc_vps` |H.265/HEVC&nbsp;Video&nbsp;Parameter&nbsp;Set |<sub></sub>|
|[`html`](#html) |HyperText&nbsp;Markup&nbsp;Language |<sub></sub>|
|`icc_profile` |International&nbsp;Color&nbsp;Consortium&nbsp;profile |<sub></sub>|
|`icmp` |Internet&nbsp;Control&nbsp;Message&nbsp;Protocol |<sub></sub>|
|`icmpv6` |Internet&nbsp;Control&nbsp;Message&nbsp;Protocol&nbsp;v6 |<sub></sub>|
|`id3v1` |ID3v1&nbsp;metadata |<sub></sub>|
|`id3v11` |ID3v1.1&nbsp;metadata |<sub></sub>|
|`id3v2` |ID3v2&nbsp;metadata |<sub>`image`</sub>|
|`ipv4_packet` |Internet&nbsp;protocol&nbsp;v4&nbsp;packet |<sub>`ip_packet`</sub>|
|`ipv6_packet` |Internet&nbsp;protocol&nbsp;v6&nbsp;packet |<sub>`ip_packet`</sub>|
|`jpeg` |Joint&nbsp;Photographic&nbsp;Experts&nbsp;Group&nbsp;file |<sub>`exif` `icc_profile`</sub>|
|`json` |JavaScript&nbsp;Object&nbsp;Notation |<sub></sub>|
|`jsonl` |JavaScript&nbsp;Object&nbsp;Notation&nbsp;Lines |<sub></sub>|
|[`macho`](#macho) |Mach-O&nbsp;macOS&nbsp;executable |<sub></sub>|
|`macho_fat` |Fat&nbsp;Mach-O&nbsp;macOS&nbsp;executable&nbsp;(multi-architecture) |<sub>`macho`</sub>|
|[`matroska`](#matroska) |Matroska&nbsp;file |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `image` `mp3_frame` `mpeg_asc` `mpeg_pes_packet` `mpeg_spu` `opus_packet` `vorbis_packet` `vp8_frame` `vp9_cfm` `vp9_frame`</sub>|
|[`mp3`](#mp3) |MP3&nbsp;file |<sub>`id3v2` `id3v1` `id3v11` `apev2` `mp3_frame`</sub>|
|`mp3_frame` |MPEG&nbsp;audio&nbsp;layer&nbsp;3&nbsp;frame |<sub>`xing`</sub>|
|[`mp4`](#mp4) |ISOBMFF&nbsp;MPEG-4&nbsp;part&nbsp;12&nbsp;and&nbsp;similar |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `icc_profile` `id3v2` `image` `jpeg` `mp3_frame` `mpeg_es` `mpeg_pes_packet` `opus_packet` `prores_frame` `protobuf_widevine` `pssh_playready` `vorbis_packet` `vp9_frame` `vpx_ccr`</sub>|
|`mpeg_asc` |MPEG-4&nbsp;Audio&nbsp;Specific&nbsp;Config |<sub></sub>|
|`mpeg_es` |MPEG&nbsp;Elementary&nbsp;Stream |<sub>`mpeg_asc` `vorbis_packet`</sub>|
|`mpeg_pes` |MPEG&nbsp;Packetized&nbsp;elementary&nbsp;stream |<sub>`mpeg_pes_packet` `mpeg_spu`</sub>|
|`mpeg_pes_packet` |MPEG&nbsp;Packetized&nbsp;elementary&nbsp;stream&nbsp;packet |<sub></sub>|
|`mpeg_spu` |Sub&nbsp;Picture&nbsp;Unit&nbsp;(DVD&nbsp;subtitle) |<sub></sub>|
|`mpeg_ts` |MPEG&nbsp;Transport&nbsp;Stream |<sub></sub>|
|[`msgpack`](#msgpack) |MessagePack |<sub></sub>|
|`ogg` |OGG&nbsp;file |<sub>`ogg_page` `vorbis_packet` `opus_packet` `flac_metadatablock` `flac_frame`</sub>|
|`ogg_page` |OGG&nbsp;page |<sub></sub>|
|`opus_packet` |Opus&nbsp;packet |<sub>`vorbis_comment`</sub>|
|`pcap` |PCAP&nbsp;packet&nbsp;capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|`pcapng` |PCAPNG&nbsp;packet&nbsp;capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|`png` |Portable&nbsp;Network&nbsp;Graphics&nbsp;file |<sub>`icc_profile` `exif`</sub>|
|`prores_frame` |Apple&nbsp;ProRes&nbsp;frame |<sub></sub>|
|[`protobuf`](#protobuf) |Protobuf |<sub></sub>|
|`protobuf_widevine` |Widevine&nbsp;protobuf |<sub>`protobuf`</sub>|
|`pssh_playready` |PlayReady&nbsp;PSSH |<sub></sub>|
|`raw` |Raw&nbsp;bits |<sub></sub>|
|[`rtmp`](#rtmp) |Real-Time&nbsp;Messaging&nbsp;Protocol |<sub>`amf0` `mpeg_asc`</sub>|
|`sll2_packet` |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation&nbsp;v2 |<sub>`inet_packet`</sub>|
|`sll_packet` |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation |<sub>`inet_packet`</sub>|
|`tar` |Tar&nbsp;archive |<sub>`probe`</sub>|
|`tcp_segment` |Transmission&nbsp;control&nbsp;protocol&nbsp;segment |<sub></sub>|
|`tiff` |Tag&nbsp;Image&nbsp;File&nbsp;Format |<sub>`icc_profile`</sub>|
|`toml` |Tom's&nbsp;Obvious,&nbsp;Minimal&nbsp;Language |<sub></sub>|
|`udp_datagram` |User&nbsp;datagram&nbsp;protocol |<sub>`udp_payload`</sub>|
|`vorbis_comment` |Vorbis&nbsp;comment |<sub>`flac_picture`</sub>|
|`vorbis_packet` |Vorbis&nbsp;packet |<sub>`vorbis_comment`</sub>|
|`vp8_frame` |VP8&nbsp;frame |<sub></sub>|
|`vp9_cfm` |VP9&nbsp;Codec&nbsp;Feature&nbsp;Metadata |<sub></sub>|
|`vp9_frame` |VP9&nbsp;frame |<sub></sub>|
|`vpx_ccr` |VPX&nbsp;Codec&nbsp;Configuration&nbsp;Record |<sub></sub>|
|`wasm` |WebAssembly&nbsp;Binary&nbsp;Format |<sub></sub>|
|`wav` |WAV&nbsp;file |<sub>`id3v2` `id3v1` `id3v11`</sub>|
|`webp` |WebP&nbsp;image |<sub>`vp8_frame`</sub>|
|`xing` |Xing&nbsp;header |<sub></sub>|
|[`xml`](#xml) |Extensible&nbsp;Markup&nbsp;Language |<sub></sub>|
|`yaml` |YAML&nbsp;Ain't&nbsp;Markup&nbsp;Language |<sub></sub>|
|[`zip`](#zip) |ZIP&nbsp;archive |<sub>`probe`</sub>|
|`image` |Group |<sub>`gif` `jpeg` `mp4` `png` `tiff` `webp`</sub>|
|`inet_packet` |Group |<sub>`ipv4_packet` `ipv6_packet`</sub>|
|`ip_packet` |Group |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
|`link_frame` |Group |<sub>`bsd_loopback_frame` `ether8023_frame` `sll2_packet` `sll_packet`</sub>|
|`probe` |Group |<sub>`adts` `ar` `avro_ocf` `bitcoin_blkdat` `bzip2` `elf` `flac` `gif` `gzip` `jpeg` `json` `jsonl` `macho` `macho_fat` `matroska` `mp3` `mp4` `mpeg_ts` `ogg` `pcap` `pcapng` `png` `tar` `tiff` `toml` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
|`tcp_stream` |Group |<sub>`dns_tcp` `rtmp`</sub>|
|`udp_payload` |Group |<sub>`dns`</sub>|
|Name |Description |Dependencies|
|- |- |-|
|[`aac_frame`](#aac_frame) |Advanced&nbsp;Audio&nbsp;Coding&nbsp;frame |<sub></sub>|
|`adts` |Audio&nbsp;Data&nbsp;Transport&nbsp;Stream |<sub>`adts_frame`</sub>|
|`adts_frame` |Audio&nbsp;Data&nbsp;Transport&nbsp;Stream&nbsp;frame |<sub>`aac_frame`</sub>|
|`amf0` |Action&nbsp;Message&nbsp;Format&nbsp;0 |<sub></sub>|
|`apev2` |APEv2&nbsp;metadata&nbsp;tag |<sub>`image`</sub>|
|`ar` |Unix&nbsp;archive |<sub>`probe`</sub>|
|[`asn1_ber`](#asn1_ber) |ASN1&nbsp;BER&nbsp;(basic&nbsp;encoding&nbsp;rules,&nbsp;also&nbsp;CER&nbsp;and&nbsp;DER)|<sub></sub>|
|`av1_ccr` |AV1&nbsp;Codec&nbsp;Configuration&nbsp;Record |<sub></sub>|
|`av1_frame` |AV1&nbsp;frame |<sub>`av1_obu`</sub>|
|`av1_obu` |AV1&nbsp;Open&nbsp;Bitstream&nbsp;Unit |<sub></sub>|
|`avc_annexb` |H.264/AVC&nbsp;Annex&nbsp;B |<sub>`avc_nalu`</sub>|
|[`avc_au`](#avc_au) |H.264/AVC&nbsp;Access&nbsp;Unit |<sub>`avc_nalu`</sub>|
|`avc_dcr` |H.264/AVC&nbsp;Decoder&nbsp;Configuration&nbsp;Record |<sub>`avc_nalu`</sub>|
|`avc_nalu` |H.264/AVC&nbsp;Network&nbsp;Access&nbsp;Layer&nbsp;Unit |<sub>`avc_sps` `avc_pps` `avc_sei`</sub>|
|`avc_pps` |H.264/AVC&nbsp;Picture&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`avc_sei` |H.264/AVC&nbsp;Supplemental&nbsp;Enhancement&nbsp;Information |<sub></sub>|
|`avc_sps` |H.264/AVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set |<sub></sub>|
|[`avro_ocf`](#avro_ocf) |Avro&nbsp;object&nbsp;container&nbsp;file |<sub></sub>|
|[`bencode`](#bencode) |BitTorrent&nbsp;bencoding |<sub></sub>|
|`bitcoin_blkdat` |Bitcoin&nbsp;blk.dat |<sub>`bitcoin_block`</sub>|
|[`bitcoin_block`](#bitcoin_block) |Bitcoin&nbsp;block |<sub>`bitcoin_transaction`</sub>|
|`bitcoin_script` |Bitcoin&nbsp;script |<sub></sub>|
|`bitcoin_transaction` |Bitcoin&nbsp;transaction |<sub>`bitcoin_script`</sub>|
|`bsd_loopback_frame` |BSD&nbsp;loopback&nbsp;frame |<sub>`inet_packet`</sub>|
|[`bson`](#bson) |Binary&nbsp;JSON |<sub></sub>|
|`bzip2` |bzip2&nbsp;compression |<sub>`probe`</sub>|
|[`cbor`](#cbor) |Concise&nbsp;Binary&nbsp;Object&nbsp;Representation |<sub></sub>|
|[`csv`](#csv) |Comma&nbsp;separated&nbsp;values |<sub></sub>|
|`dns` |DNS&nbsp;packet |<sub></sub>|
|`dns_tcp` |DNS&nbsp;packet&nbsp;(TCP) |<sub></sub>|
|`elf` |Executable&nbsp;and&nbsp;Linkable&nbsp;Format |<sub></sub>|
|`ether8023_frame` |Ethernet&nbsp;802.3&nbsp;frame |<sub>`inet_packet`</sub>|
|`exif` |Exchangeable&nbsp;Image&nbsp;File&nbsp;Format |<sub></sub>|
|`fairplay_spc` |FairPlay&nbsp;Server&nbsp;Playback&nbsp;Context |<sub></sub>|
|`flac` |Free&nbsp;Lossless&nbsp;Audio&nbsp;Codec&nbsp;file |<sub>`flac_metadatablocks` `flac_frame`</sub>|
|[`flac_frame`](#flac_frame) |FLAC&nbsp;frame |<sub></sub>|
|`flac_metadatablock` |FLAC&nbsp;metadatablock |<sub>`flac_streaminfo` `flac_picture` `vorbis_comment`</sub>|
|`flac_metadatablocks` |FLAC&nbsp;metadatablocks |<sub>`flac_metadatablock`</sub>|
|`flac_picture` |FLAC&nbsp;metadatablock&nbsp;picture |<sub>`image`</sub>|
|`flac_streaminfo` |FLAC&nbsp;streaminfo |<sub></sub>|
|`gif` |Graphics&nbsp;Interchange&nbsp;Format |<sub></sub>|
|`gzip` |gzip&nbsp;compression |<sub>`probe`</sub>|
|`hevc_annexb` |H.265/HEVC&nbsp;Annex&nbsp;B |<sub>`hevc_nalu`</sub>|
|[`hevc_au`](#hevc_au) |H.265/HEVC&nbsp;Access&nbsp;Unit |<sub>`hevc_nalu`</sub>|
|`hevc_dcr` |H.265/HEVC&nbsp;Decoder&nbsp;Configuration&nbsp;Record |<sub>`hevc_nalu`</sub>|
|`hevc_nalu` |H.265/HEVC&nbsp;Network&nbsp;Access&nbsp;Layer&nbsp;Unit |<sub>`hevc_vps` `hevc_pps` `hevc_sps`</sub>|
|`hevc_pps` |H.265/HEVC&nbsp;Picture&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`hevc_sps` |H.265/HEVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`hevc_vps` |H.265/HEVC&nbsp;Video&nbsp;Parameter&nbsp;Set |<sub></sub>|
|[`html`](#html) |HyperText&nbsp;Markup&nbsp;Language |<sub></sub>|
|`icc_profile` |International&nbsp;Color&nbsp;Consortium&nbsp;profile |<sub></sub>|
|`icmp` |Internet&nbsp;Control&nbsp;Message&nbsp;Protocol |<sub></sub>|
|`icmpv6` |Internet&nbsp;Control&nbsp;Message&nbsp;Protocol&nbsp;v6 |<sub></sub>|
|`id3v1` |ID3v1&nbsp;metadata |<sub></sub>|
|`id3v11` |ID3v1.1&nbsp;metadata |<sub></sub>|
|`id3v2` |ID3v2&nbsp;metadata |<sub>`image`</sub>|
|`ipv4_packet` |Internet&nbsp;protocol&nbsp;v4&nbsp;packet |<sub>`ip_packet`</sub>|
|`ipv6_packet` |Internet&nbsp;protocol&nbsp;v6&nbsp;packet |<sub>`ip_packet`</sub>|
|`jpeg` |Joint&nbsp;Photographic&nbsp;Experts&nbsp;Group&nbsp;file |<sub>`exif` `icc_profile`</sub>|
|`json` |JavaScript&nbsp;Object&nbsp;Notation |<sub></sub>|
|`jsonl` |JavaScript&nbsp;Object&nbsp;Notation&nbsp;Lines |<sub></sub>|
|[`macho`](#macho) |Mach-O&nbsp;macOS&nbsp;executable |<sub></sub>|
|`macho_fat` |Fat&nbsp;Mach-O&nbsp;macOS&nbsp;executable&nbsp;(multi-architecture) |<sub>`macho`</sub>|
|`markdown` |Markdown |<sub></sub>|
|[`matroska`](#matroska) |Matroska&nbsp;file |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `image` `mp3_frame` `mpeg_asc` `mpeg_pes_packet` `mpeg_spu` `opus_packet` `vorbis_packet` `vp8_frame` `vp9_cfm` `vp9_frame`</sub>|
|[`mp3`](#mp3) |MP3&nbsp;file |<sub>`id3v2` `id3v1` `id3v11` `apev2` `mp3_frame`</sub>|
|`mp3_frame` |MPEG&nbsp;audio&nbsp;layer&nbsp;3&nbsp;frame |<sub>`xing`</sub>|
|[`mp4`](#mp4) |ISOBMFF,&nbsp;QuickTime&nbsp;and&nbsp;similar |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `icc_profile` `id3v2` `image` `jpeg` `mp3_frame` `mpeg_es` `mpeg_pes_packet` `opus_packet` `prores_frame` `protobuf_widevine` `pssh_playready` `vorbis_packet` `vp9_frame` `vpx_ccr`</sub>|
|`mpeg_asc` |MPEG-4&nbsp;Audio&nbsp;Specific&nbsp;Config |<sub></sub>|
|`mpeg_es` |MPEG&nbsp;Elementary&nbsp;Stream |<sub>`mpeg_asc` `vorbis_packet`</sub>|
|`mpeg_pes` |MPEG&nbsp;Packetized&nbsp;elementary&nbsp;stream |<sub>`mpeg_pes_packet` `mpeg_spu`</sub>|
|`mpeg_pes_packet` |MPEG&nbsp;Packetized&nbsp;elementary&nbsp;stream&nbsp;packet |<sub></sub>|
|`mpeg_spu` |Sub&nbsp;Picture&nbsp;Unit&nbsp;(DVD&nbsp;subtitle) |<sub></sub>|
|`mpeg_ts` |MPEG&nbsp;Transport&nbsp;Stream |<sub></sub>|
|[`msgpack`](#msgpack) |MessagePack |<sub></sub>|
|`ogg` |OGG&nbsp;file |<sub>`ogg_page` `vorbis_packet` `opus_packet` `flac_metadatablock` `flac_frame`</sub>|
|`ogg_page` |OGG&nbsp;page |<sub></sub>|
|`opus_packet` |Opus&nbsp;packet |<sub>`vorbis_comment`</sub>|
|`pcap` |PCAP&nbsp;packet&nbsp;capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|`pcapng` |PCAPNG&nbsp;packet&nbsp;capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|`png` |Portable&nbsp;Network&nbsp;Graphics&nbsp;file |<sub>`icc_profile` `exif`</sub>|
|`prores_frame` |Apple&nbsp;ProRes&nbsp;frame |<sub></sub>|
|[`protobuf`](#protobuf) |Protobuf |<sub></sub>|
|`protobuf_widevine` |Widevine&nbsp;protobuf |<sub>`protobuf`</sub>|
|`pssh_playready` |PlayReady&nbsp;PSSH |<sub></sub>|
|`raw` |Raw&nbsp;bits |<sub></sub>|
|[`rtmp`](#rtmp) |Real-Time&nbsp;Messaging&nbsp;Protocol |<sub>`amf0` `mpeg_asc`</sub>|
|`sll2_packet` |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation&nbsp;v2 |<sub>`inet_packet`</sub>|
|`sll_packet` |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation |<sub>`inet_packet`</sub>|
|`tar` |Tar&nbsp;archive |<sub>`probe`</sub>|
|`tcp_segment` |Transmission&nbsp;control&nbsp;protocol&nbsp;segment |<sub></sub>|
|`tiff` |Tag&nbsp;Image&nbsp;File&nbsp;Format |<sub>`icc_profile`</sub>|
|`toml` |Tom's&nbsp;Obvious,&nbsp;Minimal&nbsp;Language |<sub></sub>|
|`udp_datagram` |User&nbsp;datagram&nbsp;protocol |<sub>`udp_payload`</sub>|
|`vorbis_comment` |Vorbis&nbsp;comment |<sub>`flac_picture`</sub>|
|`vorbis_packet` |Vorbis&nbsp;packet |<sub>`vorbis_comment`</sub>|
|`vp8_frame` |VP8&nbsp;frame |<sub></sub>|
|`vp9_cfm` |VP9&nbsp;Codec&nbsp;Feature&nbsp;Metadata |<sub></sub>|
|`vp9_frame` |VP9&nbsp;frame |<sub></sub>|
|`vpx_ccr` |VPX&nbsp;Codec&nbsp;Configuration&nbsp;Record |<sub></sub>|
|`wasm` |WebAssembly&nbsp;Binary&nbsp;Format |<sub></sub>|
|`wav` |WAV&nbsp;file |<sub>`id3v2` `id3v1` `id3v11`</sub>|
|`webp` |WebP&nbsp;image |<sub>`vp8_frame`</sub>|
|`xing` |Xing&nbsp;header |<sub></sub>|
|[`xml`](#xml) |Extensible&nbsp;Markup&nbsp;Language |<sub></sub>|
|`yaml` |YAML&nbsp;Ain't&nbsp;Markup&nbsp;Language |<sub></sub>|
|[`zip`](#zip) |ZIP&nbsp;archive |<sub>`probe`</sub>|
|`image` |Group |<sub>`gif` `jpeg` `mp4` `png` `tiff` `webp`</sub>|
|`inet_packet` |Group |<sub>`ipv4_packet` `ipv6_packet`</sub>|
|`ip_packet` |Group |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
|`link_frame` |Group |<sub>`bsd_loopback_frame` `ether8023_frame` `sll2_packet` `sll_packet`</sub>|
|`probe` |Group |<sub>`adts` `ar` `avro_ocf` `bitcoin_blkdat` `bzip2` `elf` `flac` `gif` `gzip` `jpeg` `json` `jsonl` `macho` `macho_fat` `matroska` `mp3` `mp4` `mpeg_ts` `ogg` `pcap` `pcapng` `png` `tar` `tiff` `toml` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
|`tcp_stream` |Group |<sub>`dns_tcp` `rtmp`</sub>|
|`udp_payload` |Group |<sub>`dns`</sub>|
[#]: sh-end
@ -161,35 +162,25 @@ Supports decoding BER, CER and DER (X.690).
- Does not support specifying a schema.
- Supports `torepr` but without schema all sequences and sets will be arrays.
#### Examples
#### `frompem` can be used to decode certificates etc
`frompem` and `topem` can be used to work with PEM format
```
```sh
$ fq -d raw 'frompem | asn1_ber | d' cert.pem
```
Can be used to decode nested parts
```
#### Can decode nested values
```sh
$ fq -d asn1_ber '.constructed[1].value | asn1_ber' file.ber
```
If schema is known and not complicated it can be reproduced
```
#### Manual schema
```sh
$ fq -d asn1_ber 'torepr as $r | ["version", "modulus", "private_exponent", "private_exponen", "prime1", "prime2", "exponent1", "exponent2", "coefficient"] | with_entries({key: .value, value: $r[.key]})' pkcs1.der
```
Supports `torepr`
```
$ fq -d asn1_ber torepr file
```
Supports `torepr`
```
... | asn1_ber | torepr
```
#### References and links
#### References
- https://www.itu.int/ITU-T/studygroups/com10/languages/X.690_1297.pdf
- https://en.wikipedia.org/wiki/X.690
- https://letsencrypt.org/docs/a-warm-welcome-to-asn1-and-der/
@ -222,70 +213,70 @@ Supports reading Avro Object Container Format (OCF) files based on the 1.11.0 sp
Capable of handling null, deflate, and snappy codecs for data compression.
Limitations:
- Schema does not support self-referential types, only built-in types.
- Decimal logical types are not supported for decoding, will just be treated as their primitive type
#### References and links
- Schema does not support self-referential types, only built-in types.
- Decimal logical types are not supported for decoding, will just be treated as their primitive type
#### References
- https://avro.apache.org/docs/current/spec.html#Object+Container+Files
### bencode
#### Convert represented value to JSON
```
$ fq -d bencode torepr file.torrent
```
#### References
- https://wiki.theory.org/BitTorrentSpecification#Bencoding
### bitcoin_block
#### Options
|Name |Default|Description|
|- |- |-|
|`has_header`|false |Has blkdat header|
#### Examples
Supports `torepr`
Decode file using bitcoin_block options
```
$ fq -d bencode torepr file
$ fq -d bitcoin_block -o has_header=false . file
```
Supports `torepr`
Decode value as bitcoin_block
```
... | bencode | torepr
... | bitcoin_block({has_header:false})
```
#### References and links
- https://wiki.theory.org/BitTorrentSpecification#Bencoding
### bson
#### Examples
#### Convert represented value to JSON
BSON as JSON
```
$ fq -d bson torepr file
$ fq -d bson torepr file.bson
```
Supports `torepr`
### Filter represented value
```
$ fq -d bson torepr file
$ fq -d bson 'torepr | select(.name=="bob")' file.bson
```
Supports `torepr`
```
... | bson | torepr
```
#### References and links
- https://wiki.theory.org/BitTorrentSpecification#Bencoding
#### References
- https://bsonspec.org/spec.html
### cbor
#### Examples
#### Convert represented value to JSON
Supports `torepr`
```
$ fq -d cbor torepr file
$ fq -d cbor torepr file.cbor
```
Supports `torepr`
```
... | cbor | torepr
```
#### References and links
#### References
- https://en.wikipedia.org/wiki/CBOR
- https://www.rfc-editor.org/rfc/rfc8949.html
@ -376,33 +367,30 @@ Decode value as html
Supports decoding vanilla and FAT Mach-O binaries.
#### Examples
#### Select 64bit load segments
Select 64bit load segments
```
```sh
$ fq '.load_commands[] | select(.cmd=="segment_64")' file
```
#### References and links
#### References
- https://github.com/aidansteele/osx-abi-macho-file-format-reference
### matroska
#### Examples
#### Lookup element path using `matroska_path`
Lookup element decode value using `matroska_path`
```
... | matroska_path(".Segment.Tracks[0)"
```sh
$ fq 'matroska_path(".Segment.Tracks[0)")' file.mkv
```
Return `matroska_path` string for a box decode value
```
... | grep_by(.id == "Tracks") | matroska_path
#### Get element path using `matroska_path`
```sh
$ fq 'grep_by(.id == "Tracks") | matroska_path' file.mkv
```
#### References and links
#### References
- https://tools.ietf.org/html/draft-ietf-cellar-ebml-00
- https://matroska.org/technical/specs/index.html
- https://www.matroska.org/technical/basics.html
@ -432,8 +420,6 @@ Decode value as mp3
### mp4
Support `mp4_path`
#### Options
|Name |Default|Description|
@ -443,16 +429,6 @@ Support `mp4_path`
#### Examples
Lookup box decode value using `mp4_path`
```
... | mp4_path(".moov.trak[1]")
```
Return `mp4_path` string for a box decode value
```
... | grep_by(.type == "trak") | mp4_path
```
Decode file using mp4 options
```
$ fq -d mp4 -o allow_truncated=false -o decode_samples=true . file
@ -463,48 +439,70 @@ Decode value as mp4
... | mp4({allow_truncated:false,decode_samples:true})
```
#### References and links
#### `mp4_path($path)` - Lookup mp4 box using a mp4 box path.
```sh
# <decode value box> | mp4_path($path) -> <decode value box>
$ fq 'mp4_path(".moov.trak[1]")' file.mp4
```
#### `mp4_path` - Return mp4 box path for a decode value box.
```sh
# <decode value box> | mp4_path -> string
$ fq 'grep_by(.type == "trak") | mp4_path' file.mp4
```
#### Force decode a single box
```sh
$ fq -n '"AAAAHGVsc3QAAAAAAAAAAQAAADIAAAQAAAEAAA==" | frombase64 | mp4({force:true}) | d'
```
#### Don't decode samples and manually decode first sample for first track as a `aac_frame`
```sh
$ fq -o decode_samples=false '.tracks[0].samples[0] | aac_frame | d' file.mp4
```
#### Entries for first edit list as values
```sh
$ fq 'first(grep_by(.type=="elst").entries) | tovalue' file.mp4
```
#### References
- [ISO/IEC base media file format (MPEG-4 Part 12)](https://en.wikipedia.org/wiki/ISO/IEC_base_media_file_format)
- [Quicktime file format](https://developer.apple.com/standards/qtff-2001.pdf)
### msgpack
#### Examples
#### Convert represented value to JSON
Supports `torepr`
```
$ fq -d msgpack torepr file
$ fq -d msgpack torepr file.msgpack
```
Supports `torepr`
```
... | msgpack | torepr
```
#### References and links
#### References
- https://github.com/msgpack/msgpack/blob/master/spec.md
### protobuf
#### Examples
#### Can decode sub messages
Can be used to decode sub messages
```
$ fq -d protobuf '.fields[6].wire_value | protobuf | d'
```sh
$ fq -d protobuf '.fields[6].wire_value | protobuf | d' file
```
#### References and links
#### References
- https://developers.google.com/protocol-buffers/docs/encoding
### rtmp
Current only supports plain RTMP (not RTMPT or encrypted variants etc) with AMF0 (not AMF3).
#### References and links
#### References
- https://rtmp.veriskope.com/docs/spec/
- https://rtmp.veriskope.com/pdf/video_file_format_spec_v10.pdf
@ -530,14 +528,11 @@ Decode value as xml
... | xml({array:false,attribute_prefix:"@",seq:false})
```
#### References and links
#### References
- [xml.com's Converting Between XML and JSON](https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html)
### zip
Supports ZIP64.
#### Options
|Name |Default|Description|
@ -556,9 +551,11 @@ Decode value as zip
... | zip({uncompress:true})
```
#### References and links
Supports ZIP64.
#### References
- https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
- https://opensource.apple.com/source/zip/zip-6/unzip/unzip/proginfo/extra.fld
[#]: sh-end

18
doc/formats.svg
View File

@ -1919,17 +1919,23 @@
<polygon fill="paleturquoise" stroke="transparent" points="3858.81,-2025.5 3858.81,-2051.5 3899.81,-2051.5 3899.81,-2025.5 3858.81,-2025.5"/>
<text text-anchor="start" x="3861.31" y="-2033.5" font-family="Times,serif" font-size="20.00">html</text>
</g>
<!-- msgpack -->
<!-- markdown -->
<g id="node112" class="node">
<title>markdown</title>
<polygon fill="paleturquoise" stroke="transparent" points="3929.81,-2025.5 3929.81,-2051.5 4018.81,-2051.5 4018.81,-2025.5 3929.81,-2025.5"/>
<text text-anchor="start" x="3932.31" y="-2033.5" font-family="Times,serif" font-size="20.00">markdown</text>
</g>
<!-- msgpack -->
<g id="node113" class="node">
<title>msgpack</title>
<polygon fill="paleturquoise" stroke="transparent" points="3929.81,-2025.5 3929.81,-2051.5 4004.81,-2051.5 4004.81,-2025.5 3929.81,-2025.5"/>
<text text-anchor="start" x="3932.31" y="-2033.5" font-family="Times,serif" font-size="20.00">msgpack</text>
<polygon fill="paleturquoise" stroke="transparent" points="4048.81,-2025.5 4048.81,-2051.5 4123.81,-2051.5 4123.81,-2025.5 4048.81,-2025.5"/>
<text text-anchor="start" x="4051.31" y="-2033.5" font-family="Times,serif" font-size="20.00">msgpack</text>
</g>
<!-- raw -->
<g id="node113" class="node">
<g id="node114" class="node">
<title>raw</title>
<polygon fill="paleturquoise" stroke="transparent" points="4036.81,-2025.5 4036.81,-2051.5 4071.81,-2051.5 4071.81,-2025.5 4036.81,-2025.5"/>
<text text-anchor="start" x="4039.31" y="-2033.5" font-family="Times,serif" font-size="20.00">raw</text>
<polygon fill="paleturquoise" stroke="transparent" points="4155.81,-2025.5 4155.81,-2051.5 4190.81,-2051.5 4190.81,-2025.5 4155.81,-2025.5"/>
<text text-anchor="start" x="4158.31" y="-2033.5" font-family="Times,serif" font-size="20.00">raw</text>
</g>
</g>
</svg>

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 134 KiB

After

Width:  |  Height:  |  Size: 135 KiB

798
format/all/help.fqtest
View File

File diff suppressed because it is too large Load Diff

3
format/asn1/asn1_ber.go
View File

@ -29,6 +29,7 @@ import (
)
//go:embed asn1_ber.jq
//go:embed asn1_ber.md
var asn1FS embed.FS
func init() {
@ -36,7 +37,7 @@ func init() {
Name: format.ASN1_BER,
Description: "ASN1 BER (basic encoding rules, also CER and DER)",
DecodeFn: decodeASN1BER,
Functions: []string{"torepr", "_help"},
Functions: []string{"torepr"},
})
interp.RegisterFS(asn1FS)
}

21
format/asn1/asn1_ber.jq
View File

@ -6,24 +6,3 @@ def _asn1_ber_torepr:
end
else .constructed | map(_asn1_ber_torepr)
end;
def _asn1_ber__help:
{ notes: "Supports decoding BER, CER and DER (X.690).
- Currently no extra validation is done for CER and DER.
- Does not support specifying a schema.
- Supports `torepr` but without schema all sequences and sets will be arrays.",
examples: [
{comment: "`frompem` and `topem` can be used to work with PEM format", shell: "fq -d raw 'frompem | asn1_ber | d' cert.pem"},
{comment: "Can be used to decode nested parts", shell: "fq -d asn1_ber '.constructed[1].value | asn1_ber' file.ber"},
{ comment: "If schema is known and not complicated it can be reproduced",
shell: "fq -d asn1_ber 'torepr as $r | [\"version\", \"modulus\", \"private_exponent\", \"private_exponen\", \"prime1\", \"prime2\", \"exponent1\", \"exponent2\", \"coefficient\"] | with_entries({key: .value, value: $r[.key]})' pkcs1.der"
}
],
links: [
{url: "https://www.itu.int/ITU-T/studygroups/com10/languages/X.690_1297.pdf"},
{url: "https://en.wikipedia.org/wiki/X.690"},
{url: "https://letsencrypt.org/docs/a-warm-welcome-to-asn1-and-der/"},
{url: "https://lapo.it/asn1js/"}
]
};

29
format/asn1/asn1_ber.md Normal file
View File

@ -0,0 +1,29 @@
Supports decoding BER, CER and DER (X.690).
- Currently no extra validation is done for CER and DER.
- Does not support specifying a schema.
- Supports `torepr` but without schema all sequences and sets will be arrays.
#### `frompem` can be used to decode certificates etc
```sh
$ fq -d raw 'frompem | asn1_ber | d' cert.pem
```
#### Can decode nested values
```sh
$ fq -d asn1_ber '.constructed[1].value | asn1_ber' file.ber
```
#### Manual schema
```sh
$ fq -d asn1_ber 'torepr as $r | ["version", "modulus", "private_exponent", "private_exponen", "prime1", "prime2", "exponent1", "exponent2", "coefficient"] | with_entries({key: .value, value: $r[.key]})' pkcs1.der
```
#### References
- https://www.itu.int/ITU-T/studygroups/com10/languages/X.690_1297.pdf
- https://en.wikipedia.org/wiki/X.690
- https://letsencrypt.org/docs/a-warm-welcome-to-asn1-and-der/
- https://lapo.it/asn1js/

3
format/avro/avro_ocf.go
View File

@ -16,7 +16,7 @@ import (
"github.com/wader/fq/pkg/scalar"
)
//go:embed avro_ocf.jq
//go:embed avro_ocf.md
var avroOcfFS embed.FS
func init() {
@ -25,7 +25,6 @@ func init() {
Description: "Avro object container file",
Groups: []string{format.PROBE},
DecodeFn: decodeAvroOCF,
Functions: []string{"_help"},
})
interp.RegisterFS(avroOcfFS)
}

12
format/avro/avro_ocf.jq
View File

@ -1,12 +0,0 @@
def _avro_ocf__help:
{ notes: "Supports reading Avro Object Container Format (OCF) files based on the 1.11.0 specification.
Capable of handling null, deflate, and snappy codecs for data compression.
Limitations:
- Schema does not support self-referential types, only built-in types.
- Decimal logical types are not supported for decoding, will just be treated as their primitive type",
links: [
{url: "https://avro.apache.org/docs/current/spec.html#Object+Container+Files"}
]
};

11
format/avro/avro_ocf.md Normal file
View File

@ -0,0 +1,11 @@
Supports reading Avro Object Container Format (OCF) files based on the 1.11.0 specification.
Capable of handling null, deflate, and snappy codecs for data compression.
Limitations:
- Schema does not support self-referential types, only built-in types.
- Decimal logical types are not supported for decoding, will just be treated as their primitive type
#### References
- https://avro.apache.org/docs/current/spec.html#Object+Container+Files

3
format/bencode/bencode.go
View File

@ -13,6 +13,7 @@ import (
)
//go:embed bencode.jq
//go:embed bencode.md
var bencodeFS embed.FS
func init() {
@ -20,7 +21,7 @@ func init() {
Name: format.BENCODE,
Description: "BitTorrent bencoding",
DecodeFn: decodeBencode,
Functions: []string{"torepr", "_help"},
Functions: []string{"torepr"},
})
interp.RegisterFS(bencodeFS)
}

6
format/bencode/bencode.jq
View File

@ -9,9 +9,3 @@ def _bencode_torepr:
)
else error("unknown type \(.type)")
end;
def _bencode__help:
{ links: [
{url: "https://wiki.theory.org/BitTorrentSpecification#Bencoding"}
]
};

8
format/bencode/bencode.md Normal file
View File

@ -0,0 +1,8 @@
#### Convert represented value to JSON
```
$ fq -d bencode torepr file.torrent
```
#### References
- https://wiki.theory.org/BitTorrentSpecification#Bencoding

3
format/bson/bson.go
View File

@ -13,6 +13,7 @@ import (
)
//go:embed bson.jq
//go:embed bson.md
var bsonFS embed.FS
func init() {
@ -20,7 +21,7 @@ func init() {
Name: format.BSON,
Description: "Binary JSON",
DecodeFn: decodeBSON,
Functions: []string{"torepr", "_help"},
Functions: []string{"torepr"},
})
interp.RegisterFS(bsonFS)
}

8
format/bson/bson.jq
View File

@ -17,11 +17,3 @@ def _bson_torepr:
| _f
);
def _bson__help:
{ examples: [
{comment: "BSON as JSON", shell: "fq -d bson torepr file"}
],
links: [
{url: "https://wiki.theory.org/BitTorrentSpecification#Bencoding"}
]
};

14
format/bson/bson.md Normal file
View File

@ -0,0 +1,14 @@
#### Convert represented value to JSON
```
$ fq -d bson torepr file.bson
```
### Filter represented value
```
$ fq -d bson 'torepr | select(.name=="bob")' file.bson
```
#### References
- https://bsonspec.org/spec.html

3
format/cbor/cbor.go
View File

@ -21,6 +21,7 @@ import (
)
//go:embed cbor.jq
//go:embed cbor.md
var cborFS embed.FS
func init() {
@ -28,7 +29,7 @@ func init() {
Name: format.CBOR,
Description: "Concise Binary Object Representation",
DecodeFn: decodeCBOR,
Functions: []string{"torepr", "_help"},
Functions: []string{"torepr"},
})
interp.RegisterFS(cborFS)
}

7
format/cbor/cbor.jq
View File

@ -8,10 +8,3 @@ def _cbor_torepr:
elif .major_type == "bytes" then .value | tostring
else .value | tovalue
end;
def _cbor__help:
{ links: [
{url: "https://en.wikipedia.org/wiki/CBOR"},
{url: "https://www.rfc-editor.org/rfc/rfc8949.html"}
]
};

9
format/cbor/cbor.md Normal file
View File

@ -0,0 +1,9 @@
#### Convert represented value to JSON
```
$ fq -d cbor torepr file.cbor
```
#### References
- https://en.wikipedia.org/wiki/CBOR
- https://www.rfc-editor.org/rfc/rfc8949.html

3
format/macho/macho.go
View File

@ -14,7 +14,7 @@ import (
"github.com/wader/fq/pkg/scalar"
)
//go:embed macho.jq
//go:embed macho.md
var machoFS embed.FS
func init() {
@ -23,7 +23,6 @@ func init() {
Description: "Mach-O macOS executable",
Groups: []string{format.PROBE},
DecodeFn: machoDecode,
Functions: []string{"_help"},
})
interp.RegisterFS(machoFS)
}

9
format/macho/macho.jq
View File

@ -1,9 +0,0 @@
def _macho__help:
{ notes: "Supports decoding vanilla and FAT Mach-O binaries.",
examples: [
{comment: "Select 64bit load segments", shell: "fq '.load_commands[] | select(.cmd==\"segment_64\")' file"}
],
links: [
{url: "https://github.com/aidansteele/osx-abi-macho-file-format-reference"}
]
};

10
format/macho/macho.md Normal file
View File

@ -0,0 +1,10 @@
Supports decoding vanilla and FAT Mach-O binaries.
#### Select 64bit load segments
```sh
$ fq '.load_commands[] | select(.cmd=="segment_64")' file
```
#### References
- https://github.com/aidansteele/osx-abi-macho-file-format-reference

4
format/markdown/markdown.go
View File

@ -3,7 +3,7 @@ package markdown
import (
"embed"
"fmt"
"io/ioutil"
"io"
"github.com/gomarkdown/markdown"
"github.com/gomarkdown/markdown/ast"
@ -28,7 +28,7 @@ func init() {
}
func decodeMarkdown(d *decode.D, _ any) any {
b, err := ioutil.ReadAll(bitio.NewIOReader(d.RawLen(d.Len())))
b, err := io.ReadAll(bitio.NewIOReader(d.RawLen(d.Len())))
if err != nil {
panic(err)
}

52
format/markdown/markdown.jq Normal file
View File

@ -0,0 +1,52 @@
def _markdown__todisplay: tovalue;
def word_break($width):
def _f($a; $acc; $l):
( $a[0] as $w
| ($w // "" | length) as $wl
| if $w == null then $acc
elif ($l + $wl) >= $width then
( $acc
, _f($a[1:]; [$w]; $wl)
)
else _f($a[1:]; $acc+[$w]; $l+$wl)
end
);
( [_f([splits("\\s{1,}")]; []; 0)]
| map(join(" "))
);
def _markdown_to_text($width; $header_depth):
def lb: if $width > 0 then word_break($width) | join("\n") end;
def _f($pln):
if type == "string" then gsub("\n"; " ")
elif .type == "document" then .children[] | _f("\n\n")
elif .type == "heading" then "#" * (.level+$header_depth), " ", (.children[] | _f("\n\n")), "\n"
elif .type == "paragraph" then
( ( [.children[] | _f("\n\n")]
| join("")
| lb
)
, $pln
)
elif .type == "link" then
( ( [ .children[]
| _f("")
]
| join("")
) as $text
| $text
, if $text != .destination then " (", .destination, ")"
else empty
end
)
elif .type == "code_block" then "\n", (" ", .literal | split("\n") | join("\n ")), "\n"
elif .type == "code" then .literal
elif .type == "list" then (.children[] | _f("\n\n")), "\n" # TODO: delim
elif .type == "list_item" then .bullet_char, " ", (.children[] | _f("\n"))
elif .type == "html_span" then .literal | gsub("<br>"; "\n") # TODO: more?
else empty
end;
[_f("\n\n")] | join("");
def _markdown_to_text:
_markdown_to_text(-1; 0);

2
format/matroska/matroska.go
View File

@ -27,6 +27,7 @@ import (
)
//go:embed matroska.jq
//go:embed matroska.md
var matroskaFS embed.FS
var aacFrameFormat decode.Group
@ -78,7 +79,6 @@ func init() {
{Names: []string{format.VP9_CFM}, Group: &vp9CFMFormat},
{Names: []string{format.VP9_FRAME}, Group: &vp9FrameFormat},
},
Functions: []string{"_help"},
})
interp.RegisterFS(matroskaFS)

14
format/matroska/matroska.jq
View File

@ -12,17 +12,3 @@ def matroska_path:
| format_root
| matroska_path($c)
);
def _matroska__help:
{ examples: [
{comment: "Lookup element decode value using `matroska_path`", expr: "matroska_path(\".Segment.Tracks[0)\""},
{comment: "Return `matroska_path` string for a box decode value", expr: "grep_by(.id == \"Tracks\") | matroska_path"}
],
links: [
{url: "https://tools.ietf.org/html/draft-ietf-cellar-ebml-00"},
{url: "https://matroska.org/technical/specs/index.html"},
{url: "https://www.matroska.org/technical/basics.html"},
{url: "https://www.matroska.org/technical/codec_specs.html"},
{url: "https://wiki.xiph.org/MatroskaOpus"}
]
};

18
format/matroska/matroska.md Normal file
View File

@ -0,0 +1,18 @@
#### Lookup element path using `matroska_path`
```sh
$ fq 'matroska_path(".Segment.Tracks[0)")' file.mkv
```
#### Get element path using `matroska_path`
```sh
$ fq 'grep_by(.id == "Tracks") | matroska_path' file.mkv
```
#### References
- https://tools.ietf.org/html/draft-ietf-cellar-ebml-00
- https://matroska.org/technical/specs/index.html
- https://www.matroska.org/technical/basics.html
- https://www.matroska.org/technical/codec_specs.html
- https://wiki.xiph.org/MatroskaOpus

4
format/mp4/mp4.go
View File

@ -26,6 +26,7 @@ import (
)
//go:embed mp4.jq
//go:embed mp4.md
var mp4FS embed.FS
var aacFrameFormat decode.Group
@ -55,7 +56,7 @@ var vpxCCRFormat decode.Group
func init() {
interp.RegisterFormat(decode.Format{
Name: format.MP4,
Description: "ISOBMFF MPEG-4 part 12 and similar",
Description: "ISOBMFF, QuickTime and similar",
Groups: []string{
format.PROBE,
format.IMAGE, // avif
@ -90,7 +91,6 @@ func init() {
{Names: []string{format.VP9_FRAME}, Group: &vp9FrameFormat},
{Names: []string{format.VPX_CCR}, Group: &vpxCCRFormat},
},
Functions: []string{"_help"},
})
interp.RegisterFS(mp4FS)
}

12
format/mp4/mp4.jq
View File

@ -12,15 +12,3 @@ def mp4_path:
| format_root
| mp4_path($c)
);
def _mp4__help:
{ notes: "Support `mp4_path`",
examples: [
{comment: "Lookup box decode value using `mp4_path`", expr: "mp4_path(\".moov.trak[1]\")"},
{comment: "Return `mp4_path` string for a box decode value", expr: "grep_by(.type == \"trak\") | mp4_path"}
],
links: [
{title: "ISO/IEC base media file format (MPEG-4 Part 12)", url: "https://en.wikipedia.org/wiki/ISO/IEC_base_media_file_format"},
{title: "Quicktime file format", url: "https://developer.apple.com/standards/qtff-2001.pdf"}
]
};

36
format/mp4/mp4.md Normal file
View File

@ -0,0 +1,36 @@
#### `mp4_path($path)` - Lookup mp4 box using a mp4 box path.
```sh
# <decode value box> | mp4_path($path) -> <decode value box>
$ fq 'mp4_path(".moov.trak[1]")' file.mp4
```
#### `mp4_path` - Return mp4 box path for a decode value box.
```sh
# <decode value box> | mp4_path -> string
$ fq 'grep_by(.type == "trak") | mp4_path' file.mp4
```
#### Force decode a single box
```sh
$ fq -n '"AAAAHGVsc3QAAAAAAAAAAQAAADIAAAQAAAEAAA==" | frombase64 | mp4({force:true}) | d'
```
#### Don't decode samples and manually decode first sample for first track as a `aac_frame`
```sh
$ fq -o decode_samples=false '.tracks[0].samples[0] | aac_frame | d' file.mp4
```
#### Entries for first edit list as values
```sh
$ fq 'first(grep_by(.type=="elst").entries) | tovalue' file.mp4
```
#### References
- [ISO/IEC base media file format (MPEG-4 Part 12)](https://en.wikipedia.org/wiki/ISO/IEC_base_media_file_format)
- [Quicktime file format](https://developer.apple.com/standards/qtff-2001.pdf)

3
format/msgpack/msgpack.go
View File

@ -14,6 +14,7 @@ import (
)
//go:embed msgpack.jq
//go:embed msgpack.md
var msgPackFS embed.FS
func init() {
@ -21,7 +22,7 @@ func init() {
Name: format.MSGPACK,
Description: "MessagePack",
DecodeFn: decodeMsgPack,
Functions: []string{"torepr", "_help"},
Functions: []string{"torepr"},
})
interp.RegisterFS(msgPackFS)
}

5
format/msgpack/msgpack.jq
View File

@ -9,8 +9,3 @@ def _msgpack_torepr:
else .value | tovalue
end;
def _msgpack__help:
{ links: [
{url: "https://github.com/msgpack/msgpack/blob/master/spec.md"}
]
};

8
format/msgpack/msgpack.md Normal file
View File

@ -0,0 +1,8 @@
#### Convert represented value to JSON
```
$ fq -d msgpack torepr file.msgpack
```
#### References
- https://github.com/msgpack/msgpack/blob/master/spec.md

3
format/protobuf/protobuf.go
View File

@ -12,7 +12,7 @@ import (
"github.com/wader/fq/pkg/scalar"
)
//go:embed protobuf.jq
//go:embed protobuf.md
var protobufFS embed.FS
func init() {
@ -20,7 +20,6 @@ func init() {
Name: format.PROTOBUF,
Description: "Protobuf",
DecodeFn: protobufDecode,
Functions: []string{"_help"},
})
interp.RegisterFS(protobufFS)
}

8
format/protobuf/protobuf.jq
View File

@ -1,8 +0,0 @@
def _protobuf__help:
{ examples: [
{comment: "Can be used to decode sub messages", shell: "fq -d protobuf '.fields[6].wire_value | protobuf | d'"}
],
links: [
{url: "https://developers.google.com/protocol-buffers/docs/encoding"}
]
};

8
format/protobuf/protobuf.md Normal file
View File

@ -0,0 +1,8 @@
#### Can decode sub messages
```sh
$ fq -d protobuf '.fields[6].wire_value | protobuf | d' file
```
#### References
- https://developers.google.com/protocol-buffers/docs/encoding

3
format/rtmp/rtmp.go
View File

@ -21,7 +21,7 @@ import (
var rtmpAmf0Group decode.Group
var rtmpMpegASCFormat decode.Group
//go:embed rtmp.jq
//go:embed rtmp.md
var rtmpFS embed.FS
func init() {
@ -36,7 +36,6 @@ func init() {
{Names: []string{format.AMF0}, Group: &rtmpAmf0Group},
{Names: []string{format.MPEG_ASC}, Group: &rtmpMpegASCFormat},
},
Functions: []string{"_help"},
})
interp.RegisterFS(rtmpFS)
}

7
format/rtmp/rtmp.jq
View File

@ -1,7 +0,0 @@
def _rtmp__help:
{ notes: "Current only supports plain RTMP (not RTMPT or encrypted variants etc) with AMF0 (not AMF3).",
links: [
{url: "https://rtmp.veriskope.com/docs/spec/"},
{url: "https://rtmp.veriskope.com/pdf/video_file_format_spec_v10.pdf"}
]
};

5
format/rtmp/rtmp.md Normal file
View File

@ -0,0 +1,5 @@
Current only supports plain RTMP (not RTMPT or encrypted variants etc) with AMF0 (not AMF3).
#### References
- https://rtmp.veriskope.com/docs/spec/
- https://rtmp.veriskope.com/pdf/video_file_format_spec_v10.pdf

3
format/xml/xml.go
View File

@ -29,6 +29,7 @@ import (
)
//go:embed xml.jq
//go:embed xml.md
var xmlFS embed.FS
func init() {
@ -43,7 +44,7 @@ func init() {
Array: false,
AttributePrefix: "@",
},
Functions: []string{"_todisplay", "_help"},
Functions: []string{"_todisplay"},
})
interp.RegisterFS(xmlFS)
interp.RegisterFunc1("toxml", toXML)

6
format/xml/xml.jq
View File

@ -1,8 +1,2 @@
def toxml: toxml(null);
def _xml__todisplay: tovalue;
def _xml__help:
{ links: [
{title: "xml.com's Converting Between XML and JSON", url: "https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html"}
]
};

2
format/xml/xml.md Normal file
View File

@ -0,0 +1,2 @@
#### References
- [xml.com's Converting Between XML and JSON](https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html)

3
format/zip/zip.go
View File

@ -15,7 +15,7 @@ import (
"github.com/wader/fq/pkg/scalar"
)
//go:embed zip.jq
//go:embed zip.md
var zipFS embed.FS
var probeFormat decode.Group
@ -32,7 +32,6 @@ func init() {
Dependencies: []decode.Dependency{
{Names: []string{format.PROBE}, Group: &probeFormat},
},
Functions: []string{"_help"},
})
interp.RegisterFS(zipFS)
}

6
format/zip/zip.jq
View File

@ -1,6 +0,0 @@
def _zip__help:
{ notes: "Supports ZIP64.",
links: [
{url: "https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT"}
]
};

5
format/zip/zip.md Normal file
View File

@ -0,0 +1,5 @@
Supports ZIP64.
#### References
- https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
- https://opensource.apple.com/source/zip/zip-6/unzip/unzip/proginfo/extra.fld

17
pkg/interp/funcs.jq
View File

@ -153,23 +153,6 @@ def paste:
)
end;
# very simple markdown to text converter
# assumes very basic markdown as input
def _markdown_to_text:
( .
# ```
# code
# ```
# -> code
| gsub("\\n```\\n"; "\n"; "m")
# #, ##, ###, ... -> #
| gsub("(?<line>\\n)?#+(?<title>.*)\\n"; "\(.line // "")#\(.title)\n"; "m")
# [title](url) -> title (url)
| gsub("\\[(?<title>.*)\\]\\((?<url>.*)\\)"; "\(.title) (\(.url))")
# `code` -> code
| gsub("`(?<code>.*)`"; .code)
);
def expr_to_path: _expr_to_path;
def path_to_expr: _path_to_expr;

36
pkg/interp/help.jq
View File

@ -64,14 +64,6 @@ def _help_format_enrich($arg0; $f; $include_basic):
, {comment: "Decode value as \($f.name)", expr: "\($f.name)"}
]
end
| (($f.functions // []) | map(select(startswith("_") | not))) as $public_functions
| if ($public_functions | length) > 0 then
.examples +=
[ $public_functions[]
| {comment: "Supports `\(.)`", shell: "fq -d \($f.name) torepr file"}
, {comment: "Supports `\(.)`", expr: "\($f.name) | torepr"}
]
end
| if $f.decode_in_arg then
.examples +=
[ { comment: "Decode file using \($f.name) options"
@ -146,13 +138,14 @@ def _help($arg0; $topic):
elif _registry.formats | has($topic) then
( _registry.formats[$topic] as $f
| (_format_func($f.name; "_help")? // {} | _help_format_enrich($arg0; $f; true)) as $fhelp
| ((_registry.files[][] | select(.name=="\($topic).md").data) // false) as $doc
| "\($f.name): \($f.description) decoder"
, ($fhelp.notes | if . then _markdown_to_text else empty end)
, if $f.decode_in_arg then
( $f.decode_in_arg
| to_entries
| map([" \(.key)=\(.value) ", $f.decode_in_arg_doc[.key]])
| "Options:"
| "# Options"
, ""
, table(
.;
map(
@ -167,28 +160,19 @@ def _help($arg0; $topic):
)
else empty
end
, "Examples:"
, ""
, "# Decode examples"
, ""
, ( $fhelp.examples[]
| " # \(.comment | _markdown_to_text)"
| " # \(.comment)"
, if .shell then " $ \(.shell)"
elif .expr then " ... | \(.expr)"
else empty
end
)
, if isempty($f.functions | select(. == "torepr")) | not then
( "Supports torepr:"
, " ... | \($f.name) | torepr"
)
else empty
end
, if $fhelp.links then
( "References and links"
, ( $fhelp.links[]
| if .title then " \(.title) \(.url)"
else " \(.url)"
end
)
)
, ""
# TODO: [:-1] hack to remove extra newline as we use println later
, if $doc then $doc | markdown | _markdown_to_text(options.width; -3)[:-1]
else empty
end
)

2
pkg/interp/testdata/args.fqtest vendored
View File

@ -166,7 +166,7 @@ markdown Markdown
matroska Matroska file
mp3 MP3 file
mp3_frame MPEG audio layer 3 frame
mp4 ISOBMFF MPEG-4 part 12 and similar
mp4 ISOBMFF, QuickTime and similar
mpeg_asc MPEG-4 Audio Specific Config
mpeg_es MPEG Elementary Stream
mpeg_pes MPEG Packetized elementary stream