wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-12-25 14:23:18 +03:00
Code Issues Projects Releases Wiki Activity

Merge pull request #422 from wader/markdown

Browse Source 
Move format documentation to markdown again
This commit is contained in:
Mattias Wadman 2022-09-11 00:59:49 +02:00 committed by GitHub
commit 577ea7665c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
56 changed files with 1771 additions and 633 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitattributes vendored
View File

@ -3,3 +3,4 @@
*.json eol=lf
*.jq eol=lf
*.xml eol=lf
*.md eol=lf

3
README.md
View File

@ -48,7 +48,7 @@ avc_sps,
[avro_ocf](doc/formats.md#avro_ocf),
[bencode](doc/formats.md#bencode),
bitcoin_blkdat,
bitcoin_block,
[bitcoin_block](doc/formats.md#bitcoin_block),
bitcoin_script,
bitcoin_transaction,
bsd_loopback_frame,
@ -91,6 +91,7 @@ json,
jsonl,
[macho](doc/formats.md#macho),
macho_fat,
markdown,
[matroska](doc/formats.md#matroska),
[mp3](doc/formats.md#mp3),
mp3_frame,

27
doc/formats.jq
View File

@ -2,14 +2,13 @@
def code: "`\(.)`";
def nbsp: gsub(" "; " ");
def has_section($f; $fhelp): $fhelp.notes or $fhelp.examples or $fhelp.links or $f.decode_in_arg;
def has_section($f; $fhelp): $fhelp.notes or $fhelp.examples or $f.decode_in_arg or ((_registry.files[][] | select(.name=="\($f.name).md").data) // false);
def formats_list:
[ formats
| to_entries[] as {$key, $value}
| (_format_func($key; "_help")? // {}) as $fhelp
| if has_section($value; $fhelp) then "[\($key)](doc/formats.md#\($key))"
else $key
[ formats[] as $f
| ({} | _help_format_enrich("fq"; $f; false)) as $fhelp
| if has_section($f; $fhelp) then "[\($f.name)](doc/formats.md#\($f.name))"
else $f.name
end
] | join(",\n");
@ -71,7 +70,8 @@ def formats_table:
def formats_sections:
( formats[] as $f
| (_format_func($f.name; "_help")? // {} | _help_format_enrich("fq"; $f; false)) as $fhelp
| ((_registry.files[][] | select(.name=="\($f.name).md").data) // false) as $doc
| ({} | _help_format_enrich("fq"; $f; false)) as $fhelp
| select(has_section($f; $fhelp))
| "### \($f.name)"
, ""
@ -134,16 +134,5 @@ def formats_sections:
)
else empty
end
, if $fhelp.links then
( "#### References and links"
, ""
, ( $fhelp.links[]
| if .title then "- [\(.title)](\(.url))"
else "- \(.url)"
end
)
, ""
)
else empty
end
, ($doc // empty)
);

433
doc/formats.md
View File

@ -2,121 +2,122 @@
[fq -rn -L . 'include "formats"; formats_table']: sh-start
|Name |Description |Dependencies|
|- |- |-|
|[`aac_frame`](#aac_frame) |Advanced&nbsp;Audio&nbsp;Coding&nbsp;frame |<sub></sub>|
|`adts` |Audio&nbsp;Data&nbsp;Transport&nbsp;Stream |<sub>`adts_frame`</sub>|
|`adts_frame` |Audio&nbsp;Data&nbsp;Transport&nbsp;Stream&nbsp;frame |<sub>`aac_frame`</sub>|
|`amf0` |Action&nbsp;Message&nbsp;Format&nbsp;0 |<sub></sub>|
|`apev2` |APEv2&nbsp;metadata&nbsp;tag |<sub>`image`</sub>|
|`ar` |Unix&nbsp;archive |<sub>`probe`</sub>|
|[`asn1_ber`](#asn1_ber) |ASN1&nbsp;BER&nbsp;(basic&nbsp;encoding&nbsp;rules,&nbsp;also&nbsp;CER&nbsp;and&nbsp;DER)|<sub></sub>|
|`av1_ccr` |AV1&nbsp;Codec&nbsp;Configuration&nbsp;Record |<sub></sub>|
|`av1_frame` |AV1&nbsp;frame |<sub>`av1_obu`</sub>|
|`av1_obu` |AV1&nbsp;Open&nbsp;Bitstream&nbsp;Unit |<sub></sub>|
|`avc_annexb` |H.264/AVC&nbsp;Annex&nbsp;B |<sub>`avc_nalu`</sub>|
|[`avc_au`](#avc_au) |H.264/AVC&nbsp;Access&nbsp;Unit |<sub>`avc_nalu`</sub>|
|`avc_dcr` |H.264/AVC&nbsp;Decoder&nbsp;Configuration&nbsp;Record |<sub>`avc_nalu`</sub>|
|`avc_nalu` |H.264/AVC&nbsp;Network&nbsp;Access&nbsp;Layer&nbsp;Unit |<sub>`avc_sps` `avc_pps` `avc_sei`</sub>|
|`avc_pps` |H.264/AVC&nbsp;Picture&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`avc_sei` |H.264/AVC&nbsp;Supplemental&nbsp;Enhancement&nbsp;Information |<sub></sub>|
|`avc_sps` |H.264/AVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set |<sub></sub>|
|[`avro_ocf`](#avro_ocf) |Avro&nbsp;object&nbsp;container&nbsp;file |<sub></sub>|
|[`bencode`](#bencode) |BitTorrent&nbsp;bencoding |<sub></sub>|
|`bitcoin_blkdat` |Bitcoin&nbsp;blk.dat |<sub>`bitcoin_block`</sub>|
|`bitcoin_block` |Bitcoin&nbsp;block |<sub>`bitcoin_transaction`</sub>|
|`bitcoin_script` |Bitcoin&nbsp;script |<sub></sub>|
|`bitcoin_transaction` |Bitcoin&nbsp;transaction |<sub>`bitcoin_script`</sub>|
|`bsd_loopback_frame` |BSD&nbsp;loopback&nbsp;frame |<sub>`inet_packet`</sub>|
|[`bson`](#bson) |Binary&nbsp;JSON |<sub></sub>|
|`bzip2` |bzip2&nbsp;compression |<sub>`probe`</sub>|
|[`cbor`](#cbor) |Concise&nbsp;Binary&nbsp;Object&nbsp;Representation |<sub></sub>|
|[`csv`](#csv) |Comma&nbsp;separated&nbsp;values |<sub></sub>|
|`dns` |DNS&nbsp;packet |<sub></sub>|
|`dns_tcp` |DNS&nbsp;packet&nbsp;(TCP) |<sub></sub>|
|`elf` |Executable&nbsp;and&nbsp;Linkable&nbsp;Format |<sub></sub>|
|`ether8023_frame` |Ethernet&nbsp;802.3&nbsp;frame |<sub>`inet_packet`</sub>|
|`exif` |Exchangeable&nbsp;Image&nbsp;File&nbsp;Format |<sub></sub>|
|`fairplay_spc` |FairPlay&nbsp;Server&nbsp;Playback&nbsp;Context |<sub></sub>|
|`flac` |Free&nbsp;Lossless&nbsp;Audio&nbsp;Codec&nbsp;file |<sub>`flac_metadatablocks` `flac_frame`</sub>|
|[`flac_frame`](#flac_frame) |FLAC&nbsp;frame |<sub></sub>|
|`flac_metadatablock` |FLAC&nbsp;metadatablock |<sub>`flac_streaminfo` `flac_picture` `vorbis_comment`</sub>|
|`flac_metadatablocks` |FLAC&nbsp;metadatablocks |<sub>`flac_metadatablock`</sub>|
|`flac_picture` |FLAC&nbsp;metadatablock&nbsp;picture |<sub>`image`</sub>|
|`flac_streaminfo` |FLAC&nbsp;streaminfo |<sub></sub>|
|`gif` |Graphics&nbsp;Interchange&nbsp;Format |<sub></sub>|
|`gzip` |gzip&nbsp;compression |<sub>`probe`</sub>|
|`hevc_annexb` |H.265/HEVC&nbsp;Annex&nbsp;B |<sub>`hevc_nalu`</sub>|
|[`hevc_au`](#hevc_au) |H.265/HEVC&nbsp;Access&nbsp;Unit |<sub>`hevc_nalu`</sub>|
|`hevc_dcr` |H.265/HEVC&nbsp;Decoder&nbsp;Configuration&nbsp;Record |<sub>`hevc_nalu`</sub>|
|`hevc_nalu` |H.265/HEVC&nbsp;Network&nbsp;Access&nbsp;Layer&nbsp;Unit |<sub>`hevc_vps` `hevc_pps` `hevc_sps`</sub>|
|`hevc_pps` |H.265/HEVC&nbsp;Picture&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`hevc_sps` |H.265/HEVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`hevc_vps` |H.265/HEVC&nbsp;Video&nbsp;Parameter&nbsp;Set |<sub></sub>|
|[`html`](#html) |HyperText&nbsp;Markup&nbsp;Language |<sub></sub>|
|`icc_profile` |International&nbsp;Color&nbsp;Consortium&nbsp;profile |<sub></sub>|
|`icmp` |Internet&nbsp;Control&nbsp;Message&nbsp;Protocol |<sub></sub>|
|`icmpv6` |Internet&nbsp;Control&nbsp;Message&nbsp;Protocol&nbsp;v6 |<sub></sub>|
|`id3v1` |ID3v1&nbsp;metadata |<sub></sub>|
|`id3v11` |ID3v1.1&nbsp;metadata |<sub></sub>|
|`id3v2` |ID3v2&nbsp;metadata |<sub>`image`</sub>|
|`ipv4_packet` |Internet&nbsp;protocol&nbsp;v4&nbsp;packet |<sub>`ip_packet`</sub>|
|`ipv6_packet` |Internet&nbsp;protocol&nbsp;v6&nbsp;packet |<sub>`ip_packet`</sub>|
|`jpeg` |Joint&nbsp;Photographic&nbsp;Experts&nbsp;Group&nbsp;file |<sub>`exif` `icc_profile`</sub>|
|`json` |JavaScript&nbsp;Object&nbsp;Notation |<sub></sub>|
|`jsonl` |JavaScript&nbsp;Object&nbsp;Notation&nbsp;Lines |<sub></sub>|
|[`macho`](#macho) |Mach-O&nbsp;macOS&nbsp;executable |<sub></sub>|
|`macho_fat` |Fat&nbsp;Mach-O&nbsp;macOS&nbsp;executable&nbsp;(multi-architecture) |<sub>`macho`</sub>|
|[`matroska`](#matroska) |Matroska&nbsp;file |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `image` `mp3_frame` `mpeg_asc` `mpeg_pes_packet` `mpeg_spu` `opus_packet` `vorbis_packet` `vp8_frame` `vp9_cfm` `vp9_frame`</sub>|
|[`mp3`](#mp3) |MP3&nbsp;file |<sub>`id3v2` `id3v1` `id3v11` `apev2` `mp3_frame`</sub>|
|`mp3_frame` |MPEG&nbsp;audio&nbsp;layer&nbsp;3&nbsp;frame |<sub>`xing`</sub>|
|[`mp4`](#mp4) |ISOBMFF&nbsp;MPEG-4&nbsp;part&nbsp;12&nbsp;and&nbsp;similar |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `icc_profile` `id3v2` `image` `jpeg` `mp3_frame` `mpeg_es` `mpeg_pes_packet` `opus_packet` `prores_frame` `protobuf_widevine` `pssh_playready` `vorbis_packet` `vp9_frame` `vpx_ccr`</sub>|
|`mpeg_asc` |MPEG-4&nbsp;Audio&nbsp;Specific&nbsp;Config |<sub></sub>|
|`mpeg_es` |MPEG&nbsp;Elementary&nbsp;Stream |<sub>`mpeg_asc` `vorbis_packet`</sub>|
|`mpeg_pes` |MPEG&nbsp;Packetized&nbsp;elementary&nbsp;stream |<sub>`mpeg_pes_packet` `mpeg_spu`</sub>|
|`mpeg_pes_packet` |MPEG&nbsp;Packetized&nbsp;elementary&nbsp;stream&nbsp;packet |<sub></sub>|
|`mpeg_spu` |Sub&nbsp;Picture&nbsp;Unit&nbsp;(DVD&nbsp;subtitle) |<sub></sub>|
|`mpeg_ts` |MPEG&nbsp;Transport&nbsp;Stream |<sub></sub>|
|[`msgpack`](#msgpack) |MessagePack |<sub></sub>|
|`ogg` |OGG&nbsp;file |<sub>`ogg_page` `vorbis_packet` `opus_packet` `flac_metadatablock` `flac_frame`</sub>|
|`ogg_page` |OGG&nbsp;page |<sub></sub>|
|`opus_packet` |Opus&nbsp;packet |<sub>`vorbis_comment`</sub>|
|`pcap` |PCAP&nbsp;packet&nbsp;capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|`pcapng` |PCAPNG&nbsp;packet&nbsp;capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|`png` |Portable&nbsp;Network&nbsp;Graphics&nbsp;file |<sub>`icc_profile` `exif`</sub>|
|`prores_frame` |Apple&nbsp;ProRes&nbsp;frame |<sub></sub>|
|[`protobuf`](#protobuf) |Protobuf |<sub></sub>|
|`protobuf_widevine` |Widevine&nbsp;protobuf |<sub>`protobuf`</sub>|
|`pssh_playready` |PlayReady&nbsp;PSSH |<sub></sub>|
|`raw` |Raw&nbsp;bits |<sub></sub>|
|[`rtmp`](#rtmp) |Real-Time&nbsp;Messaging&nbsp;Protocol |<sub>`amf0` `mpeg_asc`</sub>|
|`sll2_packet` |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation&nbsp;v2 |<sub>`inet_packet`</sub>|
|`sll_packet` |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation |<sub>`inet_packet`</sub>|
|`tar` |Tar&nbsp;archive |<sub>`probe`</sub>|
|`tcp_segment` |Transmission&nbsp;control&nbsp;protocol&nbsp;segment |<sub></sub>|
|`tiff` |Tag&nbsp;Image&nbsp;File&nbsp;Format |<sub>`icc_profile`</sub>|
|`toml` |Tom's&nbsp;Obvious,&nbsp;Minimal&nbsp;Language |<sub></sub>|
|`udp_datagram` |User&nbsp;datagram&nbsp;protocol |<sub>`udp_payload`</sub>|
|`vorbis_comment` |Vorbis&nbsp;comment |<sub>`flac_picture`</sub>|
|`vorbis_packet` |Vorbis&nbsp;packet |<sub>`vorbis_comment`</sub>|
|`vp8_frame` |VP8&nbsp;frame |<sub></sub>|
|`vp9_cfm` |VP9&nbsp;Codec&nbsp;Feature&nbsp;Metadata |<sub></sub>|
|`vp9_frame` |VP9&nbsp;frame |<sub></sub>|
|`vpx_ccr` |VPX&nbsp;Codec&nbsp;Configuration&nbsp;Record |<sub></sub>|
|`wasm` |WebAssembly&nbsp;Binary&nbsp;Format |<sub></sub>|
|`wav` |WAV&nbsp;file |<sub>`id3v2` `id3v1` `id3v11`</sub>|
|`webp` |WebP&nbsp;image |<sub>`vp8_frame`</sub>|
|`xing` |Xing&nbsp;header |<sub></sub>|
|[`xml`](#xml) |Extensible&nbsp;Markup&nbsp;Language |<sub></sub>|
|`yaml` |YAML&nbsp;Ain't&nbsp;Markup&nbsp;Language |<sub></sub>|
|[`zip`](#zip) |ZIP&nbsp;archive |<sub>`probe`</sub>|
|`image` |Group |<sub>`gif` `jpeg` `mp4` `png` `tiff` `webp`</sub>|
|`inet_packet` |Group |<sub>`ipv4_packet` `ipv6_packet`</sub>|
|`ip_packet` |Group |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
|`link_frame` |Group |<sub>`bsd_loopback_frame` `ether8023_frame` `sll2_packet` `sll_packet`</sub>|
|`probe` |Group |<sub>`adts` `ar` `avro_ocf` `bitcoin_blkdat` `bzip2` `elf` `flac` `gif` `gzip` `jpeg` `json` `jsonl` `macho` `macho_fat` `matroska` `mp3` `mp4` `mpeg_ts` `ogg` `pcap` `pcapng` `png` `tar` `tiff` `toml` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
|`tcp_stream` |Group |<sub>`dns_tcp` `rtmp`</sub>|
|`udp_payload` |Group |<sub>`dns`</sub>|
|Name |Description |Dependencies|
|- |- |-|
|[`aac_frame`](#aac_frame) |Advanced&nbsp;Audio&nbsp;Coding&nbsp;frame |<sub></sub>|
|`adts` |Audio&nbsp;Data&nbsp;Transport&nbsp;Stream |<sub>`adts_frame`</sub>|
|`adts_frame` |Audio&nbsp;Data&nbsp;Transport&nbsp;Stream&nbsp;frame |<sub>`aac_frame`</sub>|
|`amf0` |Action&nbsp;Message&nbsp;Format&nbsp;0 |<sub></sub>|
|`apev2` |APEv2&nbsp;metadata&nbsp;tag |<sub>`image`</sub>|
|`ar` |Unix&nbsp;archive |<sub>`probe`</sub>|
|[`asn1_ber`](#asn1_ber) |ASN1&nbsp;BER&nbsp;(basic&nbsp;encoding&nbsp;rules,&nbsp;also&nbsp;CER&nbsp;and&nbsp;DER)|<sub></sub>|
|`av1_ccr` |AV1&nbsp;Codec&nbsp;Configuration&nbsp;Record |<sub></sub>|
|`av1_frame` |AV1&nbsp;frame |<sub>`av1_obu`</sub>|
|`av1_obu` |AV1&nbsp;Open&nbsp;Bitstream&nbsp;Unit |<sub></sub>|
|`avc_annexb` |H.264/AVC&nbsp;Annex&nbsp;B |<sub>`avc_nalu`</sub>|
|[`avc_au`](#avc_au) |H.264/AVC&nbsp;Access&nbsp;Unit |<sub>`avc_nalu`</sub>|
|`avc_dcr` |H.264/AVC&nbsp;Decoder&nbsp;Configuration&nbsp;Record |<sub>`avc_nalu`</sub>|
|`avc_nalu` |H.264/AVC&nbsp;Network&nbsp;Access&nbsp;Layer&nbsp;Unit |<sub>`avc_sps` `avc_pps` `avc_sei`</sub>|
|`avc_pps` |H.264/AVC&nbsp;Picture&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`avc_sei` |H.264/AVC&nbsp;Supplemental&nbsp;Enhancement&nbsp;Information |<sub></sub>|
|`avc_sps` |H.264/AVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set |<sub></sub>|
|[`avro_ocf`](#avro_ocf) |Avro&nbsp;object&nbsp;container&nbsp;file |<sub></sub>|
|[`bencode`](#bencode) |BitTorrent&nbsp;bencoding |<sub></sub>|
|`bitcoin_blkdat` |Bitcoin&nbsp;blk.dat |<sub>`bitcoin_block`</sub>|
|[`bitcoin_block`](#bitcoin_block) |Bitcoin&nbsp;block |<sub>`bitcoin_transaction`</sub>|
|`bitcoin_script` |Bitcoin&nbsp;script |<sub></sub>|
|`bitcoin_transaction` |Bitcoin&nbsp;transaction |<sub>`bitcoin_script`</sub>|
|`bsd_loopback_frame` |BSD&nbsp;loopback&nbsp;frame |<sub>`inet_packet`</sub>|
|[`bson`](#bson) |Binary&nbsp;JSON |<sub></sub>|
|`bzip2` |bzip2&nbsp;compression |<sub>`probe`</sub>|
|[`cbor`](#cbor) |Concise&nbsp;Binary&nbsp;Object&nbsp;Representation |<sub></sub>|
|[`csv`](#csv) |Comma&nbsp;separated&nbsp;values |<sub></sub>|
|`dns` |DNS&nbsp;packet |<sub></sub>|
|`dns_tcp` |DNS&nbsp;packet&nbsp;(TCP) |<sub></sub>|
|`elf` |Executable&nbsp;and&nbsp;Linkable&nbsp;Format |<sub></sub>|
|`ether8023_frame` |Ethernet&nbsp;802.3&nbsp;frame |<sub>`inet_packet`</sub>|
|`exif` |Exchangeable&nbsp;Image&nbsp;File&nbsp;Format |<sub></sub>|
|`fairplay_spc` |FairPlay&nbsp;Server&nbsp;Playback&nbsp;Context |<sub></sub>|
|`flac` |Free&nbsp;Lossless&nbsp;Audio&nbsp;Codec&nbsp;file |<sub>`flac_metadatablocks` `flac_frame`</sub>|
|[`flac_frame`](#flac_frame) |FLAC&nbsp;frame |<sub></sub>|
|`flac_metadatablock` |FLAC&nbsp;metadatablock |<sub>`flac_streaminfo` `flac_picture` `vorbis_comment`</sub>|
|`flac_metadatablocks` |FLAC&nbsp;metadatablocks |<sub>`flac_metadatablock`</sub>|
|`flac_picture` |FLAC&nbsp;metadatablock&nbsp;picture |<sub>`image`</sub>|
|`flac_streaminfo` |FLAC&nbsp;streaminfo |<sub></sub>|
|`gif` |Graphics&nbsp;Interchange&nbsp;Format |<sub></sub>|
|`gzip` |gzip&nbsp;compression |<sub>`probe`</sub>|
|`hevc_annexb` |H.265/HEVC&nbsp;Annex&nbsp;B |<sub>`hevc_nalu`</sub>|
|[`hevc_au`](#hevc_au) |H.265/HEVC&nbsp;Access&nbsp;Unit |<sub>`hevc_nalu`</sub>|
|`hevc_dcr` |H.265/HEVC&nbsp;Decoder&nbsp;Configuration&nbsp;Record |<sub>`hevc_nalu`</sub>|
|`hevc_nalu` |H.265/HEVC&nbsp;Network&nbsp;Access&nbsp;Layer&nbsp;Unit |<sub>`hevc_vps` `hevc_pps` `hevc_sps`</sub>|
|`hevc_pps` |H.265/HEVC&nbsp;Picture&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`hevc_sps` |H.265/HEVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`hevc_vps` |H.265/HEVC&nbsp;Video&nbsp;Parameter&nbsp;Set |<sub></sub>|
|[`html`](#html) |HyperText&nbsp;Markup&nbsp;Language |<sub></sub>|
|`icc_profile` |International&nbsp;Color&nbsp;Consortium&nbsp;profile |<sub></sub>|
|`icmp` |Internet&nbsp;Control&nbsp;Message&nbsp;Protocol |<sub></sub>|
|`icmpv6` |Internet&nbsp;Control&nbsp;Message&nbsp;Protocol&nbsp;v6 |<sub></sub>|
|`id3v1` |ID3v1&nbsp;metadata |<sub></sub>|
|`id3v11` |ID3v1.1&nbsp;metadata |<sub></sub>|
|`id3v2` |ID3v2&nbsp;metadata |<sub>`image`</sub>|
|`ipv4_packet` |Internet&nbsp;protocol&nbsp;v4&nbsp;packet |<sub>`ip_packet`</sub>|
|`ipv6_packet` |Internet&nbsp;protocol&nbsp;v6&nbsp;packet |<sub>`ip_packet`</sub>|
|`jpeg` |Joint&nbsp;Photographic&nbsp;Experts&nbsp;Group&nbsp;file |<sub>`exif` `icc_profile`</sub>|
|`json` |JavaScript&nbsp;Object&nbsp;Notation |<sub></sub>|
|`jsonl` |JavaScript&nbsp;Object&nbsp;Notation&nbsp;Lines |<sub></sub>|
|[`macho`](#macho) |Mach-O&nbsp;macOS&nbsp;executable |<sub></sub>|
|`macho_fat` |Fat&nbsp;Mach-O&nbsp;macOS&nbsp;executable&nbsp;(multi-architecture) |<sub>`macho`</sub>|
|`markdown` |Markdown |<sub></sub>|
|[`matroska`](#matroska) |Matroska&nbsp;file |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `image` `mp3_frame` `mpeg_asc` `mpeg_pes_packet` `mpeg_spu` `opus_packet` `vorbis_packet` `vp8_frame` `vp9_cfm` `vp9_frame`</sub>|
|[`mp3`](#mp3) |MP3&nbsp;file |<sub>`id3v2` `id3v1` `id3v11` `apev2` `mp3_frame`</sub>|
|`mp3_frame` |MPEG&nbsp;audio&nbsp;layer&nbsp;3&nbsp;frame |<sub>`xing`</sub>|
|[`mp4`](#mp4) |ISOBMFF,&nbsp;QuickTime&nbsp;and&nbsp;similar |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `icc_profile` `id3v2` `image` `jpeg` `mp3_frame` `mpeg_es` `mpeg_pes_packet` `opus_packet` `prores_frame` `protobuf_widevine` `pssh_playready` `vorbis_packet` `vp9_frame` `vpx_ccr`</sub>|
|`mpeg_asc` |MPEG-4&nbsp;Audio&nbsp;Specific&nbsp;Config |<sub></sub>|
|`mpeg_es` |MPEG&nbsp;Elementary&nbsp;Stream |<sub>`mpeg_asc` `vorbis_packet`</sub>|
|`mpeg_pes` |MPEG&nbsp;Packetized&nbsp;elementary&nbsp;stream |<sub>`mpeg_pes_packet` `mpeg_spu`</sub>|
|`mpeg_pes_packet` |MPEG&nbsp;Packetized&nbsp;elementary&nbsp;stream&nbsp;packet |<sub></sub>|
|`mpeg_spu` |Sub&nbsp;Picture&nbsp;Unit&nbsp;(DVD&nbsp;subtitle) |<sub></sub>|
|`mpeg_ts` |MPEG&nbsp;Transport&nbsp;Stream |<sub></sub>|
|[`msgpack`](#msgpack) |MessagePack |<sub></sub>|
|`ogg` |OGG&nbsp;file |<sub>`ogg_page` `vorbis_packet` `opus_packet` `flac_metadatablock` `flac_frame`</sub>|
|`ogg_page` |OGG&nbsp;page |<sub></sub>|
|`opus_packet` |Opus&nbsp;packet |<sub>`vorbis_comment`</sub>|
|`pcap` |PCAP&nbsp;packet&nbsp;capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|`pcapng` |PCAPNG&nbsp;packet&nbsp;capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|`png` |Portable&nbsp;Network&nbsp;Graphics&nbsp;file |<sub>`icc_profile` `exif`</sub>|
|`prores_frame` |Apple&nbsp;ProRes&nbsp;frame |<sub></sub>|
|[`protobuf`](#protobuf) |Protobuf |<sub></sub>|
|`protobuf_widevine` |Widevine&nbsp;protobuf |<sub>`protobuf`</sub>|
|`pssh_playready` |PlayReady&nbsp;PSSH |<sub></sub>|
|`raw` |Raw&nbsp;bits |<sub></sub>|
|[`rtmp`](#rtmp) |Real-Time&nbsp;Messaging&nbsp;Protocol |<sub>`amf0` `mpeg_asc`</sub>|
|`sll2_packet` |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation&nbsp;v2 |<sub>`inet_packet`</sub>|
|`sll_packet` |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation |<sub>`inet_packet`</sub>|
|`tar` |Tar&nbsp;archive |<sub>`probe`</sub>|
|`tcp_segment` |Transmission&nbsp;control&nbsp;protocol&nbsp;segment |<sub></sub>|
|`tiff` |Tag&nbsp;Image&nbsp;File&nbsp;Format |<sub>`icc_profile`</sub>|
|`toml` |Tom's&nbsp;Obvious,&nbsp;Minimal&nbsp;Language |<sub></sub>|
|`udp_datagram` |User&nbsp;datagram&nbsp;protocol |<sub>`udp_payload`</sub>|
|`vorbis_comment` |Vorbis&nbsp;comment |<sub>`flac_picture`</sub>|
|`vorbis_packet` |Vorbis&nbsp;packet |<sub>`vorbis_comment`</sub>|
|`vp8_frame` |VP8&nbsp;frame |<sub></sub>|
|`vp9_cfm` |VP9&nbsp;Codec&nbsp;Feature&nbsp;Metadata |<sub></sub>|
|`vp9_frame` |VP9&nbsp;frame |<sub></sub>|
|`vpx_ccr` |VPX&nbsp;Codec&nbsp;Configuration&nbsp;Record |<sub></sub>|
|`wasm` |WebAssembly&nbsp;Binary&nbsp;Format |<sub></sub>|
|`wav` |WAV&nbsp;file |<sub>`id3v2` `id3v1` `id3v11`</sub>|
|`webp` |WebP&nbsp;image |<sub>`vp8_frame`</sub>|
|`xing` |Xing&nbsp;header |<sub></sub>|
|[`xml`](#xml) |Extensible&nbsp;Markup&nbsp;Language |<sub></sub>|
|`yaml` |YAML&nbsp;Ain't&nbsp;Markup&nbsp;Language |<sub></sub>|
|[`zip`](#zip) |ZIP&nbsp;archive |<sub>`probe`</sub>|
|`image` |Group |<sub>`gif` `jpeg` `mp4` `png` `tiff` `webp`</sub>|
|`inet_packet` |Group |<sub>`ipv4_packet` `ipv6_packet`</sub>|
|`ip_packet` |Group |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
|`link_frame` |Group |<sub>`bsd_loopback_frame` `ether8023_frame` `sll2_packet` `sll_packet`</sub>|
|`probe` |Group |<sub>`adts` `ar` `avro_ocf` `bitcoin_blkdat` `bzip2` `elf` `flac` `gif` `gzip` `jpeg` `json` `jsonl` `macho` `macho_fat` `matroska` `mp3` `mp4` `mpeg_ts` `ogg` `pcap` `pcapng` `png` `tar` `tiff` `toml` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
|`tcp_stream` |Group |<sub>`dns_tcp` `rtmp`</sub>|
|`udp_payload` |Group |<sub>`dns`</sub>|
[#]: sh-end
@ -161,35 +162,25 @@ Supports decoding BER, CER and DER (X.690).
- Does not support specifying a schema.
- Supports `torepr` but without schema all sequences and sets will be arrays.
#### Examples
#### `frompem` can be used to decode certificates etc
`frompem` and `topem` can be used to work with PEM format
```
```sh
$ fq -d raw 'frompem | asn1_ber | d' cert.pem
```
Can be used to decode nested parts
```
#### Can decode nested values
```sh
$ fq -d asn1_ber '.constructed[1].value | asn1_ber' file.ber
```
If schema is known and not complicated it can be reproduced
```
#### Manual schema
```sh
$ fq -d asn1_ber 'torepr as $r | ["version", "modulus", "private_exponent", "private_exponen", "prime1", "prime2", "exponent1", "exponent2", "coefficient"] | with_entries({key: .value, value: $r[.key]})' pkcs1.der
```
Supports `torepr`
```
$ fq -d asn1_ber torepr file
```
Supports `torepr`
```
... | asn1_ber | torepr
```
#### References and links
#### References
- https://www.itu.int/ITU-T/studygroups/com10/languages/X.690_1297.pdf
- https://en.wikipedia.org/wiki/X.690
- https://letsencrypt.org/docs/a-warm-welcome-to-asn1-and-der/
@ -222,70 +213,70 @@ Supports reading Avro Object Container Format (OCF) files based on the 1.11.0 sp
Capable of handling null, deflate, and snappy codecs for data compression.
Limitations:
- Schema does not support self-referential types, only built-in types.
- Decimal logical types are not supported for decoding, will just be treated as their primitive type
#### References and links
- Schema does not support self-referential types, only built-in types.
- Decimal logical types are not supported for decoding, will just be treated as their primitive type
#### References
- https://avro.apache.org/docs/current/spec.html#Object+Container+Files
### bencode
#### Convert represented value to JSON
```
$ fq -d bencode torepr file.torrent
```
#### References
- https://wiki.theory.org/BitTorrentSpecification#Bencoding
### bitcoin_block
#### Options
|Name |Default|Description|
|- |- |-|
|`has_header`|false |Has blkdat header|
#### Examples
Supports `torepr`
Decode file using bitcoin_block options
```
$ fq -d bencode torepr file
$ fq -d bitcoin_block -o has_header=false . file
```
Supports `torepr`
Decode value as bitcoin_block
```
... | bencode | torepr
... | bitcoin_block({has_header:false})
```
#### References and links
- https://wiki.theory.org/BitTorrentSpecification#Bencoding
### bson
#### Examples
#### Convert represented value to JSON
BSON as JSON
```
$ fq -d bson torepr file
$ fq -d bson torepr file.bson
```
Supports `torepr`
### Filter represented value
```
$ fq -d bson torepr file
$ fq -d bson 'torepr | select(.name=="bob")' file.bson
```
Supports `torepr`
```
... | bson | torepr
```
#### References and links
- https://wiki.theory.org/BitTorrentSpecification#Bencoding
#### References
- https://bsonspec.org/spec.html
### cbor
#### Examples
#### Convert represented value to JSON
Supports `torepr`
```
$ fq -d cbor torepr file
$ fq -d cbor torepr file.cbor
```
Supports `torepr`
```
... | cbor | torepr
```
#### References and links
#### References
- https://en.wikipedia.org/wiki/CBOR
- https://www.rfc-editor.org/rfc/rfc8949.html
@ -376,33 +367,30 @@ Decode value as html
Supports decoding vanilla and FAT Mach-O binaries.
#### Examples
#### Select 64bit load segments
Select 64bit load segments
```
```sh
$ fq '.load_commands[] | select(.cmd=="segment_64")' file
```
#### References and links
#### References
- https://github.com/aidansteele/osx-abi-macho-file-format-reference
### matroska
#### Examples
#### Lookup element path using `matroska_path`
Lookup element decode value using `matroska_path`
```
... | matroska_path(".Segment.Tracks[0)"
```sh
$ fq 'matroska_path(".Segment.Tracks[0)")' file.mkv
```
Return `matroska_path` string for a box decode value
```
... | grep_by(.id == "Tracks") | matroska_path
#### Get element path using `matroska_path`
```sh
$ fq 'grep_by(.id == "Tracks") | matroska_path' file.mkv
```
#### References and links
#### References
- https://tools.ietf.org/html/draft-ietf-cellar-ebml-00
- https://matroska.org/technical/specs/index.html
- https://www.matroska.org/technical/basics.html
@ -432,8 +420,6 @@ Decode value as mp3
### mp4
Support `mp4_path`
#### Options
|Name |Default|Description|
@ -443,16 +429,6 @@ Support `mp4_path`
#### Examples
Lookup box decode value using `mp4_path`
```
... | mp4_path(".moov.trak[1]")
```
Return `mp4_path` string for a box decode value
```
... | grep_by(.type == "trak") | mp4_path
```
Decode file using mp4 options
```
$ fq -d mp4 -o allow_truncated=false -o decode_samples=true . file
@ -463,48 +439,70 @@ Decode value as mp4
... | mp4({allow_truncated:false,decode_samples:true})
```
#### References and links
#### `mp4_path($path)` - Lookup mp4 box using a mp4 box path.
```sh
# <decode value box> | mp4_path($path) -> <decode value box>
$ fq 'mp4_path(".moov.trak[1]")' file.mp4
```
#### `mp4_path` - Return mp4 box path for a decode value box.
```sh
# <decode value box> | mp4_path -> string
$ fq 'grep_by(.type == "trak") | mp4_path' file.mp4
```
#### Force decode a single box
```sh
$ fq -n '"AAAAHGVsc3QAAAAAAAAAAQAAADIAAAQAAAEAAA==" | frombase64 | mp4({force:true}) | d'
```
#### Don't decode samples and manually decode first sample for first track as a `aac_frame`
```sh
$ fq -o decode_samples=false '.tracks[0].samples[0] | aac_frame | d' file.mp4
```
#### Entries for first edit list as values
```sh
$ fq 'first(grep_by(.type=="elst").entries) | tovalue' file.mp4
```
#### References
- [ISO/IEC base media file format (MPEG-4 Part 12)](https://en.wikipedia.org/wiki/ISO/IEC_base_media_file_format)
- [Quicktime file format](https://developer.apple.com/standards/qtff-2001.pdf)
### msgpack
#### Examples
#### Convert represented value to JSON
Supports `torepr`
```
$ fq -d msgpack torepr file
$ fq -d msgpack torepr file.msgpack
```
Supports `torepr`
```
... | msgpack | torepr
```
#### References and links
#### References
- https://github.com/msgpack/msgpack/blob/master/spec.md
### protobuf
#### Examples
#### Can decode sub messages
Can be used to decode sub messages
```
$ fq -d protobuf '.fields[6].wire_value | protobuf | d'
```sh
$ fq -d protobuf '.fields[6].wire_value | protobuf | d' file
```
#### References and links
#### References
- https://developers.google.com/protocol-buffers/docs/encoding
### rtmp
Current only supports plain RTMP (not RTMPT or encrypted variants etc) with AMF0 (not AMF3).
#### References and links
#### References
- https://rtmp.veriskope.com/docs/spec/
- https://rtmp.veriskope.com/pdf/video_file_format_spec_v10.pdf
@ -530,14 +528,11 @@ Decode value as xml
... | xml({array:false,attribute_prefix:"@",seq:false})
```
#### References and links
#### References
- [xml.com's Converting Between XML and JSON](https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html)
### zip
Supports ZIP64.
#### Options
|Name |Default|Description|
@ -556,9 +551,11 @@ Decode value as zip
... | zip({uncompress:true})
```
#### References and links
Supports ZIP64.
#### References
- https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
- https://opensource.apple.com/source/zip/zip-6/unzip/unzip/proginfo/extra.fld
[#]: sh-end

18
doc/formats.svg
View File

@ -1919,17 +1919,23 @@
<polygon fill="paleturquoise" stroke="transparent" points="3858.81,-2025.5 3858.81,-2051.5 3899.81,-2051.5 3899.81,-2025.5 3858.81,-2025.5"/>
<text text-anchor="start" x="3861.31" y="-2033.5" font-family="Times,serif" font-size="20.00">html</text>
</g>
<!-- msgpack -->
<!-- markdown -->
<g id="node112" class="node">
<title>markdown</title>
<polygon fill="paleturquoise" stroke="transparent" points="3929.81,-2025.5 3929.81,-2051.5 4018.81,-2051.5 4018.81,-2025.5 3929.81,-2025.5"/>
<text text-anchor="start" x="3932.31" y="-2033.5" font-family="Times,serif" font-size="20.00">markdown</text>
</g>
<!-- msgpack -->
<g id="node113" class="node">
<title>msgpack</title>
<polygon fill="paleturquoise" stroke="transparent" points="3929.81,-2025.5 3929.81,-2051.5 4004.81,-2051.5 4004.81,-2025.5 3929.81,-2025.5"/>
<text text-anchor="start" x="3932.31" y="-2033.5" font-family="Times,serif" font-size="20.00">msgpack</text>
<polygon fill="paleturquoise" stroke="transparent" points="4048.81,-2025.5 4048.81,-2051.5 4123.81,-2051.5 4123.81,-2025.5 4048.81,-2025.5"/>
<text text-anchor="start" x="4051.31" y="-2033.5" font-family="Times,serif" font-size="20.00">msgpack</text>
</g>
<!-- raw -->
<g id="node113" class="node">
<g id="node114" class="node">
<title>raw</title>
<polygon fill="paleturquoise" stroke="transparent" points="4036.81,-2025.5 4036.81,-2051.5 4071.81,-2051.5 4071.81,-2025.5 4036.81,-2025.5"/>
<text text-anchor="start" x="4039.31" y="-2033.5" font-family="Times,serif" font-size="20.00">raw</text>
<polygon fill="paleturquoise" stroke="transparent" points="4155.81,-2025.5 4155.81,-2051.5 4190.81,-2051.5 4190.81,-2025.5 4155.81,-2025.5"/>
<text text-anchor="start" x="4158.31" y="-2033.5" font-family="Times,serif" font-size="20.00">raw</text>
</g>
</g>
</svg>

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 134 KiB

After

Width:  |  Height:  |  Size: 135 KiB

1
format/all/all.go
View File

@ -28,6 +28,7 @@ import (
_ "github.com/wader/fq/format/jpeg"
_ "github.com/wader/fq/format/json"
_ "github.com/wader/fq/format/macho"
_ "github.com/wader/fq/format/markdown"
_ "github.com/wader/fq/format/math"
_ "github.com/wader/fq/format/matroska"
_ "github.com/wader/fq/format/mp3"

798
format/all/help.fqtest
View File

File diff suppressed because it is too large Load Diff

3
format/asn1/asn1_ber.go
View File

@ -29,6 +29,7 @@ import (
)
//go:embed asn1_ber.jq
//go:embed asn1_ber.md
var asn1FS embed.FS
func init() {
@ -36,7 +37,7 @@ func init() {
Name: format.ASN1_BER,
Description: "ASN1 BER (basic encoding rules, also CER and DER)",
DecodeFn: decodeASN1BER,
Functions: []string{"torepr", "_help"},
Functions: []string{"torepr"},
})
interp.RegisterFS(asn1FS)
}

21
format/asn1/asn1_ber.jq
View File

@ -6,24 +6,3 @@ def _asn1_ber_torepr:
end
else .constructed | map(_asn1_ber_torepr)
end;
def _asn1_ber__help:
{ notes: "Supports decoding BER, CER and DER (X.690).
- Currently no extra validation is done for CER and DER.
- Does not support specifying a schema.
- Supports `torepr` but without schema all sequences and sets will be arrays.",
examples: [
{comment: "`frompem` and `topem` can be used to work with PEM format", shell: "fq -d raw 'frompem | asn1_ber | d' cert.pem"},
{comment: "Can be used to decode nested parts", shell: "fq -d asn1_ber '.constructed[1].value | asn1_ber' file.ber"},
{ comment: "If schema is known and not complicated it can be reproduced",
shell: "fq -d asn1_ber 'torepr as $r | [\"version\", \"modulus\", \"private_exponent\", \"private_exponen\", \"prime1\", \"prime2\", \"exponent1\", \"exponent2\", \"coefficient\"] | with_entries({key: .value, value: $r[.key]})' pkcs1.der"
}
],
links: [
{url: "https://www.itu.int/ITU-T/studygroups/com10/languages/X.690_1297.pdf"},
{url: "https://en.wikipedia.org/wiki/X.690"},
{url: "https://letsencrypt.org/docs/a-warm-welcome-to-asn1-and-der/"},
{url: "https://lapo.it/asn1js/"}
]
};

29
format/asn1/asn1_ber.md Normal file
View File

@ -0,0 +1,29 @@
Supports decoding BER, CER and DER (X.690).
- Currently no extra validation is done for CER and DER.
- Does not support specifying a schema.
- Supports `torepr` but without schema all sequences and sets will be arrays.
#### `frompem` can be used to decode certificates etc
```sh
$ fq -d raw 'frompem | asn1_ber | d' cert.pem
```
#### Can decode nested values
```sh
$ fq -d asn1_ber '.constructed[1].value | asn1_ber' file.ber
```
#### Manual schema
```sh
$ fq -d asn1_ber 'torepr as $r | ["version", "modulus", "private_exponent", "private_exponen", "prime1", "prime2", "exponent1", "exponent2", "coefficient"] | with_entries({key: .value, value: $r[.key]})' pkcs1.der
```
#### References
- https://www.itu.int/ITU-T/studygroups/com10/languages/X.690_1297.pdf
- https://en.wikipedia.org/wiki/X.690
- https://letsencrypt.org/docs/a-warm-welcome-to-asn1-and-der/
- https://lapo.it/asn1js/

3
format/avro/avro_ocf.go
View File

@ -16,7 +16,7 @@ import (
"github.com/wader/fq/pkg/scalar"
)
//go:embed avro_ocf.jq
//go:embed avro_ocf.md
var avroOcfFS embed.FS
func init() {
@ -25,7 +25,6 @@ func init() {
Description: "Avro object container file",
Groups: []string{format.PROBE},
DecodeFn: decodeAvroOCF,
Functions: []string{"_help"},
})
interp.RegisterFS(avroOcfFS)
}

12
format/avro/avro_ocf.jq
View File

@ -1,12 +0,0 @@
def _avro_ocf__help:
{ notes: "Supports reading Avro Object Container Format (OCF) files based on the 1.11.0 specification.
Capable of handling null, deflate, and snappy codecs for data compression.
Limitations:
- Schema does not support self-referential types, only built-in types.
- Decimal logical types are not supported for decoding, will just be treated as their primitive type",
links: [
{url: "https://avro.apache.org/docs/current/spec.html#Object+Container+Files"}
]
};

11
format/avro/avro_ocf.md Normal file
View File

@ -0,0 +1,11 @@
Supports reading Avro Object Container Format (OCF) files based on the 1.11.0 specification.
Capable of handling null, deflate, and snappy codecs for data compression.
Limitations:
- Schema does not support self-referential types, only built-in types.
- Decimal logical types are not supported for decoding, will just be treated as their primitive type
#### References
- https://avro.apache.org/docs/current/spec.html#Object+Container+Files

3
format/bencode/bencode.go
View File

@ -13,6 +13,7 @@ import (
)
//go:embed bencode.jq
//go:embed bencode.md
var bencodeFS embed.FS
func init() {
@ -20,7 +21,7 @@ func init() {
Name: format.BENCODE,
Description: "BitTorrent bencoding",
DecodeFn: decodeBencode,
Functions: []string{"torepr", "_help"},
Functions: []string{"torepr"},
})
interp.RegisterFS(bencodeFS)
}

6
format/bencode/bencode.jq
View File

@ -9,9 +9,3 @@ def _bencode_torepr:
)
else error("unknown type \(.type)")
end;
def _bencode__help:
{ links: [
{url: "https://wiki.theory.org/BitTorrentSpecification#Bencoding"}
]
};

8
format/bencode/bencode.md Normal file
View File

@ -0,0 +1,8 @@
#### Convert represented value to JSON
```
$ fq -d bencode torepr file.torrent
```
#### References
- https://wiki.theory.org/BitTorrentSpecification#Bencoding

3
format/bson/bson.go
View File

@ -13,6 +13,7 @@ import (
)
//go:embed bson.jq
//go:embed bson.md
var bsonFS embed.FS
func init() {
@ -20,7 +21,7 @@ func init() {
Name: format.BSON,
Description: "Binary JSON",
DecodeFn: decodeBSON,
Functions: []string{"torepr", "_help"},
Functions: []string{"torepr"},
})
interp.RegisterFS(bsonFS)
}

8
format/bson/bson.jq
View File

@ -17,11 +17,3 @@ def _bson_torepr:
| _f
);
def _bson__help:
{ examples: [
{comment: "BSON as JSON", shell: "fq -d bson torepr file"}
],
links: [
{url: "https://wiki.theory.org/BitTorrentSpecification#Bencoding"}
]
};

14
format/bson/bson.md Normal file
View File

@ -0,0 +1,14 @@
#### Convert represented value to JSON
```
$ fq -d bson torepr file.bson
```
### Filter represented value
```
$ fq -d bson 'torepr | select(.name=="bob")' file.bson
```
#### References
- https://bsonspec.org/spec.html

3
format/cbor/cbor.go
View File

@ -21,6 +21,7 @@ import (
)
//go:embed cbor.jq
//go:embed cbor.md
var cborFS embed.FS
func init() {
@ -28,7 +29,7 @@ func init() {
Name: format.CBOR,
Description: "Concise Binary Object Representation",
DecodeFn: decodeCBOR,
Functions: []string{"torepr", "_help"},
Functions: []string{"torepr"},
})
interp.RegisterFS(cborFS)
}

7
format/cbor/cbor.jq
View File

@ -8,10 +8,3 @@ def _cbor_torepr:
elif .major_type == "bytes" then .value | tostring
else .value | tovalue
end;
def _cbor__help:
{ links: [
{url: "https://en.wikipedia.org/wiki/CBOR"},
{url: "https://www.rfc-editor.org/rfc/rfc8949.html"}
]
};

9
format/cbor/cbor.md Normal file
View File

@ -0,0 +1,9 @@
#### Convert represented value to JSON
```
$ fq -d cbor torepr file.cbor
```
#### References
- https://en.wikipedia.org/wiki/CBOR
- https://www.rfc-editor.org/rfc/rfc8949.html

1
format/format.go
View File

@ -86,6 +86,7 @@ const (
JSONL = "jsonl"
MACHO = "macho"
MACHO_FAT = "macho_fat"
MARKDOWN = "markdown"
MATROSKA = "matroska"
MP3 = "mp3"
MP3_FRAME = "mp3_frame"

3
format/macho/macho.go
View File

@ -14,7 +14,7 @@ import (
"github.com/wader/fq/pkg/scalar"
)
//go:embed macho.jq
//go:embed macho.md
var machoFS embed.FS
func init() {
@ -23,7 +23,6 @@ func init() {
Description: "Mach-O macOS executable",
Groups: []string{format.PROBE},
DecodeFn: machoDecode,
Functions: []string{"_help"},
})
interp.RegisterFS(machoFS)
}

9
format/macho/macho.jq
View File

@ -1,9 +0,0 @@
def _macho__help:
{ notes: "Supports decoding vanilla and FAT Mach-O binaries.",
examples: [
{comment: "Select 64bit load segments", shell: "fq '.load_commands[] | select(.cmd==\"segment_64\")' file"}
],
links: [
{url: "https://github.com/aidansteele/osx-abi-macho-file-format-reference"}
]
};

10
format/macho/macho.md Normal file
View File

@ -0,0 +1,10 @@
Supports decoding vanilla and FAT Mach-O binaries.
#### Select 64bit load segments
```sh
$ fq '.load_commands[] | select(.cmd=="segment_64")' file
```
#### References
- https://github.com/aidansteele/osx-abi-macho-file-format-reference

319
format/markdown/markdown.go Normal file
View File

@ -0,0 +1,319 @@
package markdown
import (
"embed"
"fmt"
"io"
"github.com/gomarkdown/markdown"
"github.com/gomarkdown/markdown/ast"
"github.com/wader/fq/format"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
"github.com/wader/fq/pkg/scalar"
)
//go:embed markdown.jq
var markdownFS embed.FS
func init() {
interp.RegisterFormat(decode.Format{
Name: format.MARKDOWN,
Description: "Markdown",
DecodeFn: decodeMarkdown,
Functions: []string{"_todisplay"},
})
interp.RegisterFS(markdownFS)
}
func decodeMarkdown(d *decode.D, _ any) any {
b, err := io.ReadAll(bitio.NewIOReader(d.RawLen(d.Len())))
if err != nil {
panic(err)
}
var s scalar.S
s.Actual = node(markdown.Parse(b, nil))
d.Value.V = &s
d.Value.Range.Len = d.Len()
return nil
}
func stringSlice[T string | []byte](ss []T) []any {
var vs []any
for _, e := range ss {
vs = append(vs, string(e))
}
return vs
}
func sliceMap[F, T any](vs []F, fn func(F) T) []T {
ts := make([]T, len(vs))
for i, v := range vs {
ts[i] = fn(v)
}
return ts
}
func intSlice[T ~int](ss []T) []any {
var vs []any
for _, e := range ss {
vs = append(vs, e)
}
return vs
}
func attr(v map[string]any, attr *ast.Attribute) {
if attr == nil {
return
}
v["id"] = string(attr.ID)
var as []any
for _, a := range attr.Attrs {
as = append(as, string(a))
}
v["attrs"] = as
var cs []any
for _, a := range attr.Classes {
cs = append(cs, string(a))
}
v["classes"] = cs
}
func leaf(v map[string]any, typ string, l ast.Leaf) {
v["type"] = typ
v["literal"] = string(l.Literal)
attr(v, l.Attribute)
}
func container(v map[string]any, typ string, c ast.Container) {
v["type"] = typ
v["literal"] = string(c.Literal)
var cs []any
children := c.GetChildren()
for _, n := range children {
cv := node(n)
if cv != nil {
cs = append(cs, node(n))
}
}
v["children"] = cs
attr(v, c.Attribute)
}
func listType(t ast.ListType) []any {
var vs []any
if t&ast.ListTypeOrdered == ast.ListTypeOrdered {
vs = append(vs, "ordered")
}
if t%ast.ListTypeOrdered == ast.ListTypeOrdered {
vs = append(vs, "ordered")
}
if t%ast.ListTypeDefinition == ast.ListTypeDefinition {
vs = append(vs, "definition")
}
if t%ast.ListTypeTerm == ast.ListTypeTerm {
vs = append(vs, "term")
}
if t%ast.ListItemContainsBlock == ast.ListItemContainsBlock {
vs = append(vs, "contains_block")
}
if t%ast.ListItemBeginningOfList == ast.ListItemBeginningOfList {
vs = append(vs, "beginning_of_list")
}
if t%ast.ListItemEndOfList == ast.ListItemEndOfList {
vs = append(vs, "end_of_list")
}
return vs
}
func node(n ast.Node) any {
v := map[string]any{}
switch n := n.(type) {
case *ast.Text:
if n.Leaf.Attribute == nil {
if len(n.Leaf.Literal) > 0 {
return string(n.Leaf.Literal)
}
// skip
return nil
}
case *ast.Softbreak:
leaf(v, "softbreak", n.Leaf)
case *ast.Hardbreak:
leaf(v, "hardbreak", n.Leaf)
case *ast.NonBlockingSpace:
leaf(v, "nbsp", n.Leaf)
case *ast.Emph:
container(v, "em", n.Container)
case *ast.Strong:
container(v, "strong", n.Container)
case *ast.Del:
container(v, "del", n.Container)
case *ast.BlockQuote:
container(v, "blockquote", n.Container)
case *ast.Aside:
container(v, "aside", n.Container)
case *ast.Link:
container(v, "link", n.Container)
v["destination"] = string(n.Destination)
v["title"] = string(n.Title)
v["note_id"] = n.NoteID
v["deferred_id"] = string(n.DeferredID)
v["additional_attributes"] = stringSlice(n.AdditionalAttributes)
case *ast.CrossReference:
container(v, "cross_reference", n.Container)
v["destination"] = string(n.Destination)
case *ast.Citation:
leaf(v, "citation", n.Leaf)
v["destination"] = stringSlice(n.Destination)
v["type"] = sliceMap(n.Type, func(v ast.CitationTypes) string {
switch v {
case ast.CitationTypeNone:
return "none"
case ast.CitationTypeSuppressed:
return "suppressed"
case ast.CitationTypeInformative:
return "informative"
case ast.CitationTypeNormative:
return "normative"
default:
return "unknown"
}
})
v["type"] = intSlice(n.Type)
v["suffix"] = stringSlice(n.Suffix)
case *ast.Image:
container(v, "image", n.Container)
v["destination"] = string(n.Destination)
v["title"] = string(n.Title)
case *ast.Code:
leaf(v, "code", n.Leaf)
case *ast.CodeBlock:
leaf(v, "code_block", n.Leaf)
v["is_fenced"] = n.IsFenced
v["info"] = string(n.Info)
if n.FenceChar != 0 {
v["fence_char"] = string(n.FenceChar)
}
v["fence_length"] = n.FenceLength
v["fence_offset"] = n.FenceOffset
case *ast.Caption:
container(v, "caption", n.Container)
case *ast.CaptionFigure:
container(v, "caption_figure", n.Container)
v["heading_id"] = n.HeadingID
case *ast.Document:
container(v, "document", n.Container)
case *ast.Paragraph:
container(v, "paragraph", n.Container)
case *ast.HTMLSpan:
leaf(v, "html_span", n.Leaf)
case *ast.HTMLBlock:
leaf(v, "html_block", n.Leaf)
case *ast.Heading:
container(v, "heading", n.Container)
v["level"] = n.Level
v["heading_id"] = n.HeadingID
v["is_titleblock"] = n.IsTitleblock
v["is_special"] = n.IsSpecial
case *ast.HorizontalRule:
leaf(v, "hr", n.Leaf)
case *ast.List:
container(v, "list", n.Container)
v["list_flags"] = listType(n.ListFlags)
v["tight"] = n.Tight
if n.BulletChar != 0 {
v["bullet_char"] = string(n.BulletChar)
}
if n.Delimiter != 0 {
v["delimiter"] = string(n.Delimiter)
}
v["start"] = n.Start
v["ref_link"] = string(n.RefLink)
v["is_footnotes_list"] = n.IsFootnotesList
case *ast.ListItem:
container(v, "list_item", n.Container)
v["list_flags"] = listType(n.ListFlags)
v["tight"] = n.Tight
if n.BulletChar != 0 {
v["bullet_char"] = string(n.BulletChar)
}
if n.Delimiter != 0 {
v["delimiter"] = string(n.Delimiter)
}
v["ref_link"] = string(n.RefLink)
v["is_footnotes_list"] = n.IsFootnotesList
case *ast.Table:
container(v, "table", n.Container)
case *ast.TableCell:
container(v, "table_cell", n.Container)
v["is_header"] = n.IsHeader
v["align"] = n.Align.String()
v["col_span"] = n.ColSpan
case *ast.TableHeader:
container(v, "table_header", n.Container)
case *ast.TableBody:
container(v, "table_body", n.Container)
case *ast.TableRow:
container(v, "table_row", n.Container)
case *ast.TableFooter:
container(v, "table_footer", n.Container)
case *ast.Math:
leaf(v, "math", n.Leaf)
case *ast.MathBlock:
container(v, "math_block", n.Container)
case *ast.DocumentMatter:
container(v, "document_matter", n.Container)
v["matter"] = func(v ast.DocumentMatters) string {
switch v {
case ast.DocumentMatterNone:
return "none"
case ast.DocumentMatterFront:
return "front"
case ast.DocumentMatterMain:
return "main"
case ast.DocumentMatterBack:
return "back"
default:
return "unknown"
}
}(n.Matter)
case *ast.Callout:
leaf(v, "callout", n.Leaf)
v["id"] = string(n.ID)
case *ast.Index:
leaf(v, "index", n.Leaf)
v["primary"] = n.Primary
v["item"] = string(n.Item)
v["subitem"] = string(n.Subitem)
v["id"] = n.ID
case *ast.Subscript:
leaf(v, "subscript", n.Leaf)
case *ast.Superscript:
leaf(v, "superscript", n.Leaf)
case *ast.Footnotes:
container(v, "footnotes", n.Container)
default:
panic(fmt.Sprintf("unknown node %T", node))
}
for k, e := range v {
if s, ok := e.(string); ok && s == "" {
delete(v, k)
}
}
return v
}

52
format/markdown/markdown.jq Normal file
View File

@ -0,0 +1,52 @@
def _markdown__todisplay: tovalue;
def word_break($width):
def _f($a; $acc; $l):
( $a[0] as $w
| ($w // "" | length) as $wl
| if $w == null then $acc
elif ($l + $wl) >= $width then
( $acc
, _f($a[1:]; [$w]; $wl)
)
else _f($a[1:]; $acc+[$w]; $l+$wl)
end
);
( [_f([splits("\\s{1,}")]; []; 0)]
| map(join(" "))
);
def _markdown_to_text($width; $header_depth):
def lb: if $width > 0 then word_break($width) | join("\n") end;
def _f($pln):
if type == "string" then gsub("\n"; " ")
elif .type == "document" then .children[] | _f("\n\n")
elif .type == "heading" then "#" * (.level+$header_depth), " ", (.children[] | _f("\n\n")), "\n"
elif .type == "paragraph" then
( ( [.children[] | _f("\n\n")]
| join("")
| lb
)
, $pln
)
elif .type == "link" then
( ( [ .children[]
| _f("")
]
| join("")
) as $text
| $text
, if $text != .destination then " (", .destination, ")"
else empty
end
)
elif .type == "code_block" then "\n", (" ", .literal | split("\n") | join("\n ")), "\n"
elif .type == "code" then .literal
elif .type == "list" then (.children[] | _f("\n\n")), "\n" # TODO: delim
elif .type == "list_item" then .bullet_char, " ", (.children[] | _f("\n"))
elif .type == "html_span" then .literal | gsub("<br>"; "\n") # TODO: more?
else empty
end;
[_f("\n\n")] | join("");
def _markdown_to_text:
_markdown_to_text(-1; 0);

316
format/markdown/testdata/test.fqtest vendored Normal file
View File

@ -0,0 +1,316 @@
$ fq -d markdown . test.md
{
"children": [
{
"children": [
"Before"
],
"type": "paragraph"
},
{
"children": [
"header 1"
],
"is_special": false,
"is_titleblock": false,
"level": 1,
"type": "heading"
},
{
"children": [
"Paragraph with ",
{
"children": [
"bold"
],
"type": "strong"
},
" and ",
{
"children": [
"italic"
],
"type": "em"
},
"\non\nmultiple\nlines."
],
"type": "paragraph"
},
{
"children": [
{
"children": [
"Some citation"
],
"type": "paragraph"
}
],
"type": "blockquote"
},
{
"children": [
"A footnote",
{
"additional_attributes": [],
"children": [
"^1"
],
"destination": "footnote1",
"note_id": 0,
"type": "link"
},
" and this also",
{
"additional_attributes": [],
"children": [
"^note"
],
"destination": "footnote2",
"note_id": 0,
"type": "link"
}
],
"type": "paragraph"
},
{
"children": [
"header 2"
],
"is_special": false,
"is_titleblock": false,
"level": 2,
"type": "heading"
},
{
"fence_length": 0,
"fence_offset": 0,
"info": "jq",
"is_fenced": true,
"literal": "code\nblock\n",
"type": "code_block"
},
{
"fence_length": 0,
"fence_offset": 0,
"is_fenced": false,
"literal": "also\ncode\n",
"type": "code_block"
},
{
"children": [
"header 3"
],
"is_special": false,
"is_titleblock": false,
"level": 3,
"type": "heading"
},
{
"children": [
"Some text with ",
{
"literal": "code",
"type": "code"
}
],
"type": "paragraph"
},
{
"children": [
"header 4"
],
"is_special": false,
"is_titleblock": false,
"level": 4,
"type": "heading"
},
{
"children": [
"Some text ",
{
"additional_attributes": [],
"children": [
"with a link"
],
"destination": "http://host/path",
"note_id": 0,
"type": "link"
}
],
"type": "paragraph"
},
{
"children": [
"An image ",
{
"children": [
"img alt text"
],
"destination": "path/image.png",
"type": "image"
}
],
"type": "paragraph"
},
{
"children": [
"header 5"
],
"is_special": false,
"is_titleblock": false,
"level": 5,
"type": "heading"
},
{
"children": [
{
"bullet_char": "-",
"children": [
{
"children": [
"list of"
],
"type": "paragraph"
}
],
"delimiter": ".",
"is_footnotes_list": false,
"list_flags": [],
"tight": false,
"type": "list_item"
},
{
"bullet_char": "-",
"children": [
{
"children": [
"things"
],
"type": "paragraph"
}
],
"delimiter": ".",
"is_footnotes_list": false,
"list_flags": [],
"tight": false,
"type": "list_item"
}
],
"delimiter": ".",
"is_footnotes_list": false,
"list_flags": [],
"start": 0,
"tight": true,
"type": "list"
},
{
"children": [
"a table"
],
"type": "paragraph"
},
{
"children": [
{
"children": [
{
"children": [
{
"children": [
"a"
],
"col_span": 0,
"is_header": true,
"type": "table_cell"
},
{
"children": [
"b"
],
"col_span": 0,
"is_header": true,
"type": "table_cell"
},
{
"children": [
"c"
],
"col_span": 0,
"is_header": true,
"type": "table_cell"
}
],
"type": "table_row"
}
],
"type": "table_header"
},
{
"children": [
{
"children": [
{
"children": [
"1"
],
"col_span": 0,
"is_header": false,
"type": "table_cell"
},
{
"children": [
"2"
],
"col_span": 0,
"is_header": false,
"type": "table_cell"
},
{
"children": [
"3"
],
"col_span": 0,
"is_header": false,
"type": "table_cell"
}
],
"type": "table_row"
}
],
"type": "table_body"
}
],
"type": "table"
},
{
"children": [
"header 6"
],
"is_special": false,
"is_titleblock": false,
"level": 6,
"type": "heading"
},
{
"children": [
"Some text with line ",
{
"literal": "<br>",
"type": "html_span"
},
" break and ",
{
"literal": "<b>",
"type": "html_span"
},
"bold",
{
"literal": "</b>",
"type": "html_span"
}
],
"type": "paragraph"
}
],
"type": "document"
}

50
format/markdown/testdata/test.md vendored Normal file
View File

@ -0,0 +1,50 @@
Before
# header 1
Paragraph with **bold** and *italic*
on
multiple
lines.
> Some citation
A footnote[^1] and this also[^note]
## header 2
```jq
code
block
```
also
code
### header 3
Some text with `code`
#### header 4
Some text [with a link](http://host/path)
An image ![img alt text](path/image.png)
##### header 5
- list of
- things
a table
| a | b | c |
| --- | --- | --- |
| 1 | 2 | 3 |
###### header 6
Some text with line <br> break and <b>bold</b>
[^1]: footnote1
[^note]: footnote2

2
format/matroska/matroska.go
View File

@ -27,6 +27,7 @@ import (
)
//go:embed matroska.jq
//go:embed matroska.md
var matroskaFS embed.FS
var aacFrameFormat decode.Group
@ -78,7 +79,6 @@ func init() {
{Names: []string{format.VP9_CFM}, Group: &vp9CFMFormat},
{Names: []string{format.VP9_FRAME}, Group: &vp9FrameFormat},
},
Functions: []string{"_help"},
})
interp.RegisterFS(matroskaFS)

14
format/matroska/matroska.jq
View File

@ -12,17 +12,3 @@ def matroska_path:
| format_root
| matroska_path($c)
);
def _matroska__help:
{ examples: [
{comment: "Lookup element decode value using `matroska_path`", expr: "matroska_path(\".Segment.Tracks[0)\""},
{comment: "Return `matroska_path` string for a box decode value", expr: "grep_by(.id == \"Tracks\") | matroska_path"}
],
links: [
{url: "https://tools.ietf.org/html/draft-ietf-cellar-ebml-00"},
{url: "https://matroska.org/technical/specs/index.html"},
{url: "https://www.matroska.org/technical/basics.html"},
{url: "https://www.matroska.org/technical/codec_specs.html"},
{url: "https://wiki.xiph.org/MatroskaOpus"}
]
};

18
format/matroska/matroska.md Normal file
View File

@ -0,0 +1,18 @@
#### Lookup element path using `matroska_path`
```sh
$ fq 'matroska_path(".Segment.Tracks[0)")' file.mkv
```
#### Get element path using `matroska_path`
```sh
$ fq 'grep_by(.id == "Tracks") | matroska_path' file.mkv
```
#### References
- https://tools.ietf.org/html/draft-ietf-cellar-ebml-00
- https://matroska.org/technical/specs/index.html
- https://www.matroska.org/technical/basics.html
- https://www.matroska.org/technical/codec_specs.html
- https://wiki.xiph.org/MatroskaOpus

4
format/mp4/mp4.go
View File

@ -26,6 +26,7 @@ import (
)
//go:embed mp4.jq
//go:embed mp4.md
var mp4FS embed.FS
var aacFrameFormat decode.Group
@ -55,7 +56,7 @@ var vpxCCRFormat decode.Group
func init() {
interp.RegisterFormat(decode.Format{
Name: format.MP4,
Description: "ISOBMFF MPEG-4 part 12 and similar",
Description: "ISOBMFF, QuickTime and similar",
Groups: []string{
format.PROBE,
format.IMAGE, // avif
@ -90,7 +91,6 @@ func init() {
{Names: []string{format.VP9_FRAME}, Group: &vp9FrameFormat},
{Names: []string{format.VPX_CCR}, Group: &vpxCCRFormat},
},
Functions: []string{"_help"},
})
interp.RegisterFS(mp4FS)
}

12
format/mp4/mp4.jq
View File

@ -12,15 +12,3 @@ def mp4_path:
| format_root
| mp4_path($c)
);
def _mp4__help:
{ notes: "Support `mp4_path`",
examples: [
{comment: "Lookup box decode value using `mp4_path`", expr: "mp4_path(\".moov.trak[1]\")"},
{comment: "Return `mp4_path` string for a box decode value", expr: "grep_by(.type == \"trak\") | mp4_path"}
],
links: [
{title: "ISO/IEC base media file format (MPEG-4 Part 12)", url: "https://en.wikipedia.org/wiki/ISO/IEC_base_media_file_format"},
{title: "Quicktime file format", url: "https://developer.apple.com/standards/qtff-2001.pdf"}
]
};

36
format/mp4/mp4.md Normal file
View File

@ -0,0 +1,36 @@
#### `mp4_path($path)` - Lookup mp4 box using a mp4 box path.
```sh
# <decode value box> | mp4_path($path) -> <decode value box>
$ fq 'mp4_path(".moov.trak[1]")' file.mp4
```
#### `mp4_path` - Return mp4 box path for a decode value box.
```sh
# <decode value box> | mp4_path -> string
$ fq 'grep_by(.type == "trak") | mp4_path' file.mp4
```
#### Force decode a single box
```sh
$ fq -n '"AAAAHGVsc3QAAAAAAAAAAQAAADIAAAQAAAEAAA==" | frombase64 | mp4({force:true}) | d'
```
#### Don't decode samples and manually decode first sample for first track as a `aac_frame`
```sh
$ fq -o decode_samples=false '.tracks[0].samples[0] | aac_frame | d' file.mp4
```
#### Entries for first edit list as values
```sh
$ fq 'first(grep_by(.type=="elst").entries) | tovalue' file.mp4
```
#### References
- [ISO/IEC base media file format (MPEG-4 Part 12)](https://en.wikipedia.org/wiki/ISO/IEC_base_media_file_format)
- [Quicktime file format](https://developer.apple.com/standards/qtff-2001.pdf)

3
format/msgpack/msgpack.go
View File

@ -14,6 +14,7 @@ import (
)
//go:embed msgpack.jq
//go:embed msgpack.md
var msgPackFS embed.FS
func init() {
@ -21,7 +22,7 @@ func init() {
Name: format.MSGPACK,
Description: "MessagePack",
DecodeFn: decodeMsgPack,
Functions: []string{"torepr", "_help"},
Functions: []string{"torepr"},
})
interp.RegisterFS(msgPackFS)
}

5
format/msgpack/msgpack.jq
View File

@ -9,8 +9,3 @@ def _msgpack_torepr:
else .value | tovalue
end;
def _msgpack__help:
{ links: [
{url: "https://github.com/msgpack/msgpack/blob/master/spec.md"}
]
};

8
format/msgpack/msgpack.md Normal file
View File

@ -0,0 +1,8 @@
#### Convert represented value to JSON
```
$ fq -d msgpack torepr file.msgpack
```
#### References
- https://github.com/msgpack/msgpack/blob/master/spec.md

3
format/protobuf/protobuf.go
View File

@ -12,7 +12,7 @@ import (
"github.com/wader/fq/pkg/scalar"
)
//go:embed protobuf.jq
//go:embed protobuf.md
var protobufFS embed.FS
func init() {
@ -20,7 +20,6 @@ func init() {
Name: format.PROTOBUF,
Description: "Protobuf",
DecodeFn: protobufDecode,
Functions: []string{"_help"},
})
interp.RegisterFS(protobufFS)
}

8
format/protobuf/protobuf.jq
View File

@ -1,8 +0,0 @@
def _protobuf__help:
{ examples: [
{comment: "Can be used to decode sub messages", shell: "fq -d protobuf '.fields[6].wire_value | protobuf | d'"}
],
links: [
{url: "https://developers.google.com/protocol-buffers/docs/encoding"}
]
};

8
format/protobuf/protobuf.md Normal file
View File

@ -0,0 +1,8 @@
#### Can decode sub messages
```sh
$ fq -d protobuf '.fields[6].wire_value | protobuf | d' file
```
#### References
- https://developers.google.com/protocol-buffers/docs/encoding

3
format/rtmp/rtmp.go
View File

@ -21,7 +21,7 @@ import (
var rtmpAmf0Group decode.Group
var rtmpMpegASCFormat decode.Group
//go:embed rtmp.jq
//go:embed rtmp.md
var rtmpFS embed.FS
func init() {
@ -36,7 +36,6 @@ func init() {
{Names: []string{format.AMF0}, Group: &rtmpAmf0Group},
{Names: []string{format.MPEG_ASC}, Group: &rtmpMpegASCFormat},
},
Functions: []string{"_help"},
})
interp.RegisterFS(rtmpFS)
}

7
format/rtmp/rtmp.jq
View File

@ -1,7 +0,0 @@
def _rtmp__help:
{ notes: "Current only supports plain RTMP (not RTMPT or encrypted variants etc) with AMF0 (not AMF3).",
links: [
{url: "https://rtmp.veriskope.com/docs/spec/"},
{url: "https://rtmp.veriskope.com/pdf/video_file_format_spec_v10.pdf"}
]
};

5
format/rtmp/rtmp.md Normal file
View File

@ -0,0 +1,5 @@
Current only supports plain RTMP (not RTMPT or encrypted variants etc) with AMF0 (not AMF3).
#### References
- https://rtmp.veriskope.com/docs/spec/
- https://rtmp.veriskope.com/pdf/video_file_format_spec_v10.pdf

3
format/xml/xml.go
View File

@ -29,6 +29,7 @@ import (
)
//go:embed xml.jq
//go:embed xml.md
var xmlFS embed.FS
func init() {
@ -43,7 +44,7 @@ func init() {
Array: false,
AttributePrefix: "@",
},
Functions: []string{"_todisplay", "_help"},
Functions: []string{"_todisplay"},
})
interp.RegisterFS(xmlFS)
interp.RegisterFunc1("toxml", toXML)

6
format/xml/xml.jq
View File

@ -1,8 +1,2 @@
def toxml: toxml(null);
def _xml__todisplay: tovalue;
def _xml__help:
{ links: [
{title: "xml.com's Converting Between XML and JSON", url: "https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html"}
]
};

2
format/xml/xml.md Normal file
View File

@ -0,0 +1,2 @@
#### References
- [xml.com's Converting Between XML and JSON](https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html)

3
format/zip/zip.go
View File

@ -15,7 +15,7 @@ import (
"github.com/wader/fq/pkg/scalar"
)
//go:embed zip.jq
//go:embed zip.md
var zipFS embed.FS
var probeFormat decode.Group
@ -32,7 +32,6 @@ func init() {
Dependencies: []decode.Dependency{
{Names: []string{format.PROBE}, Group: &probeFormat},
},
Functions: []string{"_help"},
})
interp.RegisterFS(zipFS)
}

6
format/zip/zip.jq
View File

@ -1,6 +0,0 @@
def _zip__help:
{ notes: "Supports ZIP64.",
links: [
{url: "https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT"}
]
};

5
format/zip/zip.md Normal file
View File

@ -0,0 +1,5 @@
Supports ZIP64.
#### References
- https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
- https://opensource.apple.com/source/zip/zip-6/unzip/unzip/proginfo/extra.fld

4
go.mod
View File

@ -25,6 +25,10 @@ require (
// bump: gomod-golang-snappy link "Source diff $CURRENT..$LATEST" https://github.com/golang/snappy/compare/v$CURRENT..v$LATEST
github.com/golang/snappy v0.0.4
// has no tags
// go get -d github.com/gomarkdown/markdown@master && go mod tidy
github.com/gomarkdown/markdown v0.0.0-20220627144906-e9a81102ebeb
// has no tags yet
// bump-disabled: gomod-gopacket /github\.com\/gopacket\/gopacket v(.*)/ https://github.com/gopacket/gopacket.git|^1
// bump-disabled: gomod-gopacket command go get -d github.com/gopacket/gopacket@v$LATEST && go mod tidy

2
go.sum
View File

@ -4,6 +4,8 @@ github.com/creasty/defaults v1.6.0 h1:ltuE9cfphUtlrBeomuu8PEyISTXnxqkBIoQfXgv7BS
github.com/creasty/defaults v1.6.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomarkdown/markdown v0.0.0-20220627144906-e9a81102ebeb h1:5b/eFaSaKPFG9ygDBaPKkydKU5nFJYk08g9jPIVogMg=
github.com/gomarkdown/markdown v0.0.0-20220627144906-e9a81102ebeb/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
github.com/gopacket/gopacket v0.0.0-20220819214934-ee81b8c880da h1:AAwDU9N39fQNYUtg270aiU6N7U2ZVsGZKiRwsCMsWEo=
github.com/gopacket/gopacket v0.0.0-20220819214934-ee81b8c880da/go.mod h1:DlRRfaM/QjAu2ADqraIure1Eif0HpNL8hmyVQ+qci5Y=
github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=

17
pkg/interp/funcs.jq
View File

@ -153,23 +153,6 @@ def paste:
)
end;
# very simple markdown to text converter
# assumes very basic markdown as input
def _markdown_to_text:
( .
# ```
# code
# ```
# -> code
| gsub("\\n```\\n"; "\n"; "m")
# #, ##, ###, ... -> #
| gsub("(?<line>\\n)?#+(?<title>.*)\\n"; "\(.line // "")#\(.title)\n"; "m")
# [title](url) -> title (url)
| gsub("\\[(?<title>.*)\\]\\((?<url>.*)\\)"; "\(.title) (\(.url))")
# `code` -> code
| gsub("`(?<code>.*)`"; .code)
);
def expr_to_path: _expr_to_path;
def path_to_expr: _path_to_expr;

36
pkg/interp/help.jq
View File

@ -64,14 +64,6 @@ def _help_format_enrich($arg0; $f; $include_basic):
, {comment: "Decode value as \($f.name)", expr: "\($f.name)"}
]
end
| (($f.functions // []) | map(select(startswith("_") | not))) as $public_functions
| if ($public_functions | length) > 0 then
.examples +=
[ $public_functions[]
| {comment: "Supports `\(.)`", shell: "fq -d \($f.name) torepr file"}
, {comment: "Supports `\(.)`", expr: "\($f.name) | torepr"}
]
end
| if $f.decode_in_arg then
.examples +=
[ { comment: "Decode file using \($f.name) options"
@ -146,13 +138,14 @@ def _help($arg0; $topic):
elif _registry.formats | has($topic) then
( _registry.formats[$topic] as $f
| (_format_func($f.name; "_help")? // {} | _help_format_enrich($arg0; $f; true)) as $fhelp
| ((_registry.files[][] | select(.name=="\($topic).md").data) // false) as $doc
| "\($f.name): \($f.description) decoder"
, ($fhelp.notes | if . then _markdown_to_text else empty end)
, if $f.decode_in_arg then
( $f.decode_in_arg
| to_entries
| map([" \(.key)=\(.value) ", $f.decode_in_arg_doc[.key]])
| "Options:"
| "# Options"
, ""
, table(
.;
map(
@ -167,28 +160,19 @@ def _help($arg0; $topic):
)
else empty
end
, "Examples:"
, ""
, "# Decode examples"
, ""
, ( $fhelp.examples[]
| " # \(.comment | _markdown_to_text)"
| " # \(.comment)"
, if .shell then " $ \(.shell)"
elif .expr then " ... | \(.expr)"
else empty
end
)
, if isempty($f.functions | select(. == "torepr")) | not then
( "Supports torepr:"
, " ... | \($f.name) | torepr"
)
else empty
end
, if $fhelp.links then
( "References and links"
, ( $fhelp.links[]
| if .title then " \(.title) \(.url)"
else " \(.url)"
end
)
)
, ""
# TODO: [:-1] hack to remove extra newline as we use println later
, if $doc then $doc | markdown | _markdown_to_text(options.width; -3)[:-1]
else empty
end
)

3
pkg/interp/testdata/args.fqtest vendored
View File

@ -162,10 +162,11 @@ json JavaScript Object Notation
jsonl JavaScript Object Notation Lines
macho Mach-O macOS executable
macho_fat Fat Mach-O macOS executable (multi-architecture)
markdown Markdown
matroska Matroska file
mp3 MP3 file
mp3_frame MPEG audio layer 3 frame
mp4 ISOBMFF MPEG-4 part 12 and similar
mp4 ISOBMFF, QuickTime and similar
mpeg_asc MPEG-4 Audio Specific Config
mpeg_es MPEG Elementary Stream
mpeg_pes MPEG Packetized elementary stream