Merge branch 'wader_master' into postgres

doc/formats.md

@@ -163,7 +163,7 @@ Supports decoding BER, CER and DER (X.690).
 - Does not support specifying a schema.
 - Supports `torepr` but without schema all sequences and sets will be arrays.
 
-### `frompem` can be used to decode certificates etc
+### Can be used to decode certificates etc
 
 ```sh
 $ fq -d raw 'frompem | asn1_ber | d' cert.pem
@@ -221,6 +221,11 @@ Limitations:
 ### References
 - https://avro.apache.org/docs/current/spec.html#Object+Container+Files
 
+### Authors
+- Xentripetal
+xentripetal@fastmail.com
+[@xentripetal](https://github.com/xentripetal)
+
 ## bencode
 
 ### Convert represented value to JSON
@@ -429,14 +434,10 @@ $ '<a href="url">text</a>' | fq -d html -o array=true
     ]
   ]
 ]
-```
 
-```sh
-# Decode html files to a {file: "title", ...} object
+# decode html files to a {file: "title", ...} object
 $ fq -n -d html '[inputs | {key: input_filename, value: .html.head.title?}] | from_entries' *.html
-```
 
-```sh
 # <a> href:s in file
 $ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href' file.html
 ```
@@ -454,15 +455,20 @@ $ fq '.load_commands[] | select(.cmd=="segment_64")' file
 ### References
 - https://github.com/aidansteele/osx-abi-macho-file-format-reference
 
+### Authors
+- Sıddık AÇIL
+acils@itu.edu.tr
+[@Akaame](https://github.com/Akaame)
+
 ## matroska
 
-### Lookup element path using `matroska_path`
+### Lookup element using path
 
 ```sh
 $ fq 'matroska_path(".Segment.Tracks[0)")' file.mkv
 ```
 
-### Get element path using `matroska_path`
+### Get path to element
 
 ```sh
 $ fq 'grep_by(.id == "Tracks") | matroska_path' file.mkv
@@ -517,14 +523,14 @@ Decode value as mp4
 ... | mp4({allow_truncated:false,decode_samples:true})
 ```
 
-### `mp4_path($path)` - Lookup mp4 box using a mp4 box path.
+### Lookup mp4 box using a mp4 box path.
 
 ```sh
 # <decode value box> | mp4_path($path) -> <decode value box>
 $ fq 'mp4_path(".moov.trak[1]")' file.mp4
 ```
 
-### `mp4_path` - Return mp4 box path for a decode value box.
+### Get mp4 box path for a decode value box.
 
 ```sh
 # <decode value box> | mp4_path -> string
@@ -537,9 +543,10 @@ $ fq 'grep_by(.type == "trak") | mp4_path' file.mp4
 $ fq -n '"AAAAHGVsc3QAAAAAAAAAAQAAADIAAAQAAAEAAA==" | frombase64 | mp4({force:true}) | d'
 ```
 
-### Don't decode samples and manually decode first sample for first track as a `aac_frame`
+### Speed up decoding by not decoding samples
 
 ```sh
+# manually decode first sample as a aac_frame
 $ fq -o decode_samples=false '.tracks[0].samples[0] | aac_frame | d' file.mp4
 ```
 
@@ -725,9 +732,7 @@ $ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -d xml -o seq=true
 # access text of the <c> element
 $ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq '.a.c["#text"]'
 "ccc"
-```
 
-```sh
 # decode to object and encode to xml
 $ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -r -d xml -o seq=true 'toxml({indent:2})'
 <a>
@@ -770,11 +775,9 @@ Elements are arrays of the shape `["#text": "body text", "attr_name", {key: "att
     ]
   ]
 ]
-```
 
-```sh
 # decode to array and encode to xml
-✗ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -r -d xml -o array=true -o seq=true 'toxml({indent:2})'
+$ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -r -d xml -o array=true -o seq=true 'toxml({indent:2})'
 <a>
   <b></b>
   <b>bbb</b>

format/all/all.fqtest

@@ -32,3 +32,111 @@ $ fq -n _registry.groups.probe
   "xml",
   "yaml"
 ]
+$ fq --help formats
+aac_frame            Advanced Audio Coding frame
+adts                 Audio Data Transport Stream
+adts_frame           Audio Data Transport Stream frame
+amf0                 Action Message Format 0
+apev2                APEv2 metadata tag
+ar                   Unix archive
+asn1_ber             ASN1 BER (basic encoding rules, also CER and DER)
+av1_ccr              AV1 Codec Configuration Record
+av1_frame            AV1 frame
+av1_obu              AV1 Open Bitstream Unit
+avc_annexb           H.264/AVC Annex B
+avc_au               H.264/AVC Access Unit
+avc_dcr              H.264/AVC Decoder Configuration Record
+avc_nalu             H.264/AVC Network Access Layer Unit
+avc_pps              H.264/AVC Picture Parameter Set
+avc_sei              H.264/AVC Supplemental Enhancement Information
+avc_sps              H.264/AVC Sequence Parameter Set
+avro_ocf             Avro object container file
+bencode              BitTorrent bencoding
+bitcoin_blkdat       Bitcoin blk.dat
+bitcoin_block        Bitcoin block
+bitcoin_script       Bitcoin script
+bitcoin_transaction  Bitcoin transaction
+bsd_loopback_frame   BSD loopback frame
+bson                 Binary JSON
+bzip2                bzip2 compression
+cbor                 Concise Binary Object Representation
+csv                  Comma separated values
+dns                  DNS packet
+dns_tcp              DNS packet (TCP)
+elf                  Executable and Linkable Format
+ether8023_frame      Ethernet 802.3 frame
+exif                 Exchangeable Image File Format
+fairplay_spc         FairPlay Server Playback Context
+flac                 Free Lossless Audio Codec file
+flac_frame           FLAC frame
+flac_metadatablock   FLAC metadatablock
+flac_metadatablocks  FLAC metadatablocks
+flac_picture         FLAC metadatablock picture
+flac_streaminfo      FLAC streaminfo
+gif                  Graphics Interchange Format
+gzip                 gzip compression
+hevc_annexb          H.265/HEVC Annex B
+hevc_au              H.265/HEVC Access Unit
+hevc_dcr             H.265/HEVC Decoder Configuration Record
+hevc_nalu            H.265/HEVC Network Access Layer Unit
+hevc_pps             H.265/HEVC Picture Parameter Set
+hevc_sps             H.265/HEVC Sequence Parameter Set
+hevc_vps             H.265/HEVC Video Parameter Set
+html                 HyperText Markup Language
+icc_profile          International Color Consortium profile
+icmp                 Internet Control Message Protocol
+icmpv6               Internet Control Message Protocol v6
+id3v1                ID3v1 metadata
+id3v11               ID3v1.1 metadata
+id3v2                ID3v2 metadata
+ipv4_packet          Internet protocol v4 packet
+ipv6_packet          Internet protocol v6 packet
+jpeg                 Joint Photographic Experts Group file
+json                 JavaScript Object Notation
+jsonl                JavaScript Object Notation Lines
+macho                Mach-O macOS executable
+macho_fat            Fat Mach-O macOS executable (multi-architecture)
+markdown             Markdown
+matroska             Matroska file
+mp3                  MP3 file
+mp3_frame            MPEG audio layer 3 frame
+mp4                  ISOBMFF, QuickTime and similar
+mpeg_asc             MPEG-4 Audio Specific Config
+mpeg_es              MPEG Elementary Stream
+mpeg_pes             MPEG Packetized elementary stream
+mpeg_pes_packet      MPEG Packetized elementary stream packet
+mpeg_spu             Sub Picture Unit (DVD subtitle)
+mpeg_ts              MPEG Transport Stream
+msgpack              MessagePack
+ogg                  OGG file
+ogg_page             OGG page
+opus_packet          Opus packet
+pcap                 PCAP packet capture
+pcapng               PCAPNG packet capture
+png                  Portable Network Graphics file
+prores_frame         Apple ProRes frame
+protobuf             Protobuf
+protobuf_widevine    Widevine protobuf
+pssh_playready       PlayReady PSSH
+raw                  Raw bits
+rtmp                 Real-Time Messaging Protocol
+sll2_packet          Linux cooked capture encapsulation v2
+sll_packet           Linux cooked capture encapsulation
+tar                  Tar archive
+tcp_segment          Transmission control protocol segment
+tiff                 Tag Image File Format
+toml                 Tom's Obvious, Minimal Language
+udp_datagram         User datagram protocol
+vorbis_comment       Vorbis comment
+vorbis_packet        Vorbis packet
+vp8_frame            VP8 frame
+vp9_cfm              VP9 Codec Feature Metadata
+vp9_frame            VP9 frame
+vpx_ccr              VPX Codec Configuration Record
+wasm                 WebAssembly Binary Format
+wav                  WAV file
+webp                 WebP image
+xing                 Xing header
+xml                  Extensible Markup Language
+yaml                 YAML Ain't Markup Language
+zip                  ZIP archive

format/asn1/asn1_ber.md

@@ -4,7 +4,7 @@ Supports decoding BER, CER and DER (X.690).
 - Does not support specifying a schema.
 - Supports `torepr` but without schema all sequences and sets will be arrays.
 
-### `frompem` can be used to decode certificates etc
+### Can be used to decode certificates etc
 
 ```sh
 $ fq -d raw 'frompem | asn1_ber | d' cert.pem

format/asn1/testdata/help_asn1_ber.fqtest

@@ -0,0 +1,39 @@
+$ fq -h asn1_ber
+asn1_ber: ASN1 BER (basic encoding rules, also CER and DER) decoder
+
+Decode examples
+===============
+
+  # Decode file as asn1_ber
+  $ fq -d asn1_ber . file
+  # Decode value as asn1_ber
+  ... | asn1_ber
+
+Supports decoding BER, CER and DER (X.690).
+
+- Currently no extra validation is done for CER and DER.
+- Does not support specifying a schema.
+- Supports torepr but without schema all sequences and sets will be arrays.
+
+Can be used to decode certificates etc
+======================================
+
+  $ fq -d raw 'frompem | asn1_ber | d' cert.pem
+  
+Can decode nested values
+========================
+
+  $ fq -d asn1_ber '.constructed[1].value | asn1_ber' file.ber
+  
+Manual schema
+=============
+
+  $ fq -d asn1_ber 'torepr as $r | ["version", "modulus", "private_exponent", "private_exponen", "prime1", "prime2", "exponent1", "exponent2", "coefficient"] | with_entries({key: .value, value: $r[.key]})' pkcs1.der
+  
+References
+==========
+- https://www.itu.int/ITU-T/studygroups/com10/languages/X.690_1297.pdf
+- https://en.wikipedia.org/wiki/X.690
+- https://letsencrypt.org/docs/a-warm-welcome-to-asn1-and-der/
+- https://lapo.it/asn1js/
+

format/avro/avro_ocf.md

@@ -9,3 +9,8 @@ Limitations:
 
 ### References
 - https://avro.apache.org/docs/current/spec.html#Object+Container+Files
+
+### Authors
+- Xentripetal
+xentripetal@fastmail.com
+[@xentripetal](https://github.com/xentripetal)

format/avro/testdata/help_avro_ocf.fqtest

@@ -0,0 +1,28 @@
+$ fq -h avro_ocf
+avro_ocf: Avro object container file decoder
+
+Decode examples
+===============
+
+  # Decode file as avro_ocf
+  $ fq -d avro_ocf . file
+  # Decode value as avro_ocf
+  ... | avro_ocf
+
+Supports reading Avro Object Container Format (OCF) files based on the 1.11.0 specification.
+
+Capable of handling null, deflate, and snappy codecs for data compression.
+
+Limitations:
+
+- Schema does not support self-referential types, only built-in types.
+- Decimal logical types are not supported for decoding, will just be treated as their primitive type
+
+References
+==========
+- https://avro.apache.org/docs/current/spec.html#Object+Container+Files
+
+Authors
+=======
+- Xentripetal xentripetal@fastmail.com @xentripetal (https://github.com/xentripetal)
+

format/bencode/testdata/help_bencode.fqtest

@@ -0,0 +1,20 @@
+$ fq -h bencode
+bencode: BitTorrent bencoding decoder
+
+Decode examples
+===============
+
+  # Decode file as bencode
+  $ fq -d bencode . file
+  # Decode value as bencode
+  ... | bencode
+
+Convert represented value to JSON
+=================================
+
+  $ fq -d bencode torepr file.torrent
+  
+References
+==========
+- https://wiki.theory.org/BitTorrentSpecification#Bencoding
+

format/bitcoin/testdata/help_bitcoin_block.fqtest

@@ -0,0 +1,20 @@
+$ fq -h bitcoin_block
+bitcoin_block: Bitcoin block decoder
+
+Options
+=======
+
+  has_header=false  Has blkdat header
+
+Decode examples
+===============
+
+  # Decode file as bitcoin_block
+  $ fq -d bitcoin_block . file
+  # Decode value as bitcoin_block
+  ... | bitcoin_block
+  # Decode file using bitcoin_block options
+  $ fq -d bitcoin_block -o has_header=false . file
+  # Decode value as bitcoin_block
+  ... | bitcoin_block({has_header:false})
+

format/bson/testdata/help_bson.fqtest

@@ -0,0 +1,25 @@
+$ fq -h bson
+bson: Binary JSON decoder
+
+Decode examples
+===============
+
+  # Decode file as bson
+  $ fq -d bson . file
+  # Decode value as bson
+  ... | bson
+
+Convert represented value to JSON
+=================================
+
+  $ fq -d bson torepr file.bson
+  
+Filter represented value
+========================
+
+  $ fq -d bson 'torepr | select(.name=="bob")' file.bson
+  
+References
+==========
+- https://bsonspec.org/spec.html
+

format/cbor/testdata/help_cbor.fqtest

@@ -0,0 +1,21 @@
+$ fq -h cbor
+cbor: Concise Binary Object Representation decoder
+
+Decode examples
+===============
+
+  # Decode file as cbor
+  $ fq -d cbor . file
+  # Decode value as cbor
+  ... | cbor
+
+Convert represented value to JSON
+=================================
+
+  $ fq -d cbor torepr file.cbor
+  
+References
+==========
+- https://en.wikipedia.org/wiki/CBOR
+- https://www.rfc-editor.org/rfc/rfc8949.html
+

format/csv/testdata/help_csv.fqtest

@@ -0,0 +1,31 @@
+$ fq -h csv
+csv: Comma separated values decoder
+
+Options
+=======
+
+  comma=,    Separator character
+  comment=#  Comment line character
+
+Decode examples
+===============
+
+  # Decode file as csv
+  $ fq -d csv . file
+  # Decode value as csv
+  ... | csv
+  # Decode file using csv options
+  $ fq -d csv -o comma="," -o comment="#" . file
+  # Decode value as csv
+  ... | csv({comma:",",comment:"#"})
+
+TSV to CSV
+==========
+
+  $ fq -d csv -o comma="\t" tocsv file.tsv
+  
+Convert rows to objects based on header row
+===========================================
+
+  $ fq -d csv '.[0] as $t | .[1:] | map(with_entries(.key = $t[.key]))' file.csv
+  

format/flac/testdata/help_flac_frame.fqtest

@@ -0,0 +1,20 @@
+$ fq -h flac_frame
+flac_frame: FLAC frame decoder
+
+Options
+=======
+
+  bits_per_sample=16  Bits per sample
+
+Decode examples
+===============
+
+  # Decode file as flac_frame
+  $ fq -d flac_frame . file
+  # Decode value as flac_frame
+  ... | flac_frame
+  # Decode file using flac_frame options
+  $ fq -d flac_frame -o bits_per_sample=16 . file
+  # Decode value as flac_frame
+  ... | flac_frame({bits_per_sample:16})
+

format/macho/macho.md

@@ -8,3 +8,8 @@ $ fq '.load_commands[] | select(.cmd=="segment_64")' file
 
 ### References
 - https://github.com/aidansteele/osx-abi-macho-file-format-reference
+
+### Authors
+- Sıddık AÇIL
+acils@itu.edu.tr
+[@Akaame](https://github.com/Akaame)

format/macho/testdata/help_macho.fqtest

@@ -0,0 +1,26 @@
+$ fq -h macho
+macho: Mach-O macOS executable decoder
+
+Decode examples
+===============
+
+  # Decode file as macho
+  $ fq -d macho . file
+  # Decode value as macho
+  ... | macho
+
+Supports decoding vanilla and FAT Mach-O binaries.
+
+Select 64bit load segments
+==========================
+
+  $ fq '.load_commands[] | select(.cmd=="segment_64")' file
+  
+References
+==========
+- https://github.com/aidansteele/osx-abi-macho-file-format-reference
+
+Authors
+=======
+- Sıddık AÇIL acils@itu.edu.tr @Akaame (https://github.com/Akaame)
+

format/markdown/markdown.jq

@@ -21,7 +21,13 @@ def _markdown_to_text($width; $header_depth):
   def _f($pln):
     if type == "string" then gsub("\n"; " ")
     elif .type == "document" then .children[] | _f("\n\n")
-    elif .type == "heading" then "#" * (.level+$header_depth), " ", (.children[] | _f("\n\n")), "\n"
+    elif .type == "heading" then
+      ( (.children[] | _f("\n\n")) as $title
+      | $title
+      , "\n"
+      , ("=" * ($title | length))
+      , "\n"
+      )
     elif .type == "paragraph" then
       ( ( [.children[] | _f("\n\n")]
         | join("")

format/matroska/matroska.md

@@ -1,10 +1,10 @@
-### Lookup element path using `matroska_path`
+### Lookup element using path
 
 ```sh
 $ fq 'matroska_path(".Segment.Tracks[0)")' file.mkv
 ```
 
-### Get element path using `matroska_path`
+### Get path to element
 
 ```sh
 $ fq 'grep_by(.id == "Tracks") | matroska_path' file.mkv

format/matroska/testdata/help_matroska.fqtest

@@ -0,0 +1,29 @@
+$ fq -h matroska
+matroska: Matroska file decoder
+
+Decode examples
+===============
+
+  # Decode file as matroska
+  $ fq -d matroska . file
+  # Decode value as matroska
+  ... | matroska
+
+Lookup element using path
+=========================
+
+  $ fq 'matroska_path(".Segment.Tracks[0)")' file.mkv
+  
+Get path to element
+===================
+
+  $ fq 'grep_by(.id == "Tracks") | matroska_path' file.mkv
+  
+References
+==========
+- https://tools.ietf.org/html/draft-ietf-cellar-ebml-00
+- https://matroska.org/technical/specs/index.html
+- https://www.matroska.org/technical/basics.html
+- https://www.matroska.org/technical/codec_specs.html
+- https://wiki.xiph.org/MatroskaOpus
+

format/mp3/testdata/help_mp3.fqtest

@@ -0,0 +1,21 @@
+$ fq -h mp3
+mp3: MP3 file decoder
+
+Options
+=======
+
+  max_sync_seek=32768          Max byte distance to next sync
+  max_unique_header_configs=5  Max number of unique frame header configs allowed
+
+Decode examples
+===============
+
+  # Decode file as mp3
+  $ fq -d mp3 . file
+  # Decode value as mp3
+  ... | mp3
+  # Decode file using mp3 options
+  $ fq -d mp3 -o max_sync_seek=32768 -o max_unique_header_configs=5 . file
+  # Decode value as mp3
+  ... | mp3({max_sync_seek:32768,max_unique_header_configs:5})
+

format/mp4/mp4.md

@@ -1,11 +1,11 @@
-### `mp4_path($path)` - Lookup mp4 box using a mp4 box path.
+### Lookup mp4 box using a mp4 box path.
 
 ```sh
 # <decode value box> | mp4_path($path) -> <decode value box>
 $ fq 'mp4_path(".moov.trak[1]")' file.mp4
 ```
 
-### `mp4_path` - Return mp4 box path for a decode value box.
+### Get mp4 box path for a decode value box.
 
 ```sh
 # <decode value box> | mp4_path -> string
@@ -18,9 +18,10 @@ $ fq 'grep_by(.type == "trak") | mp4_path' file.mp4
 $ fq -n '"AAAAHGVsc3QAAAAAAAAAAQAAADIAAAQAAAEAAA==" | frombase64 | mp4({force:true}) | d'
 ```
 
-### Don't decode samples and manually decode first sample for first track as a `aac_frame`
+### Speed up decoding by not decoding samples
 
 ```sh
+# manually decode first sample as a aac_frame
 $ fq -o decode_samples=false '.tracks[0].samples[0] | aac_frame | d' file.mp4
 ```
 

format/mp4/testdata/help_mp4.fqtest

@@ -0,0 +1,54 @@
+$ fq -h mp4
+mp4: ISOBMFF, QuickTime and similar decoder
+
+Options
+=======
+
+  allow_truncated=false  Allow box to be truncated
+  decode_samples=true    Decode supported media samples
+
+Decode examples
+===============
+
+  # Decode file as mp4
+  $ fq -d mp4 . file
+  # Decode value as mp4
+  ... | mp4
+  # Decode file using mp4 options
+  $ fq -d mp4 -o allow_truncated=false -o decode_samples=true . file
+  # Decode value as mp4
+  ... | mp4({allow_truncated:false,decode_samples:true})
+
+Lookup mp4 box using a mp4 box path.
+====================================
+
+  # <decode value box> | mp4_path($path) -> <decode value box>
+  $ fq 'mp4_path(".moov.trak[1]")' file.mp4
+  
+Get mp4 box path for a decode value box.
+========================================
+
+  # <decode value box> | mp4_path -> string
+  $ fq 'grep_by(.type == "trak") | mp4_path' file.mp4
+  
+Force decode a single box
+=========================
+
+  $ fq -n '"AAAAHGVsc3QAAAAAAAAAAQAAADIAAAQAAAEAAA==" | frombase64 | mp4({force:true}) | d'
+  
+Speed up decoding by not decoding samples
+=========================================
+
+  # manually decode first sample as a aac_frame
+  $ fq -o decode_samples=false '.tracks[0].samples[0] | aac_frame | d' file.mp4
+  
+Entries for first edit list as values
+=====================================
+
+  $ fq 'first(grep_by(.type=="elst").entries) | tovalue' file.mp4
+  
+References
+==========
+- ISO/IEC base media file format (MPEG-4 Part 12) (https://en.wikipedia.org/wiki/ISO/IEC_base_media_file_format)
+- Quicktime file format (https://developer.apple.com/standards/qtff-2001.pdf)
+

format/mpeg/testdata/help_aac_frame.fqtest

@@ -0,0 +1,20 @@
+$ fq -h aac_frame
+aac_frame: Advanced Audio Coding frame decoder
+
+Options
+=======
+
+  object_type=1  Audio object type
+
+Decode examples
+===============
+
+  # Decode file as aac_frame
+  $ fq -d aac_frame . file
+  # Decode value as aac_frame
+  ... | aac_frame
+  # Decode file using aac_frame options
+  $ fq -d aac_frame -o object_type=1 . file
+  # Decode value as aac_frame
+  ... | aac_frame({object_type:1})
+

format/mpeg/testdata/help_avc_au.fqtest

@@ -0,0 +1,20 @@
+$ fq -h avc_au
+avc_au: H.264/AVC Access Unit decoder
+
+Options
+=======
+
+  length_size=4  Length value size
+
+Decode examples
+===============
+
+  # Decode file as avc_au
+  $ fq -d avc_au . file
+  # Decode value as avc_au
+  ... | avc_au
+  # Decode file using avc_au options
+  $ fq -d avc_au -o length_size=4 . file
+  # Decode value as avc_au
+  ... | avc_au({length_size:4})
+

format/mpeg/testdata/help_hevc_au.fqtest

@@ -0,0 +1,20 @@
+$ fq -h hevc_au
+hevc_au: H.265/HEVC Access Unit decoder
+
+Options
+=======
+
+  length_size=4  Length value size
+
+Decode examples
+===============
+
+  # Decode file as hevc_au
+  $ fq -d hevc_au . file
+  # Decode value as hevc_au
+  ... | hevc_au
+  # Decode file using hevc_au options
+  $ fq -d hevc_au -o length_size=4 . file
+  # Decode value as hevc_au
+  ... | hevc_au({length_size:4})
+

format/msgpack/testdata/help_msgpack.fqtest

@@ -0,0 +1,20 @@
+$ fq -h msgpack
+msgpack: MessagePack decoder
+
+Decode examples
+===============
+
+  # Decode file as msgpack
+  $ fq -d msgpack . file
+  # Decode value as msgpack
+  ... | msgpack
+
+Convert represented value to JSON
+=================================
+
+  $ fq -d msgpack torepr file.msgpack
+  
+References
+==========
+- https://github.com/msgpack/msgpack/blob/master/spec.md
+

format/protobuf/testdata/help_protobuf.fqtest

@@ -0,0 +1,20 @@
+$ fq -h protobuf
+protobuf: Protobuf decoder
+
+Decode examples
+===============
+
+  # Decode file as protobuf
+  $ fq -d protobuf . file
+  # Decode value as protobuf
+  ... | protobuf
+
+Can decode sub messages
+=======================
+
+  $ fq -d protobuf '.fields[6].wire_value | protobuf | d' file
+  
+References
+==========
+- https://developers.google.com/protocol-buffers/docs/encoding
+

format/rtmp/testdata/help_rtmp.fqtest

@@ -0,0 +1,18 @@
+$ fq -h rtmp
+rtmp: Real-Time Messaging Protocol decoder
+
+Decode examples
+===============
+
+  # Decode file as rtmp
+  $ fq -d rtmp . file
+  # Decode value as rtmp
+  ... | rtmp
+
+Current only supports plain RTMP (not RTMPT or encrypted variants etc) with AMF0 (not AMF3).
+
+References
+==========
+- https://rtmp.veriskope.com/docs/spec/
+- https://rtmp.veriskope.com/pdf/video_file_format_spec_v10.pdf
+

format/wasm/testdata/help_wasm.fqtest

@@ -0,0 +1,29 @@
+$ fq -h wasm
+wasm: WebAssembly Binary Format decoder
+
+Decode examples
+===============
+
+  # Decode file as wasm
+  $ fq -d wasm . file
+  # Decode value as wasm
+  ... | wasm
+
+Count opcode usage
+==================
+
+  $ fq '.sections[] | select(.id == "code_section") | [.. | .opcode? // empty] | count | map({key: .[0], value: .[1]}) | from_entries' file.wasm
+  
+List exports and imports
+========================
+
+  $ fq '.sections | {import: map(select(.id == "import_section").content.im.x[].nm.b), export: map(select(.id == "export_section").content.ex.x[].nm.b)}' file.wasm
+  
+Authors
+=======
+- Takashi Oguma @bitbears-dev (https://github.com/bitbears-dev) @0xb17bea125 (https://twitter.com/0xb17bea125)
+
+References
+==========
+- https://webassembly.github.io/spec/core/
+

format/xml/html.md

@@ -51,14 +51,10 @@ $ '<a href="url">text</a>' | fq -d html -o array=true
     ]
   ]
 ]
-```
 
-```sh
-# Decode html files to a {file: "title", ...} object
+# decode html files to a {file: "title", ...} object
 $ fq -n -d html '[inputs | {key: input_filename, value: .html.head.title?}] | from_entries' *.html
-```
 
-```sh
 # <a> href:s in file
 $ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href' file.html
 ```

format/xml/testdata/help_html.fqtest

@@ -0,0 +1,81 @@
+$ fq -h html
+html: HyperText Markup Language decoder
+
+Options
+=======
+
+  array=false         Decode as nested arrays
+  attribute_prefix=@  Prefix for attribute keys
+  seq=false           Use seq attribute to preserve element order
+
+Decode examples
+===============
+
+  # Decode file as html
+  $ fq -d html . file
+  # Decode value as html
+  ... | html
+  # Decode file using html options
+  $ fq -d html -o array=false -o attribute_prefix="@" -o seq=false . file
+  # Decode value as html
+  ... | html({array:false,attribute_prefix:"@",seq:false})
+
+HTML is decoded in HTML5 mode and will always include <html>, <body> and <head> element.
+
+See xml format for more examples and how to preserve element order and how to encode to xml.
+
+There is no tohtml function, see toxml instead.
+
+Element as object
+=================
+
+  # decode as object is the default
+  $ echo '<a href="url">text</a>' | fq -d html
+  {
+    "html": {
+      "body": {
+        "a": {
+          "#text": "text",
+          "@href": "url"
+        }
+      },
+      "head": ""
+    }
+  }
+  
+Element as array
+================
+
+  $ '<a href="url">text</a>' | fq -d html -o array=true
+  [
+    "html",
+    null,
+    [
+      [
+        "head",
+        null,
+        []
+      ],
+      [
+        "body",
+        null,
+        [
+          [
+            "a",
+            {
+              "#text": "text",
+              "href": "url"
+            },
+            []
+          ]
+        ]
+      ]
+    ]
+  ]
+  
+  # decode html files to a {file: "title", ...} object
+  $ fq -n -d html '[inputs | {key: input_filename, value: .html.head.title?}] | from_entries' *.html
+  
+  # <a> href:s in file
+  $ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href' file.html
+  

format/xml/testdata/help_xml.fqtest

@@ -0,0 +1,124 @@
+$ fq -h xml
+xml: Extensible Markup Language decoder
+
+Options
+=======
+
+  array=false         Decode as nested arrays
+  attribute_prefix=@  Prefix for attribute keys
+  seq=false           Use seq attribute to preserve element order
+
+Decode examples
+===============
+
+  # Decode file as xml
+  $ fq -d xml . file
+  # Decode value as xml
+  ... | xml
+  # Decode file using xml options
+  $ fq -d xml -o array=false -o attribute_prefix="@" -o seq=false . file
+  # Decode value as xml
+  ... | xml({array:false,attribute_prefix:"@",seq:false})
+
+XML can be decoded and encoded into jq values in two ways, elements as object or array. Which variant to use depends a bit what you want to do. The object variant
+might be easier to query for a specific value but array might be easier to use to generate xml or to query after all elements of some kind etc.
+
+Encoding is done using the toxml function and it will figure what variant that is used based on the input value. Is has two optional options indent and
+attribute_prefix.
+
+Elements as object
+==================
+Element can have different shapes depending on body text, attributes and children:
+
+- <a key="value">text</a> is {"a":{"#text":"text","@key":"value"}}, has text (#text) and attributes (@key)
+- <a>text</a> is {"a":"text"}
+- <a><b>text</b></a> is {"a":{"b":"text"}} one child with only text and no attributes
+- <a><b/><b>text</b></a> is {"a":{"b":["","text"]}} two children with same name end up in an array
+- <a><b/><b key="value">text</b></a> is {"a":{"b":["",{"#text":"text","@key":"value"}]}}
+
+If there is #seq attribute it encodes the child element order. Use -o seq=true to include sequence number when decoding, otherwise order might be lost.
+
+
+  # decode as object is the default
+  $ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -d xml -o seq=true
+  {
+    "a": {
+      "b": [
+        {
+          "#seq": 0
+        },
+        {
+          "#seq": 1,
+          "#text": "bbb"
+        }
+      ],
+      "c": {
+        "#seq": 2,
+        "#text": "ccc",
+        "@attr": "value"
+      }
+    }
+  }
+  
+  # access text of the <c> element
+  $ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq '.a.c["#text"]'
+  "ccc"
+  
+  # decode to object and encode to xml
+  $ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -r -d xml -o seq=true 'toxml({indent:2})'
+  <a>
+    <b></b>
+    <b>bbb</b>
+    <c attr="value">ccc</c>
+  </a>
+  
+Elements as array
+=================
+Elements are arrays of the shape ["#text": "body text", "attr_name", {key: "attr value"}|null, [<child element>, ...]].
+
+
+  # decode as array
+  ✗ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -d xml -o array=true
+  [
+    "a",
+    null,
+    [
+      [
+        "b",
+        null,
+        []
+      ],
+      [
+        "b",
+        {
+          "#text": "bbb"
+        },
+        []
+      ],
+      [
+        "c",
+        {
+          "#text": "ccc",
+          "attr": "value"
+        },
+        []
+      ]
+    ]
+  ]
+  
+  # decode to array and encode to xml
+  $ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -r -d xml -o array=true -o seq=true 'toxml({indent:2})'
+  <a>
+    <b></b>
+    <b>bbb</b>
+    <c attr="value">ccc</c>
+  </a>
+  
+  # access text of the <c> element, the object variant above is probably easier to use
+  $ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -o array=true '.[2][2][1]["#text"]'
+  "ccc"
+  
+References
+==========
+- xml.com's Converting Between XML and JSON (https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html)
+

format/xml/xml.md

@@ -44,9 +44,7 @@ $ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -d xml -o seq=true
 # access text of the <c> element
 $ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq '.a.c["#text"]'
 "ccc"
-```
 
-```sh
 # decode to object and encode to xml
 $ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -r -d xml -o seq=true 'toxml({indent:2})'
 <a>
@@ -89,11 +87,9 @@ Elements are arrays of the shape `["#text": "body text", "attr_name", {key: "att
     ]
   ]
 ]
-```
 
-```sh
 # decode to array and encode to xml
-✗ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -r -d xml -o array=true -o seq=true 'toxml({indent:2})'
+$ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -r -d xml -o array=true -o seq=true 'toxml({indent:2})'
 <a>
   <b></b>
   <b>bbb</b>

format/zip/testdata/help_zip.fqtest

@@ -0,0 +1,27 @@
+$ fq -h zip
+zip: ZIP archive decoder
+
+Options
+=======
+
+  uncompress=true  Uncompress and probe files
+
+Decode examples
+===============
+
+  # Decode file as zip
+  $ fq -d zip . file
+  # Decode value as zip
+  ... | zip
+  # Decode file using zip options
+  $ fq -d zip -o uncompress=true . file
+  # Decode value as zip
+  ... | zip({uncompress:true})
+
+Supports ZIP64.
+
+References
+==========
+- https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
+- https://opensource.apple.com/source/zip/zip-6/unzip/unzip/proginfo/extra.fld
+

pkg/interp/help.jq

@@ -140,11 +140,13 @@ def _help($arg0; $topic):
       | (_format_func($f.name; "_help")? // {} | _help_format_enrich($arg0; $f; true)) as $fhelp
       | ((_registry.files[][] | select(.name=="\($topic).md").data) // false) as $doc
       | "\($f.name): \($f.description) decoder"
+      , ""
       , if $f.decode_in_arg then
           ( $f.decode_in_arg
           | to_entries
           | map(["  \(.key)=\(.value)  ", $f.decode_in_arg_doc[.key]])
-          | "# Options"
+          | "Options"
+          , "======="
           , ""
           , table(
               .;
@@ -157,11 +159,12 @@ def _help($arg0; $topic):
                 )
               ) | join("")
             )
+          , ""
           )
         else empty
         end
-      , ""
-      , "# Decode examples"
+      , "Decode examples"
+      , "==============="
       , ""
       , ( $fhelp.examples[]
         | "  # \(.comment)"

pkg/interp/testdata/args.fqtest

@@ -98,114 +98,6 @@ string_input        false
 unicode             false
 verbose             false
 width               135
-$ fq --help formats
-aac_frame            Advanced Audio Coding frame
-adts                 Audio Data Transport Stream
-adts_frame           Audio Data Transport Stream frame
-amf0                 Action Message Format 0
-apev2                APEv2 metadata tag
-ar                   Unix archive
-asn1_ber             ASN1 BER (basic encoding rules, also CER and DER)
-av1_ccr              AV1 Codec Configuration Record
-av1_frame            AV1 frame
-av1_obu              AV1 Open Bitstream Unit
-avc_annexb           H.264/AVC Annex B
-avc_au               H.264/AVC Access Unit
-avc_dcr              H.264/AVC Decoder Configuration Record
-avc_nalu             H.264/AVC Network Access Layer Unit
-avc_pps              H.264/AVC Picture Parameter Set
-avc_sei              H.264/AVC Supplemental Enhancement Information
-avc_sps              H.264/AVC Sequence Parameter Set
-avro_ocf             Avro object container file
-bencode              BitTorrent bencoding
-bitcoin_blkdat       Bitcoin blk.dat
-bitcoin_block        Bitcoin block
-bitcoin_script       Bitcoin script
-bitcoin_transaction  Bitcoin transaction
-bsd_loopback_frame   BSD loopback frame
-bson                 Binary JSON
-bzip2                bzip2 compression
-cbor                 Concise Binary Object Representation
-csv                  Comma separated values
-dns                  DNS packet
-dns_tcp              DNS packet (TCP)
-elf                  Executable and Linkable Format
-ether8023_frame      Ethernet 802.3 frame
-exif                 Exchangeable Image File Format
-fairplay_spc         FairPlay Server Playback Context
-flac                 Free Lossless Audio Codec file
-flac_frame           FLAC frame
-flac_metadatablock   FLAC metadatablock
-flac_metadatablocks  FLAC metadatablocks
-flac_picture         FLAC metadatablock picture
-flac_streaminfo      FLAC streaminfo
-gif                  Graphics Interchange Format
-gzip                 gzip compression
-hevc_annexb          H.265/HEVC Annex B
-hevc_au              H.265/HEVC Access Unit
-hevc_dcr             H.265/HEVC Decoder Configuration Record
-hevc_nalu            H.265/HEVC Network Access Layer Unit
-hevc_pps             H.265/HEVC Picture Parameter Set
-hevc_sps             H.265/HEVC Sequence Parameter Set
-hevc_vps             H.265/HEVC Video Parameter Set
-html                 HyperText Markup Language
-icc_profile          International Color Consortium profile
-icmp                 Internet Control Message Protocol
-icmpv6               Internet Control Message Protocol v6
-id3v1                ID3v1 metadata
-id3v11               ID3v1.1 metadata
-id3v2                ID3v2 metadata
-ipv4_packet          Internet protocol v4 packet
-ipv6_packet          Internet protocol v6 packet
-jpeg                 Joint Photographic Experts Group file
-json                 JavaScript Object Notation
-jsonl                JavaScript Object Notation Lines
-macho                Mach-O macOS executable
-macho_fat            Fat Mach-O macOS executable (multi-architecture)
-markdown             Markdown
-matroska             Matroska file
-mp3                  MP3 file
-mp3_frame            MPEG audio layer 3 frame
-mp4                  ISOBMFF, QuickTime and similar
-mpeg_asc             MPEG-4 Audio Specific Config
-mpeg_es              MPEG Elementary Stream
-mpeg_pes             MPEG Packetized elementary stream
-mpeg_pes_packet      MPEG Packetized elementary stream packet
-mpeg_spu             Sub Picture Unit (DVD subtitle)
-mpeg_ts              MPEG Transport Stream
-msgpack              MessagePack
-ogg                  OGG file
-ogg_page             OGG page
-opus_packet          Opus packet
-pcap                 PCAP packet capture
-pcapng               PCAPNG packet capture
-png                  Portable Network Graphics file
-prores_frame         Apple ProRes frame
-protobuf             Protobuf
-protobuf_widevine    Widevine protobuf
-pssh_playready       PlayReady PSSH
-raw                  Raw bits
-rtmp                 Real-Time Messaging Protocol
-sll2_packet          Linux cooked capture encapsulation v2
-sll_packet           Linux cooked capture encapsulation
-tar                  Tar archive
-tcp_segment          Transmission control protocol segment
-tiff                 Tag Image File Format
-toml                 Tom's Obvious, Minimal Language
-udp_datagram         User datagram protocol
-vorbis_comment       Vorbis comment
-vorbis_packet        Vorbis packet
-vp8_frame            VP8 frame
-vp9_cfm              VP9 Codec Feature Metadata
-vp9_frame            VP9 frame
-vpx_ccr              VPX Codec Configuration Record
-wasm                 WebAssembly Binary Format
-wav                  WAV file
-webp                 WebP image
-xing                 Xing header
-xml                  Extensible Markup Language
-yaml                 YAML Ain't Markup Language
-zip                  ZIP archive
 $ fq -X
 exitcode: 2
 stderr: