help,markdown: Rewrote and made text rendering nicer

Plan is to use it for man page etc and also some ansi renderer

format/apple/bookmark/testdata/help_applebookmark.fqtest

@@ -29,12 +29,10 @@ converting them to readable JSON with torepr:
 
 Authors
 =======
-
 - David McDonald @dgmcdona (https://github.com/dgmcdona) @river_rat_504 (https://twitter.com/river_rat_504)
 
 References
 ==========
-
 - https://developer.apple.com/documentation/foundation/url/2143023-bookmarkdata
 - https://mac-alias.readthedocs.io/en/latest/bookmark_fmt.html
 - 

format/apple/macho/testdata/help_macho.fqtest

@@ -13,15 +13,12 @@ Supports decoding vanilla and FAT Mach-O binaries.
 
 Select 64bit load segments
 ==========================
-
   $ fq '.load_commands[] | select(.cmd=="segment_64")' file
 
 References
 ==========
-
 - https://github.com/aidansteele/osx-abi-macho-file-format-reference
 
 Authors
 =======
-
 - Sıddık AÇIL acils@itu.edu.tr @Akaame (https://github.com/Akaame)

format/asn1/testdata/help_asn1_ber.fqtest

@@ -17,22 +17,18 @@ Supports decoding BER, CER and DER (X.690).
 
 Can be used to decode certificates etc
 ======================================
-
   $ fq -d bytes 'from_pem | asn1_ber | d' cert.pem
 
 Can decode nested values
 ========================
-
   $ fq -d asn1_ber '.constructed[1].value | asn1_ber' file.ber
 
 Manual schema
 =============
-
   $ fq -d asn1_ber 'torepr as $r | ["version", "modulus", "private_exponent", "private_exponen", "prime1", "prime2", "exponent1", "exponent2", "coefficient"] | with_entries({key: .value, value: $r[.key]})' pkcs1.der
 
 References
 ==========
-
 - https://www.itu.int/ITU-T/studygroups/com10/languages/X.690_1297.pdf
 - https://en.wikipedia.org/wiki/X.690
 - https://letsencrypt.org/docs/a-warm-welcome-to-asn1-and-der/

format/avro/testdata/help_avro_ocf.fqtest

@@ -20,10 +20,8 @@ Limitations:
 
 References
 ==========
-
 - https://avro.apache.org/docs/current/spec.html#Object+Container+Files
 
 Authors
 =======
-
 - Xentripetal xentripetal@fastmail.com @xentripetal (https://github.com/xentripetal)

format/bencode/testdata/help_bencode.fqtest

@@ -11,10 +11,8 @@ Decode examples
 
 Convert represented value to JSON
 =================================
-
   $ fq -d bencode torepr file.torrent
 
 References
 ==========
-
 - https://wiki.theory.org/BitTorrentSpecification#Bencoding

format/bson/testdata/help_bson.fqtest

@@ -11,26 +11,21 @@ Decode examples
 
 Limitations
 ===========
-
 - The decimal128 type is not supported for decoding, will just be treated as binary
 
 Convert represented value to JSON
 =================================
-
   $ fq -d bson torepr file.bson
 
 Filter represented value
 ========================
-
   $ fq -d bson 'torepr | select(.name=="bob")' file.bson
 
 Authors
 =======
-
 - Mattias Wadman mattias.wadman@gmail.com, original author
 - Matt Dale @matthewdale (https://github.com/matthewdale), additional types and bug fixes
 
 References
 ==========
-
 - https://bsonspec.org/spec.html

format/cbor/testdata/help_cbor.fqtest

@@ -11,11 +11,9 @@ Decode examples
 
 Convert represented value to JSON
 =================================
-
   $ fq -d cbor torepr file.cbor
 
 References
 ==========
-
 - https://en.wikipedia.org/wiki/CBOR
 - https://www.rfc-editor.org/rfc/rfc8949.html

format/csv/testdata/help_csv.fqtest

@@ -21,10 +21,8 @@ Decode examples
 
 TSV to CSV
 ==========
-
   $ fq -d csv -o comma="\t" to_csv file.tsv
 
 Convert rows to objects based on header row
 ===========================================
-
   $ fq -d csv '.[0] as $t | .[1:] | map(with_entries(.key = $t[.key]))' file.csv

format/markdown/markdown.jq

@@ -16,15 +16,35 @@ def _word_break($width):
   | map(join(" "))
   );
 
-def _markdown_to_text($width; $header_depth):
+
+# for a document output {heading: <heading>, children: [<children until next heading>]}
+# heading can be null for a document with children before a heading
+def _markdown_split_headings:
+  foreach
+    ( .children[]
+    , {type:"heading"} # dummy heading to flush
+    ) as $c
+  (
+    {heading: null, children: null, extract: null};
+    if $c.type == "heading" then
+      ( .extract = {heading,children}
+      | .heading = $c
+      | .children = null
+      )
+    else
+      ( .children += [$c]
+      | .extract = null
+      )
+    end;
+    .extract | select(.heading or .children)
+  );
+
+def _markdown_children_to_text($width):
   def lb: if $width > 0 then _word_break($width) | join("\n") end;
   def _f:
     if type == "string" then gsub("\n"; " ")
     elif .type == "document" then .children[] | _f
-    elif .type == "heading" then
-      ( (.children[] | _f) as $title
-      | "\($title)\n\("=" * ($title | length))"
-      )
+    elif .type == "heading" then .children[] | _f
     elif .type == "paragraph" then
       ( [.children[] | _f]
       | join("")
@@ -47,6 +67,28 @@ def _markdown_to_text($width; $header_depth):
     elif .type == "html_span" then .literal | gsub("<br>"; "\n") # TODO: more?
     else empty
     end;
-  [_f] | join("\n\n");
+  [_f] | join("\n");
+
+def _markdown_to_text($width; $header_depth):
+  [ _markdown_split_headings
+  | if .heading then
+      ( (.heading | _markdown_children_to_text($width)) as $h
+      | $h
+      , ("=" * ($h | length))
+      )
+    else empty
+    end
+  , ( .children
+    | if length == 0 then ""
+      else
+        ( .[]
+        | _markdown_children_to_text($width)
+        | select(. != "")
+        | .
+        , ""
+        )
+      end
+    )
+  ][:-1] | join("\n");
 def _markdown_to_text:
   _markdown_to_text(-1; 0);

format/markdown/testdata/help_markdown.fqtest

@@ -11,5 +11,4 @@ Decode examples
 
 Array with all level 1 and 2 headers
 ====================================
-
   $ fq -d markdown '[.. | select(.type=="heading" and .level<=2)?.children[0]]' file.md

format/matroska/testdata/help_matroska.fqtest

@@ -20,17 +20,14 @@ Decode examples
 
 Lookup element using path
 =========================
-
   $ fq 'matroska_path(".Segment.Tracks[0)")' file.mkv
 
 Get path to element
 ===================
-
   $ fq 'grep_by(.id == "Tracks") | matroska_path' file.mkv
 
 References
 ==========
-
 - https://tools.ietf.org/html/draft-ietf-cellar-ebml-00
 - https://matroska.org/technical/specs/index.html
 - https://www.matroska.org/technical/basics.html

format/mp4/testdata/help_mp4.fqtest

@@ -21,39 +21,32 @@ Decode examples
 
 Speed up decoding by not decoding samples
 =========================================
-
   # manually decode first sample as a aac_frame
   $ fq -o decode_samples=false '.tracks[0].samples[0] | aac_frame | d' file.mp4
 
 Entries for first edit list as values
 =====================================
-
   $ fq 'first(grep_by(.type=="elst").entries) | tovalue' file.mp4
 
 Whole box tree as JSON (exclude mdat data and tracks)
 =====================================================
-
   $ fq 'del(.tracks) | grep_by(.type=="mdat").data = "<excluded>" | tovalue' file.mp4
 
 Force decode a single box
 =========================
-
   $ fq -n '"AAAAHGVsc3QAAAAAAAAAAQAAADIAAAQAAAEAAA==" | from_base64 | mp4({force:true}) | d'
 
 Lookup mp4 box using a mp4 box path.
 ====================================
-
   # <decode value box> | mp4_path($path) -> <decode value box>
   $ fq 'mp4_path(".moov.trak[1]")' file.mp4
 
 Get mp4 box path for a decode value box.
 ========================================
-
   # <decode value box> | mp4_path -> string
   $ fq 'grep_by(.type == "trak") | mp4_path' file.mp4
 
 References
 ==========
-
 - ISO/IEC base media file format (MPEG-4 Part 12) (https://en.wikipedia.org/wiki/ISO/IEC_base_media_file_format)
 - Quicktime file format (https://developer.apple.com/standards/qtff-2001.pdf)

format/msgpack/testdata/help_msgpack.fqtest

@@ -11,10 +11,8 @@ Decode examples
 
 Convert represented value to JSON
 =================================
-
   $ fq -d msgpack torepr file.msgpack
 
 References
 ==========
-
 - https://github.com/msgpack/msgpack/blob/master/spec.md

format/pcap/testdata/help_pcap.fqtest

@@ -11,7 +11,6 @@ Decode examples
 
 Build object with number of (reassembled) TCP bytes sent to/from client IP
 ==========================================================================
-
   # for a pcapng file you would use .[0].tcp_connections for first section
   $ fq '.tcp_connections | group_by(.client.ip) | map({key: .[0].client.ip, value: map(.client.stream, .server.stream | tobytes.size) | add}) | from_entries'
   {

format/protobuf/testdata/help_protobuf.fqtest

@@ -11,10 +11,8 @@ Decode examples
 
 Can decode sub messages
 =======================
-
   $ fq -d protobuf '.fields[6].wire_value | protobuf | d' file
 
 References
 ==========
-
 - https://developers.google.com/protocol-buffers/docs/encoding

format/riff/testdata/help_avi.fqtest

@@ -20,22 +20,18 @@ Decode examples
 
 Samples
 =======
-
 AVI has many redundant ways to index samples so currently .streams[].samples will only include samples the most "modern" way used in
 the file. That is in order of stream super index, movi ix index then idx1 index.
 
 Extract samples for stream 1
 ============================
-
   $ fq '.streams[1].samples[] | tobytes' file.avi > stream01.mp3
 
 Show stream summary
 ===================
-
   $ fq -o decode_samples=false '[.chunks[0] | grep_by(.id=="LIST" and .type=="strl") | grep_by(.id=="strh") as {$type} | grep_by(.id=="strf") as {$format_tag, $compression} | {$type,$format_tag,$compression}]' *.avi
 
 References
 ==========
-
 - AVI RIFF File Reference (https://learn.microsoft.com/en-us/windows/win32/directshow/avi-riff-file-reference)
 - OpenDML AVI File Format Extensions (http://www.jmcgowan.com/odmlff2.pdf)

format/rtmp/testdata/help_rtmp.fqtest

@@ -13,11 +13,9 @@ Current only supports plain RTMP (not RTMPT or encrypted variants etc) with AMF0
 
 Show rtmp streams in PCAP file
 ==============================
-
   fq '.tcp_connections[] | select(.server.port=="rtmp") | d' file.cap
 
 References
 ==========
-
 - https://rtmp.veriskope.com/docs/spec/
 - https://rtmp.veriskope.com/pdf/video_file_format_spec_v10.pdf

format/tls/testdata/help_tls.fqtest

@@ -23,7 +23,6 @@ traffic in both directions if a NSS key log is provided.
 
 Decode and decrypt provding a PCAP and key log
 ==============================================
-
 Write traffic to a PCAP file:
 
   $ tcpdump -i <iface> -w traffic.pcap
@@ -47,7 +46,6 @@ Decode, decrypt and query. Uses keylog=@<path> to read option value from keylog
 
 Supported cipher suites for decryption
 ======================================
-
 TLS_DH_ANON_EXPORT_WITH_DES40_CBC_SHA, TLS_DH_ANON_EXPORT_WITH_RC4_40_MD5, TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA,
 TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA, TLS_DHE_DSS_WITH_AES_128_CBC_SHA, TLS_DHE_DSS_WITH_AES_128_CBC_SHA256,
 TLS_DHE_DSS_WITH_AES_128_GCM_SHA256, TLS_DHE_DSS_WITH_AES_256_CBC_SHA, TLS_DHE_DSS_WITH_AES_256_CBC_SHA256,
@@ -80,6 +78,5 @@ TLS_RSA_WITH_RC4_128_SHA
 
 References
 ==========
-
 - RFC 5246: The Transport Layer Security (TLS) Protocol (https://www.rfc-editor.org/rfc/rfc5246)
 - RFC 6101: The Secure Sockets Layer (SSL) Protocol Version 3.0 (https://www.rfc-editor.org/rfc/rfc)

format/tzif/testdata/help_tzif.fqtest

@@ -11,20 +11,16 @@ Decode examples
 
 Get last transition time
 ========================
-
   fq '.v2plusdatablock.transition_times[-1] | tovalue' tziffile
 
 Count leap second records
 =========================
-
   fq '.v2plusdatablock.leap_second_records | length' tziffile
 
 Authors
 =======
-
 - Takashi Oguma @bitbears-dev (https://github.com/bitbears-dev) @0xb17bea125 (https://twitter.com/0xb17bea125)
 
 References
 ==========
-
 - https://datatracker.ietf.org/doc/html/rfc8536

format/wasm/testdata/help_wasm.fqtest

@@ -11,20 +11,16 @@ Decode examples
 
 Count opcode usage
 ==================
-
   $ fq '.sections[] | select(.id == "code_section") | [.. | .opcode? // empty] | count | map({key: .[0], value: .[1]}) | from_entries' file.wasm
 
 List exports and imports
 ========================
-
   $ fq '.sections | {import: map(select(.id == "import_section").content.im.x[].nm.b), export: map(select(.id == "export_section").content.ex.x[].nm.b)}' file.wasm
 
 Authors
 =======
-
 - Takashi Oguma @bitbears-dev (https://github.com/bitbears-dev) @0xb17bea125 (https://twitter.com/0xb17bea125)
 
 References
 ==========
-
 - https://webassembly.github.io/spec/core/

format/xml/testdata/help_html.fqtest

@@ -28,7 +28,6 @@ There is no to_html function, see to_xml instead.
 
 Element as object
 =================
-
   # decode as object is the default
   $ echo '<a href="url">text</a>' | fq -d html
   {
@@ -45,7 +44,6 @@ Element as object
 
 Element as array
 ================
-
   $ '<a href="url">text</a>' | fq -d html -o array=true
   [
     "html",

format/xml/testdata/help_xml.fqtest

@@ -29,7 +29,6 @@ options indent and attribute_prefix.
 
 Elements as object
 ==================
-
 Element can have different shapes depending on body text, attributes and children:
 
 - <a key="value">text</a> is {"a":{"#text":"text","@key":"value"}}, has text (#text) and attributes (@key)
@@ -76,7 +75,6 @@ order might be lost.
 
 Elements as array
 =================
-
 Elements are arrays of the shape ["#text": "body text", "attr_name", {key: "attr value"}|null, [<child element>, ...]].
 
   # decode as array
@@ -122,5 +120,4 @@ Elements are arrays of the shape ["#text": "body text", "attr_name", {key: "attr
 
 References
 ==========
-
 - xml.com's Converting Between XML and JSON (https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html)

format/zip/testdata/help_zip.fqtest

@@ -22,6 +22,5 @@ Supports ZIP64.
 
 References
 ==========
-
 - https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
 - https://opensource.apple.com/source/zip/zip-6/unzip/unzip/proginfo/extra.fld