mirror of
https://github.com/wader/fq.git
synced 2024-09-11 12:05:39 +03:00
help,markdown: Rewrote and made text rendering nicer
Plan is to use it for man page etc and also some ansi renderer
This commit is contained in:
parent
a200d3eee0
commit
2c505feec8
@ -29,12 +29,10 @@ converting them to readable JSON with torepr:
|
||||
|
||||
Authors
|
||||
=======
|
||||
|
||||
- David McDonald @dgmcdona (https://github.com/dgmcdona) @river_rat_504 (https://twitter.com/river_rat_504)
|
||||
|
||||
References
|
||||
==========
|
||||
|
||||
- https://developer.apple.com/documentation/foundation/url/2143023-bookmarkdata
|
||||
- https://mac-alias.readthedocs.io/en/latest/bookmark_fmt.html
|
||||
-
|
||||
|
@ -13,15 +13,12 @@ Supports decoding vanilla and FAT Mach-O binaries.
|
||||
|
||||
Select 64bit load segments
|
||||
==========================
|
||||
|
||||
$ fq '.load_commands[] | select(.cmd=="segment_64")' file
|
||||
|
||||
References
|
||||
==========
|
||||
|
||||
- https://github.com/aidansteele/osx-abi-macho-file-format-reference
|
||||
|
||||
Authors
|
||||
=======
|
||||
|
||||
- Sıddık AÇIL acils@itu.edu.tr @Akaame (https://github.com/Akaame)
|
||||
|
4
format/asn1/testdata/help_asn1_ber.fqtest
vendored
4
format/asn1/testdata/help_asn1_ber.fqtest
vendored
@ -17,22 +17,18 @@ Supports decoding BER, CER and DER (X.690).
|
||||
|
||||
Can be used to decode certificates etc
|
||||
======================================
|
||||
|
||||
$ fq -d bytes 'from_pem | asn1_ber | d' cert.pem
|
||||
|
||||
Can decode nested values
|
||||
========================
|
||||
|
||||
$ fq -d asn1_ber '.constructed[1].value | asn1_ber' file.ber
|
||||
|
||||
Manual schema
|
||||
=============
|
||||
|
||||
$ fq -d asn1_ber 'torepr as $r | ["version", "modulus", "private_exponent", "private_exponen", "prime1", "prime2", "exponent1", "exponent2", "coefficient"] | with_entries({key: .value, value: $r[.key]})' pkcs1.der
|
||||
|
||||
References
|
||||
==========
|
||||
|
||||
- https://www.itu.int/ITU-T/studygroups/com10/languages/X.690_1297.pdf
|
||||
- https://en.wikipedia.org/wiki/X.690
|
||||
- https://letsencrypt.org/docs/a-warm-welcome-to-asn1-and-der/
|
||||
|
2
format/avro/testdata/help_avro_ocf.fqtest
vendored
2
format/avro/testdata/help_avro_ocf.fqtest
vendored
@ -20,10 +20,8 @@ Limitations:
|
||||
|
||||
References
|
||||
==========
|
||||
|
||||
- https://avro.apache.org/docs/current/spec.html#Object+Container+Files
|
||||
|
||||
Authors
|
||||
=======
|
||||
|
||||
- Xentripetal xentripetal@fastmail.com @xentripetal (https://github.com/xentripetal)
|
||||
|
2
format/bencode/testdata/help_bencode.fqtest
vendored
2
format/bencode/testdata/help_bencode.fqtest
vendored
@ -11,10 +11,8 @@ Decode examples
|
||||
|
||||
Convert represented value to JSON
|
||||
=================================
|
||||
|
||||
$ fq -d bencode torepr file.torrent
|
||||
|
||||
References
|
||||
==========
|
||||
|
||||
- https://wiki.theory.org/BitTorrentSpecification#Bencoding
|
||||
|
5
format/bson/testdata/help_bson.fqtest
vendored
5
format/bson/testdata/help_bson.fqtest
vendored
@ -11,26 +11,21 @@ Decode examples
|
||||
|
||||
Limitations
|
||||
===========
|
||||
|
||||
- The decimal128 type is not supported for decoding, will just be treated as binary
|
||||
|
||||
Convert represented value to JSON
|
||||
=================================
|
||||
|
||||
$ fq -d bson torepr file.bson
|
||||
|
||||
Filter represented value
|
||||
========================
|
||||
|
||||
$ fq -d bson 'torepr | select(.name=="bob")' file.bson
|
||||
|
||||
Authors
|
||||
=======
|
||||
|
||||
- Mattias Wadman mattias.wadman@gmail.com, original author
|
||||
- Matt Dale @matthewdale (https://github.com/matthewdale), additional types and bug fixes
|
||||
|
||||
References
|
||||
==========
|
||||
|
||||
- https://bsonspec.org/spec.html
|
||||
|
2
format/cbor/testdata/help_cbor.fqtest
vendored
2
format/cbor/testdata/help_cbor.fqtest
vendored
@ -11,11 +11,9 @@ Decode examples
|
||||
|
||||
Convert represented value to JSON
|
||||
=================================
|
||||
|
||||
$ fq -d cbor torepr file.cbor
|
||||
|
||||
References
|
||||
==========
|
||||
|
||||
- https://en.wikipedia.org/wiki/CBOR
|
||||
- https://www.rfc-editor.org/rfc/rfc8949.html
|
||||
|
2
format/csv/testdata/help_csv.fqtest
vendored
2
format/csv/testdata/help_csv.fqtest
vendored
@ -21,10 +21,8 @@ Decode examples
|
||||
|
||||
TSV to CSV
|
||||
==========
|
||||
|
||||
$ fq -d csv -o comma="\t" to_csv file.tsv
|
||||
|
||||
Convert rows to objects based on header row
|
||||
===========================================
|
||||
|
||||
$ fq -d csv '.[0] as $t | .[1:] | map(with_entries(.key = $t[.key]))' file.csv
|
||||
|
@ -16,15 +16,35 @@ def _word_break($width):
|
||||
| map(join(" "))
|
||||
);
|
||||
|
||||
def _markdown_to_text($width; $header_depth):
|
||||
|
||||
# for a document output {heading: <heading>, children: [<children until next heading>]}
|
||||
# heading can be null for a document with children before a heading
|
||||
def _markdown_split_headings:
|
||||
foreach
|
||||
( .children[]
|
||||
, {type:"heading"} # dummy heading to flush
|
||||
) as $c
|
||||
(
|
||||
{heading: null, children: null, extract: null};
|
||||
if $c.type == "heading" then
|
||||
( .extract = {heading,children}
|
||||
| .heading = $c
|
||||
| .children = null
|
||||
)
|
||||
else
|
||||
( .children += [$c]
|
||||
| .extract = null
|
||||
)
|
||||
end;
|
||||
.extract | select(.heading or .children)
|
||||
);
|
||||
|
||||
def _markdown_children_to_text($width):
|
||||
def lb: if $width > 0 then _word_break($width) | join("\n") end;
|
||||
def _f:
|
||||
if type == "string" then gsub("\n"; " ")
|
||||
elif .type == "document" then .children[] | _f
|
||||
elif .type == "heading" then
|
||||
( (.children[] | _f) as $title
|
||||
| "\($title)\n\("=" * ($title | length))"
|
||||
)
|
||||
elif .type == "heading" then .children[] | _f
|
||||
elif .type == "paragraph" then
|
||||
( [.children[] | _f]
|
||||
| join("")
|
||||
@ -47,6 +67,28 @@ def _markdown_to_text($width; $header_depth):
|
||||
elif .type == "html_span" then .literal | gsub("<br>"; "\n") # TODO: more?
|
||||
else empty
|
||||
end;
|
||||
[_f] | join("\n\n");
|
||||
[_f] | join("\n");
|
||||
|
||||
def _markdown_to_text($width; $header_depth):
|
||||
[ _markdown_split_headings
|
||||
| if .heading then
|
||||
( (.heading | _markdown_children_to_text($width)) as $h
|
||||
| $h
|
||||
, ("=" * ($h | length))
|
||||
)
|
||||
else empty
|
||||
end
|
||||
, ( .children
|
||||
| if length == 0 then ""
|
||||
else
|
||||
( .[]
|
||||
| _markdown_children_to_text($width)
|
||||
| select(. != "")
|
||||
| .
|
||||
, ""
|
||||
)
|
||||
end
|
||||
)
|
||||
][:-1] | join("\n");
|
||||
def _markdown_to_text:
|
||||
_markdown_to_text(-1; 0);
|
||||
|
@ -11,5 +11,4 @@ Decode examples
|
||||
|
||||
Array with all level 1 and 2 headers
|
||||
====================================
|
||||
|
||||
$ fq -d markdown '[.. | select(.type=="heading" and .level<=2)?.children[0]]' file.md
|
||||
|
@ -20,17 +20,14 @@ Decode examples
|
||||
|
||||
Lookup element using path
|
||||
=========================
|
||||
|
||||
$ fq 'matroska_path(".Segment.Tracks[0)")' file.mkv
|
||||
|
||||
Get path to element
|
||||
===================
|
||||
|
||||
$ fq 'grep_by(.id == "Tracks") | matroska_path' file.mkv
|
||||
|
||||
References
|
||||
==========
|
||||
|
||||
- https://tools.ietf.org/html/draft-ietf-cellar-ebml-00
|
||||
- https://matroska.org/technical/specs/index.html
|
||||
- https://www.matroska.org/technical/basics.html
|
||||
|
7
format/mp4/testdata/help_mp4.fqtest
vendored
7
format/mp4/testdata/help_mp4.fqtest
vendored
@ -21,39 +21,32 @@ Decode examples
|
||||
|
||||
Speed up decoding by not decoding samples
|
||||
=========================================
|
||||
|
||||
# manually decode first sample as a aac_frame
|
||||
$ fq -o decode_samples=false '.tracks[0].samples[0] | aac_frame | d' file.mp4
|
||||
|
||||
Entries for first edit list as values
|
||||
=====================================
|
||||
|
||||
$ fq 'first(grep_by(.type=="elst").entries) | tovalue' file.mp4
|
||||
|
||||
Whole box tree as JSON (exclude mdat data and tracks)
|
||||
=====================================================
|
||||
|
||||
$ fq 'del(.tracks) | grep_by(.type=="mdat").data = "<excluded>" | tovalue' file.mp4
|
||||
|
||||
Force decode a single box
|
||||
=========================
|
||||
|
||||
$ fq -n '"AAAAHGVsc3QAAAAAAAAAAQAAADIAAAQAAAEAAA==" | from_base64 | mp4({force:true}) | d'
|
||||
|
||||
Lookup mp4 box using a mp4 box path.
|
||||
====================================
|
||||
|
||||
# <decode value box> | mp4_path($path) -> <decode value box>
|
||||
$ fq 'mp4_path(".moov.trak[1]")' file.mp4
|
||||
|
||||
Get mp4 box path for a decode value box.
|
||||
========================================
|
||||
|
||||
# <decode value box> | mp4_path -> string
|
||||
$ fq 'grep_by(.type == "trak") | mp4_path' file.mp4
|
||||
|
||||
References
|
||||
==========
|
||||
|
||||
- ISO/IEC base media file format (MPEG-4 Part 12) (https://en.wikipedia.org/wiki/ISO/IEC_base_media_file_format)
|
||||
- Quicktime file format (https://developer.apple.com/standards/qtff-2001.pdf)
|
||||
|
2
format/msgpack/testdata/help_msgpack.fqtest
vendored
2
format/msgpack/testdata/help_msgpack.fqtest
vendored
@ -11,10 +11,8 @@ Decode examples
|
||||
|
||||
Convert represented value to JSON
|
||||
=================================
|
||||
|
||||
$ fq -d msgpack torepr file.msgpack
|
||||
|
||||
References
|
||||
==========
|
||||
|
||||
- https://github.com/msgpack/msgpack/blob/master/spec.md
|
||||
|
1
format/pcap/testdata/help_pcap.fqtest
vendored
1
format/pcap/testdata/help_pcap.fqtest
vendored
@ -11,7 +11,6 @@ Decode examples
|
||||
|
||||
Build object with number of (reassembled) TCP bytes sent to/from client IP
|
||||
==========================================================================
|
||||
|
||||
# for a pcapng file you would use .[0].tcp_connections for first section
|
||||
$ fq '.tcp_connections | group_by(.client.ip) | map({key: .[0].client.ip, value: map(.client.stream, .server.stream | tobytes.size) | add}) | from_entries'
|
||||
{
|
||||
|
@ -11,10 +11,8 @@ Decode examples
|
||||
|
||||
Can decode sub messages
|
||||
=======================
|
||||
|
||||
$ fq -d protobuf '.fields[6].wire_value | protobuf | d' file
|
||||
|
||||
References
|
||||
==========
|
||||
|
||||
- https://developers.google.com/protocol-buffers/docs/encoding
|
||||
|
4
format/riff/testdata/help_avi.fqtest
vendored
4
format/riff/testdata/help_avi.fqtest
vendored
@ -20,22 +20,18 @@ Decode examples
|
||||
|
||||
Samples
|
||||
=======
|
||||
|
||||
AVI has many redundant ways to index samples so currently .streams[].samples will only include samples the most "modern" way used in
|
||||
the file. That is in order of stream super index, movi ix index then idx1 index.
|
||||
|
||||
Extract samples for stream 1
|
||||
============================
|
||||
|
||||
$ fq '.streams[1].samples[] | tobytes' file.avi > stream01.mp3
|
||||
|
||||
Show stream summary
|
||||
===================
|
||||
|
||||
$ fq -o decode_samples=false '[.chunks[0] | grep_by(.id=="LIST" and .type=="strl") | grep_by(.id=="strh") as {$type} | grep_by(.id=="strf") as {$format_tag, $compression} | {$type,$format_tag,$compression}]' *.avi
|
||||
|
||||
References
|
||||
==========
|
||||
|
||||
- AVI RIFF File Reference (https://learn.microsoft.com/en-us/windows/win32/directshow/avi-riff-file-reference)
|
||||
- OpenDML AVI File Format Extensions (http://www.jmcgowan.com/odmlff2.pdf)
|
||||
|
2
format/rtmp/testdata/help_rtmp.fqtest
vendored
2
format/rtmp/testdata/help_rtmp.fqtest
vendored
@ -13,11 +13,9 @@ Current only supports plain RTMP (not RTMPT or encrypted variants etc) with AMF0
|
||||
|
||||
Show rtmp streams in PCAP file
|
||||
==============================
|
||||
|
||||
fq '.tcp_connections[] | select(.server.port=="rtmp") | d' file.cap
|
||||
|
||||
References
|
||||
==========
|
||||
|
||||
- https://rtmp.veriskope.com/docs/spec/
|
||||
- https://rtmp.veriskope.com/pdf/video_file_format_spec_v10.pdf
|
||||
|
3
format/tls/testdata/help_tls.fqtest
vendored
3
format/tls/testdata/help_tls.fqtest
vendored
@ -23,7 +23,6 @@ traffic in both directions if a NSS key log is provided.
|
||||
|
||||
Decode and decrypt provding a PCAP and key log
|
||||
==============================================
|
||||
|
||||
Write traffic to a PCAP file:
|
||||
|
||||
$ tcpdump -i <iface> -w traffic.pcap
|
||||
@ -47,7 +46,6 @@ Decode, decrypt and query. Uses keylog=@<path> to read option value from keylog
|
||||
|
||||
Supported cipher suites for decryption
|
||||
======================================
|
||||
|
||||
TLS_DH_ANON_EXPORT_WITH_DES40_CBC_SHA, TLS_DH_ANON_EXPORT_WITH_RC4_40_MD5, TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA,
|
||||
TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA, TLS_DHE_DSS_WITH_AES_128_CBC_SHA, TLS_DHE_DSS_WITH_AES_128_CBC_SHA256,
|
||||
TLS_DHE_DSS_WITH_AES_128_GCM_SHA256, TLS_DHE_DSS_WITH_AES_256_CBC_SHA, TLS_DHE_DSS_WITH_AES_256_CBC_SHA256,
|
||||
@ -80,6 +78,5 @@ TLS_RSA_WITH_RC4_128_SHA
|
||||
|
||||
References
|
||||
==========
|
||||
|
||||
- RFC 5246: The Transport Layer Security (TLS) Protocol (https://www.rfc-editor.org/rfc/rfc5246)
|
||||
- RFC 6101: The Secure Sockets Layer (SSL) Protocol Version 3.0 (https://www.rfc-editor.org/rfc/rfc)
|
||||
|
4
format/tzif/testdata/help_tzif.fqtest
vendored
4
format/tzif/testdata/help_tzif.fqtest
vendored
@ -11,20 +11,16 @@ Decode examples
|
||||
|
||||
Get last transition time
|
||||
========================
|
||||
|
||||
fq '.v2plusdatablock.transition_times[-1] | tovalue' tziffile
|
||||
|
||||
Count leap second records
|
||||
=========================
|
||||
|
||||
fq '.v2plusdatablock.leap_second_records | length' tziffile
|
||||
|
||||
Authors
|
||||
=======
|
||||
|
||||
- Takashi Oguma @bitbears-dev (https://github.com/bitbears-dev) @0xb17bea125 (https://twitter.com/0xb17bea125)
|
||||
|
||||
References
|
||||
==========
|
||||
|
||||
- https://datatracker.ietf.org/doc/html/rfc8536
|
||||
|
4
format/wasm/testdata/help_wasm.fqtest
vendored
4
format/wasm/testdata/help_wasm.fqtest
vendored
@ -11,20 +11,16 @@ Decode examples
|
||||
|
||||
Count opcode usage
|
||||
==================
|
||||
|
||||
$ fq '.sections[] | select(.id == "code_section") | [.. | .opcode? // empty] | count | map({key: .[0], value: .[1]}) | from_entries' file.wasm
|
||||
|
||||
List exports and imports
|
||||
========================
|
||||
|
||||
$ fq '.sections | {import: map(select(.id == "import_section").content.im.x[].nm.b), export: map(select(.id == "export_section").content.ex.x[].nm.b)}' file.wasm
|
||||
|
||||
Authors
|
||||
=======
|
||||
|
||||
- Takashi Oguma @bitbears-dev (https://github.com/bitbears-dev) @0xb17bea125 (https://twitter.com/0xb17bea125)
|
||||
|
||||
References
|
||||
==========
|
||||
|
||||
- https://webassembly.github.io/spec/core/
|
||||
|
2
format/xml/testdata/help_html.fqtest
vendored
2
format/xml/testdata/help_html.fqtest
vendored
@ -28,7 +28,6 @@ There is no to_html function, see to_xml instead.
|
||||
|
||||
Element as object
|
||||
=================
|
||||
|
||||
# decode as object is the default
|
||||
$ echo '<a href="url">text</a>' | fq -d html
|
||||
{
|
||||
@ -45,7 +44,6 @@ Element as object
|
||||
|
||||
Element as array
|
||||
================
|
||||
|
||||
$ '<a href="url">text</a>' | fq -d html -o array=true
|
||||
[
|
||||
"html",
|
||||
|
3
format/xml/testdata/help_xml.fqtest
vendored
3
format/xml/testdata/help_xml.fqtest
vendored
@ -29,7 +29,6 @@ options indent and attribute_prefix.
|
||||
|
||||
Elements as object
|
||||
==================
|
||||
|
||||
Element can have different shapes depending on body text, attributes and children:
|
||||
|
||||
- <a key="value">text</a> is {"a":{"#text":"text","@key":"value"}}, has text (#text) and attributes (@key)
|
||||
@ -76,7 +75,6 @@ order might be lost.
|
||||
|
||||
Elements as array
|
||||
=================
|
||||
|
||||
Elements are arrays of the shape ["#text": "body text", "attr_name", {key: "attr value"}|null, [<child element>, ...]].
|
||||
|
||||
# decode as array
|
||||
@ -122,5 +120,4 @@ Elements are arrays of the shape ["#text": "body text", "attr_name", {key: "attr
|
||||
|
||||
References
|
||||
==========
|
||||
|
||||
- xml.com's Converting Between XML and JSON (https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html)
|
||||
|
1
format/zip/testdata/help_zip.fqtest
vendored
1
format/zip/testdata/help_zip.fqtest
vendored
@ -22,6 +22,5 @@ Supports ZIP64.
|
||||
|
||||
References
|
||||
==========
|
||||
|
||||
- https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
|
||||
- https://opensource.apple.com/source/zip/zip-6/unzip/unzip/proginfo/extra.fld
|
||||
|
Loading…
Reference in New Issue
Block a user