Merge eae6a43d23 into 2bc6c768c7

Update make-golangci-lint to 1.59.1 from 1.59.0

.github/workflows/ci.yml

@@ -7,7 +7,7 @@ on:
   pull_request:
 
 env:
-  GOLANGCILINT_VERSION: "1.58.2"
+  GOLANGCILINT_VERSION: "1.59.1"
 
 jobs:
   lint:
@@ -15,7 +15,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: "1.22.3"
+          go-version: "1.22.4"
       - uses: actions/checkout@v3
       - uses: golangci/golangci-lint-action@v3
         with:
@@ -47,7 +47,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: "1.22.3"
+          go-version: "1.22.4"
       - name: Test
         env:
           GOARCH: ${{ matrix.goarch }}

.github/workflows/release.yml

@@ -15,7 +15,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: "1.22.3"
+          go-version: "1.22.4"
       - uses: goreleaser/goreleaser-action@v5
         with:
           distribution: goreleaser

Dockerfile

@@ -1,5 +1,5 @@
 # bump: docker-golang /FROM golang:([\d.]+)/ docker:golang|^1
-FROM golang:1.22.3-bookworm AS base
+FROM golang:1.22.4-bookworm AS base
 
 # expect is used to test cli
 RUN \

Makefile

@@ -61,7 +61,7 @@ gogenerate: always
 lint: always
 # bump: make-golangci-lint /golangci-lint@v([\d.]+)/ git:https://github.com/golangci/golangci-lint.git|^1
 # bump: make-golangci-lint link "Release notes" https://github.com/golangci/golangci-lint/releases/tag/v$LATEST
-	go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.58.2 run
+	go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.59.1 run
 
 depgraph.svg: always
 	go run github.com/kisielk/godepgraph@latest github.com/wader/fq | dot -Tsvg -o godepgraph.svg

README.md

@@ -169,6 +169,7 @@ For details see [formats.md](doc/formats.md) and [usage.md](doc/usage.md).
 
 ## Presentations
 
+- [Kodsnack 585 - Polymorfisk JSON](https://kodsnack.se/585/) - Swedish podcast episode about jq and fq
 - "fq - jq for binary formats" at [FOSDEM 2023](https://fosdem.org/2023/) - [video & slides](https://fosdem.org/2023/schedule/event/bintools_fq/)
 - "fq - jq for binary formats" at [No time to wait 6](https://mediaarea.net/NoTimeToWait6) - [video](https://www.youtube.com/watch?v=-Pwt5KL-xRs&t=1450s) - [slides](doc/presentations/nttw6/fq-nttw6-slides.pdf)
 - "fq - jq for binary formats" at [Binary Tools Summit 2022](https://binary-tools.net/summit.html) - [video](https://www.youtube.com/watch?v=GJOq_b0eb-s&list=PLTj8twuHdQz-JcX7k6eOwyVPDB8CyfZc8&index=1) - [slides](doc/presentations/bts2022/fq-bts2022-v1.pdf)

doc/formats.jq

@@ -70,6 +70,7 @@ def formats_sections:
   | ({} | _help_format_enrich("fq"; $f; false)) as $fhelp
   | select(has_section($f; $fhelp))
   | "## \($f.name)"
+  , $f.description + "."
   , ""
   , ($fhelp.notes | if . then ., "" else empty end)
   , if $f.decode_in_arg then

doc/formats.md

@@ -157,6 +157,7 @@ fq -d bytes 'mp4({force: true})' file.mp4
 [fq -rn -L . 'include "formats"; formats_sections']: sh-start
 
 ## aac_frame
+Advanced Audio Coding frame.
 
 ### Options
 
@@ -177,6 +178,7 @@ Decode value as aac_frame
 ```
 
 ## apple_bookmark
+Apple BookmarkData.
 
 Apple's `bookmarkData` format is used to encode information that can be resolved
 into a `URL` object for a file even if the user moves or renames it. Can also
@@ -218,6 +220,7 @@ torepr' <sfl2 file>
 - https://michaellynn.github.io/2015/10/24/apples-bookmarkdata-exposed/
 
 ## asn1_ber
+ASN1 BER (basic encoding rules, also CER and DER).
 
 Supports decoding BER, CER and DER (X.690).
 
@@ -250,6 +253,7 @@ $ fq -d asn1_ber 'torepr as $r | ["version", "modulus", "private_exponent", "pri
 - https://lapo.it/asn1js/
 
 ## avc_au
+H.264/AVC Access Unit.
 
 ### Options
 
@@ -270,6 +274,7 @@ Decode value as avc_au
 ```
 
 ## avi
+Audio Video Interleaved.
 
 ### Options
 
@@ -318,6 +323,7 @@ $ fq -o decode_samples=false -o decode_extended_chunks=false d file.avi
 - [OpenDML AVI File Format Extensions](http://www.jmcgowan.com/odmlff2.pdf)
 
 ## avro_ocf
+Avro object container file.
 
 Supports reading Avro Object Container Format (OCF) files based on the 1.11.0 specification.
 
@@ -337,6 +343,7 @@ xentripetal@fastmail.com
 [@xentripetal](https://github.com/xentripetal)
 
 ## bencode
+BitTorrent bencoding.
 
 ### Convert represented value to JSON
 
@@ -348,6 +355,7 @@ $ fq -d bencode torepr file.torrent
 - https://wiki.theory.org/BitTorrentSpecification#Bencoding
 
 ## bitcoin_block
+Bitcoin block.
 
 ### Options
 
@@ -368,6 +376,7 @@ Decode value as bitcoin_block
 ```
 
 ## bits
+Raw bits.
 
 Decode to a slice and indexable binary of bits.
 
@@ -390,6 +399,7 @@ $ echo 'hello' | fq -c -d bits '[.[range(8)]]'
 ```
 
 ## bplist
+Apple Binary Property List.
 
 ### Show full decoding
 ```sh
@@ -454,6 +464,7 @@ bplist> from_ns_keyed_archiver(1)
 - https://opensource.apple.com/source/CF/CF-550/CFBinaryPList.c
 
 ## bson
+Binary JSON.
 
 ### Limitations
 
@@ -479,6 +490,7 @@ $ fq -d bson 'torepr | select(.name=="bob")' file.bson
 - https://bsonspec.org/spec.html
 
 ## bytes
+Raw bytes.
 
 Decode to a slice and indexable binary of bytes.
 
@@ -506,6 +518,7 @@ $ echo 'hello' | fq -d bytes '.[1]'
 ```
 
 ## caff
+Live2D Cubism archive.
 
 ### Options
 
@@ -529,6 +542,7 @@ Decode value as caff
 - [@ronsor](https://github.com/ronsor)
 
 ## cbor
+Concise Binary Object Representation.
 
 ### Convert represented value to JSON
 
@@ -541,6 +555,7 @@ $ fq -d cbor torepr file.cbor
 - https://www.rfc-editor.org/rfc/rfc8949.html
 
 ## csv
+Comma separated values.
 
 ### Options
 
@@ -574,6 +589,7 @@ $ fq -d csv '.[0] as $t | .[1:] | map(with_entries(.key = $t[.key]))' file.csv
 ```
 
 ## fit
+Garmin Flexible and Interoperable Data Transfer.
 
 ### Limitations
 
@@ -596,6 +612,7 @@ $ fq '[.data_records[] | select(.record_header.message_type == "data").data_mess
 - https://developer.garmin.com/fit/cookbook/decoding-activity-files/
 
 ## flac_frame
+FLAC frame.
 
 ### Options
 
@@ -616,6 +633,7 @@ Decode value as flac_frame
 ```
 
 ## hevc_au
+H.265/HEVC Access Unit.
 
 ### Options
 
@@ -636,6 +654,7 @@ Decode value as hevc_au
 ```
 
 ## html
+HyperText Markup Language.
 
 ### Options
 
@@ -719,6 +738,7 @@ $ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href'
 ```
 
 ## leveldb_descriptor
+LevelDB Descriptor.
 
 ### Limitations
 
@@ -735,6 +755,7 @@ $ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href'
 - https://github.com/google/leveldb/blob/main/db/version_edit.cc
 
 ## leveldb_log
+LevelDB Log.
 
 ### Limitations
 
@@ -751,6 +772,7 @@ $ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href'
 - https://github.com/google/leveldb/blob/main/db/write_batch.cc
 
 ## leveldb_table
+LevelDB Table.
 
 ### Limitations
 
@@ -768,6 +790,7 @@ $ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href'
 - https://github.com/google/leveldb/blob/main/doc/index.md
 
 ## luajit
+LuaJIT 2.0 bytecode.
 
 ### Authors
 - [@dlatchx](https://github.com/dlatchx)
@@ -777,6 +800,7 @@ $ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href'
 - http://scm.zoomquiet.top/data/20131216145900/index.html
 
 ## macho
+Mach-O macOS executable.
 
 Supports decoding vanilla and FAT Mach-O binaries.
 
@@ -795,12 +819,14 @@ acils@itu.edu.tr
 [@Akaame](https://github.com/Akaame)
 
 ## markdown
+Markdown.
 
 ### Array with all level 1 and 2 headers
 ```sh
 $ fq -d markdown '[.. | select(.type=="heading" and .level<=2)?.children[0]]' file.md
 ```
 ## matroska
+Matroska file.
 
 ### Options
 
@@ -840,11 +866,13 @@ $ fq 'grep_by(.id == "Tracks") | matroska_path' file.mkv
 - https://wiki.xiph.org/MatroskaOpus
 
 ## moc3
+MOC3 file.
 
 ### Authors
 - [@ronsor](https://github.com/ronsor)
 
 ## mp3
+MP3 file.
 
 ### Options
 
@@ -867,6 +895,7 @@ Decode value as mp3
 ```
 
 ## mp4
+ISOBMFF, QuickTime and similar.
 
 ### Options
 
@@ -932,6 +961,7 @@ $ fq 'grep_by(.type == "trak") | mp4_path' file.mp4
 - [Quicktime file format](https://developer.apple.com/standards/qtff-2001.pdf)
 
 ## msgpack
+MessagePack.
 
 ### Convert represented value to JSON
 
@@ -943,6 +973,7 @@ $ fq -d msgpack torepr file.msgpack
 - https://github.com/msgpack/msgpack/blob/master/spec.md
 
 ## nes
+iNES/NES 2.0 cartridge ROM format.
 
 ### Limitations
 
@@ -979,6 +1010,7 @@ $ for line in $(fq -r '.chr_rom[] | nes_tokitty(5)' file.nes);do printf "%b%s" "
 - https://bugzmanov.github.io/nes_ebook/chapter_6_3.html
 
 ## opentimestamps
+OpenTimestamps file.
 
 ### View a full OpenTimestamps file
 
@@ -1006,6 +1038,7 @@ $ fq '.operations | map(select(.attestation_type == "bitcoin")) | length > 0' fi
 - https://github.com/opentimestamps/python-opentimestamps
 
 ## pcap
+PCAP packet capture.
 
 ### Build object with number of (reassembled) TCP bytes sent to/from client IP
 ```sh
@@ -1018,6 +1051,7 @@ $ fq '.tcp_connections | group_by(.client.ip) | map({key: .[0].client.ip, value:
 }
 ```
 ## pg_btree
+PostgreSQL btree index file.
 
 ### Options
 
@@ -1057,6 +1091,7 @@ p.n.safonov@gmail.com
 ### References
 - https://www.postgresql.org/docs/current/storage-page-layout.html
 ## pg_control
+PostgreSQL control file.
 
 ### Options
 
@@ -1096,6 +1131,7 @@ p.n.safonov@gmail.com
 ### References
 - https://github.com/postgres/postgres/blob/REL_14_2/src/include/catalog/pg_control.h
 ## pg_heap
+PostgreSQL heap file.
 
 ### Options
 
@@ -1148,6 +1184,7 @@ p.n.safonov@gmail.com
 ### References
 - https://www.postgresql.org/docs/current/storage-page-layout.html
 ## protobuf
+Protobuf.
 
 ### Can decode sub messages
 
@@ -1159,6 +1196,7 @@ $ fq -d protobuf '.fields[6].wire_value | protobuf | d' file
 - https://developers.google.com/protocol-buffers/docs/encoding
 
 ## rtmp
+Real-Time Messaging Protocol.
 
 Current only supports plain RTMP (not RTMPT or encrypted variants etc) with AMF0 (not AMF3).
 
@@ -1172,6 +1210,7 @@ fq '.tcp_connections[] | select(.server.port=="rtmp") | d' file.cap
 - https://rtmp.veriskope.com/pdf/video_file_format_spec_v10.pdf
 
 ## tls
+Transport layer security.
 
 ### Options
 
@@ -1319,6 +1358,7 @@ $ fq -o keylog=@traffic.keylog  'first(grep_by(.server.stream | format == "tls")
 - [RFC 6101: The Secure Sockets Layer (SSL) Protocol Version 3.0](https://www.rfc-editor.org/rfc/rfc)
 
 ## tzif
+Time Zone Information Format.
 
 ### Get last transition time
 ```sh
@@ -1339,6 +1379,7 @@ fq '.v2plusdatablock.leap_second_records | length' tziffile
 - https://datatracker.ietf.org/doc/html/rfc8536
 
 ## wasm
+WebAssembly Binary Format.
 
 ### Count opcode usage
 ```sh
@@ -1359,6 +1400,7 @@ $ fq '.sections | {import: map(select(.id == "import_section").content.im.x[].nm
 - https://webassembly.github.io/spec/core/
 
 ## xml
+Extensible Markup Language.
 
 ### Options
 
@@ -1487,6 +1529,7 @@ $ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -o array=true '.[2][2
 - [xml.com's Converting Between XML and JSON](https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html)
 
 ## zip
+ZIP archive.
 
 ### Options
 

doc/fq.1

@@ -0,0 +1,410 @@
+.TH FQ "1" "July 2023"
+
+.SH NAME
+fq \- jq for binary formats
+
+.SH SYNOPSIS
+\fBfq\fR [\fI\,OPTION\/\fR]... [--] \fI\,EXPR [\fI\,FILE\/\fR]...
+
+\fBfq\fR . <FILE>
+
+\fBfq\fR d <FILE>
+
+\fBfq\fR -V '.path[1].value' <FILE>
+
+\fBfq\fR tovalue <FILE>
+
+\fBfq\fR -r to_toml file.yml
+
+\fBfq\fR -s -d html 'map(.html.head.title?)' *.html
+
+\fBfq\fR file.cbor | fq -d cbor torepr
+
+\fBfq\fR 'grep("^main$") | parent' /bin/ls
+
+\fBfq\fR -i
+
+.SH DESCRIPTION
+\fBfq\fR is inspired by the well known jq tool and language that allows you to work with binary formats the same way you would using jq. In addition it can present data like a hex viewer, transform, slice and concatenate binary data. It also supports nested formats and has an interactive REPL with auto-completion.
+.P
+It was originally designed to query, inspect and debug media codecs and containers like mp4, flac, mp3, jpeg. But since then it has been extended to support a variety of formats like executables, packet captures (with TCP reassembly) and serialization formats like JSON, YAML, XML, ASN1 BER, Avro, CBOR, protobuf. In addition it also has functions to work with URLs, convert to/from hex, number bases, search for things etc.
+.P
+In summary it aims to be jq, hexdump, dd and gdb for files combined into one.
+
+.SH INVOKING FQ
+.SS Command Line Arguments
+Most of jq's CLI arguments work with fq. But there are some additional ones specific to fq:
+.TP
+\fB-d\fR, \fB--decode\fR \fI\,FORMAT\/\fR
+Use \fI\,FORMAT\/\fR to decode instead of trying to auto-detect format.
+.TP
+\fB-i\fR, \fB--repl\fR [\fI\,INPUT\/\fR]...
+Use interactive REPL.
+
+Can be used with no input, one and multiple inputs, for example just \fBfq -i\fR starts a REPL with null input, \fBfq -i 123\fR with the number 123 as input, \fBfq -i . a b\fR with two files as input. This also works with \fB--slurp\fR. In the REPL it is also possible to start a sub-REPL(s) by ending a query with \fB<query> | repl\fR, use ctrl-D to exit the sub-REPL. The sub-REPL will evaluate separately on each output from the query it was started. Use \fB[<query>] | repl\fR if you want to "slurp" into an array.
+.TP
+\fB-o\fR, \fB--options\fR <\fIKEY\fR=\fIVALUE\fR|@\fIPATH\fR>
+\fIKEY\fR is name of option
+
+\fIVALUE\fR will be interpreted as a JSON value if possible otherwise a string, for example \fB-o name=abc\fR and \fB-o name='"abc"'\fR are equivalent.
+
+@\fBPATH\fR will read string from file at \fBPATH\fR.
+
+Specifying a global option or a format option, for example, \fB-o decode_samples=false\fR would, for some container decoders like mp4 and matroska, disable decoding of samples.
+.TP
+\fB-V\fR, \fB--value-output\fR
+Output JSON value instead of decode tree. Use \fB-Vr\fR if you want raw string (no quotes).
+.TP
+\fB-M\fR
+Disable coloured output.
+.TP
+\fB-o <NAME>=<VALUE>\fR
+Set a decoding/encoding option. A list of supported options are -
+.RS
+.P
+\fBbits_format\fR - string, md5, hex, base64, truncate, snippet
+.RS
+Controls the representation of raw binary in the output JSON
+.RE
+.RE
+.SS Supported Formats
+Use \fBfq --help formats\fR to get a list of supported formats
+
+aac_frame, adts, adts_frame, aiff, amf0, apev2, apple_bookmark, ar, asn1_ber, av1_ccr, av1_frame, av1_obu, avc_annexb, avc_au, avc_dcr, avc_nalu, avc_pps, avc_sei, avc_sps, avi, avro_ocf, bencode, bitcoin_blkdat, bitcoin_block, bitcoin_script, bitcoin_transaction, bits, bplist, bsd_loopback_frame, bson, bytes, bzip2, cbor, csv, dns, dns_tcp, elf, ether8023_frame, exif, fairplay_spc, flac, flac_frame, flac_metadatablock, flac_metadatablocks, flac_picture, flac_streaminfo, gif, gzip, hevc_annexb, hevc_au, hevc_dcr, hevc_nalu, hevc_pps, hevc_sps, hevc_vps, html, icc_profile, icmp, icmpv6, id3v1, id3v11, id3v2, ipv4_packet, ipv6_packet, jpeg, json, jsonl, luajit, macho, macho_fat, markdown, matroska, mp3, mp3_frame, mp3_frame_vbri, mp3_frame_xing, mp4, mpeg_asc, mpeg_es, mpeg_pes, mpeg_pes_packet, mpeg_spu, mpeg_ts, msgpack, ogg, ogg_page, opus_packet, pcap, pcapng, pg_btree, pg_control, pg_heap, png, prores_frame, protobuf, protobuf_widevine, pssh_playready, rtmp, sll2_packet, sll_packet, tar, tcp_segment, tiff, tls, toml, tzif, udp_datagram, vorbis_comment, vorbis_packet, vp8_frame, vp9_cfm, vp9_frame, vpx_ccr, wasm, wav, webp, xml, yaml, zip
+.TP
+It can also work with some common text formats like URLs, hex, base64, PEM etc and for some serialization formats like XML, YAML, etc. it can transform both from and to jq values.
+.TP
+Use \fBfq\fR -h formats to get details about each supported file format
+.SH CONFIGURATION
+Custom functions can be defined in an init.fq which is searched for in the following locations -
+.P
+macOS - $HOME/Library/Application Support/fq/init.jq
+.P
+Linux, BSD, etc - $HOME/.config/fq/init.jq
+.P
+Windows - %AppData%\fq\init.jq
+.SS Unicode output
+Defining the environment variable \fICLIUNICODE\fR will enable the use of unicode characters for improved output
+.SH THE FQ LANGUAGE
+\fBfq\fR is based on the \fBjq\fR language and for basic usage its syntax is similar to how object and array access looks in JavaScript or JSON path, .food[10] etc. but it can do much more and is a very expressive language. To get the most out of fq it's recommended to learn more about jq, this manual assumes some familiarity with the jq language.
+.SS Types Specific to fq
+\fBfq\fR has two additional types compared to jq, \fIdecode value\fR and \fIbinary\fR. In standard jq expressions they will in most cases behave as some standard jq type.
+.TP
+.B Decode Value
+
+This type is returned by decoders and it is used to represent parts of the decoded input. It can act as all standard jq types, object, array, number, string etc.
+
+Each decode value has an associated bit range in the input which can be accessed as a binary using \fBtobits\fR and \fBtobytes\fR.
+
+Each non-compound decode value has these properties:
+.RS
+An actual value - This is the decoded representation of the bits, a number, string, bool etc. Can be accessed using \fBtoactual\fR.
+
+An optional symbolic value - Is usually a mapping of the actual to symbolic value, ex: map number to a string value. Can be accessed using \fBtosym\fR.
+
+An optional description - Can be accessed using \fBtodescription\fR
+
+\fBparent\fR is the parent decode value
+
+\fBparents\fR is the all parent decode values
+
+\fBtopath\fR is the jq path for the decode value
+
+\fBtorepr\fR convert decode value to its representation if possible
+
+The value of a decode value is the symbolic value if available and otherwise the actual value. To explicitly access the value use tovalue. In most expressions this is not needed as it will be done automatically.
+.RE
+.TP
+.B Binary
+
+Binaries are raw bits with a unit size, 1 (bits) or 8 (bytes), that can have a non-byte aligned size. Will act as byte padded strings in standard jq expressions.
+
+Use \fBtobits\fR and \fBtobytes\fR to create them from \fIdecode values\fR, \fIstrings\fR, \fInumbers\fR or \fIbinary arrays\fR. \fBtobytes\fR will, if needed zero pad most significant bits to be byte aligned.
+
+There is also \fBtobitsrange\fR and \fBtobytesrange\fR which does the same thing but will preserve its source range when displayed. \" TODO: tobytesrange, padding
+.TP
+.B Binary Array
+
+Binary arrays are arrays of \fInumbers\fR, \fIstrings\fR, \fIbinaries\fR or other nested \fIbinary arrays\fR. When used as input to \fBtobits\fR/\fBtobytes\fR the following rules are used:
+
+\fBNumber\fR is a byte with value be 0-255
+\fBString\fR it's UTF8 codepoint bytes
+\fBBinary\fR as is
+\fBBinary array\fR used recursively
+
+Binary arrays are similar to and inspired by Erlang iolist.
+.SS Functions
+All decode functions are available in two forms, just \fI<format>\fR (like mp3) that returns a decode value on error and \fIfrom_<format>\fR which throws error on decode error.
+
+Note that jq sometimes uses the notation name/0, name/1 etc in error messages and documentation which means \fI<function-name>\fR/\fI<arity>\fR, Same function names with different arity are treated as separate functions, but are usually related in some way in practice.
+.TP
+.B
+Functions Added in fq
+
+All standard library functions from jq
+
+A few new general functions:
+.RS
+.RS
+\fBdisplay\fR/\fBd\fR displays a value
+
+\fBprint\fR, \fBprintln\fR, \fBprinterr\fR, \fBprinterrln\fR prints to stdout and stderr.
+
+\fBgroup\fR group values, same as \fBgroup_by(.)\fR.
+
+\fBstreaks\fR, \fBstreaks_by(f)\fR like \fBgroup\fR but groups streaks based on condition.
+
+\fBcount\fR, \fBcount_by(f)\fR like \fBgroup\fR but counts groups lengths.
+
+\fBdebug(f)\fR like \fBdebug\fR but uses \fIarg\fR to produce a debug message.
+
+\fBpath_to_expr\fR convert \fI["key", 1]\fR to \fI".key[1]\fR".
+
+\fBexpr_to_path\fR from \fI".key[1]"\fR to \fI["key", 1]\fR.
+
+\fBdiff($a; $b)\fR produce diff object between two values.
+
+\fBdelta\fR, \fBdelta_by(f)\fR, array with difference between all consecutive pairs.
+
+\fBchunk(f)\fR, split array or string into even chunks
+
+.RE
+\fBdisplay\fR is the main function for displaying values and is also the function that will be used if no other output function is explicitly used. If its input is a decode value it will output a dump and tree structure or otherwise it will output as JSON.
+\" There are some examples with images in usage.md. TODO: Find a way to accomodate them in the man page
+Bitwise functions \fBband\fR, \fBbor\fR, \fBbxor\fR, \fBbsl\fR, \fBbsr\fR and \fBbnot\fR. Works the same as jq math functions, unary uses input and if more than one argument all as arguments ignoring the input.
+
+Some decode value specific functions:
+.RS
+\fBroot\fR tree root for value
+
+\fBbuffer_root\fR root value of buffer for value
+
+\fBformat_root\fR root value of format for value
+
+\fBparent\fR parent value
+
+\fBparents\fR output parents of value
+
+\fBtopath\fR path of value. Use \fBpath_to_expr\fR to get a string representation.
+
+\fBtovalue\fR, \fBtovalue($opts)\fR symbolic value if available otherwise actual value
+
+\fBtoactual\fR, \fBtoactual($opts)\fR actual value (usually the decoded value)
+
+\fBtosym\fR, \fBtosym($opts)\fR symbolic value (mapped etc)
+
+\fBtodescription\fR description of value
+
+\fBtorepr\fR converts decode value into what it represents. For example convert msgpack decode value into a value representing its JSON representation.
+
+All regexp functions work with binary as input and pattern argument with these differences compared to when using string input:
+.RS
+All offset and length will be in bytes.
+
+For \fBcapture\fR the \fB.string\fR value is a binary.
+
+If pattern is a binary it will be matched literally and not as a regexp.
+
+If pattern is a binary or flags include "b" each input byte will be read as separate code points
+
+.RE
+String functions are not overloaded to support binary for now as some of them might have behaviors that might be confusing.
+
+\fBexplode\fR is overloaded to work with binary. Will explode into array of the unit of the binary. end of binary. instead of possibly multi-byte UTF-8 codepoints. This allows to match raw bytes. Ex: \fBmatch("\u00ff"; "b")\fR will match the byte 0xff and not the UTF-8 encoded codepoint for 255, \fBmatch("[^\u00ff]"; "b")\fR will match all non-0xff bytes.
+
+\fBgrep\fR functions take 1 or 2 arguments. First is a scalar to match, where a string is treated as a regexp. A binary will match exact bytes. Second argument are regexp flags with addition that "b" will treat each byte in the input binary as a code point, this makes it possible to match exact bytes.
+.RS
+\fBgrep($v)\fR, \fBgrep($v; $flags)\fR recursively match value and binary
+
+\fBvgrep($v)\fR, \fBvgrep($v; $flags)\fR recursively match value
+
+\fBbgrep($v)\fR, \fBbgrep($v; $flags)\fR recursively match binary
+
+\fBfgrep($v)\fR, \fBfgrep($v; $flags)\fR recursively match field name
+
+\fBgrep_by(f)\fR recursively match using a filter. Ex: \fBgrep_by(. > 180 and . < 200)\fR equivalent to \fBfirst(grep_by(format == "id3v2"))\fR.
+
+.RE
+Binary:
+.RS
+\fBtobits\fR - Transform input to binary with bit as unit, does not preserve source range, will start at zero.
+
+\fBtobitsrange\fR - Transform input to binary with bit as unit, preserves source range if possible.
+
+\fBtobytes\fR - Transform input to binary with byte as unit, does not preserve source range, will start at zero.
+
+\fBtobytesrange\fR - Transform input binary with byte as unit, preserves source range if possible.
+
+\fB.[start:end]\fR, \fB.[:end]\fR, \fB.[start:]\fR - Slice binary from start to end preserve source range.
+
+.RE
+\fBopen\fR open file for reading
+
+All decode functions take an optional option argument. The only option currently is \fBforce\fR to ignore decoder asserts. For example to decode as mp3 and ignore assets do \fBmp3({force: true})\fR or \fBdecode("mp3"; {force: true})\fR, from command line you currently have to do \fBfq -d bytes 'mp3({force: true})\fR' file.
+
+\fBdecode\fR, \fBdecode("<format>")\fR, \fBdecode("<format>"; $opts)\fR decode format
+
+\fBprobe\fR, \fBprobe($opts)\fR probe and decode format
+
+\fBmp3\fR, \fBmp3($opts)\fR, ..., \fB<format>\fR, \fB<format>($opts)\fR same as \fBdecode("<format>")\fR, \fBdecode("<format>"; $opts)\fR decode as format and return decode value even on decode error.
+
+\fBfrom_mp3\fR, \fBfrom_mp3($opts)\fR, ..., \fBfrom_<format>\fR, \fBfrom_<format>($opts)\fR same as \fBdecode("<format>")\fR, \fBdecode("<format>"; $opts)\fR decode as format but throw error on decode error.
+
+Display shows hexdump/ASCII/tree for decode values and jq value for other types.
+.RS
+\fBd\fR/\fBd($opts)\fR display value and truncate long arrays and binaries
+
+\fBda\fR/\fBda($opts)\fR display value and don't truncate arrays
+
+\fBdd\fR/\fBdd($opts)\fR display value and don't truncate arrays or binaries
+
+\fBdv\fR/\fBdv($opts)\fR verbosely display value and don't truncate arrays but truncate binaries
+
+\fBddv\fR/\fBddv($opts)\fR verbosely display value and don't truncate arrays or binaries
+
+.RE
+\fBhd\fR/\fBhexdump\fR hexdump value
+
+\fBrepl\fR/\fBrepl($opts)\fR nested REPL, must be last in a pipeline. \fB1 | repl\fR, can "slurp" outputs. Ex: \fB1, 2, 3 | repl, [1,2,3] | repl({compact: true})\fR.
+
+\fBslurp("<name>")\fR slurp outputs and save them to \fBRname\fR, must be last in the pipeline. Will be available as a global array \fB$name\fR. Ex \fB1,2,3 | slurp("a"), $a[]\fR same as \fBspew("a")\fR.
+
+\fBspew\fR/\fBspew("<name>")\fR output previously slurped values. \fBspew\fR outputs all slurps as an object, \fBspew("<name>")\fR outputs one slurp. Ex: \fBspew("a")\fR.
+
+\fBpaste\fR read string from stdin until EOF. Useful for pasting text.
+.RS
+Ex: \fBpaste | from_pem | asn1_ber | repl\fR read from stdin then decode and start a new sub-REPL with result.
+.RE
+.RE
+.RE
+
+.B Naming Inconsistencies
+jq's naming conversion is a bit inconsistent, some standard library functions are named tojson while others from_entries. fq follows this tradition a bit by but tries to use snake_case unless there is a good reason.
+
+Here are all the non-snake_case functions added by \fBfq\fR. Most of them deal with decode and binary values which are new "primitive" types:
+.RS
+\fBtoactual\fR
+
+\fBtobits\fR
+
+\fBtobitsrange\fR
+
+\fBtobytes\fR
+
+\fBtobytesrange\fR
+
+\fBtodescription\fR
+
+\fBtopath\fR
+
+\fBtorepr\fR
+
+\fBtosym\fR
+
+\fBtovalue\fR
+
+.RE
+.SH ENCODINGS, SERIALIZATION, & HASHES
+In addition to binary formats \fBfq\fR also support reading to and from encodings and serialization formats.
+
+At the moment \fBfq\fR does not have any dedicated argument for serialization formats but raw string input -R slurp -s and raw string output -r can make things easier. The combination -Rs will read all inputs into one string (same as jq).
+
+Note that from* functions output \fBjq\fR values and to* takes \fBjq\fR values as input so in some cases not all information will be properly preserved. For example, for the element and attribute order might change and text and comment nodes might move or be merged. \fByq\fR might be a better tool if that is needed.
+.SH USE AS SCRIPT INTERPRETER
+\fBfq\fR can be used as a script interpreter using a shebang
+.SH EXAMPLES
+.TP
+\fBfq\fR '.frames[1].header | tovalue' file.mp3
+Get the second mp3 frame header as JSON
+.TP
+\fBfq\fR '.frames[0:10] | map(tobytesrange.start)' file.mp3
+Get the byte start position for the first 10 mp3 frames in an array
+.TP
+\fBfq\fR -d bytes '.[100:] | mp3_frame | d' file.mp3
+Decode byte range 100 to end as mp3_frame
+.TP
+\fBfq\fR '.somefield | tobytesrange[10:] | mp3_frame | d' file.mp3
+
+decode byte range 10 bytes from .somefield and preserve relative position in file
+.TP
+
+\fBfq\fR 'def f: .. | select(format=="avc_sps"); diff(input|f; input|f)' a.mp4 b.mp4
+
+Show AVC SPS difference between two mp4 files. \fB-n\fR tells \fBfq\fR to not have an implicit input, \fBf\fR is a function to select out some interesting value, call \fBdiff\fR with two arguments, decoded value for a.mp4 and b.mp4 filtered thru \fBf\fR.
+.TP
+
+\fBfq\fR 'first(.. | select(format=="jpeg")) | tobytes' file > file.jpeg
+
+Recursively look for the first value that is a jpeg decode value root. Use tobytes to get bytes for value. Redirect bytes to a file.
+.TP
+
+\fBfq\fR '.. | select(.type=="stsz")? as $stsz | .entries | count | max_by(.[1])[1] as $m | ($stsz | topath | path_to_expr), (.[] | "\(.[0]): \((100*.[1]/$m)*"=") \(.[1])") | println' file.mp4
+
+Recursively look for a all sample size boxes "stsz" and use ? to ignore errors when doing .type on arrays etc. Save reference to box, count unique values, save the max, output the path to the box and output a histogram scaled to 0-100.
+.TP
+
+\fBfq\fR '.tcp_connections | grep("GET /.* HTTP/1.?")' file.pcap
+
+Find TCP streams that looks like HTTP GET requests in a PCAP file. Use \fBgrep\fR to recursively find strings matching a regexp.
+.TP
+
+\fBfq\fR -rn '[inputs | [input_filename, first(.chunks[] | select(.type=="IHDR") | .width)]] | max_by(.[1]) | .[0]' *.png
+
+Find the widest PNG in a directory
+.TP
+
+\fBfq\fR '.. | select(scalars and in_bytes_range(0x123))' file
+
+Which values include the bytes at position 0x123 \" TODO: Check if this is a typo
+
+.SH BUGS (+ TRICKS)
+\fBselect\fR fails with \fBexpected an ... but got: ...\fR
+.RS
+Try adding \fBselect(...)?\fR to catch and ignore type errors in the select expression.
+.RE
+
+Run interactive mode with no input
+.RS
+fq -i
+.RE
+
+Manual Decode - Sometimes fq fails to decode or you know there is valid data buried inside some binary or maybe you know the format of some gap field. Then you can decode manually:
+.RS
+# try decode a `mp3_frame` that failed to decode
+
+$ fq -d mp3 '.gap0 | mp3_frame' file.mp3
+
+# skip first 10 bytes then decode as `mp3_frame`
+
+$ fq -d bytes '.[10:] | mp3_frame' file.mp3
+.RE
+Use . as input and in a positional argument
+
+.RS
+The expression \fB.a | f(.b)\fR might not work as expected. \fB.\fR is \fB.a\fR when evaluating the arguments so the positional argument will end up being \fB.a.b\fR. Instead do \fB. as $c | .a | f($c.b)\fR.
+.RE
+
+Building array is slow
+.RS
+Try to use \fBmap\fR or \fBforeach\fR to avoid rebuilding the whole array for each append.
+.RE
+
+Use print and println to produce more friendly compact output
+
+repl argument using function or variable causes variable not defined
+.RS
+\fBtrue as $verbose | repl({verbose: $verbose})\fR will currently fail as repl is implemented by rewriting the query to \fBmap(true as $verbose | .) | repl({verbose: $verbose})\fR.
+.RE
+
+\fBerror\fR produces no output
+.RS
+\fBnull | error\fR behaves as empty
+.RE
+
+.SH AUTHOR
+\fBfq\fR is written by Mattias Wadman
+
+.SH SEE ALSO
+\fBjq\fR - lightweight and flexible command-line JSON processor
+
+\fByq\fR - lightweight and portable command-line YAML, JSON and XML processor

doc/usage.md

@@ -406,8 +406,6 @@ There is also `tobitsrange` and `tobytesrange` which does the same thing but wil
 
 Both `.[index]` and `.[start:end]` support negative indices to index from end.
 
-TODO: tobytesrange, padding
-
 #### Binary array
 
 Is an array of numbers, strings, binaries or other nested binary arrays. When used as input to `tobits`/`tobytes` the following rules are used:
@@ -428,92 +426,175 @@ Some examples:
 
 `[(.a | tobytes[-10:]), 255, (.b | tobits[:10])] | tobytes` the concatenation of the last 10 bytes of `.a`, a byte with value 255 and the first 10 bits of `.b`.
 
-The difference between `tobits` and `tobytes` is
-
-TODO: padding and alignment
-
 ## Functions
 
-All decode functions are available in two forms, just `<format>` (like `mp3`) that returns a decode value on error and `from_<format>` which throws error on decode error.
+fq has all the same standard library jq functions and in addition some new ones.
+
+### Additional generic functions
+
+#### `grep_by(f)`
+Recursively select using a filter. Ex: `grep_by(. > 180 and . < 200)`, `first(grep_by(format == "id3v2"))`.
+
+#### `group`
+Group values, same as `group_by(.)`.
+
+#### `streaks`, `streaks_by(f)`
+Like `group` but groups streaks based on condition.
+
+#### `count`, `count_by(f)`
+Like `group` but counts groups lengths based on condition.
+
+#### `delta`, `delta_by(f)`
+Array with difference between consecutive. `delta` is same as `delta_by(.b - .a)`.
+
+#### `chunk($size)`
+Split array or string into `$size` length chunks. Last chunk might be shorter.
+
+#### `path_to_expr`
+Converts a path value `["key", 1]` to a string `".key[1]"`.
+
+#### `expr_to_path`
+Converts from a string `".key[1]"` to path value `["key", 1]`.
+
+#### `diff($a; $b)`
+Produce a diff between `$a` and `$b`. Differences are represented as a object `{a: <value from a>, b: <value from b>}`.
+
+#### `band`, `bor`, `bxor`, `bsl`, `bsr`, `bnot`.
+Bitwise functions. Works the same as jq math functions. Functions with no arguments like `1 | bnot` uses only input, functions with more than one argument ignores input, `bsl(1; 3)`.
+
+#### `repl`/`repl($opts)`
+Nested REPL. Must be last in a pipeline. `1 | repl`, can "slurp" outputs. Ex: `1, 2, 3 | repl`, `[1,2,3] | repl({compact: true})`.
+
+#### `slurp("<name>")`
+Slurp outputs and save them to `$name`. Must be last in the pipeline. Will be available as a global array `$name`. Ex `1,2,3 | slurp("a")`, `$a[]` same as `spew("a")`.
+
+#### `spew`/`spew("<name>")`
+Output previously slurped values.
+
+#### `spew`
+Outputs all slurps as an object. `spew("<name>")` outputs one slurp. Ex: `spew("a")`.
+
+#### `paste`
+Read string from stdin until ^D. Useful for pasting text. Ex: `paste | from_pem | asn1_ber | repl` read from stdin then decode and start a new sub-REPL with result.
+
+### Format decode functions
+
+Format decode functions are available in two forms, just `mp3` or `mp3($opts)` that returns a decode value even on error and `from_mp3` or `from_mp3($opts)` which throws error on decode error.
+
+The the only general format option currently is `force` to ignore decoder asserts.
+For example to decode as mp3 and ignore assets do `mp3({force: true})` or `decode("mp3"; {force: true})`. From command line you can either do `fq -d mp3 -o force=true . file.mp3` or `fq -d bytes 'mp3({force: true})' file.mp3`.
+
+Some formats has own options that can be specificed as part of `$opts` or as `-o name=value`. Too see options for a format do `fq -h mp3` or `help(mp3)` in a REPL. From command line you can either do `fq -d mp3 -o max_sync_seek=100 . file.mp3` or `fq -d bytes 'mp3({max_sync_seek: 100})' file.mp3`.
+
+#### `decode`, `decode("<format>")`, `decode("<format>"; $opts)`
+Decode format.
+
+#### `probe`, `probe($opts)`
+Probe and decode format.
+
+#### `<format>`, `<format>($opts)`
+Same as `decode("<format>")` and `decode("<format>"; $opts)`. Decode as format and return decode value even on decode error.
+
+#### `from_<format>`, `from_<format>($opts)`
+Same as `decode("<format>")` and `decode("<format>"; $opts)` decode as format but throw error on decode error.
 
 Note that jq sometimes uses the notation `name/0`, `name/1` etc in error messages and documentation which means `<function-name>/<arity>`. Same function names with different arity are treated as separate functions, but are usually related in some way in practice.
 
-### Function added in fq
+#### `print`, `println`, `printerr`, `printerrln`
+Print string or if not a string compact JSON value to stdout or stderr.
 
-- All standard library functions from jq
-- Adds a few new general functions:
-  - `print`, `println`, `printerr`, `printerrln` prints to stdout and stderr.
-  - `group` group values, same as `group_by(.)`.
-  - `streaks`, `streaks_by(f)` like `group` but groups streaks based on condition.
-  - `count`, `count_by(f)` like `group` but counts groups lengths.
-  - `debug(f)` like `debug` but uses arg to produce a debug message. `{a: 123} | debug({a}) | ...`.
-  - `path_to_expr` from `["key", 1]` to `".key[1]"`.
-  - `expr_to_path` from `".key[1]"` to `["key", 1]`.
-  - `diff($a; $b)` produce diff object between two values.
-  - `delta`, `delta_by(f)`, array with difference between all consecutive pairs.
-  - `chunk(f)`, split array or string into even chunks
-- Bitwise functions `band`, `bor`, `bxor`, `bsl`, `bsr` and `bnot`. Works the same as jq math functions,
-unary uses input and if more than one argument all as arguments ignoring the input. Ex: `1 | bnot` `bsl(1; 3)`
-- Adds some decode value specific functions:
-  - `root` tree root for value
-  - `buffer_root` root value of buffer for value
-  - `format_root` root value of format for value
-  - `parent` parent value
-  - `parents` output parents of value
-  - `topath` path of value. Use `path_to_expr` to get a string representation.
-  - `tovalue`, `tovalue($opts)` symbolic value if available otherwise actual value
-  - `toactual`, `toactual($opts)` actual value (usually the decoded value)
-  - `tosym`, `tosym($opts)` symbolic value (mapped etc)
-  - `todescription` description of value
-  - `torepr` converts decode value into what it represents. For example convert msgpack decode value
-  into a value representing its JSON representation.
-  - All regexp functions work with binary as input and pattern argument with these differences
-  compared to when using string input:
-    - All offset and length will be in bytes.
-    - For `capture` the `.string` value is a binary.
-    - If pattern is a binary it will be matched literally and not as a regexp.
-    - If pattern is a binary or flags include "b" each input byte will be read as separate code points
-  - String functions are not overloaded to support binary for now as some of them might have behaviors that might be confusing.
-  - `explode` is overloaded to work with binary. Will explode into array of the unit of the binary.
-  end of binary.
-  instead of possibly multi-byte UTF-8 codepoints. This allows to match raw bytes. Ex: `match("\u00ff"; "b")`
-  will match the byte `0xff` and not the UTF-8 encoded codepoint for 255, `match("[^\u00ff]"; "b")` will match
-  all non-`0xff` bytes.
-  - `grep` functions take 1 or 2 arguments. First is a scalar to match, where a string is
-  treated as a regexp. A binary will match exact bytes. Second argument are regexp
-  flags with addition that "b" will treat each byte in the input binary as a code point, this
-  makes it possible to match exact bytes.
-    - `grep($v)`, `grep($v; $flags)` recursively match value and binary
-    - `vgrep($v)`, `vgrep($v; $flags)` recursively match value
-    - `bgrep($v)`, `bgrep($v; $flags)` recursively match binary
-    - `fgrep($v)`, `fgrep($v; $flags)` recursively match field name
-  - `grep_by(f)` recursively match using a filter. Ex: `grep_by(. > 180 and . < 200)`, `first(grep_by(format == "id3v2"))`.
-  - Binary:
-    - `tobits` - Transform input to binary with bit as unit, does not preserve source range, will start at zero.
-    - `tobitsrange` - Transform input to binary with bit as unit, preserves source range if possible.
-    - `tobytes` - Transform input to binary with byte as unit, does not preserve source range, will start at zero.
-    - `tobytesrange` - Transform input binary with byte as unit, preserves source range if possible.
-    - `.[start:end]`, `.[:end]`, `.[start:]` - Slice binary from start to end preserve source range.
-- `open` open file for reading
-- All decode functions take an optional option argument. The only option currently is `force` to ignore decoder asserts.
-For example to decode as mp3 and ignore assets do `mp3({force: true})` or `decode("mp3"; {force: true})`, from command line
-you currently have to do `fq -d bytes 'mp3({force: true})' file`.
-- `decode`, `decode("<format>")`, `decode("<format>"; $opts)` decode format
-- `probe`, `probe($opts)` probe and decode format
-- `mp3`, `mp3($opts)`, ..., `<format>`, `<format>($opts)` same as `decode("<format>")`, `decode("<format>"; $opts)` decode as format and return decode value even on decode error.
-- `from_mp3`, `from_mp3($opts)`, ..., `from_<format>`, `from_<format>($opts)` same as `decode("<format>")`, `decode("<format>"; $opts)` decode as format but throw error on decode error.
-- Display shows hexdump/ASCII/tree for decode values and jq value for other types.
-  - `d`/`d($opts)` display value and truncate long arrays and binaries
-  - `da`/`da($opts)` display value and don't truncate arrays
-  - `dd`/`dd($opts)` display value and don't truncate arrays or binaries
-  - `dv`/`dv($opts)` verbosely display value and don't truncate arrays but truncate binaries
-  - `ddv`/`ddv($opts)` verbosely display value and don't truncate arrays or binaries
-- `hd`/`hexdump` hexdump value
-- `repl`/`repl($opts)` nested REPL, must be last in a pipeline. `1 | repl`, can "slurp" outputs. Ex: `1, 2, 3 | repl`, `[1,2,3] | repl({compact: true})`.
-- `slurp("<name>")` slurp outputs and save them to `$name`, must be last in the pipeline. Will be available as a global array `$name`. Ex `1,2,3 | slurp("a")`, `$a[]` same as `spew("a")`.
-- `spew`/`spew("<name>")` output previously slurped values. `spew` outputs all slurps as an object, `spew("<name>")` outputs one slurp. Ex: `spew("a")`.
-- `paste` read string from stdin until ^D. Useful for pasting text.
-    - Ex: `paste | from_pem | asn1_ber | repl` read from stdin then decode and start a new sub-REPL with result.
+#### `root`
+Root decode value for decode value.
+
+#### `buffer_root`
+Root decode value of buffer for decode value.
+
+#### `format_root`
+Root decode value of format for decode value.
+
+#### `parent`
+Parent decode value for decode value.
+
+#### `parents`
+Outputs all parent decode values from decode value.
+
+#### `topath`
+Path for decode value. Use `path_to_expr` to get a string representation.
+
+#### `tovalue`, `tovalue($opts)`
+Symbolic, if available, or actual value for decode value.
+
+#### `toactual`, `toactual($opts)`
+Actual value for decode value.
+
+#### `tosym`, `tosym($opts)`
+Symbolic value for decode value.
+
+#### `todescription`
+Description for decode value.
+
+#### `torepr`
+Converts decode value into what it represents. For example converts msgpack decode value into a value representing its JSON representation.
+
+### Display functions
+
+Display shows hexdump, ASCII and tree column dump for decode values and jq value for other types.
+
+#### `d`/`d($opts)`
+display value and truncate long arrays and binaries.
+
+#### `da`/`da($opts)`
+Display value and don't truncate arrays.
+
+#### `dd`/`dd($opts)`
+Display value and don't truncate arrays or binaries.
+
+#### `dv`/`dv($opts)`
+Verbosely display value and don't truncate arrays but truncate binaries.
+
+#### `ddv`/`ddv($opts)`
+Verbosely display value and don't truncate arrays or binaries.
+
+#### `hd`/`hexdump`
+Hexdump value.
+
+### Binary values
+
+Binary values represents raw bits or bytes. When used in standard jq expressions they will behave as strings (UTF-8) with some exceptions listed below.
+
+- All regexp functions work with binary as input and pattern argument with these differences
+compared to when using string input:
+  - All offset and length will be in bytes.
+  - For `capture` the `.string` value is a binary.
+  - If pattern is a binary it will be matched literally and not as a regexp.
+  - If pattern is a binary or flags include "b" each input byte will be read as separate code points
+- `explode` is overloaded to work with binary. Will explode into array of the unit of the binary.
+- `.[start:end]`, `.[:end]`, `.[start:]` - Slice binary from start to end preserve source range.
+
+#### `grep($v)`, `grep($v; $flags)`, `vgrep($v)`, `vgrep($v; $flags)`, `bgrep($v)`, `bgrep($v; $flags)`
+Recursively match `$v`.
+
+`$v` is a scalar to match, where a string is treated as a regexp. A binary will match exact bytes.
+`$flags` argument are regexp flags with additional flag "b" that will treat each byte in the input binary
+as a code point. This makes it possible to match exact bytes.
+
+#### `fgrep($v)`, `fgrep($v; $flags)`
+Recursively match field name in for decode value.
+
+#### `tobits`
+Transform input to binary with bit as unit and don't preserve source range.
+
+#### `tobitsrange`
+Transform input to binary with bit as unit and preserve source range.
+
+#### `tobytes`
+Transform input to binary with byte as unit and don't preserve source range.
+
+#### `tobytesrange`
+Transform input to binary with byte as unit and preserve source range.
+
+#### `open`
+Open file for reading.
 
 ### Naming inconsistencies
 

go.mod

@@ -9,7 +9,7 @@ require (
 	// bump: gomod-BurntSushi/toml /github\.com\/BurntSushi\/toml v(.*)/ https://github.com/BurntSushi/toml.git|^1
 	// bump: gomod-BurntSushi/toml command go get -d github.com/BurntSushi/toml@v$LATEST && go mod tidy
 	// bump: gomod-BurntSushi/toml link "Source diff $CURRENT..$LATEST" https://github.com/BurntSushi/toml/compare/v$CURRENT..v$LATEST
-	github.com/BurntSushi/toml v1.3.2
+	github.com/BurntSushi/toml v1.4.0
 
 	// bump: gomod-creasty-defaults /github\.com\/creasty\/defaults v(.*)/ https://github.com/creasty/defaults.git|^1
 	// bump: gomod-creasty-defaults command go get -d github.com/creasty/defaults@v$LATEST && go mod tidy
@@ -48,7 +48,7 @@ require (
 	// bump: gomod-golang-x-crypto /golang\.org\/x\/crypto v(.*)/ https://github.com/golang/crypto.git|^0
 	// bump: gomod-golang-x-crypto command go get -d golang.org/x/crypto@v$LATEST && go mod tidy
 	// bump: gomod-golang-x-crypto link "Tags" https://github.com/golang/crypto/tags
-	golang.org/x/crypto v0.23.0
+	golang.org/x/crypto v0.24.0
 
 	// has no tags
 	// go get -d golang.org/x/exp@master && go mod tidy
@@ -57,17 +57,17 @@ require (
 	// bump: gomod-golang-x-net /golang\.org\/x\/net v(.*)/ https://github.com/golang/net.git|^0
 	// bump: gomod-golang-x-net command go get -d golang.org/x/net@v$LATEST && go mod tidy
 	// bump: gomod-golang-x-net link "Tags" https://github.com/golang/net/tags
-	golang.org/x/net v0.25.0
+	golang.org/x/net v0.26.0
 
 	// bump: gomod-golang-x-term /golang\.org\/x\/term v(.*)/ https://github.com/golang/term.git|^0
 	// bump: gomod-golang-x-term command go get -d golang.org/x/term@v$LATEST && go mod tidy
 	// bump: gomod-golang-x-term link "Tags" https://github.com/golang/term/tags
-	golang.org/x/term v0.20.0
+	golang.org/x/term v0.21.0
 
 	// bump: gomod-golang/text /golang\.org\/x\/text v(.*)/ https://github.com/golang/text.git|^0
 	// bump: gomod-golang/text command go get -d golang.org/x/text@v$LATEST && go mod tidy
 	// bump: gomod-golang/text link "Source diff $CURRENT..$LATEST" https://github.com/golang/text/compare/v$CURRENT..v$LATEST
-	golang.org/x/text v0.15.0
+	golang.org/x/text v0.16.0
 
 	// bump: gomod-gopkg.in/yaml.v3 /gopkg\.in\/yaml\.v3 v(.*)/ https://github.com/go-yaml/yaml.git|^3
 	// bump: gomod-gopkg.in/yaml.v3 command go get -d gopkg.in/yaml.v3@v$LATEST && go mod tidy
@@ -79,6 +79,6 @@ require (
 	github.com/itchyny/timefmt-go v0.1.5 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
-	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 )

go.sum

@@ -1,5 +1,5 @@
-github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
-github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
+github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA=
 github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
 github.com/ergochat/readline v0.1.1 h1:C8Uuo3ybB23GWOt0uxmHbGzKM9owmtXary6Clrj84s0=
@@ -25,18 +25,18 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/wader/gojq v0.12.1-0.20240401131232-6c6bc364201a h1:P881Oecjt9FEXrwkGJ6UObJksxejJaF/fKq1ZfXpiVE=
 github.com/wader/gojq v0.12.1-0.20240401131232-6c6bc364201a/go.mod h1:qVrzkUdnBtJvM4twyRQ6xdziPSnSp35dLm4s/DN2iP4=
-golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
+golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
 golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw=
 golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ=
-golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
-golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
-golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
-golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
+golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
+golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=