Merge e9226226d3 into 2bc6c768c7

Update make-golangci-lint to 1.59.1 from 1.59.0

.github/workflows/ci.yml

@@ -7,7 +7,7 @@ on:
   pull_request:
 
 env:
-  GOLANGCILINT_VERSION: "1.58.2"
+  GOLANGCILINT_VERSION: "1.59.1"
 
 jobs:
   lint:
@@ -15,7 +15,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: "1.22.3"
+          go-version: "1.22.4"
       - uses: actions/checkout@v3
       - uses: golangci/golangci-lint-action@v3
         with:
@@ -47,7 +47,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: "1.22.3"
+          go-version: "1.22.4"
       - name: Test
         env:
           GOARCH: ${{ matrix.goarch }}

.github/workflows/release.yml

@@ -15,7 +15,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: "1.22.3"
+          go-version: "1.22.4"
       - uses: goreleaser/goreleaser-action@v5
         with:
           distribution: goreleaser

Dockerfile

@@ -1,5 +1,5 @@
 # bump: docker-golang /FROM golang:([\d.]+)/ docker:golang|^1
-FROM golang:1.22.3-bookworm AS base
+FROM golang:1.22.4-bookworm AS base
 
 # expect is used to test cli
 RUN \

Makefile

@@ -61,7 +61,7 @@ gogenerate: always
 lint: always
 # bump: make-golangci-lint /golangci-lint@v([\d.]+)/ git:https://github.com/golangci/golangci-lint.git|^1
 # bump: make-golangci-lint link "Release notes" https://github.com/golangci/golangci-lint/releases/tag/v$LATEST
-	go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.58.2 run
+	go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.59.1 run
 
 depgraph.svg: always
 	go run github.com/kisielk/godepgraph@latest github.com/wader/fq | dot -Tsvg -o godepgraph.svg

README.md

@@ -169,6 +169,7 @@ For details see [formats.md](doc/formats.md) and [usage.md](doc/usage.md).
 
 ## Presentations
 
+- [Kodsnack 585 - Polymorfisk JSON](https://kodsnack.se/585/) - Swedish podcast episode about jq and fq
 - "fq - jq for binary formats" at [FOSDEM 2023](https://fosdem.org/2023/) - [video & slides](https://fosdem.org/2023/schedule/event/bintools_fq/)
 - "fq - jq for binary formats" at [No time to wait 6](https://mediaarea.net/NoTimeToWait6) - [video](https://www.youtube.com/watch?v=-Pwt5KL-xRs&t=1450s) - [slides](doc/presentations/nttw6/fq-nttw6-slides.pdf)
 - "fq - jq for binary formats" at [Binary Tools Summit 2022](https://binary-tools.net/summit.html) - [video](https://www.youtube.com/watch?v=GJOq_b0eb-s&list=PLTj8twuHdQz-JcX7k6eOwyVPDB8CyfZc8&index=1) - [slides](doc/presentations/bts2022/fq-bts2022-v1.pdf)

doc/formats.jq

@@ -70,6 +70,7 @@ def formats_sections:
   | ({} | _help_format_enrich("fq"; $f; false)) as $fhelp
   | select(has_section($f; $fhelp))
   | "## \($f.name)"
+  , $f.description + "."
   , ""
   , ($fhelp.notes | if . then ., "" else empty end)
   , if $f.decode_in_arg then

doc/formats.md

@@ -157,6 +157,7 @@ fq -d bytes 'mp4({force: true})' file.mp4
 [fq -rn -L . 'include "formats"; formats_sections']: sh-start
 
 ## aac_frame
+Advanced Audio Coding frame.
 
 ### Options
 
@@ -177,6 +178,7 @@ Decode value as aac_frame
 ```
 
 ## apple_bookmark
+Apple BookmarkData.
 
 Apple's `bookmarkData` format is used to encode information that can be resolved
 into a `URL` object for a file even if the user moves or renames it. Can also
@@ -218,6 +220,7 @@ torepr' <sfl2 file>
 - https://michaellynn.github.io/2015/10/24/apples-bookmarkdata-exposed/
 
 ## asn1_ber
+ASN1 BER (basic encoding rules, also CER and DER).
 
 Supports decoding BER, CER and DER (X.690).
 
@@ -250,6 +253,7 @@ $ fq -d asn1_ber 'torepr as $r | ["version", "modulus", "private_exponent", "pri
 - https://lapo.it/asn1js/
 
 ## avc_au
+H.264/AVC Access Unit.
 
 ### Options
 
@@ -270,6 +274,7 @@ Decode value as avc_au
 ```
 
 ## avi
+Audio Video Interleaved.
 
 ### Options
 
@@ -318,6 +323,7 @@ $ fq -o decode_samples=false -o decode_extended_chunks=false d file.avi
 - [OpenDML AVI File Format Extensions](http://www.jmcgowan.com/odmlff2.pdf)
 
 ## avro_ocf
+Avro object container file.
 
 Supports reading Avro Object Container Format (OCF) files based on the 1.11.0 specification.
 
@@ -337,6 +343,7 @@ xentripetal@fastmail.com
 [@xentripetal](https://github.com/xentripetal)
 
 ## bencode
+BitTorrent bencoding.
 
 ### Convert represented value to JSON
 
@@ -348,6 +355,7 @@ $ fq -d bencode torepr file.torrent
 - https://wiki.theory.org/BitTorrentSpecification#Bencoding
 
 ## bitcoin_block
+Bitcoin block.
 
 ### Options
 
@@ -368,6 +376,7 @@ Decode value as bitcoin_block
 ```
 
 ## bits
+Raw bits.
 
 Decode to a slice and indexable binary of bits.
 
@@ -390,6 +399,7 @@ $ echo 'hello' | fq -c -d bits '[.[range(8)]]'
 ```
 
 ## bplist
+Apple Binary Property List.
 
 ### Show full decoding
 ```sh
@@ -454,6 +464,7 @@ bplist> from_ns_keyed_archiver(1)
 - https://opensource.apple.com/source/CF/CF-550/CFBinaryPList.c
 
 ## bson
+Binary JSON.
 
 ### Limitations
 
@@ -479,6 +490,7 @@ $ fq -d bson 'torepr | select(.name=="bob")' file.bson
 - https://bsonspec.org/spec.html
 
 ## bytes
+Raw bytes.
 
 Decode to a slice and indexable binary of bytes.
 
@@ -506,6 +518,7 @@ $ echo 'hello' | fq -d bytes '.[1]'
 ```
 
 ## caff
+Live2D Cubism archive.
 
 ### Options
 
@@ -529,6 +542,7 @@ Decode value as caff
 - [@ronsor](https://github.com/ronsor)
 
 ## cbor
+Concise Binary Object Representation.
 
 ### Convert represented value to JSON
 
@@ -541,6 +555,7 @@ $ fq -d cbor torepr file.cbor
 - https://www.rfc-editor.org/rfc/rfc8949.html
 
 ## csv
+Comma separated values.
 
 ### Options
 
@@ -574,6 +589,7 @@ $ fq -d csv '.[0] as $t | .[1:] | map(with_entries(.key = $t[.key]))' file.csv
 ```
 
 ## fit
+Garmin Flexible and Interoperable Data Transfer.
 
 ### Limitations
 
@@ -596,6 +612,7 @@ $ fq '[.data_records[] | select(.record_header.message_type == "data").data_mess
 - https://developer.garmin.com/fit/cookbook/decoding-activity-files/
 
 ## flac_frame
+FLAC frame.
 
 ### Options
 
@@ -616,6 +633,7 @@ Decode value as flac_frame
 ```
 
 ## hevc_au
+H.265/HEVC Access Unit.
 
 ### Options
 
@@ -636,6 +654,7 @@ Decode value as hevc_au
 ```
 
 ## html
+HyperText Markup Language.
 
 ### Options
 
@@ -719,6 +738,7 @@ $ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href'
 ```
 
 ## leveldb_descriptor
+LevelDB Descriptor.
 
 ### Limitations
 
@@ -735,6 +755,7 @@ $ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href'
 - https://github.com/google/leveldb/blob/main/db/version_edit.cc
 
 ## leveldb_log
+LevelDB Log.
 
 ### Limitations
 
@@ -751,6 +772,7 @@ $ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href'
 - https://github.com/google/leveldb/blob/main/db/write_batch.cc
 
 ## leveldb_table
+LevelDB Table.
 
 ### Limitations
 
@@ -768,6 +790,7 @@ $ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href'
 - https://github.com/google/leveldb/blob/main/doc/index.md
 
 ## luajit
+LuaJIT 2.0 bytecode.
 
 ### Authors
 - [@dlatchx](https://github.com/dlatchx)
@@ -777,6 +800,7 @@ $ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href'
 - http://scm.zoomquiet.top/data/20131216145900/index.html
 
 ## macho
+Mach-O macOS executable.
 
 Supports decoding vanilla and FAT Mach-O binaries.
 
@@ -795,12 +819,14 @@ acils@itu.edu.tr
 [@Akaame](https://github.com/Akaame)
 
 ## markdown
+Markdown.
 
 ### Array with all level 1 and 2 headers
 ```sh
 $ fq -d markdown '[.. | select(.type=="heading" and .level<=2)?.children[0]]' file.md
 ```
 ## matroska
+Matroska file.
 
 ### Options
 
@@ -840,11 +866,13 @@ $ fq 'grep_by(.id == "Tracks") | matroska_path' file.mkv
 - https://wiki.xiph.org/MatroskaOpus
 
 ## moc3
+MOC3 file.
 
 ### Authors
 - [@ronsor](https://github.com/ronsor)
 
 ## mp3
+MP3 file.
 
 ### Options
 
@@ -867,6 +895,7 @@ Decode value as mp3
 ```
 
 ## mp4
+ISOBMFF, QuickTime and similar.
 
 ### Options
 
@@ -932,6 +961,7 @@ $ fq 'grep_by(.type == "trak") | mp4_path' file.mp4
 - [Quicktime file format](https://developer.apple.com/standards/qtff-2001.pdf)
 
 ## msgpack
+MessagePack.
 
 ### Convert represented value to JSON
 
@@ -943,6 +973,7 @@ $ fq -d msgpack torepr file.msgpack
 - https://github.com/msgpack/msgpack/blob/master/spec.md
 
 ## nes
+iNES/NES 2.0 cartridge ROM format.
 
 ### Limitations
 
@@ -979,6 +1010,7 @@ $ for line in $(fq -r '.chr_rom[] | nes_tokitty(5)' file.nes);do printf "%b%s" "
 - https://bugzmanov.github.io/nes_ebook/chapter_6_3.html
 
 ## opentimestamps
+OpenTimestamps file.
 
 ### View a full OpenTimestamps file
 
@@ -1006,6 +1038,7 @@ $ fq '.operations | map(select(.attestation_type == "bitcoin")) | length > 0' fi
 - https://github.com/opentimestamps/python-opentimestamps
 
 ## pcap
+PCAP packet capture.
 
 ### Build object with number of (reassembled) TCP bytes sent to/from client IP
 ```sh
@@ -1018,6 +1051,7 @@ $ fq '.tcp_connections | group_by(.client.ip) | map({key: .[0].client.ip, value:
 }
 ```
 ## pg_btree
+PostgreSQL btree index file.
 
 ### Options
 
@@ -1057,6 +1091,7 @@ p.n.safonov@gmail.com
 ### References
 - https://www.postgresql.org/docs/current/storage-page-layout.html
 ## pg_control
+PostgreSQL control file.
 
 ### Options
 
@@ -1096,6 +1131,7 @@ p.n.safonov@gmail.com
 ### References
 - https://github.com/postgres/postgres/blob/REL_14_2/src/include/catalog/pg_control.h
 ## pg_heap
+PostgreSQL heap file.
 
 ### Options
 
@@ -1148,6 +1184,7 @@ p.n.safonov@gmail.com
 ### References
 - https://www.postgresql.org/docs/current/storage-page-layout.html
 ## protobuf
+Protobuf.
 
 ### Can decode sub messages
 
@@ -1159,6 +1196,7 @@ $ fq -d protobuf '.fields[6].wire_value | protobuf | d' file
 - https://developers.google.com/protocol-buffers/docs/encoding
 
 ## rtmp
+Real-Time Messaging Protocol.
 
 Current only supports plain RTMP (not RTMPT or encrypted variants etc) with AMF0 (not AMF3).
 
@@ -1172,6 +1210,7 @@ fq '.tcp_connections[] | select(.server.port=="rtmp") | d' file.cap
 - https://rtmp.veriskope.com/pdf/video_file_format_spec_v10.pdf
 
 ## tls
+Transport layer security.
 
 ### Options
 
@@ -1319,6 +1358,7 @@ $ fq -o keylog=@traffic.keylog  'first(grep_by(.server.stream | format == "tls")
 - [RFC 6101: The Secure Sockets Layer (SSL) Protocol Version 3.0](https://www.rfc-editor.org/rfc/rfc)
 
 ## tzif
+Time Zone Information Format.
 
 ### Get last transition time
 ```sh
@@ -1339,6 +1379,7 @@ fq '.v2plusdatablock.leap_second_records | length' tziffile
 - https://datatracker.ietf.org/doc/html/rfc8536
 
 ## wasm
+WebAssembly Binary Format.
 
 ### Count opcode usage
 ```sh
@@ -1359,6 +1400,7 @@ $ fq '.sections | {import: map(select(.id == "import_section").content.im.x[].nm
 - https://webassembly.github.io/spec/core/
 
 ## xml
+Extensible Markup Language.
 
 ### Options
 
@@ -1487,6 +1529,7 @@ $ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -o array=true '.[2][2
 - [xml.com's Converting Between XML and JSON](https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html)
 
 ## zip
+ZIP archive.
 
 ### Options
 

doc/usage.md

@@ -406,8 +406,6 @@ There is also `tobitsrange` and `tobytesrange` which does the same thing but wil
 
 Both `.[index]` and `.[start:end]` support negative indices to index from end.
 
-TODO: tobytesrange, padding
-
 #### Binary array
 
 Is an array of numbers, strings, binaries or other nested binary arrays. When used as input to `tobits`/`tobytes` the following rules are used:
@@ -428,92 +426,175 @@ Some examples:
 
 `[(.a | tobytes[-10:]), 255, (.b | tobits[:10])] | tobytes` the concatenation of the last 10 bytes of `.a`, a byte with value 255 and the first 10 bits of `.b`.
 
-The difference between `tobits` and `tobytes` is
-
-TODO: padding and alignment
-
 ## Functions
 
-All decode functions are available in two forms, just `<format>` (like `mp3`) that returns a decode value on error and `from_<format>` which throws error on decode error.
+fq has all the same standard library jq functions and in addition some new ones.
+
+### Additional generic functions
+
+#### `grep_by(f)`
+Recursively select using a filter. Ex: `grep_by(. > 180 and . < 200)`, `first(grep_by(format == "id3v2"))`.
+
+#### `group`
+Group values, same as `group_by(.)`.
+
+#### `streaks`, `streaks_by(f)`
+Like `group` but groups streaks based on condition.
+
+#### `count`, `count_by(f)`
+Like `group` but counts groups lengths based on condition.
+
+#### `delta`, `delta_by(f)`
+Array with difference between consecutive. `delta` is same as `delta_by(.b - .a)`.
+
+#### `chunk($size)`
+Split array or string into `$size` length chunks. Last chunk might be shorter.
+
+#### `path_to_expr`
+Converts a path value `["key", 1]` to a string `".key[1]"`.
+
+#### `expr_to_path`
+Converts from a string `".key[1]"` to path value `["key", 1]`.
+
+#### `diff($a; $b)`
+Produce a diff between `$a` and `$b`. Differences are represented as a object `{a: <value from a>, b: <value from b>}`.
+
+#### `band`, `bor`, `bxor`, `bsl`, `bsr`, `bnot`.
+Bitwise functions. Works the same as jq math functions. Functions with no arguments like `1 | bnot` uses only input, functions with more than one argument ignores input, `bsl(1; 3)`.
+
+#### `repl`/`repl($opts)`
+Nested REPL. Must be last in a pipeline. `1 | repl`, can "slurp" outputs. Ex: `1, 2, 3 | repl`, `[1,2,3] | repl({compact: true})`.
+
+#### `slurp("<name>")`
+Slurp outputs and save them to `$name`. Must be last in the pipeline. Will be available as a global array `$name`. Ex `1,2,3 | slurp("a")`, `$a[]` same as `spew("a")`.
+
+#### `spew`/`spew("<name>")`
+Output previously slurped values.
+
+#### `spew`
+Outputs all slurps as an object. `spew("<name>")` outputs one slurp. Ex: `spew("a")`.
+
+#### `paste`
+Read string from stdin until ^D. Useful for pasting text. Ex: `paste | from_pem | asn1_ber | repl` read from stdin then decode and start a new sub-REPL with result.
+
+### Format decode functions
+
+Format decode functions are available in two forms, just `mp3` or `mp3($opts)` that returns a decode value even on error and `from_mp3` or `from_mp3($opts)` which throws error on decode error.
+
+The the only general format option currently is `force` to ignore decoder asserts.
+For example to decode as mp3 and ignore assets do `mp3({force: true})` or `decode("mp3"; {force: true})`. From command line you can either do `fq -d mp3 -o force=true . file.mp3` or `fq -d bytes 'mp3({force: true})' file.mp3`.
+
+Some formats has own options that can be specificed as part of `$opts` or as `-o name=value`. Too see options for a format do `fq -h mp3` or `help(mp3)` in a REPL. From command line you can either do `fq -d mp3 -o max_sync_seek=100 . file.mp3` or `fq -d bytes 'mp3({max_sync_seek: 100})' file.mp3`.
+
+#### `decode`, `decode("<format>")`, `decode("<format>"; $opts)`
+Decode format.
+
+#### `probe`, `probe($opts)`
+Probe and decode format.
+
+#### `<format>`, `<format>($opts)`
+Same as `decode("<format>")` and `decode("<format>"; $opts)`. Decode as format and return decode value even on decode error.
+
+#### `from_<format>`, `from_<format>($opts)`
+Same as `decode("<format>")` and `decode("<format>"; $opts)` decode as format but throw error on decode error.
 
 Note that jq sometimes uses the notation `name/0`, `name/1` etc in error messages and documentation which means `<function-name>/<arity>`. Same function names with different arity are treated as separate functions, but are usually related in some way in practice.
 
-### Function added in fq
+#### `print`, `println`, `printerr`, `printerrln`
+Print string or if not a string compact JSON value to stdout or stderr.
 
-- All standard library functions from jq
+#### `root`
-- Adds a few new general functions:
+Root decode value for decode value.
-  - `print`, `println`, `printerr`, `printerrln` prints to stdout and stderr.
+
-  - `group` group values, same as `group_by(.)`.
+#### `buffer_root`
-  - `streaks`, `streaks_by(f)` like `group` but groups streaks based on condition.
+Root decode value of buffer for decode value.
-  - `count`, `count_by(f)` like `group` but counts groups lengths.
+
-  - `debug(f)` like `debug` but uses arg to produce a debug message. `{a: 123} | debug({a}) | ...`.
+#### `format_root`
-  - `path_to_expr` from `["key", 1]` to `".key[1]"`.
+Root decode value of format for decode value.
-  - `expr_to_path` from `".key[1]"` to `["key", 1]`.
+
-  - `diff($a; $b)` produce diff object between two values.
+#### `parent`
-  - `delta`, `delta_by(f)`, array with difference between all consecutive pairs.
+Parent decode value for decode value.
-  - `chunk(f)`, split array or string into even chunks
+
-- Bitwise functions `band`, `bor`, `bxor`, `bsl`, `bsr` and `bnot`. Works the same as jq math functions,
+#### `parents`
-unary uses input and if more than one argument all as arguments ignoring the input. Ex: `1 | bnot` `bsl(1; 3)`
+Outputs all parent decode values from decode value.
-- Adds some decode value specific functions:
+
-  - `root` tree root for value
+#### `topath`
-  - `buffer_root` root value of buffer for value
+Path for decode value. Use `path_to_expr` to get a string representation.
-  - `format_root` root value of format for value
+
-  - `parent` parent value
+#### `tovalue`, `tovalue($opts)`
-  - `parents` output parents of value
+Symbolic, if available, or actual value for decode value.
-  - `topath` path of value. Use `path_to_expr` to get a string representation.
+
-  - `tovalue`, `tovalue($opts)` symbolic value if available otherwise actual value
+#### `toactual`, `toactual($opts)`
-  - `toactual`, `toactual($opts)` actual value (usually the decoded value)
+Actual value for decode value.
-  - `tosym`, `tosym($opts)` symbolic value (mapped etc)
+
-  - `todescription` description of value
+#### `tosym`, `tosym($opts)`
-  - `torepr` converts decode value into what it represents. For example convert msgpack decode value
+Symbolic value for decode value.
-  into a value representing its JSON representation.
+
-  - All regexp functions work with binary as input and pattern argument with these differences
+#### `todescription`
-  compared to when using string input:
+Description for decode value.
-    - All offset and length will be in bytes.
+
-    - For `capture` the `.string` value is a binary.
+#### `torepr`
-    - If pattern is a binary it will be matched literally and not as a regexp.
+Converts decode value into what it represents. For example converts msgpack decode value into a value representing its JSON representation.
-    - If pattern is a binary or flags include "b" each input byte will be read as separate code points
+
-  - String functions are not overloaded to support binary for now as some of them might have behaviors that might be confusing.
+### Display functions
-  - `explode` is overloaded to work with binary. Will explode into array of the unit of the binary.
+
-  end of binary.
+Display shows hexdump, ASCII and tree column dump for decode values and jq value for other types.
-  instead of possibly multi-byte UTF-8 codepoints. This allows to match raw bytes. Ex: `match("\u00ff"; "b")`
+
-  will match the byte `0xff` and not the UTF-8 encoded codepoint for 255, `match("[^\u00ff]"; "b")` will match
+#### `d`/`d($opts)`
-  all non-`0xff` bytes.
+display value and truncate long arrays and binaries.
-  - `grep` functions take 1 or 2 arguments. First is a scalar to match, where a string is
+
-  treated as a regexp. A binary will match exact bytes. Second argument are regexp
+#### `da`/`da($opts)`
-  flags with addition that "b" will treat each byte in the input binary as a code point, this
+Display value and don't truncate arrays.
-  makes it possible to match exact bytes.
+
-    - `grep($v)`, `grep($v; $flags)` recursively match value and binary
+#### `dd`/`dd($opts)`
-    - `vgrep($v)`, `vgrep($v; $flags)` recursively match value
+Display value and don't truncate arrays or binaries.
-    - `bgrep($v)`, `bgrep($v; $flags)` recursively match binary
+
-    - `fgrep($v)`, `fgrep($v; $flags)` recursively match field name
+#### `dv`/`dv($opts)`
-  - `grep_by(f)` recursively match using a filter. Ex: `grep_by(. > 180 and . < 200)`, `first(grep_by(format == "id3v2"))`.
+Verbosely display value and don't truncate arrays but truncate binaries.
-  - Binary:
+
-    - `tobits` - Transform input to binary with bit as unit, does not preserve source range, will start at zero.
+#### `ddv`/`ddv($opts)`
-    - `tobitsrange` - Transform input to binary with bit as unit, preserves source range if possible.
+Verbosely display value and don't truncate arrays or binaries.
-    - `tobytes` - Transform input to binary with byte as unit, does not preserve source range, will start at zero.
+
-    - `tobytesrange` - Transform input binary with byte as unit, preserves source range if possible.
+#### `hd`/`hexdump`
-    - `.[start:end]`, `.[:end]`, `.[start:]` - Slice binary from start to end preserve source range.
+Hexdump value.
-- `open` open file for reading
+
-- All decode functions take an optional option argument. The only option currently is `force` to ignore decoder asserts.
+### Binary values
-For example to decode as mp3 and ignore assets do `mp3({force: true})` or `decode("mp3"; {force: true})`, from command line
+
-you currently have to do `fq -d bytes 'mp3({force: true})' file`.
+Binary values represents raw bits or bytes. When used in standard jq expressions they will behave as strings (UTF-8) with some exceptions listed below.
-- `decode`, `decode("<format>")`, `decode("<format>"; $opts)` decode format
+
-- `probe`, `probe($opts)` probe and decode format
+- All regexp functions work with binary as input and pattern argument with these differences
-- `mp3`, `mp3($opts)`, ..., `<format>`, `<format>($opts)` same as `decode("<format>")`, `decode("<format>"; $opts)` decode as format and return decode value even on decode error.
+compared to when using string input:
-- `from_mp3`, `from_mp3($opts)`, ..., `from_<format>`, `from_<format>($opts)` same as `decode("<format>")`, `decode("<format>"; $opts)` decode as format but throw error on decode error.
+  - All offset and length will be in bytes.
-- Display shows hexdump/ASCII/tree for decode values and jq value for other types.
+  - For `capture` the `.string` value is a binary.
-  - `d`/`d($opts)` display value and truncate long arrays and binaries
+  - If pattern is a binary it will be matched literally and not as a regexp.
-  - `da`/`da($opts)` display value and don't truncate arrays
+  - If pattern is a binary or flags include "b" each input byte will be read as separate code points
-  - `dd`/`dd($opts)` display value and don't truncate arrays or binaries
+- `explode` is overloaded to work with binary. Will explode into array of the unit of the binary.
-  - `dv`/`dv($opts)` verbosely display value and don't truncate arrays but truncate binaries
+- `.[start:end]`, `.[:end]`, `.[start:]` - Slice binary from start to end preserve source range.
-  - `ddv`/`ddv($opts)` verbosely display value and don't truncate arrays or binaries
+
-- `hd`/`hexdump` hexdump value
+#### `grep($v)`, `grep($v; $flags)`, `vgrep($v)`, `vgrep($v; $flags)`, `bgrep($v)`, `bgrep($v; $flags)`
-- `repl`/`repl($opts)` nested REPL, must be last in a pipeline. `1 | repl`, can "slurp" outputs. Ex: `1, 2, 3 | repl`, `[1,2,3] | repl({compact: true})`.
+Recursively match `$v`.
-- `slurp("<name>")` slurp outputs and save them to `$name`, must be last in the pipeline. Will be available as a global array `$name`. Ex `1,2,3 | slurp("a")`, `$a[]` same as `spew("a")`.
+
-- `spew`/`spew("<name>")` output previously slurped values. `spew` outputs all slurps as an object, `spew("<name>")` outputs one slurp. Ex: `spew("a")`.
+`$v` is a scalar to match, where a string is treated as a regexp. A binary will match exact bytes.
-- `paste` read string from stdin until ^D. Useful for pasting text.
+`$flags` argument are regexp flags with additional flag "b" that will treat each byte in the input binary
-    - Ex: `paste | from_pem | asn1_ber | repl` read from stdin then decode and start a new sub-REPL with result.
+as a code point. This makes it possible to match exact bytes.
+
+#### `fgrep($v)`, `fgrep($v; $flags)`
+Recursively match field name in for decode value.
+
+#### `tobits`
+Transform input to binary with bit as unit and don't preserve source range.
+
+#### `tobitsrange`
+Transform input to binary with bit as unit and preserve source range.
+
+#### `tobytes`
+Transform input to binary with byte as unit and don't preserve source range.
+
+#### `tobytesrange`
+Transform input to binary with byte as unit and preserve source range.
+
+#### `open`
+Open file for reading.
 
 ### Naming inconsistencies
 

format/bits/bits.go

@@ -2,14 +2,21 @@ package bits
 
 import (
 	"embed"
+	"encoding/binary"
+	"fmt"
+	"math"
 
 	"github.com/wader/fq/format"
+	"github.com/wader/fq/internal/gojqx"
+	"github.com/wader/fq/internal/mathx"
+	"github.com/wader/fq/pkg/bitio"
 	"github.com/wader/fq/pkg/decode"
 	"github.com/wader/fq/pkg/interp"
 	"github.com/wader/fq/pkg/scalar"
 )
 
 //go:embed bits.md
+//go:embed bits.jq
 //go:embed bytes.md
 var bitsFS embed.FS
 
@@ -39,5 +46,74 @@ func init() {
 			DecodeFn:           decodeBits(8),
 			SkipDecodeFunction: true, // skip add bytes and frombytes function
 		})
+	interp.RegisterFunc2("_from_float", func(_ *interp.Interp, c any, nBits int, isLE bool) any {
+		switch nBits {
+		case 16, 32, 64:
+		default:
+			return fmt.Errorf("unsupported bit size %d, must be 16, 32 or 64", nBits)
+		}
+
+		br, err := interp.ToBitReader(c)
+		if err != nil {
+			return err
+		}
+		var b [8]byte
+		bs := b[:][0 : nBits/8]
+		_, err = br.ReadBits(bs[:], int64(nBits))
+		if err != nil {
+			return err
+		}
+		if isLE {
+			decode.ReverseBytes(bs[:])
+		}
+
+		switch nBits {
+		case 64:
+			return math.Float64frombits(binary.BigEndian.Uint64(bs[:]))
+		case 32:
+			return float64(math.Float32frombits(binary.BigEndian.Uint32(bs[:])))
+		case 16:
+			return float64(mathx.Float16(binary.BigEndian.Uint16(bs[:])).Float32())
+		default:
+			panic("unreachable")
+		}
+	})
+	interp.RegisterFunc2("_to_float", func(_ *interp.Interp, c any, nBits int, isLE bool) any {
+		switch nBits {
+		case 16, 32, 64:
+		default:
+			return fmt.Errorf("unsupported bit size %d, must be 16, 32 or 64", nBits)
+		}
+
+		v, ok := gojqx.Cast[float64](c)
+		if !ok {
+			return gojqx.FuncTypeError{Name: "_to_float", V: v}
+		}
+
+		var b [8]byte
+		bs := b[:][0 : nBits/8]
+		switch nBits {
+		case 64:
+			binary.BigEndian.PutUint64(bs, math.Float64bits(v))
+		case 32:
+			binary.BigEndian.PutUint32(bs, math.Float32bits(float32(v)))
+		case 16:
+			binary.BigEndian.PutUint16(bs, uint16(mathx.NewFloat16(float32(v))))
+		default:
+			panic("unreachable")
+		}
+		if isLE {
+			decode.ReverseBytes(bs[:])
+		}
+
+		br, err := interp.NewBinaryFromBitReader(bitio.NewBitReader(bs, -1), 8, 0)
+		if err != nil {
+			return err
+		}
+
+		return br
+
+	})
+
 	interp.RegisterFS(bitsFS)
 }

format/bits/bits.jq

@@ -0,0 +1,12 @@
+def from_f16be: _from_float(16; false);
+def from_f32be: _from_float(32; false);
+def from_f64be: _from_float(64; false);
+def from_f16le: _from_float(16; true);
+def from_f32le: _from_float(32; true);
+def from_f64le: _from_float(64; true);
+def to_f16le: _to_float(16; true);
+def to_f32le: _to_float(32; true);
+def to_f64le: _to_float(64; true);
+def to_f16be: _to_float(16; false);
+def to_f32be: _to_float(32; false);
+def to_f64be: _to_float(64; false);

go.mod

@@ -9,7 +9,7 @@ require (
 	// bump: gomod-BurntSushi/toml /github\.com\/BurntSushi\/toml v(.*)/ https://github.com/BurntSushi/toml.git|^1
 	// bump: gomod-BurntSushi/toml command go get -d github.com/BurntSushi/toml@v$LATEST && go mod tidy
 	// bump: gomod-BurntSushi/toml link "Source diff $CURRENT..$LATEST" https://github.com/BurntSushi/toml/compare/v$CURRENT..v$LATEST
-	github.com/BurntSushi/toml v1.3.2
+	github.com/BurntSushi/toml v1.4.0
 
 	// bump: gomod-creasty-defaults /github\.com\/creasty\/defaults v(.*)/ https://github.com/creasty/defaults.git|^1
 	// bump: gomod-creasty-defaults command go get -d github.com/creasty/defaults@v$LATEST && go mod tidy
@@ -48,7 +48,7 @@ require (
 	// bump: gomod-golang-x-crypto /golang\.org\/x\/crypto v(.*)/ https://github.com/golang/crypto.git|^0
 	// bump: gomod-golang-x-crypto command go get -d golang.org/x/crypto@v$LATEST && go mod tidy
 	// bump: gomod-golang-x-crypto link "Tags" https://github.com/golang/crypto/tags
-	golang.org/x/crypto v0.23.0
+	golang.org/x/crypto v0.24.0
 
 	// has no tags
 	// go get -d golang.org/x/exp@master && go mod tidy
@@ -57,17 +57,17 @@ require (
 	// bump: gomod-golang-x-net /golang\.org\/x\/net v(.*)/ https://github.com/golang/net.git|^0
 	// bump: gomod-golang-x-net command go get -d golang.org/x/net@v$LATEST && go mod tidy
 	// bump: gomod-golang-x-net link "Tags" https://github.com/golang/net/tags
-	golang.org/x/net v0.25.0
+	golang.org/x/net v0.26.0
 
 	// bump: gomod-golang-x-term /golang\.org\/x\/term v(.*)/ https://github.com/golang/term.git|^0
 	// bump: gomod-golang-x-term command go get -d golang.org/x/term@v$LATEST && go mod tidy
 	// bump: gomod-golang-x-term link "Tags" https://github.com/golang/term/tags
-	golang.org/x/term v0.20.0
+	golang.org/x/term v0.21.0
 
 	// bump: gomod-golang/text /golang\.org\/x\/text v(.*)/ https://github.com/golang/text.git|^0
 	// bump: gomod-golang/text command go get -d golang.org/x/text@v$LATEST && go mod tidy
 	// bump: gomod-golang/text link "Source diff $CURRENT..$LATEST" https://github.com/golang/text/compare/v$CURRENT..v$LATEST
-	golang.org/x/text v0.15.0
+	golang.org/x/text v0.16.0
 
 	// bump: gomod-gopkg.in/yaml.v3 /gopkg\.in\/yaml\.v3 v(.*)/ https://github.com/go-yaml/yaml.git|^3
 	// bump: gomod-gopkg.in/yaml.v3 command go get -d gopkg.in/yaml.v3@v$LATEST && go mod tidy
@@ -79,6 +79,6 @@ require (
 	github.com/itchyny/timefmt-go v0.1.5 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
-	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 )

go.sum

@@ -1,5 +1,5 @@
-github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
+github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
-github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA=
 github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
 github.com/ergochat/readline v0.1.1 h1:C8Uuo3ybB23GWOt0uxmHbGzKM9owmtXary6Clrj84s0=
@@ -25,18 +25,18 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/wader/gojq v0.12.1-0.20240401131232-6c6bc364201a h1:P881Oecjt9FEXrwkGJ6UObJksxejJaF/fKq1ZfXpiVE=
 github.com/wader/gojq v0.12.1-0.20240401131232-6c6bc364201a/go.mod h1:qVrzkUdnBtJvM4twyRQ6xdziPSnSp35dLm4s/DN2iP4=
-golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
+golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
 golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw=
 golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ=
-golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
+golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
-golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
+golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
-golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
-golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=