Merge eae6a43d23 into 6ace338d58

doc: Update formats.svg

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: "1.22.5"
+          go-version: "1.22.6"
       - uses: actions/checkout@v3
       - uses: golangci/golangci-lint-action@v3
         with:
@@ -47,7 +47,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: "1.22.5"
+          go-version: "1.22.6"
       - name: Test
         env:
           GOARCH: ${{ matrix.goarch }}

.github/workflows/release.yml

@@ -15,7 +15,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: "1.22.5"
+          go-version: "1.22.6"
       - uses: goreleaser/goreleaser-action@v5
         with:
           distribution: goreleaser

Dockerfile

@@ -1,5 +1,5 @@
 # bump: docker-golang /FROM golang:([\d.]+)/ docker:golang|^1
-FROM golang:1.22.5-bookworm AS base
+FROM golang:1.22.6-bookworm AS base
 
 # expect is used to test cli
 RUN \

README.md

@@ -141,12 +141,14 @@ pssh_playready,
 [rtmp](doc/formats.md#rtmp),
 sll2_packet,
 sll_packet,
+[tap](doc/formats.md#tap),
 tar,
 tcp_segment,
 tiff,
 [tls](doc/formats.md#tls),
 toml,
 [tzif](doc/formats.md#tzif),
+[tzx](doc/formats.md#tzx),
 udp_datagram,
 vorbis_comment,
 vorbis_packet,
@@ -167,8 +169,9 @@ It can also work with some common text formats like URLs, hex, base64, PEM etc a
 
 For details see [formats.md](doc/formats.md) and [usage.md](doc/usage.md).
 
-## Presentations
+## Presentations and media
 
+- [PBS Tidbit 8 of Y: Interview with jq Maintainer Mattias Wadman](https://pbs.bartificer.net/tidbit8) - English podcast episode about jq and some fq.
 - [Kodsnack 585 - Polymorfisk JSON](https://kodsnack.se/585/) - Swedish podcast episode about jq and fq
 - "fq - jq for binary formats" at [FOSDEM 2023](https://fosdem.org/2023/) - [video & slides](https://fosdem.org/2023/schedule/event/bintools_fq/)
 - "fq - jq for binary formats" at [No time to wait 6](https://mediaarea.net/NoTimeToWait6) - [video](https://www.youtube.com/watch?v=-Pwt5KL-xRs&t=1450s) - [slides](doc/presentations/nttw6/fq-nttw6-slides.pdf)

doc/formats.md

@@ -113,12 +113,14 @@
 |[`rtmp`](#rtmp)                                                 |Real-Time&nbsp;Messaging&nbsp;Protocol                                                                       |<sub>`amf0` `mpeg_asc`</sub>|
 |`sll2_packet`                                                   |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation&nbsp;v2                                                    |<sub>`inet_packet`</sub>|
 |`sll_packet`                                                    |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation                                                            |<sub>`inet_packet`</sub>|
+|[`tap`](#tap)                                                   |TAP&nbsp;tape&nbsp;format&nbsp;for&nbsp;ZX&nbsp;Spectrum&nbsp;computers                                      |<sub></sub>|
 |`tar`                                                           |Tar&nbsp;archive                                                                                             |<sub>`probe`</sub>|
 |`tcp_segment`                                                   |Transmission&nbsp;control&nbsp;protocol&nbsp;segment                                                         |<sub></sub>|
 |`tiff`                                                          |Tag&nbsp;Image&nbsp;File&nbsp;Format                                                                         |<sub>`icc_profile`</sub>|
 |[`tls`](#tls)                                                   |Transport&nbsp;layer&nbsp;security                                                                           |<sub>`asn1_ber`</sub>|
 |`toml`                                                          |Tom's&nbsp;Obvious,&nbsp;Minimal&nbsp;Language                                                               |<sub></sub>|
 |[`tzif`](#tzif)                                                 |Time&nbsp;Zone&nbsp;Information&nbsp;Format                                                                  |<sub></sub>|
+|[`tzx`](#tzx)                                                   |TZX&nbsp;tape&nbsp;format&nbsp;for&nbsp;ZX&nbsp;Spectrum&nbsp;computers                                      |<sub>`tap`</sub>|
 |`udp_datagram`                                                  |User&nbsp;datagram&nbsp;protocol                                                                             |<sub>`udp_payload`</sub>|
 |`vorbis_comment`                                                |Vorbis&nbsp;comment                                                                                          |<sub>`flac_picture`</sub>|
 |`vorbis_packet`                                                 |Vorbis&nbsp;packet                                                                                           |<sub>`vorbis_comment`</sub>|
@@ -137,7 +139,7 @@
 |`ip_packet`                                                     |Group                                                                                                        |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
 |`link_frame`                                                    |Group                                                                                                        |<sub>`bsd_loopback_frame` `ether8023_frame` `ipv4_packet` `ipv6_packet` `sll2_packet` `sll_packet`</sub>|
 |`mp3_frame_tags`                                                |Group                                                                                                        |<sub>`mp3_frame_vbri` `mp3_frame_xing`</sub>|
-|`probe`                                                         |Group                                                                                                        |<sub>`adts` `aiff` `apple_bookmark` `ar` `avi` `avro_ocf` `bitcoin_blkdat` `bplist` `bzip2` `caff` `elf` `fit` `flac` `gif` `gzip` `html` `jp2c` `jpeg` `json` `jsonl` `leveldb_table` `luajit` `macho` `macho_fat` `matroska` `moc3` `mp3` `mp4` `mpeg_ts` `nes` `ogg` `opentimestamps` `pcap` `pcapng` `png` `tar` `tiff` `toml` `tzif` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
+|`probe`                                                         |Group                                                                                                        |<sub>`adts` `aiff` `apple_bookmark` `ar` `avi` `avro_ocf` `bitcoin_blkdat` `bplist` `bzip2` `caff` `elf` `fit` `flac` `gif` `gzip` `html` `jp2c` `jpeg` `json` `jsonl` `leveldb_table` `luajit` `macho` `macho_fat` `matroska` `moc3` `mp3` `mp4` `mpeg_ts` `nes` `ogg` `opentimestamps` `pcap` `pcapng` `png` `tar` `tiff` `toml` `tzif` `tzx` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
 |`tcp_stream`                                                    |Group                                                                                                        |<sub>`dns_tcp` `rtmp` `tls`</sub>|
 |`udp_payload`                                                   |Group                                                                                                        |<sub>`dns`</sub>|
 
@@ -1209,6 +1211,26 @@ fq '.tcp_connections[] | select(.server.port=="rtmp") | d' file.cap
 - https://rtmp.veriskope.com/docs/spec/
 - https://rtmp.veriskope.com/pdf/video_file_format_spec_v10.pdf
 
+## tap
+TAP tape format for ZX Spectrum computers.
+
+The TAP- (and BLK-) format is nearly a direct copy of the data that is stored
+in real tapes, as it is written by the ROM save routine of the ZX-Spectrum.
+A TAP file is simply one data block or a group of 2 or more data blocks, one
+followed after the other. The TAP file may be empty.
+
+You will often find this format embedded inside the TZX tape format.
+
+The default file extension is `.tap`.
+
+### Authors
+
+- Michael R. Cook work.mrc@pm.me, original author
+
+### References
+
+- https://worldofspectrum.net/zx-modules/fileformats/tapformat.html
+
 ## tls
 Transport layer security.
 
@@ -1378,6 +1400,27 @@ fq '.v2plusdatablock.leap_second_records | length' tziffile
 ### References
 - https://datatracker.ietf.org/doc/html/rfc8536
 
+## tzx
+TZX tape format for ZX Spectrum computers.
+
+`TZX` is a file format designed to preserve cassette tapes compatible with the
+ZX Spectrum computers, although some specialized versions of the format have
+been defined for other machines such as the Amstrad CPC and C64.
+
+The format was originally created by Tomaz Kac, who was maintainer until
+`revision 1.13`, before passing it to Martijn v.d. Heide. For a brief period
+the company Ramsoft became the maintainers, and created revision `v1.20`.
+
+The default file extension is `.tzx`.
+
+### Authors
+
+- Michael R. Cook work.mrc@pm.me, original author
+
+### References
+
+- https://worldofspectrum.net/TZXformat.html
+
 ## wasm
 WebAssembly Binary Format.
 

doc/fq.1

@@ -0,0 +1,410 @@
+.TH FQ "1" "July 2023"
+
+.SH NAME
+fq \- jq for binary formats
+
+.SH SYNOPSIS
+\fBfq\fR [\fI\,OPTION\/\fR]... [--] \fI\,EXPR [\fI\,FILE\/\fR]...
+
+\fBfq\fR . <FILE>
+
+\fBfq\fR d <FILE>
+
+\fBfq\fR -V '.path[1].value' <FILE>
+
+\fBfq\fR tovalue <FILE>
+
+\fBfq\fR -r to_toml file.yml
+
+\fBfq\fR -s -d html 'map(.html.head.title?)' *.html
+
+\fBfq\fR file.cbor | fq -d cbor torepr
+
+\fBfq\fR 'grep("^main$") | parent' /bin/ls
+
+\fBfq\fR -i
+
+.SH DESCRIPTION
+\fBfq\fR is inspired by the well known jq tool and language that allows you to work with binary formats the same way you would using jq. In addition it can present data like a hex viewer, transform, slice and concatenate binary data. It also supports nested formats and has an interactive REPL with auto-completion.
+.P
+It was originally designed to query, inspect and debug media codecs and containers like mp4, flac, mp3, jpeg. But since then it has been extended to support a variety of formats like executables, packet captures (with TCP reassembly) and serialization formats like JSON, YAML, XML, ASN1 BER, Avro, CBOR, protobuf. In addition it also has functions to work with URLs, convert to/from hex, number bases, search for things etc.
+.P
+In summary it aims to be jq, hexdump, dd and gdb for files combined into one.
+
+.SH INVOKING FQ
+.SS Command Line Arguments
+Most of jq's CLI arguments work with fq. But there are some additional ones specific to fq:
+.TP
+\fB-d\fR, \fB--decode\fR \fI\,FORMAT\/\fR
+Use \fI\,FORMAT\/\fR to decode instead of trying to auto-detect format.
+.TP
+\fB-i\fR, \fB--repl\fR [\fI\,INPUT\/\fR]...
+Use interactive REPL.
+
+Can be used with no input, one and multiple inputs, for example just \fBfq -i\fR starts a REPL with null input, \fBfq -i 123\fR with the number 123 as input, \fBfq -i . a b\fR with two files as input. This also works with \fB--slurp\fR. In the REPL it is also possible to start a sub-REPL(s) by ending a query with \fB<query> | repl\fR, use ctrl-D to exit the sub-REPL. The sub-REPL will evaluate separately on each output from the query it was started. Use \fB[<query>] | repl\fR if you want to "slurp" into an array.
+.TP
+\fB-o\fR, \fB--options\fR <\fIKEY\fR=\fIVALUE\fR|@\fIPATH\fR>
+\fIKEY\fR is name of option
+
+\fIVALUE\fR will be interpreted as a JSON value if possible otherwise a string, for example \fB-o name=abc\fR and \fB-o name='"abc"'\fR are equivalent.
+
+@\fBPATH\fR will read string from file at \fBPATH\fR.
+
+Specifying a global option or a format option, for example, \fB-o decode_samples=false\fR would, for some container decoders like mp4 and matroska, disable decoding of samples.
+.TP
+\fB-V\fR, \fB--value-output\fR
+Output JSON value instead of decode tree. Use \fB-Vr\fR if you want raw string (no quotes).
+.TP
+\fB-M\fR
+Disable coloured output.
+.TP
+\fB-o <NAME>=<VALUE>\fR
+Set a decoding/encoding option. A list of supported options are -
+.RS
+.P
+\fBbits_format\fR - string, md5, hex, base64, truncate, snippet
+.RS
+Controls the representation of raw binary in the output JSON
+.RE
+.RE
+.SS Supported Formats
+Use \fBfq --help formats\fR to get a list of supported formats
+
+aac_frame, adts, adts_frame, aiff, amf0, apev2, apple_bookmark, ar, asn1_ber, av1_ccr, av1_frame, av1_obu, avc_annexb, avc_au, avc_dcr, avc_nalu, avc_pps, avc_sei, avc_sps, avi, avro_ocf, bencode, bitcoin_blkdat, bitcoin_block, bitcoin_script, bitcoin_transaction, bits, bplist, bsd_loopback_frame, bson, bytes, bzip2, cbor, csv, dns, dns_tcp, elf, ether8023_frame, exif, fairplay_spc, flac, flac_frame, flac_metadatablock, flac_metadatablocks, flac_picture, flac_streaminfo, gif, gzip, hevc_annexb, hevc_au, hevc_dcr, hevc_nalu, hevc_pps, hevc_sps, hevc_vps, html, icc_profile, icmp, icmpv6, id3v1, id3v11, id3v2, ipv4_packet, ipv6_packet, jpeg, json, jsonl, luajit, macho, macho_fat, markdown, matroska, mp3, mp3_frame, mp3_frame_vbri, mp3_frame_xing, mp4, mpeg_asc, mpeg_es, mpeg_pes, mpeg_pes_packet, mpeg_spu, mpeg_ts, msgpack, ogg, ogg_page, opus_packet, pcap, pcapng, pg_btree, pg_control, pg_heap, png, prores_frame, protobuf, protobuf_widevine, pssh_playready, rtmp, sll2_packet, sll_packet, tar, tcp_segment, tiff, tls, toml, tzif, udp_datagram, vorbis_comment, vorbis_packet, vp8_frame, vp9_cfm, vp9_frame, vpx_ccr, wasm, wav, webp, xml, yaml, zip
+.TP
+It can also work with some common text formats like URLs, hex, base64, PEM etc and for some serialization formats like XML, YAML, etc. it can transform both from and to jq values.
+.TP
+Use \fBfq\fR -h formats to get details about each supported file format
+.SH CONFIGURATION
+Custom functions can be defined in an init.fq which is searched for in the following locations -
+.P
+macOS - $HOME/Library/Application Support/fq/init.jq
+.P
+Linux, BSD, etc - $HOME/.config/fq/init.jq
+.P
+Windows - %AppData%\fq\init.jq
+.SS Unicode output
+Defining the environment variable \fICLIUNICODE\fR will enable the use of unicode characters for improved output
+.SH THE FQ LANGUAGE
+\fBfq\fR is based on the \fBjq\fR language and for basic usage its syntax is similar to how object and array access looks in JavaScript or JSON path, .food[10] etc. but it can do much more and is a very expressive language. To get the most out of fq it's recommended to learn more about jq, this manual assumes some familiarity with the jq language.
+.SS Types Specific to fq
+\fBfq\fR has two additional types compared to jq, \fIdecode value\fR and \fIbinary\fR. In standard jq expressions they will in most cases behave as some standard jq type.
+.TP
+.B Decode Value
+
+This type is returned by decoders and it is used to represent parts of the decoded input. It can act as all standard jq types, object, array, number, string etc.
+
+Each decode value has an associated bit range in the input which can be accessed as a binary using \fBtobits\fR and \fBtobytes\fR.
+
+Each non-compound decode value has these properties:
+.RS
+An actual value - This is the decoded representation of the bits, a number, string, bool etc. Can be accessed using \fBtoactual\fR.
+
+An optional symbolic value - Is usually a mapping of the actual to symbolic value, ex: map number to a string value. Can be accessed using \fBtosym\fR.
+
+An optional description - Can be accessed using \fBtodescription\fR
+
+\fBparent\fR is the parent decode value
+
+\fBparents\fR is the all parent decode values
+
+\fBtopath\fR is the jq path for the decode value
+
+\fBtorepr\fR convert decode value to its representation if possible
+
+The value of a decode value is the symbolic value if available and otherwise the actual value. To explicitly access the value use tovalue. In most expressions this is not needed as it will be done automatically.
+.RE
+.TP
+.B Binary
+
+Binaries are raw bits with a unit size, 1 (bits) or 8 (bytes), that can have a non-byte aligned size. Will act as byte padded strings in standard jq expressions.
+
+Use \fBtobits\fR and \fBtobytes\fR to create them from \fIdecode values\fR, \fIstrings\fR, \fInumbers\fR or \fIbinary arrays\fR. \fBtobytes\fR will, if needed zero pad most significant bits to be byte aligned.
+
+There is also \fBtobitsrange\fR and \fBtobytesrange\fR which does the same thing but will preserve its source range when displayed. \" TODO: tobytesrange, padding
+.TP
+.B Binary Array
+
+Binary arrays are arrays of \fInumbers\fR, \fIstrings\fR, \fIbinaries\fR or other nested \fIbinary arrays\fR. When used as input to \fBtobits\fR/\fBtobytes\fR the following rules are used:
+
+\fBNumber\fR is a byte with value be 0-255
+\fBString\fR it's UTF8 codepoint bytes
+\fBBinary\fR as is
+\fBBinary array\fR used recursively
+
+Binary arrays are similar to and inspired by Erlang iolist.
+.SS Functions
+All decode functions are available in two forms, just \fI<format>\fR (like mp3) that returns a decode value on error and \fIfrom_<format>\fR which throws error on decode error.
+
+Note that jq sometimes uses the notation name/0, name/1 etc in error messages and documentation which means \fI<function-name>\fR/\fI<arity>\fR, Same function names with different arity are treated as separate functions, but are usually related in some way in practice.
+.TP
+.B
+Functions Added in fq
+
+All standard library functions from jq
+
+A few new general functions:
+.RS
+.RS
+\fBdisplay\fR/\fBd\fR displays a value
+
+\fBprint\fR, \fBprintln\fR, \fBprinterr\fR, \fBprinterrln\fR prints to stdout and stderr.
+
+\fBgroup\fR group values, same as \fBgroup_by(.)\fR.
+
+\fBstreaks\fR, \fBstreaks_by(f)\fR like \fBgroup\fR but groups streaks based on condition.
+
+\fBcount\fR, \fBcount_by(f)\fR like \fBgroup\fR but counts groups lengths.
+
+\fBdebug(f)\fR like \fBdebug\fR but uses \fIarg\fR to produce a debug message.
+
+\fBpath_to_expr\fR convert \fI["key", 1]\fR to \fI".key[1]\fR".
+
+\fBexpr_to_path\fR from \fI".key[1]"\fR to \fI["key", 1]\fR.
+
+\fBdiff($a; $b)\fR produce diff object between two values.
+
+\fBdelta\fR, \fBdelta_by(f)\fR, array with difference between all consecutive pairs.
+
+\fBchunk(f)\fR, split array or string into even chunks
+
+.RE
+\fBdisplay\fR is the main function for displaying values and is also the function that will be used if no other output function is explicitly used. If its input is a decode value it will output a dump and tree structure or otherwise it will output as JSON.
+\" There are some examples with images in usage.md. TODO: Find a way to accomodate them in the man page
+Bitwise functions \fBband\fR, \fBbor\fR, \fBbxor\fR, \fBbsl\fR, \fBbsr\fR and \fBbnot\fR. Works the same as jq math functions, unary uses input and if more than one argument all as arguments ignoring the input.
+
+Some decode value specific functions:
+.RS
+\fBroot\fR tree root for value
+
+\fBbuffer_root\fR root value of buffer for value
+
+\fBformat_root\fR root value of format for value
+
+\fBparent\fR parent value
+
+\fBparents\fR output parents of value
+
+\fBtopath\fR path of value. Use \fBpath_to_expr\fR to get a string representation.
+
+\fBtovalue\fR, \fBtovalue($opts)\fR symbolic value if available otherwise actual value
+
+\fBtoactual\fR, \fBtoactual($opts)\fR actual value (usually the decoded value)
+
+\fBtosym\fR, \fBtosym($opts)\fR symbolic value (mapped etc)
+
+\fBtodescription\fR description of value
+
+\fBtorepr\fR converts decode value into what it represents. For example convert msgpack decode value into a value representing its JSON representation.
+
+All regexp functions work with binary as input and pattern argument with these differences compared to when using string input:
+.RS
+All offset and length will be in bytes.
+
+For \fBcapture\fR the \fB.string\fR value is a binary.
+
+If pattern is a binary it will be matched literally and not as a regexp.
+
+If pattern is a binary or flags include "b" each input byte will be read as separate code points
+
+.RE
+String functions are not overloaded to support binary for now as some of them might have behaviors that might be confusing.
+
+\fBexplode\fR is overloaded to work with binary. Will explode into array of the unit of the binary. end of binary. instead of possibly multi-byte UTF-8 codepoints. This allows to match raw bytes. Ex: \fBmatch("\u00ff"; "b")\fR will match the byte 0xff and not the UTF-8 encoded codepoint for 255, \fBmatch("[^\u00ff]"; "b")\fR will match all non-0xff bytes.
+
+\fBgrep\fR functions take 1 or 2 arguments. First is a scalar to match, where a string is treated as a regexp. A binary will match exact bytes. Second argument are regexp flags with addition that "b" will treat each byte in the input binary as a code point, this makes it possible to match exact bytes.
+.RS
+\fBgrep($v)\fR, \fBgrep($v; $flags)\fR recursively match value and binary
+
+\fBvgrep($v)\fR, \fBvgrep($v; $flags)\fR recursively match value
+
+\fBbgrep($v)\fR, \fBbgrep($v; $flags)\fR recursively match binary
+
+\fBfgrep($v)\fR, \fBfgrep($v; $flags)\fR recursively match field name
+
+\fBgrep_by(f)\fR recursively match using a filter. Ex: \fBgrep_by(. > 180 and . < 200)\fR equivalent to \fBfirst(grep_by(format == "id3v2"))\fR.
+
+.RE
+Binary:
+.RS
+\fBtobits\fR - Transform input to binary with bit as unit, does not preserve source range, will start at zero.
+
+\fBtobitsrange\fR - Transform input to binary with bit as unit, preserves source range if possible.
+
+\fBtobytes\fR - Transform input to binary with byte as unit, does not preserve source range, will start at zero.
+
+\fBtobytesrange\fR - Transform input binary with byte as unit, preserves source range if possible.
+
+\fB.[start:end]\fR, \fB.[:end]\fR, \fB.[start:]\fR - Slice binary from start to end preserve source range.
+
+.RE
+\fBopen\fR open file for reading
+
+All decode functions take an optional option argument. The only option currently is \fBforce\fR to ignore decoder asserts. For example to decode as mp3 and ignore assets do \fBmp3({force: true})\fR or \fBdecode("mp3"; {force: true})\fR, from command line you currently have to do \fBfq -d bytes 'mp3({force: true})\fR' file.
+
+\fBdecode\fR, \fBdecode("<format>")\fR, \fBdecode("<format>"; $opts)\fR decode format
+
+\fBprobe\fR, \fBprobe($opts)\fR probe and decode format
+
+\fBmp3\fR, \fBmp3($opts)\fR, ..., \fB<format>\fR, \fB<format>($opts)\fR same as \fBdecode("<format>")\fR, \fBdecode("<format>"; $opts)\fR decode as format and return decode value even on decode error.
+
+\fBfrom_mp3\fR, \fBfrom_mp3($opts)\fR, ..., \fBfrom_<format>\fR, \fBfrom_<format>($opts)\fR same as \fBdecode("<format>")\fR, \fBdecode("<format>"; $opts)\fR decode as format but throw error on decode error.
+
+Display shows hexdump/ASCII/tree for decode values and jq value for other types.
+.RS
+\fBd\fR/\fBd($opts)\fR display value and truncate long arrays and binaries
+
+\fBda\fR/\fBda($opts)\fR display value and don't truncate arrays
+
+\fBdd\fR/\fBdd($opts)\fR display value and don't truncate arrays or binaries
+
+\fBdv\fR/\fBdv($opts)\fR verbosely display value and don't truncate arrays but truncate binaries
+
+\fBddv\fR/\fBddv($opts)\fR verbosely display value and don't truncate arrays or binaries
+
+.RE
+\fBhd\fR/\fBhexdump\fR hexdump value
+
+\fBrepl\fR/\fBrepl($opts)\fR nested REPL, must be last in a pipeline. \fB1 | repl\fR, can "slurp" outputs. Ex: \fB1, 2, 3 | repl, [1,2,3] | repl({compact: true})\fR.
+
+\fBslurp("<name>")\fR slurp outputs and save them to \fBRname\fR, must be last in the pipeline. Will be available as a global array \fB$name\fR. Ex \fB1,2,3 | slurp("a"), $a[]\fR same as \fBspew("a")\fR.
+
+\fBspew\fR/\fBspew("<name>")\fR output previously slurped values. \fBspew\fR outputs all slurps as an object, \fBspew("<name>")\fR outputs one slurp. Ex: \fBspew("a")\fR.
+
+\fBpaste\fR read string from stdin until EOF. Useful for pasting text.
+.RS
+Ex: \fBpaste | from_pem | asn1_ber | repl\fR read from stdin then decode and start a new sub-REPL with result.
+.RE
+.RE
+.RE
+
+.B Naming Inconsistencies
+jq's naming conversion is a bit inconsistent, some standard library functions are named tojson while others from_entries. fq follows this tradition a bit by but tries to use snake_case unless there is a good reason.
+
+Here are all the non-snake_case functions added by \fBfq\fR. Most of them deal with decode and binary values which are new "primitive" types:
+.RS
+\fBtoactual\fR
+
+\fBtobits\fR
+
+\fBtobitsrange\fR
+
+\fBtobytes\fR
+
+\fBtobytesrange\fR
+
+\fBtodescription\fR
+
+\fBtopath\fR
+
+\fBtorepr\fR
+
+\fBtosym\fR
+
+\fBtovalue\fR
+
+.RE
+.SH ENCODINGS, SERIALIZATION, & HASHES
+In addition to binary formats \fBfq\fR also support reading to and from encodings and serialization formats.
+
+At the moment \fBfq\fR does not have any dedicated argument for serialization formats but raw string input -R slurp -s and raw string output -r can make things easier. The combination -Rs will read all inputs into one string (same as jq).
+
+Note that from* functions output \fBjq\fR values and to* takes \fBjq\fR values as input so in some cases not all information will be properly preserved. For example, for the element and attribute order might change and text and comment nodes might move or be merged. \fByq\fR might be a better tool if that is needed.
+.SH USE AS SCRIPT INTERPRETER
+\fBfq\fR can be used as a script interpreter using a shebang
+.SH EXAMPLES
+.TP
+\fBfq\fR '.frames[1].header | tovalue' file.mp3
+Get the second mp3 frame header as JSON
+.TP
+\fBfq\fR '.frames[0:10] | map(tobytesrange.start)' file.mp3
+Get the byte start position for the first 10 mp3 frames in an array
+.TP
+\fBfq\fR -d bytes '.[100:] | mp3_frame | d' file.mp3
+Decode byte range 100 to end as mp3_frame
+.TP
+\fBfq\fR '.somefield | tobytesrange[10:] | mp3_frame | d' file.mp3
+
+decode byte range 10 bytes from .somefield and preserve relative position in file
+.TP
+
+\fBfq\fR 'def f: .. | select(format=="avc_sps"); diff(input|f; input|f)' a.mp4 b.mp4
+
+Show AVC SPS difference between two mp4 files. \fB-n\fR tells \fBfq\fR to not have an implicit input, \fBf\fR is a function to select out some interesting value, call \fBdiff\fR with two arguments, decoded value for a.mp4 and b.mp4 filtered thru \fBf\fR.
+.TP
+
+\fBfq\fR 'first(.. | select(format=="jpeg")) | tobytes' file > file.jpeg
+
+Recursively look for the first value that is a jpeg decode value root. Use tobytes to get bytes for value. Redirect bytes to a file.
+.TP
+
+\fBfq\fR '.. | select(.type=="stsz")? as $stsz | .entries | count | max_by(.[1])[1] as $m | ($stsz | topath | path_to_expr), (.[] | "\(.[0]): \((100*.[1]/$m)*"=") \(.[1])") | println' file.mp4
+
+Recursively look for a all sample size boxes "stsz" and use ? to ignore errors when doing .type on arrays etc. Save reference to box, count unique values, save the max, output the path to the box and output a histogram scaled to 0-100.
+.TP
+
+\fBfq\fR '.tcp_connections | grep("GET /.* HTTP/1.?")' file.pcap
+
+Find TCP streams that looks like HTTP GET requests in a PCAP file. Use \fBgrep\fR to recursively find strings matching a regexp.
+.TP
+
+\fBfq\fR -rn '[inputs | [input_filename, first(.chunks[] | select(.type=="IHDR") | .width)]] | max_by(.[1]) | .[0]' *.png
+
+Find the widest PNG in a directory
+.TP
+
+\fBfq\fR '.. | select(scalars and in_bytes_range(0x123))' file
+
+Which values include the bytes at position 0x123 \" TODO: Check if this is a typo
+
+.SH BUGS (+ TRICKS)
+\fBselect\fR fails with \fBexpected an ... but got: ...\fR
+.RS
+Try adding \fBselect(...)?\fR to catch and ignore type errors in the select expression.
+.RE
+
+Run interactive mode with no input
+.RS
+fq -i
+.RE
+
+Manual Decode - Sometimes fq fails to decode or you know there is valid data buried inside some binary or maybe you know the format of some gap field. Then you can decode manually:
+.RS
+# try decode a `mp3_frame` that failed to decode
+
+$ fq -d mp3 '.gap0 | mp3_frame' file.mp3
+
+# skip first 10 bytes then decode as `mp3_frame`
+
+$ fq -d bytes '.[10:] | mp3_frame' file.mp3
+.RE
+Use . as input and in a positional argument
+
+.RS
+The expression \fB.a | f(.b)\fR might not work as expected. \fB.\fR is \fB.a\fR when evaluating the arguments so the positional argument will end up being \fB.a.b\fR. Instead do \fB. as $c | .a | f($c.b)\fR.
+.RE
+
+Building array is slow
+.RS
+Try to use \fBmap\fR or \fBforeach\fR to avoid rebuilding the whole array for each append.
+.RE
+
+Use print and println to produce more friendly compact output
+
+repl argument using function or variable causes variable not defined
+.RS
+\fBtrue as $verbose | repl({verbose: $verbose})\fR will currently fail as repl is implemented by rewriting the query to \fBmap(true as $verbose | .) | repl({verbose: $verbose})\fR.
+.RE
+
+\fBerror\fR produces no output
+.RS
+\fBnull | error\fR behaves as empty
+.RE
+
+.SH AUTHOR
+\fBfq\fR is written by Mattias Wadman
+
+.SH SEE ALSO
+\fBjq\fR - lightweight and flexible command-line JSON processor
+
+\fByq\fR - lightweight and portable command-line YAML, JSON and XML processor

format/all/all.fqtest

@@ -32,6 +32,7 @@ $ fq -n _registry.groups.probe
   "tar",
   "tiff",
   "tzif",
+  "tzx",
   "wasm",
   "webp",
   "zip",
@@ -156,12 +157,14 @@ pssh_playready       PlayReady PSSH
 rtmp                 Real-Time Messaging Protocol
 sll2_packet          Linux cooked capture encapsulation v2
 sll_packet           Linux cooked capture encapsulation
+tap                  TAP tape format for ZX Spectrum computers
 tar                  Tar archive
 tcp_segment          Transmission control protocol segment
 tiff                 Tag Image File Format
 tls                  Transport layer security
 toml                 Tom's Obvious, Minimal Language
 tzif                 Time Zone Information Format
+tzx                  TZX tape format for ZX Spectrum computers
 udp_datagram         User datagram protocol
 vorbis_comment       Vorbis comment
 vorbis_packet        Vorbis packet

format/all/all.go

@@ -52,12 +52,14 @@ import (
 	_ "github.com/wader/fq/format/protobuf"
 	_ "github.com/wader/fq/format/riff"
 	_ "github.com/wader/fq/format/rtmp"
+	_ "github.com/wader/fq/format/tap"
 	_ "github.com/wader/fq/format/tar"
 	_ "github.com/wader/fq/format/text"
 	_ "github.com/wader/fq/format/tiff"
 	_ "github.com/wader/fq/format/tls"
 	_ "github.com/wader/fq/format/toml"
 	_ "github.com/wader/fq/format/tzif"
+	_ "github.com/wader/fq/format/tzx"
 	_ "github.com/wader/fq/format/vorbis"
 	_ "github.com/wader/fq/format/vpx"
 	_ "github.com/wader/fq/format/wasm"

format/format.go

@@ -165,12 +165,14 @@ var (
 	RTMP                = &decode.Group{Name: "rtmp"}
 	SLL_Packet          = &decode.Group{Name: "sll_packet"}
 	SLL2_Packet         = &decode.Group{Name: "sll2_packet"}
+	TAP                 = &decode.Group{Name: "tap"}
 	TAR                 = &decode.Group{Name: "tar"}
 	TCP_Segment         = &decode.Group{Name: "tcp_segment"}
 	TIFF                = &decode.Group{Name: "tiff"}
 	TLS                 = &decode.Group{Name: "tls"}
 	TOML                = &decode.Group{Name: "toml"}
 	Tzif                = &decode.Group{Name: "tzif"}
+	TZX                 = &decode.Group{Name: "tzx"}
 	UDP_Datagram        = &decode.Group{Name: "udp_datagram"}
 	Vorbis_Comment      = &decode.Group{Name: "vorbis_comment"}
 	Vorbis_Packet       = &decode.Group{Name: "vorbis_packet"}

format/tap/tap.go

@@ -0,0 +1,158 @@
+package tzx
+
+// https://worldofspectrum.net/zx-modules/fileformats/tapformat.html
+
+import (
+	"bufio"
+	"bytes"
+	"embed"
+
+	"golang.org/x/text/encoding/charmap"
+
+	"github.com/wader/fq/format"
+	"github.com/wader/fq/pkg/bitio"
+	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
+	"github.com/wader/fq/pkg/scalar"
+)
+
+//go:embed tap.md
+var tapFS embed.FS
+
+func init() {
+	interp.RegisterFormat(
+		format.TAP,
+		&decode.Format{
+			Description: "TAP tape format for ZX Spectrum computers",
+			DecodeFn:    tapDecoder,
+		})
+
+	interp.RegisterFS(tapFS)
+}
+
+// The TAP- (and BLK-) format is nearly a direct copy of the data that is stored
+// in real tapes, as it is written by the ROM save routine of the ZX-Spectrum.
+// A TAP file is simply one data block or a group of 2 or more data blocks, one
+// followed after the other. The TAP file may be empty.
+func tapDecoder(d *decode.D) any {
+	d.Endian = decode.LittleEndian
+
+	d.FieldArray("blocks", func(d *decode.D) {
+		for !d.End() {
+			d.FieldStruct("block", func(d *decode.D) {
+				decodeTapBlock(d)
+			})
+		}
+	})
+	return nil
+}
+
+func decodeTapBlock(d *decode.D) {
+	// Length of the following data.
+	length := d.FieldU16("length")
+
+	// read header, fragment, or data block
+	switch length {
+	case 0:
+		// fragment with no data
+	case 1:
+		d.FieldRawLen("data", 8)
+	case 19:
+		d.FieldStruct("header", func(d *decode.D) {
+			decodeHeader(d)
+		})
+	default:
+		d.FieldStruct("data", func(d *decode.D) {
+			decodeDataBlock(d, length)
+		})
+	}
+}
+
+// decodes the different types of 19-byte header blocks.
+func decodeHeader(d *decode.D) {
+	blockStartPosition := d.Pos()
+
+	// Always 0: byte indicating a standard ROM loading header
+	d.FieldU8("flag", scalar.UintMapSymStr{0: "standard_speed_data"})
+	// Header type
+	dataType := d.FieldU8("data_type", scalar.UintMapSymStr{
+		0x00: "program",
+		0x01: "numeric",
+		0x02: "alphanumeric",
+		0x03: "data",
+	})
+	// Loading name of the program. Filled with spaces (0x20) to 10 characters.
+	d.FieldStr("program_name", 10, charmap.ISO8859_1)
+
+	switch dataType {
+	case 0:
+		// Length of data following the header = length of BASIC program + variables.
+		d.FieldU16("data_length")
+		// LINE parameter of SAVE command. Value 32768 means "no auto-loading".
+		// 0..9999 are valid line numbers.
+		d.FieldU16("auto_start_line")
+		// Length of BASIC program;
+		// remaining bytes ([data length] - [program length]) = offset of variables.
+		d.FieldU16("program_length")
+	case 1:
+		// Length of data following the header = length of number array * 5 + 3.
+		d.FieldU16("data_length")
+		// Unused byte.
+		d.FieldU8("unused0")
+		// (1..26 meaning A..Z) + 128.
+		d.FieldU8("variable_name", scalar.UintHex)
+		// UnusedWord: 32768.
+		d.FieldU16("unused1")
+	case 2:
+		// Length of data following the header = length of string array + 3.
+		d.FieldU16("data_length")
+		// Unused byte.
+		d.FieldU8("unused0")
+		// (1..26 meaning A$..Z$) + 192.
+		d.FieldU8("variable_name", scalar.UintHex)
+		// UnusedWord: 32768.
+		d.FieldU16("unused1")
+	case 3:
+		// Length of data following the header, in case of a SCREEN$ header = 6912.
+		d.FieldU16("data_length")
+		// In case of a SCREEN$ header = 16384.
+		d.FieldU16("start_address", scalar.UintHex)
+		//	UnusedWord: 32768.
+		d.FieldU16("unused")
+	default:
+		d.Fatalf("invalid TAP header type, got: %d", dataType)
+	}
+
+	// Simply all bytes XORed (including flag byte).
+	d.FieldU8("checksum", d.UintValidate(calculateChecksum(d, blockStartPosition, d.Pos()-blockStartPosition)), scalar.UintHex)
+}
+
+func decodeDataBlock(d *decode.D, length uint64) {
+	blockStartPosition := d.Pos()
+
+	// flag indicating the type of data block, usually 255 (standard speed data)
+	d.FieldU8("flag", scalar.UintFn(func(s scalar.Uint) (scalar.Uint, error) {
+		if s.Actual == 0xFF {
+			s.Sym = "standard_speed_data"
+		} else {
+			s.Sym = "custom_data_block"
+		}
+		return s, nil
+	}))
+	// The essential data: length minus the flag/checksum bytes (may be empty)
+	d.FieldRawLen("data", int64(length-2)*8)
+	// Simply all bytes (including flag byte) XORed
+	d.FieldU8("checksum", d.UintValidate(calculateChecksum(d, blockStartPosition, d.Pos()-blockStartPosition)), scalar.UintHex)
+}
+
+func calculateChecksum(d *decode.D, blockStartPos, blockEndPos int64) uint64 {
+	var blockSlice bytes.Buffer
+	writer := bufio.NewWriter(&blockSlice)
+	d.Copy(writer, bitio.NewIOReader(d.BitBufRange(blockStartPos, blockEndPos)))
+
+	var checksum uint8
+	for _, v := range blockSlice.Bytes() {
+		checksum ^= v
+	}
+	return uint64(checksum)
+}

format/tap/tap.md

@@ -0,0 +1,16 @@
+The TAP- (and BLK-) format is nearly a direct copy of the data that is stored
+in real tapes, as it is written by the ROM save routine of the ZX-Spectrum.
+A TAP file is simply one data block or a group of 2 or more data blocks, one
+followed after the other. The TAP file may be empty.
+
+You will often find this format embedded inside the TZX tape format.
+
+The default file extension is `.tap`.
+
+### Authors
+
+- Michael R. Cook work.mrc@pm.me, original author
+
+### References
+
+- https://worldofspectrum.net/zx-modules/fileformats/tapformat.html

format/tap/testdata/README.md

@@ -0,0 +1,22 @@
+### basic_prog1.tap
+
+The `basic_prog1.tap` test file was created directory from the FUSE emulator.
+
+Inside the emulated ZX Spectrum a BASIC program was created:
+
+```
+10 PRINT "fq is the best!"
+20 GOTO 10
+```
+
+and saved to tape:
+
+```
+SAVE "fqTestProg", LINE 10
+```
+
+Then from FUSE select the menu item `Media > Tape > Save As..`.
+
+Any BASIC, machine code, screen image, or other data, can be saved directly
+using the `SAVE` command. Further instructions can be found here:
+https://worldofspectrum.org/ZXBasicManual/zxmanchap20.html

format/tap/testdata/basic_prog1.fqtest

@@ -0,0 +1,21 @@
+$ fq -d tap dv basic_prog1.tap
+    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: basic_prog1.tap (tap) 0x0-0x3f (63)
+    |                                               |                |  blocks[0:2]: 0x0-0x3f (63)
+    |                                               |                |    [0]{}: block 0x0-0x15 (21)
+0x00|13 00                                          |..              |      length: 19 0x0-0x2 (2)
+    |                                               |                |      header{}: 0x2-0x15 (19)
+0x00|      00                                       |  .             |        flag: "standard_speed_data" (0) 0x2-0x3 (1)
+0x00|         00                                    |   .            |        data_type: "program" (0) 0x3-0x4 (1)
+0x00|            66 71 54 65 73 74 50 72 6f 67      |    fqTestProg  |        program_name: "fqTestProg" 0x4-0xe (10)
+0x00|                                          26 00|              &.|        data_length: 38 0xe-0x10 (2)
+0x10|0a 00                                          |..              |        auto_start_line: 10 0x10-0x12 (2)
+0x10|      26 00                                    |  &.            |        program_length: 38 0x12-0x14 (2)
+0x10|            01                                 |    .           |        checksum: 0x1 (valid) 0x14-0x15 (1)
+    |                                               |                |    [1]{}: block 0x15-0x3f (42)
+0x10|               28 00                           |     (.         |      length: 40 0x15-0x17 (2)
+    |                                               |                |      data{}: 0x17-0x3f (40)
+0x10|                     ff                        |       .        |        flag: "standard_speed_data" (255) 0x17-0x18 (1)
+0x10|                        00 0a 14 00 20 f5 22 66|        .... ."f|        data: raw bits 0x18-0x3e (38)
+0x20|71 20 69 73 20 74 68 65 20 62 65 73 74 21 22 0d|q is the best!".|
+0x30|00 14 0a 00 ec 31 30 0e 00 00 0a 00 00 0d      |.....10.......  |
+0x30|                                          b6|  |              .||        checksum: 0xb6 (valid) 0x3e-0x3f (1)

format/tzx/testdata/README.md

@@ -0,0 +1,28 @@
+### basic_prog1.tzx
+
+The `basic_prog1.tzx` test file was created directory from the FUSE emulator.
+
+Inside the emulated ZX Spectrum a BASIC program was created:
+
+```
+10 PRINT "fq is the best!"
+20 GOTO 10
+```
+
+and saved to tape:
+
+```
+SAVE "fqTestProg", LINE 10
+```
+
+Then from FUSE select the menu item `Media > Tape > Save As..`.
+
+Any BASIC, machine code, screen image, or other data, can be saved directly
+using the `SAVE` command. Further instructions can be found here:
+https://worldofspectrum.org/ZXBasicManual/zxmanchap20.html
+
+
+#### Archive Info
+
+The FUSE emulator is not able to add the tape metadata. As this tape block is
+very simple, it was added manually using a Hex editor.

format/tzx/testdata/basic_prog1.fqtest

@@ -0,0 +1,81 @@
+$ fq -d tzx dv basic_prog1.tzx
+    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: basic_prog1.tzx (tzx) 0x0-0xcd (205)
+0x00|5a 58 54 61 70 65 21 1a                        |ZXTape!.        |  signature: raw bits (valid) 0x0-0x8 (8)
+0x00|                        01                     |        .       |  major_version: 1 0x8-0x9 (1)
+0x00|                           14                  |         .      |  minor_version: 20 0x9-0xa (1)
+    |                                               |                |  blocks[0:3]: 0xa-0xcd (195)
+    |                                               |                |    [0]{}: block 0xa-0x88 (126)
+0x00|                              32               |          2     |      type: "archive_info" (50) 0xa-0xb (1)
+0x00|                                 7b 00         |           {.   |      length: 123 0xb-0xd (2)
+0x00|                                       09      |             .  |      count: 9 0xd-0xe (1)
+    |                                               |                |      archive_info[0:9]: 0xe-0x88 (122)
+    |                                               |                |        [0]{}: entry 0xe-0x1a (12)
+0x00|                                          00   |              . |          id: "title" (0) 0xe-0xf (1)
+0x00|                                             0a|               .|          length: 10 0xf-0x10 (1)
+0x10|66 71 74 65 73 74 70 72 6f 67                  |fqtestprog      |          value: "fqtestprog" 0x10-0x1a (10)
+    |                                               |                |        [1]{}: entry 0x1a-0x21 (7)
+0x10|                              01               |          .     |          id: "publisher" (1) 0x1a-0x1b (1)
+0x10|                                 05            |           .    |          length: 5 0x1b-0x1c (1)
+0x10|                                    77 61 64 65|            wade|          value: "wader" 0x1c-0x21 (5)
+0x20|72                                             |r               |
+    |                                               |                |        [2]{}: entry 0x21-0x32 (17)
+0x20|   02                                          | .              |          id: "author" (2) 0x21-0x22 (1)
+0x20|      0f                                       |  .             |          length: 15 0x22-0x23 (1)
+0x20|         4d 69 63 68 61 65 6c 20 52 2e 20 43 6f|   Michael R. Co|          value: "Michael R. Cook" 0x23-0x32 (15)
+0x30|6f 6b                                          |ok              |
+    |                                               |                |        [3]{}: entry 0x32-0x38 (6)
+0x30|      03                                       |  .             |          id: "year" (3) 0x32-0x33 (1)
+0x30|         04                                    |   .            |          length: 4 0x33-0x34 (1)
+0x30|            32 30 32 34                        |    2024        |          value: "2024" 0x34-0x38 (4)
+    |                                               |                |        [4]{}: entry 0x38-0x41 (9)
+0x30|                        04                     |        .       |          id: "language" (4) 0x38-0x39 (1)
+0x30|                           07                  |         .      |          length: 7 0x39-0x3a (1)
+0x30|                              45 6e 67 6c 69 73|          Englis|          value: "English" 0x3a-0x41 (7)
+0x40|68                                             |h               |
+    |                                               |                |        [5]{}: entry 0x41-0x4f (14)
+0x40|   05                                          | .              |          id: "category" (5) 0x41-0x42 (1)
+0x40|      0c                                       |  .             |          length: 12 0x42-0x43 (1)
+0x40|         54 65 73 74 20 50 72 6f 67 72 61 6d   |   Test Program |          value: "Test Program" 0x43-0x4f (12)
+    |                                               |                |        [6]{}: entry 0x4f-0x5c (13)
+0x40|                                             07|               .|          id: "loader" (7) 0x4f-0x50 (1)
+0x50|0b                                             |.               |          length: 11 0x50-0x51 (1)
+0x50|   52 4f 4d 20 74 69 6d 69 6e 67 73            | ROM timings    |          value: "ROM timings" 0x51-0x5c (11)
+    |                                               |                |        [7]{}: entry 0x5c-0x6e (18)
+0x50|                                    08         |            .   |          id: "origin" (8) 0x5c-0x5d (1)
+0x50|                                       10      |             .  |          length: 16 0x5d-0x5e (1)
+0x50|                                          4f 72|              Or|          value: "Original release" 0x5e-0x6e (16)
+0x60|69 67 69 6e 61 6c 20 72 65 6c 65 61 73 65      |iginal release  |
+    |                                               |                |        [8]{}: entry 0x6e-0x88 (26)
+0x60|                                          ff   |              . |          id: "comment" (255) 0x6e-0x6f (1)
+0x60|                                             18|               .|          length: 24 0x6f-0x70 (1)
+0x70|54 5a 58 65 64 20 62 79 20 4d 69 63 68 61 65 6c|TZXed by Michael|          value: "TZXed by Michael R. Cook" 0x70-0x88 (24)
+0x80|20 52 2e 20 43 6f 6f 6b                        | R. Cook        |
+    |                                               |                |    [1]{}: block 0x88-0xa0 (24)
+0x80|                        10                     |        .       |      type: "standard_speed_data" (16) 0x88-0x89 (1)
+0x80|                           e8 03               |         ..     |      pause: 1000 0x89-0x8b (2)
+    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|      tap{}: (tap) 0x8b-0xa0 (21)
+    |                                               |                |        blocks[0:1]: 0x8b-0xa0 (21)
+    |                                               |                |          [0]{}: block 0x8b-0xa0 (21)
+0x80|                                 13 00         |           ..   |            length: 19 0x8b-0x8d (2)
+    |                                               |                |            header{}: 0x8d-0xa0 (19)
+0x80|                                       00      |             .  |              flag: "standard_speed_data" (0) 0x8d-0x8e (1)
+0x80|                                          00   |              . |              data_type: "program" (0) 0x8e-0x8f (1)
+0x80|                                             66|               f|              program_name: "fqTestProg" 0x8f-0x99 (10)
+0x90|71 54 65 73 74 50 72 6f 67                     |qTestProg       |
+0x90|                           26 00               |         &.     |              data_length: 38 0x99-0x9b (2)
+0x90|                                 0a 00         |           ..   |              auto_start_line: 10 0x9b-0x9d (2)
+0x90|                                       26 00   |             &. |              program_length: 38 0x9d-0x9f (2)
+0x90|                                             01|               .|              checksum: 0x1 (valid) 0x9f-0xa0 (1)
+    |                                               |                |    [2]{}: block 0xa0-0xcd (45)
+0xa0|10                                             |.               |      type: "standard_speed_data" (16) 0xa0-0xa1 (1)
+0xa0|   e8 03                                       | ..             |      pause: 1000 0xa1-0xa3 (2)
+    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|      tap{}: (tap) 0xa3-0xcd (42)
+    |                                               |                |        blocks[0:1]: 0xa3-0xcd (42)
+    |                                               |                |          [0]{}: block 0xa3-0xcd (42)
+0xa0|         28 00                                 |   (.           |            length: 40 0xa3-0xa5 (2)
+    |                                               |                |            data{}: 0xa5-0xcd (40)
+0xa0|               ff                              |     .          |              flag: "standard_speed_data" (255) 0xa5-0xa6 (1)
+0xa0|                  00 0a 14 00 20 f5 22 66 71 20|      .... ."fq |              data: raw bits 0xa6-0xcc (38)
+0xb0|69 73 20 74 68 65 20 62 65 73 74 21 22 0d 00 14|is the best!"...|
+0xc0|0a 00 ec 31 30 0e 00 00 0a 00 00 0d            |...10.......    |
+0xc0|                                    b6|        |            .|  |              checksum: 0xb6 (valid) 0xcc-0xcd (1)

format/tzx/tzx.go

@@ -0,0 +1,655 @@
+package tzx
+
+// https://worldofspectrum.net/TZXformat.html
+
+import (
+	"embed"
+
+	"golang.org/x/text/encoding/charmap"
+
+	"github.com/wader/fq/format"
+	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
+	"github.com/wader/fq/pkg/scalar"
+)
+
+//go:embed tzx.md
+var tzxFS embed.FS
+
+var tapFormat decode.Group
+
+func init() {
+	interp.RegisterFormat(
+		format.TZX,
+		&decode.Format{
+			Description: "TZX tape format for ZX Spectrum computers",
+			Groups:      []*decode.Group{format.Probe},
+			DecodeFn:    tzxDecode,
+			Dependencies: []decode.Dependency{
+				{Groups: []*decode.Group{format.TAP}, Out: &tapFormat},
+			},
+		})
+	interp.RegisterFS(tzxFS)
+}
+
+func tzxDecode(d *decode.D) any {
+	d.Endian = decode.LittleEndian
+
+	d.FieldRawLen("signature", 8*8, d.AssertBitBuf([]byte("ZXTape!\x1A")))
+	d.FieldU8("major_version")
+	d.FieldU8("minor_version")
+	decodeBlocks(d)
+
+	return nil
+}
+
+func decodeBlocks(d *decode.D) {
+	d.FieldArray("blocks", func(d *decode.D) {
+		for !d.End() {
+			d.FieldStruct("block", func(d *decode.D) {
+				decodeBlock(d)
+			})
+		}
+	})
+}
+
+func decodeBlock(d *decode.D) {
+	blocks := map[uint64]func(d *decode.D){
+		// ID: 10h (16d) | Standard Speed Data
+		// This block is replayed with the standard Spectrum ROM timing values
+		// (the values in curly brackets in block ID 11). The pilot tone
+		// consists of 8063 pulses if the first data byte (the flag byte)
+		// is < 128, 3223 otherwise.
+		0x10: func(d *decode.D) {
+			// Pause after this block (ms.) {1000}
+			d.FieldU16("pause")
+
+			// A single TAP Data Block
+			peekBytes := d.PeekBytes(2)                              // get the TAP data block length
+			length := uint16(peekBytes[1])<<8 | uint16(peekBytes[0]) // bytes are stored in LittleEndian
+			length += 2                                              // include the two bytes for this value
+			d.FieldFormatLen("tap", int64(length)*8, &tapFormat, nil)
+		},
+
+		// ID: 11h (17d) | Turbo Speed Data
+		// This block is very similar to the normal TAP block but with some
+		// additional info on the timings and other important differences. The
+		// same tape encoding is used as for the standard speed data block. If
+		// a block should use some non-standard sync or pilot tones (i.e. all
+		// sorts of protection schemes) then the next three blocks describe it.
+		0x11: func(d *decode.D) {
+			d.FieldU16("pilot_pulse")  // Length of PILOT pulse {2168}
+			d.FieldU16("sync_pulse_1") // Length of SYNC first pulse {667}
+			d.FieldU16("sync_pulse_2") // Length of SYNC second pulse {735}
+			d.FieldU16("bit0_pulse")   // Length of ZERO bit pulse {855}
+			d.FieldU16("bit1_pulse")   // Length of ONE bit pulse {1710}
+
+			// Length of PILOT tone (number of pulses)
+			// {8063 header (flag<128), 3223 data (flag>=128)}
+			d.FieldU16("pilot_tone")
+
+			// Used bits in the last byte (other bits should be 0) {8}
+			// e.g. if this is 6, then the bits used (x) in the last byte are: xxxxxx00,
+			// where MSb is the leftmost bit, LSb is the rightmost bit
+			d.FieldU8("used_bits")
+
+			d.FieldU16("pause")            // Pause after this block (ms.) {1000}
+			length := d.FieldU24("length") // Length of data that follows
+
+			// Data as in .TAP files
+			d.FieldRawLen("data", int64(length*8))
+		},
+
+		// ID: 12h (18d) | Pure Tone
+		// This will produce a tone which is basically the same as the pilot
+		// tone in 10h and 11h blocks.
+		0x12: func(d *decode.D) {
+			d.FieldU16("pulse_length") // Length of one pulse in T-states
+			d.FieldU16("pulse_count")  // Number of pulses
+		},
+
+		// ID: 13h (19d) | Sequence of Pulses
+		// This will produce N pulses, each having its own timing. Up to 255
+		// pulses can be stored in this block.
+		0x13: func(d *decode.D) {
+			count := d.FieldU8("pulse_count")
+			d.FieldArray("pulses", func(d *decode.D) {
+				for i := uint64(0); i < count; i++ {
+					d.FieldU16("pulse")
+				}
+			})
+		},
+
+		// ID: 14h (20d) | Pure Data
+		// This is the same as in the turbo loading data block, except that it
+		// has no pilot or sync pulses.
+		0x14: func(d *decode.D) {
+			d.FieldU16("bit0_pulse")       // Length of ZERO bit pulse
+			d.FieldU16("bit1_pulse")       // Length of ONE bit pulse
+			d.FieldU8("used_bits")         // Used bits in last byte
+			d.FieldU16("pause")            // Pause after this block (ms.)
+			length := d.FieldU24("length") // Length of data that follows
+
+			// Data as in .TAP files
+			d.FieldRawLen("data", int64(length*8))
+		},
+
+		// ID: 15h (21d) | Direct Recording
+		// This block is used for tapes which have some parts in a format such
+		// that the turbo loader block cannot be used. This is not like a VOC
+		// file since the information is much more compact. Each sample value
+		// is represented by one bit only (0 for low, 1 for high) which means
+		// that the block will be at most 1/8 the size of the equivalent VOC.
+		// The preferred sampling frequencies are 22050 or 44100 Hz
+		// (158 or 79 T-states/sample).
+		0x15: func(d *decode.D) {
+			d.FieldU16("t_states")                 // Number of T-states per sample (bit of data)
+			d.FieldU16("pause")                    // Pause after this block in milliseconds (ms.)
+			d.FieldU8("used_bits")                 // Used bits (samples) in last byte of data (1-8)
+			length := d.FieldU24("length")         // Length of data that follows
+			d.FieldRawLen("data", int64(length*8)) // Samples data. Each bit represents a state on the EAR port
+		},
+
+		// ID: 18h (24d) | CSW Recording
+		// This block contains a sequence of raw pulses encoded in CSW format
+		// v2 (Compressed Square Wave).
+		0x18: func(d *decode.D) {
+			length := d.FieldU32("length") // Block length (without these four bytes)
+
+			// NOTE: remove these next 4 fields from the length so
+			// the data size is calculated correctly
+			length -= 2 + 3 + 1 + 4
+
+			// Pause after this block (in ms)
+			d.FieldU16("pause")
+			// Sampling rate
+			d.FieldU24("sample_rate")
+			// Compression type
+			d.FieldU8("compression_type", scalar.UintMapSymStr{0x01: "rle", 0x02: "zrle"})
+			// Number of stored pulses (after decompression)
+			d.FieldU32("stored_pulse_count")
+
+			// CSW data, encoded according to the CSW specification
+			d.FieldRawLen("data", int64(length*8))
+		},
+
+		// ID: 19h (25d) | Generalized Data
+		// This block was developed to represent an extremely wide range of data
+		// encoding techniques. Each loading component (pilot tone, sync pulses,
+		// data) is associated to a specific sequence of pulses, where each
+		// sequence (wave) can contain a different number of pulses from the
+		// others. In this way it is possible to have a situation where bit 0 is
+		// represented with 4 pulses and bit 1 with 8 pulses.
+		0x19: func(d *decode.D) {
+			length := d.FieldU32("length") // Block length (without these four bytes)
+			// TBD:
+			//	Pause        uint16     // Pause after this block (ms)
+			//	TOTP         uint32     // Total number of symbols in pilot/sync block (can be 0)
+			//	NPP          uint8      // Maximum number of pulses per pilot/sync symbol
+			//	ASP          uint8      // Number of pilot/sync symbols in the alphabet table (0=256)
+			//	TOTD         uint32     // Total number of symbols in data stream (can be 0)
+			//	NPD          uint8      // Maximum number of pulses per data symbol
+			//	ASD          uint8      // Number of data symbols in the alphabet table (0=256)
+			//	PilotSymbols []Symbol   // 0x12  SYMDEF[ASP] Pilot and sync symbols definition table
+			//	PilotStreams []PilotRLE // 0x12+ (2*NPP+1)*ASP - PRLE[TOTP]  Pilot and sync data stream
+			//	DataSymbols  []Symbol   // 0x12+ (TOTP>0)*((2*NPP+1)*ASP)+TOTP*3  - SYMDEF[ASD] Data symbols definition table
+			//	DataStreams  []uint8    // 0x12+ (TOTP>0)*((2*NPP+1)*ASP)+ TOTP*3+(2*NPD+1)*ASD - BYTE[DS]  Data stream
+			d.FieldRawLen("data", int64(length*8))
+		},
+
+		// ID: 20h (32d) | Pause Tape Command
+		// This will make a silence (low amplitude level (0)) for a given time
+		// in milliseconds. If the value is 0 then the emulator or utility should
+		// (in effect) STOP THE TAPE, until the user or emulator requests it.
+		0x20: func(d *decode.D) {
+			d.FieldU16("pause") // Pause duration in ms.
+		},
+
+		// ID: 21h (33d) | Group Start
+		// This block marks the start of a group of blocks which are to be
+		// treated as one single (composite) block. For each group start block
+		// there must be a group end block. Nesting of groups is not allowed.
+		0x21: func(d *decode.D) {
+			length := d.FieldU8("length")
+			d.FieldStr("group_name", int(length), charmap.ISO8859_1)
+		},
+
+		// ID: 22h (34d) | Group End
+		// This indicates the end of a group. This block has no body.
+		0x22: func(d *decode.D) {},
+
+		// JumpTo
+		// ID: 23h (35d)
+		// This block will allow for jumping from one block to another within
+		// the file. All blocks are included in the block count!
+		0x23: func(d *decode.D) {
+			d.FieldS16("value", scalar.SintMapSymStr{
+				0:  "loop_forever",
+				1:  "next_block",
+				2:  "skip_block",
+				-1: "prev_block",
+			})
+		},
+
+		// ID: 24h (36d) | Loop Start
+		// Indicates a sequence of identical blocks, or of identical groups of
+		// blocks. This block is the same as the FOR statement in BASIC.
+		0x24: func(d *decode.D) {
+			d.FieldU16("repetitions") // Number of repetitions (greater than 1)
+		},
+
+		// ID: 25h (37d) | Loop End
+		// This is the same as BASIC's NEXT statement. It means that the utility
+		// should jump back to the start of the loop if it hasn't been run for
+		// the specified number of times. This block has no body.
+		0x25: func(d *decode.D) {},
+
+		// ID: 26h (38d) | Call Sequence
+		// This block is an analogue of the CALL Subroutine statement. It
+		// basically executes a sequence of blocks that are somewhere else and
+		// then goes back to the next block. Because more than one call can be
+		// normally used you can include a list of sequences to be called. CALL
+		// blocks can be used in the LOOP sequences and vice versa. The value
+		// is relative so that you can add some blocks in the beginning of the
+		// file without disturbing the call values.
+		// Look at 'Jump To Block' for reference on the values.
+		0x26: func(d *decode.D) {
+			count := d.FieldU16("count")
+			d.FieldArray("call_blocks", func(d *decode.D) {
+				for i := uint64(0); i < count; i++ {
+					d.FieldS16("offset")
+				}
+			})
+		},
+
+		// ID: 27h (39d) | Return From Sequence
+		// This block indicates the end of the Called Sequence. The next block
+		// played will be the block after the last CALL block (or the next Call,
+		// if the Call block had multiple calls). This block has no body.
+		0x27: func(d *decode.D) {},
+
+		// ID: 28h (40d) | Select
+		// This block is useful when the tape consists of two or more separately
+		// loadable parts. With this block it is possible to select one of the
+		// parts and the utility/emulator will start loading from that block.
+		// All offsets are relative signed words.
+		0x28: func(d *decode.D) {
+			// Length of the whole block (without these two bytes)
+			d.FieldU16("length")
+
+			count := d.FieldU8("count")
+			d.FieldArray("selections", func(d *decode.D) {
+				for i := 0; i < int(count); i++ {
+					d.FieldStruct("selection", func(d *decode.D) {
+						d.FieldS16("offset")          // Relative Offset as `signed` value
+						length := d.FieldU8("length") // Length of description text (max 30 chars)
+						d.FieldStr("description", int(length), charmap.ISO8859_1)
+					})
+				}
+			})
+		},
+
+		// ID: 2Ah (42d) | Stop Tape When 48k Mode
+		// When this block is encountered, the tape will stop ONLY if the machine
+		// is an 48K Spectrum. This block is to be used for multi-loading games
+		// that load one level at a time in 48K mode, but load the entire tape at
+		// once if in 128K mode.
+		// This block has no body of its own, but follows the extension rule.
+		0x2A: func(d *decode.D) {
+			d.FieldU32("length") // Length of the block without these four bytes (0)
+		},
+
+		// ID: 2Bh (43d) | Set Signal Level
+		// This block sets the current signal level to the specified value
+		// (high or low). It should be used whenever it is necessary to avoid
+		// any ambiguities, e.g. with custom loaders which are level-sensitive.
+		0x2B: func(d *decode.D) {
+			d.FieldU32("length") // Block length (without these four bytes)
+			d.FieldU8("signal_level", scalar.UintMapSymStr{0: "low", 1: "high"})
+		},
+
+		// ID: 30h (48d) | Text Description
+		// This is meant to identify parts of the tape, such as where level 1
+		// starts, where to rewind to when the game ends, etc. This description
+		// is not guaranteed to be shown while the tape is playing, but can be
+		// read while browsing the tape or changing the tape pointer.
+		// The description can be up to 255 characters long.
+		0x30: func(d *decode.D) {
+			length := d.FieldU8("length")
+			d.FieldStr("description", int(length), charmap.ISO8859_1)
+		},
+
+		// ID: 31h (49d) | Message
+		// This will enable the emulators to display a message for a given time.
+		// This should not stop the tape and it should not make silence. If the
+		// time is 0 then the emulator should wait for the user to press a key.
+		0x31: func(d *decode.D) {
+			// Time (in seconds) for which the message should be displayed
+			d.FieldU8("display_time")
+			// Length of the text message
+			length := d.FieldU8("length")
+			// Message that should be displayed in ASCII format
+			d.FieldStr("message", int(length), charmap.ISO8859_1)
+		},
+
+		// ID: 32h (50d) | Archive Info
+		// This optional block is used at the beginning of the tape containing
+		// various metadata about the tape.
+		0x32: func(d *decode.D) {
+			d.FieldU16("length")        // Length of the whole block without these two bytes
+			count := d.FieldU8("count") // Number of entries in the archive info
+
+			// the archive strings
+			d.FieldArray("archive_info", func(d *decode.D) {
+				for i := uint64(0); i < count; i++ {
+					d.FieldStruct("entry", func(d *decode.D) {
+						d.FieldU8("id", scalar.UintMapSymStr{
+							0x00: "title",
+							0x01: "publisher",
+							0x02: "author",
+							0x03: "year",
+							0x04: "language",
+							0x05: "category",
+							0x06: "price",
+							0x07: "loader",
+							0x08: "origin",
+							0xFF: "comment",
+						})
+						length := d.FieldU8("length")
+						d.FieldStr("value", int(length), charmap.ISO8859_1)
+					})
+				}
+			})
+		},
+
+		// ID: 33h (51d) | Hardware Type
+		// This blocks contains information about the hardware that the programs
+		// on this tape use.
+		0x33: func(d *decode.D) {
+			// Number of machines and hardware types for which info is supplied
+			count := d.FieldU8("count")
+			d.FieldArray("hardware_info", func(d *decode.D) {
+				for i := uint64(0); i < count; i++ {
+					d.FieldStruct("info", func(d *decode.D) {
+						// Hardware Type ID (computers, printers, mice, etc.)
+						typeId := d.FieldU8("type", hwInfoTypeMapper)
+						// Hardware ID (ZX81, Kempston Joystick, etc.)
+						d.FieldU8("id", hwInfoTypeIdMapper[typeId])
+						// Hardware compatibility information
+						d.FieldU8("info_id", hwInfoIdMapper)
+					})
+				}
+			})
+		},
+
+		// ID: 35h (53d) | Custom Info
+		// This block contains various custom data. For example, it might contain
+		// some information written by a utility, extra settings required by a
+		// particular emulator, etc.
+		0x35: func(d *decode.D) {
+			d.FieldStr("identification", 10, charmap.ISO8859_1)
+			length := d.FieldU32("length")
+			d.FieldRawLen("info", int64(length*8))
+		},
+
+		// ID: 5Ah (90d) | Glue Block
+		// This block is generated when two ZX Tape files are merged together.
+		// It is here so that you can easily copy the files together and use
+		// them. Of course, this means that resulting file would be 10 bytes
+		// longer than if this block was not used. All you have to do if you
+		// encounter this block ID is to skip next 9 bytes. If you can avoid
+		// using this block for this purpose, then do so; it is preferable to
+		// use a utility to join the two files and ensure that they are both
+		// of the higher version number.
+		0x5A: func(d *decode.D) {
+			// Value: { "XTape!",0x1A,MajR,MinR }
+			// Just skip these 9 bytes and you will end up on the next ID.
+			d.FieldRawLen("value", int64(9*8))
+		},
+	}
+
+	blockType := d.FieldU8("type", blockTypeMapper)
+
+	// Deprecated block types: C64RomType, C64TurboData, EmulationInfo, Snapshot
+	if blockType == 0x16 || blockType == 0x17 || blockType == 0x34 || blockType == 0x40 {
+		d.Fatalf("deprecated block type encountered: %02x", blockType)
+	}
+
+	if fn, ok := blocks[blockType]; ok {
+		fn(d)
+	} else {
+		d.Fatalf("block type not valid, got: %02x", blockType)
+	}
+}
+
+var blockTypeMapper = scalar.UintMapSymStr{
+	0x10: "standard_speed_data",
+	0x11: "turbo_speed_data",
+	0x12: "pure_tone",
+	0x13: "sequence_of_pulses",
+	0x14: "pure_data",
+	0x15: "direct_recording", // deprecated
+	0x16: "c64_rom_type",     // deprecated
+	0x17: "c64_turbo_data",
+	0x18: "csw_recording",
+	0x19: "generalized_data",
+	0x20: "pause_tape_command",
+	0x21: "group_start",
+	0x22: "group_end",
+	0x23: "jump_to",
+	0x24: "loop_start",
+	0x25: "loop_end",
+	0x26: "call_sequence",
+	0x27: "return_from_sequence",
+	0x28: "select",
+	0x2A: "stop_tape_when_48k_mode",
+	0x2B: "set_signal_level",
+	0x30: "text_description",
+	0x31: "message",
+	0x32: "archive_info",
+	0x33: "hardware_type",
+	0x34: "emulation_info", // deprecated
+	0x35: "custom_info",
+	0x40: "snapshot", // deprecated
+	0x5A: "glue_block",
+}
+
+var hwInfoTypeMapper = scalar.UintMapDescription{
+	0x00: "Computers",
+	0x01: "External storage",
+	0x02: "ROM/RAM type add-ons",
+	0x03: "Sound devices",
+	0x04: "Joysticks",
+	0x05: "Mice",
+	0x06: "Other controllers",
+	0x07: "Serial ports",
+	0x08: "Parallel ports",
+	0x09: "Printers",
+	0x0a: "Modems",
+	0x0b: "Digitizers",
+	0x0c: "Network adapters",
+	0x0d: "Keyboards & keypads",
+	0x0e: "AD/DA converters",
+	0x0f: "EPROM programmers",
+	0x10: "Graphics",
+}
+
+var hwInfoTypeIdMapper = map[uint64]scalar.UintMapDescription{
+	0x00: { // Computers
+		0x00: "ZX Spectrum 16k",
+		0x01: "ZX Spectrum 48k, Plus",
+		0x02: "ZX Spectrum 48k ISSUE 1",
+		0x03: "ZX Spectrum 128k +(Sinclair)",
+		0x04: "ZX Spectrum 128k +2 (grey case)",
+		0x05: "ZX Spectrum 128k +2A, +3",
+		0x06: "Timex Sinclair TC-2048",
+		0x07: "Timex Sinclair TS-2068",
+		0x08: "Pentagon 128",
+		0x09: "Sam Coupe",
+		0x0a: "Didaktik M",
+		0x0b: "Didaktik Gama",
+		0x0c: "ZX-80",
+		0x0d: "ZX-81",
+		0x0e: "ZX Spectrum 128k, Spanish version",
+		0x0f: "ZX Spectrum, Arabic version",
+		0x10: "Microdigital TK 90-X",
+		0x11: "Microdigital TK 95",
+		0x12: "Byte",
+		0x13: "Elwro 800-3 ",
+		0x14: "ZS Scorpion 256",
+		0x15: "Amstrad CPC 464",
+		0x16: "Amstrad CPC 664",
+		0x17: "Amstrad CPC 6128",
+		0x18: "Amstrad CPC 464+",
+		0x19: "Amstrad CPC 6128+",
+		0x1a: "Jupiter ACE",
+		0x1b: "Enterprise",
+		0x1c: "Commodore 64",
+		0x1d: "Commodore 128",
+		0x1e: "Inves Spectrum+",
+		0x1f: "Profi",
+		0x20: "GrandRomMax",
+		0x21: "Kay 1024",
+		0x22: "Ice Felix HC 91",
+		0x23: "Ice Felix HC 2000",
+		0x24: "Amaterske RADIO Mistrum",
+		0x25: "Quorum 128",
+		0x26: "MicroART ATM",
+		0x27: "MicroART ATM Turbo 2",
+		0x28: "Chrome",
+		0x29: "ZX Badaloc",
+		0x2a: "TS-1500",
+		0x2b: "Lambda",
+		0x2c: "TK-65",
+		0x2d: "ZX-97",
+	},
+	0x01: { // External storage
+		0x00: "ZX Microdrive",
+		0x01: "Opus Discovery",
+		0x02: "MGT Disciple",
+		0x03: "MGT Plus-D",
+		0x04: "Rotronics Wafadrive",
+		0x05: "TR-DOS (BetaDisk)",
+		0x06: "Byte Drive",
+		0x07: "Watsford",
+		0x08: "FIZ",
+		0x09: "Radofin",
+		0x0a: "Didaktik disk drives",
+		0x0b: "BS-DOS (MB-02)",
+		0x0c: "ZX Spectrum +3 disk drive",
+		0x0d: "JLO (Oliger) disk interface",
+		0x0e: "Timex FDD3000",
+		0x0f: "Zebra disk drive",
+		0x10: "Ramex Millennia",
+		0x11: "Larken",
+		0x12: "Kempston disk interface",
+		0x13: "Sandy",
+		0x14: "ZX Spectrum +3e hard disk",
+		0x15: "ZXATASP",
+		0x16: "DivIDE",
+		0x17: "ZXCF",
+	},
+	0x02: { // ROM/RAM type add_ons
+		0x00: "Sam Ram",
+		0x01: "Multiface ONE",
+		0x02: "Multiface 128k",
+		0x03: "Multiface +3",
+		0x04: "MultiPrint",
+		0x05: "MB-02 ROM/RAM expansion",
+		0x06: "SoftROM",
+		0x07: "1k",
+		0x08: "16k",
+		0x09: "48k",
+		0x0a: "Memory in 8-16k used",
+	},
+	0x03: { // Sound devices
+		0x00: "Classic AY hardware (compatible with 128k ZXs)",
+		0x01: "Fuller Box AY sound hardware",
+		0x02: "Currah microSpeech",
+		0x03: "SpecDrum",
+		0x04: "AY ACB stereo (A+C=left, B+C=right); Melodik",
+		0x05: "AY ABC stereo (A+B=left, B+C=right)",
+		0x06: "RAM Music Machine",
+		0x07: "Covox",
+		0x08: "General Sound",
+		0x09: "Intec Electronics Digital Interface B8001",
+		0x0a: "Zon-X AY",
+		0x0b: "QuickSilva AY",
+		0x0c: "Jupiter ACE",
+	},
+	0x04: { // Joysticks
+		0x00: "Kempston",
+		0x01: "Cursor, Protek, AGF",
+		0x02: "Sinclair 2 Left (12345)",
+		0x03: "Sinclair 1 Right (67890)",
+		0x04: "Fuller",
+	},
+	0x05: { // Mice
+		0x00: "AMX mouse",
+		0x01: "Kempston mouse",
+	},
+	0x06: { // Other controllers
+		0x00: "Trickstick",
+		0x01: "ZX Light Gun",
+		0x02: "Zebra Graphics Tablet",
+		0x03: "Defender Light Gun",
+	},
+	0x07: { // Serial ports
+		0x00: "ZX Interface 1",
+		0x01: "ZX Spectrum 128k",
+	},
+	0x08: { // Parallel ports
+		0x00: "Kempston S",
+		0x01: "Kempston E",
+		0x02: "ZX Spectrum +3",
+		0x03: "Tasman",
+		0x04: "DK'Tronics",
+		0x05: "Hilderbay",
+		0x06: "INES Printerface",
+		0x07: "ZX LPrint Interface 3",
+		0x08: "MultiPrint",
+		0x09: "Opus Discovery",
+		0x0a: "Standard 8255 chip with ports 31,63,95",
+	},
+	0x09: { // Printers
+		0x00: "ZX Printer, Alphacom 32 & compatibles",
+		0x01: "Generic printer",
+		0x02: "EPSON compatible",
+	},
+	0x0a: { // Modems
+		0x00: "Prism VTX 5000",
+		0x01: "T/S 2050 or Westridge 2050",
+	},
+	0x0b: { // Digitizers
+		0x00: "RD Digital Tracer",
+		0x01: "DK'Tronics Light Pen",
+		0x02: "British MicroGraph Pad",
+		0x03: "Romantic Robot Videoface",
+	},
+	0x0c: { // Network adapters
+		0x00: "ZX Interface 1",
+	},
+	0x0d: { // Keyboards & keypads
+		0x00: "Keypad for ZX Spectrum 128k",
+	},
+	0x0e: { // AD/DA converters
+		0x00: "Harley Systems ADC 8.2",
+		0x01: "Blackboard Electronics",
+	},
+	0x0f: { // EPROM programmers
+		0x00: "Orme Electronics",
+	},
+	0x10: { // Graphics
+		0x00: "WRX Hi-Res",
+		0x01: "G007",
+		0x02: "Memotech",
+		0x03: "Lambda Colour",
+	},
+}
+
+var hwInfoIdMapper = scalar.UintMapDescription{
+	00: "RUNS on this machine or with this hardware, but may or may not use the hardware or special features of the machine.",
+	01: "USES the hardware or special features of the machine, such as extra memory or a sound chip.",
+	02: "RUNS but it DOESN'T use the hardware or special features of the machine.",
+	03: "DOESN'T RUN on this machine or with this hardware.",
+}

format/tzx/tzx.md

@@ -0,0 +1,17 @@
+`TZX` is a file format designed to preserve cassette tapes compatible with the
+ZX Spectrum computers, although some specialized versions of the format have
+been defined for other machines such as the Amstrad CPC and C64.
+
+The format was originally created by Tomaz Kac, who was maintainer until
+`revision 1.13`, before passing it to Martijn v.d. Heide. For a brief period
+the company Ramsoft became the maintainers, and created revision `v1.20`.
+
+The default file extension is `.tzx`.
+
+### Authors
+
+- Michael R. Cook work.mrc@pm.me, original author
+
+### References
+
+- https://worldofspectrum.net/TZXformat.html

go.mod

@@ -48,7 +48,7 @@ require (
 	// bump: gomod-golang-x-crypto /golang\.org\/x\/crypto v(.*)/ https://github.com/golang/crypto.git|^0
 	// bump: gomod-golang-x-crypto command go get -d golang.org/x/crypto@v$LATEST && go mod tidy
 	// bump: gomod-golang-x-crypto link "Tags" https://github.com/golang/crypto/tags
-	golang.org/x/crypto v0.25.0
+	golang.org/x/crypto v0.26.0
 
 	// has no tags
 	// go get -d golang.org/x/exp@master && go mod tidy
@@ -57,17 +57,17 @@ require (
 	// bump: gomod-golang-x-net /golang\.org\/x\/net v(.*)/ https://github.com/golang/net.git|^0
 	// bump: gomod-golang-x-net command go get -d golang.org/x/net@v$LATEST && go mod tidy
 	// bump: gomod-golang-x-net link "Tags" https://github.com/golang/net/tags
-	golang.org/x/net v0.27.0
+	golang.org/x/net v0.28.0
 
 	// bump: gomod-golang-x-term /golang\.org\/x\/term v(.*)/ https://github.com/golang/term.git|^0
 	// bump: gomod-golang-x-term command go get -d golang.org/x/term@v$LATEST && go mod tidy
 	// bump: gomod-golang-x-term link "Tags" https://github.com/golang/term/tags
-	golang.org/x/term v0.22.0
+	golang.org/x/term v0.23.0
 
 	// bump: gomod-golang/text /golang\.org\/x\/text v(.*)/ https://github.com/golang/text.git|^0
 	// bump: gomod-golang/text command go get -d golang.org/x/text@v$LATEST && go mod tidy
 	// bump: gomod-golang/text link "Source diff $CURRENT..$LATEST" https://github.com/golang/text/compare/v$CURRENT..v$LATEST
-	golang.org/x/text v0.16.0
+	golang.org/x/text v0.17.0
 
 	// bump: gomod-gopkg.in/yaml.v3 /gopkg\.in\/yaml\.v3 v(.*)/ https://github.com/go-yaml/yaml.git|^3
 	// bump: gomod-gopkg.in/yaml.v3 command go get -d gopkg.in/yaml.v3@v$LATEST && go mod tidy
@@ -79,6 +79,6 @@ require (
 	github.com/itchyny/timefmt-go v0.1.5 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
-	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/sys v0.23.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 )

go.sum

@@ -25,18 +25,18 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/wader/gojq v0.12.1-0.20240401131232-6c6bc364201a h1:P881Oecjt9FEXrwkGJ6UObJksxejJaF/fKq1ZfXpiVE=
 github.com/wader/gojq v0.12.1-0.20240401131232-6c6bc364201a/go.mod h1:qVrzkUdnBtJvM4twyRQ6xdziPSnSp35dLm4s/DN2iP4=
-golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
-golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
 golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw=
 golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ=
-golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
-golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
-golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
-golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
-golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
+golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
+golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
+golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
+golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
+golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
+golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=