From ebf063d1c09439911f94951ea7fee6d2b3d68522 Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Tue, 30 Apr 2024 14:16:05 +0200 Subject: [PATCH] doc: Cleanup and improve texts a bit --- README.md | 24 +++++++++++------------- doc/dev.md | 4 ++-- pkg/interp/help.jq | 8 ++++---- pkg/interp/testdata/help.fqtest | 2 +- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 00ea8880..d2b881f6 100644 --- a/README.md +++ b/README.md @@ -2,25 +2,23 @@ Tool, language and decoders for working with binary data. +TLDR: it aims to be jq, hexdump, dd and gdb for files combined into one. + ![fq demo](doc/demo.svg) -Basic usage is `fq . file` or `fq d file`. +Basic usage is `fq . file`, `fq d file` or `fq 'some query' file ...`. For details see [usage.md](doc/usage.md). ### Background -fq is inspired by the well known jq tool and language that allows you to work with binary formats the same way you would using jq. In addition it can present data like a hex viewer, transform, slice and concatenate binary data. It also supports nested formats and has an interactive REPL with auto-completion. +fq is inspired by the [jq](https://jqlang.github.io/jq/) tool and language and allows you to work with binary formats in the same way. In addition to using jq expressions it can also present decoded tree structures, transform, slice and concatenate binary data. It also supports nested formats and features an interactive REPL with auto-completion of functions and names. -It was originally designed to query, inspect and debug media codecs and containers like mp4, flac, mp3, jpeg. But since then it has been extended to support a variety of formats like executables, packet captures (with TCP reassembly) and serialization formats like JSON, YAML, XML, ASN1 BER, Avro, CBOR, protobuf. In addition it also has functions to work with URLs, convert to/from hex, number bases, search for things etc. - -In summary it aims to be jq, hexdump, dd and gdb for files combined into one. - -**NOTE:** fq is still early in development so things might change, be broken or do not make sense. That also means that there is a great opportunity to help out! +It was originally designed to query, inspect and debug media codecs and containers like MP4, FLAC and JPEG but has since been extended to support a variety of formats like executables, packet captures (with TCP reassembly) and serialization formats like JSON, YAML, XML, CBOR, protobuf. In addition it also has functions to work with URLs, convert to/from hex, number bases, search for patterns etc. ### Goals -- Make binaries accessible, queryable and sliceable. +- Make binaries more accessible, queryable and sliceable. - Nested formats and bit-oriented decoding. - Quick and comfortable CLI tool. - Bits and bytes transformations. @@ -243,7 +241,7 @@ apk add -X http://dl-cdn.alpinelinux.org/alpine/edge/testing fq Make sure you have [go](https://go.dev) 1.20 or later installed. -To install directly from git repository (no clone needed): +To install directly from git repository (no git clone needed): ```sh # build and install latest release go install github.com/wader/fq@latest @@ -267,10 +265,6 @@ go build -o fq . make test fq ``` -## TODO and ideas - -See [TODO.md](doc/TODO.md) - ## Development and adding a new decoder See [dev.md](doc/dev.md) @@ -306,6 +300,10 @@ for inventing the [jq](https://github.com/stedolan/jq) language. - [Sustainability of Digital Formats](https://www.loc.gov/preservation/digital/formats/) at Library of Congress. - [Data Format Description Language (DFDL)](https://en.wikipedia.org/wiki/Data_Format_Description_Language). +## TODO and ideas + +See [TODO.md](doc/TODO.md) + ## License `fq` is distributed under the terms of the MIT License. diff --git a/doc/dev.md b/doc/dev.md index 67295baf..5159056a 100644 --- a/doc/dev.md +++ b/doc/dev.md @@ -236,8 +236,8 @@ make test go test ./... # run all tests for one format go test -run TestFormats/mp4 ./format/ -# update all actual outputs in tests -go test ./... -update +# update all expected outputs for tests +go test ./pkg/interp ./format -update # update actual output for specific tests go run ./format -run TestFormats/elf -update # color diff diff --git a/pkg/interp/help.jq b/pkg/interp/help.jq index cb188ff2..a70c12ca 100644 --- a/pkg/interp/help.jq +++ b/pkg/interp/help.jq @@ -54,9 +54,6 @@ Same as recurse without argument. } }; -def help($_): error("help must be alone or last in pipeline. ex: help(length) or ... | help"); -def help: help(null); - def _help_format_enrich($arg0; $f; $include_basic): ( if $include_basic then .examples += @@ -76,6 +73,9 @@ def _help_format_enrich($arg0; $f; $include_basic): end ); +# trailing help gets rewritten to _help_slurp, these are here to catch other variants +def help($_): error("help must be alone or last in pipeline. ex: help(length) or ... | help"); +def help: help(null); def _help($arg0; $topic): ( $topic | if . == "usage" then @@ -236,7 +236,7 @@ def _help($arg0; $topic): | ($args | length) as $argc | if $args == null then # help - ( "Type expression to evaluate" + ( "Type jq expression to evaluate" , "help(...) Help for topic. Ex: help(mp4), help(\"mp4\")" , "\\t Completion" , "Up/Down History" diff --git a/pkg/interp/testdata/help.fqtest b/pkg/interp/testdata/help.fqtest index d2f44ed8..44354ccd 100644 --- a/pkg/interp/testdata/help.fqtest +++ b/pkg/interp/testdata/help.fqtest @@ -1,6 +1,6 @@ $ fq -ni null> help -Type expression to evaluate +Type jq expression to evaluate help(...) Help for topic. Ex: help(mp4), help("mp4") \t Completion Up/Down History