1
1
mirror of https://github.com/wader/fq.git synced 2024-09-11 20:07:11 +03:00

Compare commits

..

61 Commits

Author SHA1 Message Date
Mattias Wadman
56f2529ec1
Merge 892a42da36 into 943743dbaf 2024-05-21 15:46:50 +02:00
Mattias Wadman
943743dbaf
Merge pull request #949 from wader/bump-make-golangci-lint-1.58.2
Update make-golangci-lint to 1.58.2 from 1.58.1
2024-05-20 22:16:07 +02:00
Mattias Wadman
1b85ae622b
Merge pull request #950 from wader/bump-github-golangci-lint-1.58.2
Update github-golangci-lint to 1.58.2 from 1.58.1
2024-05-20 22:16:01 +02:00
bump
f6d7235449 Update github-golangci-lint to 1.58.2 from 1.58.1
Release notes https://github.com/golangci/golangci-lint/releases/tag/v1.58.2
2024-05-20 16:04:14 +00:00
bump
aef47df26a Update make-golangci-lint to 1.58.2 from 1.58.1
Release notes https://github.com/golangci/golangci-lint/releases/tag/v1.58.2
2024-05-20 16:04:10 +00:00
Mattias Wadman
1ec9748046
Merge pull request #947 from wader/bump-make-golangci-lint-1.58.1
Update make-golangci-lint to 1.58.1 from 1.58.0
2024-05-09 18:12:49 +02:00
Mattias Wadman
2e5514fc50
Merge pull request #948 from wader/bump-github-golangci-lint-1.58.1
Update github-golangci-lint to 1.58.1 from 1.58.0
2024-05-09 18:12:44 +02:00
bump
a59ba2a2fa Update github-golangci-lint to 1.58.1 from 1.58.0
Release notes https://github.com/golangci/golangci-lint/releases/tag/v1.58.1
2024-05-09 16:03:55 +00:00
bump
7714fcf423 Update make-golangci-lint to 1.58.1 from 1.58.0
Release notes https://github.com/golangci/golangci-lint/releases/tag/v1.58.1
2024-05-09 16:03:52 +00:00
Mattias Wadman
544cf4cc09
Merge pull request #945 from wader/bump-docker-golang-1.22.3
Update docker-golang to 1.22.3 from 1.22.2
2024-05-09 00:01:16 +02:00
Mattias Wadman
25ad5f1c8f
Merge pull request #946 from wader/bump-github-go-version-1.22.3
Update github-go-version to 1.22.3 from 1.22.2
2024-05-09 00:00:41 +02:00
bump
9ff7da12a7 Update github-go-version to 1.22.3 from 1.22.2 2024-05-08 16:03:46 +00:00
bump
94cfbc670c Update docker-golang to 1.22.3 from 1.22.2 2024-05-08 16:03:44 +00:00
Mattias Wadman
163b3b609c
Merge pull request #944 from wader/bump-gomod-golang-x-net-0.25.0
Update gomod-golang-x-net to 0.25.0 from 0.24.0
2024-05-07 18:31:45 +02:00
bump
cabb67e8ab Update gomod-golang-x-net to 0.25.0 from 0.24.0
Tags https://github.com/golang/net/tags
2024-05-07 16:03:50 +00:00
Mattias Wadman
64df8bdbf5
Merge pull request #943 from wader/bump-gomod-golang-x-crypto-0.23.0
Update gomod-golang-x-crypto to 0.23.0 from 0.22.0
2024-05-06 19:16:07 +02:00
Mattias Wadman
481ac91880
Merge pull request #942 from wader/bump-gomod-ergochat-readline-0.1.1
Update gomod-ergochat-readline to 0.1.1 from 0.1.0
2024-05-06 19:15:51 +02:00
bump
14ada50806 Update gomod-golang-x-crypto to 0.23.0 from 0.22.0
Tags https://github.com/golang/crypto/tags
2024-05-06 16:04:45 +00:00
bump
12f332064c Update gomod-ergochat-readline to 0.1.1 from 0.1.0
Release notes https://github.com/ergochat/readline/releases/tag/v0.1.1
2024-05-06 16:04:43 +00:00
Mattias Wadman
0ff3e53c5f
Merge pull request #941 from wader/bump-gomod-golang-x-term-0.20.0
Update gomod-golang-x-term to 0.20.0 from 0.19.0
2024-05-05 18:16:15 +02:00
bump
586cf142e5 Update gomod-golang-x-term to 0.20.0 from 0.19.0
Tags https://github.com/golang/term/tags
2024-05-05 16:03:41 +00:00
Mattias Wadman
fb20db5eb7
Merge pull request #937 from wader/bump-make-golangci-lint-1.58.0
Update make-golangci-lint to 1.58.0 from 1.57.2
2024-05-04 18:15:24 +02:00
Mattias Wadman
48868bd4ee
Merge pull request #939 from wader/bump-github-golangci-lint-1.58.0
Update github-golangci-lint to 1.58.0 from 1.57.2
2024-05-04 18:12:55 +02:00
Mattias Wadman
c5c8b75c56
Merge pull request #938 from wader/bump-gomod-golang/text-0.15.0
Update gomod-golang/text to 0.15.0 from 0.14.0
2024-05-04 18:12:01 +02:00
bump
a5de74cd23 Update github-golangci-lint to 1.58.0 from 1.57.2
Release notes https://github.com/golangci/golangci-lint/releases/tag/v1.58.0
2024-05-04 16:03:51 +00:00
bump
42730d7586 Update gomod-golang/text to 0.15.0 from 0.14.0
Source diff 0.14.0..0.15.0 https://github.com/golang/text/compare/v0.14.0..v0.15.0
2024-05-04 16:03:49 +00:00
bump
3a683b64f8 Update make-golangci-lint to 1.58.0 from 1.57.2
Release notes https://github.com/golangci/golangci-lint/releases/tag/v1.58.0
2024-05-04 16:03:46 +00:00
Mattias Wadman
496849daa5
Merge pull request #936 from wader/update-docs
doc: Cleanup and improve texts a bit
2024-04-30 14:25:36 +02:00
Mattias Wadman
ebf063d1c0 doc: Cleanup and improve texts a bit 2024-04-30 14:16:13 +02:00
Mattias Wadman
b8eec4078f
Merge pull request #934 from matmat/dht-patch
jpeg: Add parsing of DHT parameters
2024-04-25 13:29:44 +02:00
Mattias Iko Mattsson
6e13b4b550 jpeg: Add parsing of DHT parameters 2024-04-25 13:17:08 +02:00
Mattias Wadman
9eee65072f
Merge pull request #935 from wader/ignore-so-dotdotdot-works
build,test: Ignore some files to make ./... work
2024-04-21 11:30:37 +02:00
Mattias Wadman
6db6a54d13 build,test: Ignore some files to make ./... work 2024-04-21 09:50:02 +02:00
Mattias Wadman
08ced4515f
Merge pull request #933 from wader/native-trim
fq: Use trim from gojq
2024-04-19 18:23:15 +02:00
Mattias Wadman
6059b9ee1b fq: Use trim from gojq 2024-04-19 18:09:00 +02:00
Mattias Wadman
f746dab7b2
Merge pull request #932 from wader/jpeg-fix-weird-eoi-desc
jpeg: Fix EOI description
2024-04-19 15:34:52 +02:00
Mattias Wadman
b482556025 jpeg: Fix EOI description 2024-04-19 15:26:10 +02:00
Mattias Wadman
4674060dfe
Merge pull request #931 from wader/goreleaser-update
goreleaser: Update action and fix deprecation warning
2024-04-16 11:44:57 +02:00
Mattias Wadman
ad2c032c7e goreleaser: Update action and fix deprecation warning 2024-04-16 11:12:36 +02:00
Mattias Wadman
d30755b06e
Merge pull request #930 from wader/release-0.11.0
fq: Release 0.11.0
2024-04-14 12:51:38 +02:00
Mattias Wadman
2b12258eba jq: Release 0.11.0 2024-04-14 12:41:42 +02:00
Mattias Wadman
efa847faf1
Merge pull request #929 from wader/decode-iopanic-cleanup
decode: Cleanup io panic a bit
2024-04-12 17:33:48 +02:00
Mattias Wadman
ee2ee24dbb decode: Cleanup io panic a bit 2024-04-12 17:23:03 +02:00
Mattias Wadman
d04a846f10
Merge pull request #928 from wader/jp2c-faster-probe
jp2c: Fail probe faster
2024-04-11 00:13:37 +02:00
Mattias Wadman
79992b341a jp2c: Fail probe faster 2024-04-10 23:54:22 +02:00
Mattias Wadman
ff718d8071
Merge pull request #927 from wader/mp4-decode-uinf
mp4: Decode uinf box
2024-04-10 22:31:42 +02:00
Mattias Wadman
70b1b0d6a3 mp4: Decode uinf box 2024-04-10 22:17:39 +02:00
Mattias Wadman
a3f5e29d8f
Merge pull request #926 from wader/mp4-jp-box
mp4: Decode jP box
2024-04-10 21:59:57 +02:00
Mattias Wadman
f6609ccb0f
Merge pull request #925 from wader/jp2c-probe
jp2c: Support probe
2024-04-10 21:57:27 +02:00
Mattias Wadman
4f90a2eaae mp4: Decode jP box 2024-04-10 21:50:55 +02:00
Mattias Wadman
63f7d79c32 jp2c: Support probe 2024-04-10 21:44:14 +02:00
Mattias Wadman
240ae7f57d
Merge pull request #924 from wader/jpeg2000
jp2c: Add jpeg2000 codestream format
2024-04-10 16:32:59 +02:00
Mattias Wadman
ebffb3be7b jp2c: Add jpeg2000 codestream format 2024-04-10 16:24:14 +02:00
Mattias Wadman
ad33225f20
Merge pull request #923 from loselarry/master
chore: fix function name in comment
2024-04-10 10:36:49 +02:00
loselarry
208b3e6bf2 chore: fix function name in comment
Signed-off-by: loselarry <bikangning@yeah.net>
2024-04-10 16:16:51 +08:00
Mattias Wadman
837de9f658
Merge pull request #922 from wader/mp4-fix-jp2-test
mp4: Fix jp2 test
2024-04-09 23:12:39 +02:00
Mattias Wadman
87b6c4dd6a mp4: Fix jp2 test 2024-04-09 22:59:47 +02:00
Mattias Wadman
5cbda8fbb0
Merge pull request #921 from wader/mp4-jp2-boxes
mp4: JPEG200 boxes jp2h and ihdr
2024-04-09 20:43:26 +02:00
Mattias Wadman
8d31040fd2
Merge pull request #920 from wader/mov-avi-trim-space
mp4,avi: Trim spaces for type
2024-04-09 20:38:19 +02:00
Mattias Wadman
8009b6f661 mp4: JPEG200 boxes jp2h and ihdr 2024-04-09 20:34:53 +02:00
Mattias Wadman
1784c43824 mp4,avi: Trim spaces for type 2024-04-09 20:09:12 +02:00
63 changed files with 2705 additions and 2038 deletions

View File

@ -7,7 +7,7 @@ on:
pull_request:
env:
GOLANGCILINT_VERSION: "1.57.2"
GOLANGCILINT_VERSION: "1.58.2"
jobs:
lint:
@ -15,7 +15,7 @@ jobs:
steps:
- uses: actions/setup-go@v3
with:
go-version: "1.22.2"
go-version: "1.22.3"
- uses: actions/checkout@v3
- uses: golangci/golangci-lint-action@v3
with:
@ -47,7 +47,7 @@ jobs:
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: "1.22.2"
go-version: "1.22.3"
- name: Test
env:
GOARCH: ${{ matrix.goarch }}

View File

@ -15,8 +15,8 @@ jobs:
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: "1.22.2"
- uses: goreleaser/goreleaser-action@v2
go-version: "1.22.3"
- uses: goreleaser/goreleaser-action@v5
with:
distribution: goreleaser
version: latest

View File

@ -59,7 +59,7 @@ brews:
repository:
owner: wader
name: homebrew-tap
folder: Formula
directory: Formula
homepage: https://github.com/wader/fq
description: jq for binary formats
license: MIT

View File

@ -1,3 +1,88 @@
# 0.11.0
New iNES/NES 2.0 ROM decoder (thanks @mlofjard) and basic JPEG 2000 format support. jq language improvements and fixes from gojq. And as always various decoder improvements and fixes.
## Changes
- Add `string_truncate` option to configure how to truncate long strings when displaying a decode value tree. `dd`, `dv` etc set truncate length to zero to not truncate. #919
- gojq updates from upstream:
- Implement `ltrim`, `rtrim`, and `trim` functions
- Fix object construction with duplicate keys (`{x:0,y:1} | {a:.x,a:.y}`)
- Fix `halt` and `halt_error` functions to stop the command execution immediately
- Fix variable scope of binding syntax (`"a" as $v | def f: $v; "b" as $v | f`)
- Fix `ltrimstr` and `rtrimstr` functions to emit error on non-string input
- Fix `nearbyint` and `rint` functions to round ties to even
- Improve parser to allow `reduce`, `foreach`, `if`, `try`-`catch` syntax as object values
- Remove `pow10` in favor of `exp10`, define `scalbn` and `scalbln` by `ldexp`
- Fix issue using decode value with `ltrimstr`/`rtrimstr`.
## Format changes
- `fit`
- Skip array fields on pre read messages. #878
- Fixed subfield referencing fields below self in message. #877
- `jp2c` New JPEG 2000 codestream decoder. #928
- `icc_profile` Strip whitespace in header and tag strings. #912
- `mp4`
- Add `jp2c`, `jp2h`, `ihdr` `jP` JPEG 2000 related boxes support. #928
- Add `thmb` box support. #897
- Turns out for qt brand `hdlr` component name might be zero bytes. #896
- `nes` New iNES/NES 2.0 ROM decoder (thanks @mlofjard). #893
## Changelog
* f7b067b6 Fixed subfield referencing fields below self in message
* 9aa99b47 Update docker-golang to 1.22.1 from 1.22.0
* 0afb5b59 Update docker-golang to 1.22.2 from 1.22.1
* 2657988d Update github-go-version to 1.22.1 from 1.22.0
* 33c93918 Update github-go-version to 1.22.2 from 1.22.1
* a577c398 Update github-golangci-lint to 1.56.2 from 1.56.1
* 82d96cf9 Update github-golangci-lint to 1.57.0 from 1.56.2
* 72b4569b Update github-golangci-lint to 1.57.1 from 1.57.0
* 14aeab0b Update github-golangci-lint to 1.57.2 from 1.57.1
* 735256b9 Update gomod-golang-x-crypto to 0.20.0 from 0.19.0
* 043f067f Update gomod-golang-x-crypto to 0.21.0 from 0.20.0
* 15a7060b Update gomod-golang-x-crypto to 0.22.0 from 0.21.0
* 85f60df2 Update gomod-golang-x-net to 0.22.0 from 0.21.0
* 77c000e6 Update gomod-golang-x-net to 0.23.0 from 0.22.0
* daba6b54 Update gomod-golang-x-net to 0.24.0 from 0.23.0
* ba9ecb54 Update gomod-golang-x-term to 0.18.0 from 0.17.0
* b2aa59f7 Update gomod-golang-x-term to 0.19.0 from 0.18.0
* 1c24f64d Update make-golangci-lint to 1.56.2 from 1.56.1
* 94e80864 Update make-golangci-lint to 1.57.0 from 1.56.2
* 4f55b6af Update make-golangci-lint to 1.57.1 from 1.57.0
* a3b63b10 Update make-golangci-lint to 1.57.2 from 1.57.1
* 208b3e6b chore: fix function name in comment
* 621d7f2c decode: Align some heavily used structs to save space
* ee2ee24d decode: Cleanup io panic a bit
* e741ca78 doc: Add ImHex to related tools
* 36e8287c doc: Regenerate after nes and new ansisvg
* 225fd507 fit: Skip array fields on pre read messages
* 7500a8b7 fq: Sort formats
* bf7fa07c fq: Use go 1.20 and cleanup
* e2670404 gojq: Update fq fork
* f5fd5873 gojq: Update fq fork
* ed685116 icc_profile: Strip whitespace in header and tag strings
* c8f9cdc9 interp: Add string_truncate option
* 0db671f6 interp: Add todo test for eval error in path
* ebffb3be jp2c: Add jpeg2000 codestream format
* 79992b34 jp2c: Fail probe faster
* 63f7d79c jp2c: Support probe
* b542ff1d lint: More linters and some fixes
* c6165c0c mod: go get non-bump tracked modules
* 1784c438 mp4,avi: Trim spaces for type
* 2ea70c42 mp4: Add thmb box support
* 4f90a2ea mp4: Decode jP box
* 70b1b0d6 mp4: Decode uinf box
* 87b6c4dd mp4: Fix jp2 test
* 8009b6f6 mp4: JPEG200 boxes jp2h and ihdr
* ed3a126f mp4: Turns out for qt brand hdlr component name might be zero bytes
* f3b54042 nes: Add support for iNES/NES 2.0 ROM files
* 80bccc91 opus,vorbis: More sym snake_case
* 08df7f45 pkg/cli/test_exp.sh: use command -v
* 87052733 pssh_playready: Use snake_case sym values
* aaa43dfb test: Replace pmezard/go-difflib with go's internal diff
# 0.10.0
Adds support for various LevelDB formats (thanks @mikez) and Garmin Flexible and Interoperable Data Transfer format (FIT) (thanks @mlofjard).
@ -43,7 +128,6 @@ And as usual some small fixes and dependency updates.
$ fq -L . -r 'include "to_kml"; to_kml' file.fit > file.kml
```
- `hevc_sps` Fix some incorrect profile_tier_level decoding. #829
- `html` Fix issue parsing elements including SOLIDUS "/". #870
- Upstream issue https://github.com/golang/go/issues/63402

View File

@ -1,5 +1,5 @@
# bump: docker-golang /FROM golang:([\d.]+)/ docker:golang|^1
FROM golang:1.22.2-bookworm AS base
FROM golang:1.22.3-bookworm AS base
# expect is used to test cli
RUN \

View File

@ -61,7 +61,7 @@ gogenerate: always
lint: always
# bump: make-golangci-lint /golangci-lint@v([\d.]+)/ git:https://github.com/golangci/golangci-lint.git|^1
# bump: make-golangci-lint link "Release notes" https://github.com/golangci/golangci-lint/releases/tag/v$LATEST
go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.57.2 run
go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.58.2 run
depgraph.svg: always
go run github.com/kisielk/godepgraph@latest github.com/wader/fq | dot -Tsvg -o godepgraph.svg

View File

@ -2,25 +2,23 @@
Tool, language and decoders for working with binary data.
TLDR: it aims to be jq, hexdump, dd and gdb for files combined into one.
![fq demo](doc/demo.svg)
Basic usage is `fq . file` or `fq d file`.
Basic usage is `fq . file`, `fq d file` or `fq 'some query' file ...`.
For details see [usage.md](doc/usage.md).
### Background
fq is inspired by the well known jq tool and language that allows you to work with binary formats the same way you would using jq. In addition it can present data like a hex viewer, transform, slice and concatenate binary data. It also supports nested formats and has an interactive REPL with auto-completion.
fq is inspired by the [jq](https://jqlang.github.io/jq/) tool and language and allows you to work with binary formats in the same way. In addition to using jq expressions it can also present decoded tree structures, transform, slice and concatenate binary data. It also supports nested formats and features an interactive REPL with auto-completion of functions and names.
It was originally designed to query, inspect and debug media codecs and containers like mp4, flac, mp3, jpeg. But since then it has been extended to support a variety of formats like executables, packet captures (with TCP reassembly) and serialization formats like JSON, YAML, XML, ASN1 BER, Avro, CBOR, protobuf. In addition it also has functions to work with URLs, convert to/from hex, number bases, search for things etc.
In summary it aims to be jq, hexdump, dd and gdb for files combined into one.
**NOTE:** fq is still early in development so things might change, be broken or do not make sense. That also means that there is a great opportunity to help out!
It was originally designed to query, inspect and debug media codecs and containers like MP4, FLAC and JPEG but has since been extended to support a variety of formats like executables, packet captures (with TCP reassembly) and serialization formats like JSON, YAML, XML, CBOR, protobuf. In addition it also has functions to work with URLs, convert to/from hex, number bases, search for patterns etc.
### Goals
- Make binaries accessible, queryable and sliceable.
- Make binaries more accessible, queryable and sliceable.
- Nested formats and bit-oriented decoding.
- Quick and comfortable CLI tool.
- Bits and bytes transformations.
@ -100,6 +98,7 @@ id3v11,
id3v2,
ipv4_packet,
ipv6_packet,
jp2c,
jpeg,
json,
jsonl,
@ -242,7 +241,7 @@ apk add -X http://dl-cdn.alpinelinux.org/alpine/edge/testing fq
Make sure you have [go](https://go.dev) 1.20 or later installed.
To install directly from git repository (no clone needed):
To install directly from git repository (no git clone needed):
```sh
# build and install latest release
go install github.com/wader/fq@latest
@ -266,10 +265,6 @@ go build -o fq .
make test fq
```
## TODO and ideas
See [TODO.md](doc/TODO.md)
## Development and adding a new decoder
See [dev.md](doc/dev.md)
@ -305,6 +300,10 @@ for inventing the [jq](https://github.com/stedolan/jq) language.
- [Sustainability of Digital Formats](https://www.loc.gov/preservation/digital/formats/) at Library of Congress.
- [Data Format Description Language (DFDL)](https://en.wikipedia.org/wiki/Data_Format_Description_Language).
## TODO and ideas
See [TODO.md](doc/TODO.md)
## License
`fq` is distributed under the terms of the MIT License.

View File

@ -1,3 +1,5 @@
//go:build exclude
// tool to convert go fuzz input files to bytes
// Usage: cat format/testdata/fuzz/FuzzFormats/144bde49b40c90fd05d302ec90b6ddb2b6d6aea553bad520a8b954797e40fe72 | go run dev/fuzzbytes.go | go run .
package main

View File

@ -1,3 +1,5 @@
//go:build exclude
package main
import (

View File

@ -236,8 +236,8 @@ make test
go test ./...
# run all tests for one format
go test -run TestFormats/mp4 ./format/
# update all actual outputs in tests
go test ./... -update
# update all expected outputs for tests
go test ./pkg/interp ./format -update
# update actual output for specific tests
go run ./format -run TestFormats/elf -update
# color diff

View File

@ -70,6 +70,7 @@
|`id3v2` |ID3v2&nbsp;metadata |<sub>`image`</sub>|
|`ipv4_packet` |Internet&nbsp;protocol&nbsp;v4&nbsp;packet |<sub>`ip_packet`</sub>|
|`ipv6_packet` |Internet&nbsp;protocol&nbsp;v6&nbsp;packet |<sub>`ip_packet`</sub>|
|`jp2c` |JPEG&nbsp;2000&nbsp;codestream |<sub></sub>|
|`jpeg` |Joint&nbsp;Photographic&nbsp;Experts&nbsp;Group&nbsp;file |<sub>`exif` `icc_profile`</sub>|
|`json` |JavaScript&nbsp;Object&nbsp;Notation |<sub></sub>|
|`jsonl` |JavaScript&nbsp;Object&nbsp;Notation&nbsp;Lines |<sub></sub>|
@ -86,7 +87,7 @@
|`mp3_frame` |MPEG&nbsp;audio&nbsp;layer&nbsp;3&nbsp;frame |<sub>`mp3_frame_tags`</sub>|
|`mp3_frame_vbri` |MP3&nbsp;frame&nbsp;Fraunhofer&nbsp;encoder&nbsp;variable&nbsp;bitrate&nbsp;tag |<sub></sub>|
|`mp3_frame_xing` |MP3&nbsp;frame&nbsp;Xing/Info&nbsp;tag |<sub></sub>|
|[`mp4`](#mp4) |ISOBMFF,&nbsp;QuickTime&nbsp;and&nbsp;similar |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `icc_profile` `id3v2` `image` `jpeg` `mp3_frame` `mpeg_es` `mpeg_pes_packet` `opus_packet` `png` `prores_frame` `protobuf_widevine` `pssh_playready` `vorbis_packet` `vp9_frame` `vpx_ccr`</sub>|
|[`mp4`](#mp4) |ISOBMFF,&nbsp;QuickTime&nbsp;and&nbsp;similar |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `icc_profile` `id3v2` `image` `jp2c` `jpeg` `mp3_frame` `mpeg_es` `mpeg_pes_packet` `opus_packet` `png` `prores_frame` `protobuf_widevine` `pssh_playready` `vorbis_packet` `vp9_frame` `vpx_ccr`</sub>|
|`mpeg_asc` |MPEG-4&nbsp;Audio&nbsp;Specific&nbsp;Config |<sub></sub>|
|`mpeg_es` |MPEG&nbsp;Elementary&nbsp;Stream |<sub>`mpeg_asc` `vorbis_packet`</sub>|
|`mpeg_pes` |MPEG&nbsp;Packetized&nbsp;elementary&nbsp;stream |<sub>`mpeg_pes_packet` `mpeg_spu`</sub>|
@ -131,12 +132,12 @@
|[`xml`](#xml) |Extensible&nbsp;Markup&nbsp;Language |<sub></sub>|
|`yaml` |YAML&nbsp;Ain't&nbsp;Markup&nbsp;Language |<sub></sub>|
|[`zip`](#zip) |ZIP&nbsp;archive |<sub>`probe`</sub>|
|`image` |Group |<sub>`gif` `jpeg` `mp4` `png` `tiff` `webp`</sub>|
|`image` |Group |<sub>`gif` `jp2c` `jpeg` `mp4` `png` `tiff` `webp`</sub>|
|`inet_packet` |Group |<sub>`ipv4_packet` `ipv6_packet`</sub>|
|`ip_packet` |Group |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
|`link_frame` |Group |<sub>`bsd_loopback_frame` `ether8023_frame` `ipv4_packet` `ipv6_packet` `sll2_packet` `sll_packet`</sub>|
|`mp3_frame_tags` |Group |<sub>`mp3_frame_vbri` `mp3_frame_xing`</sub>|
|`probe` |Group |<sub>`adts` `aiff` `apple_bookmark` `ar` `avi` `avro_ocf` `bitcoin_blkdat` `bplist` `bzip2` `caff` `elf` `fit` `flac` `gif` `gzip` `html` `jpeg` `json` `jsonl` `leveldb_table` `luajit` `macho` `macho_fat` `matroska` `moc3` `mp3` `mp4` `mpeg_ts` `nes` `ogg` `opentimestamps` `pcap` `pcapng` `png` `tar` `tiff` `toml` `tzif` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
|`probe` |Group |<sub>`adts` `aiff` `apple_bookmark` `ar` `avi` `avro_ocf` `bitcoin_blkdat` `bplist` `bzip2` `caff` `elf` `fit` `flac` `gif` `gzip` `html` `jp2c` `jpeg` `json` `jsonl` `leveldb_table` `luajit` `macho` `macho_fat` `matroska` `moc3` `mp3` `mp4` `mpeg_ts` `nes` `ogg` `opentimestamps` `pcap` `pcapng` `png` `tar` `tiff` `toml` `tzif` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
|`tcp_stream` |Group |<sub>`dns_tcp` `rtmp` `tls`</sub>|
|`udp_payload` |Group |<sub>`dns`</sub>|

File diff suppressed because it is too large Load Diff

Before

Width:  |  Height:  |  Size: 155 KiB

After

Width:  |  Height:  |  Size: 156 KiB

View File

@ -1,3 +1,9 @@
//go:build exclude
package bts2022
import "github.com/wader/fq/pkg/decode"
func avcHdrParameters(d *decode.D) {
cpbCnt := d.FieldUintFn("cpb_cnt", uEV, scalar.UAdd(1))
d.FieldU4("bit_rate_scale")
@ -15,4 +21,4 @@ func avcHdrParameters(d *decode.D) {
d.FieldU5("cpb_removal_delay_length", scalar.UAdd(1))
d.FieldU5("dpb_output_delay_length", scalar.UAdd(1))
d.FieldU5("time_offset_length")
}
}

View File

@ -1,3 +1,7 @@
//go:build exclude
package bts2022
func decode(d *decode.D) any {
d.FieldArray("headers", func(d *decode.D) {
for !d.End() {

View File

@ -14,6 +14,7 @@ $ fq -n _registry.groups.probe
"flac",
"gif",
"gzip",
"jp2c",
"jpeg",
"leveldb_table",
"luajit",
@ -114,6 +115,7 @@ id3v11 ID3v1.1 metadata
id3v2 ID3v2 metadata
ipv4_packet Internet protocol v4 packet
ipv6_packet Internet protocol v6 packet
jp2c JPEG 2000 codestream
jpeg Joint Photographic Experts Group file
json JavaScript Object Notation
jsonl JavaScript Object Notation Lines

View File

@ -281,7 +281,7 @@ func decodeASN1BERValue(d *decode.D, bib *bitio.Buffer, sb *strings.Builder, par
if bib != nil {
// TODO: helper?
if _, err := bitio.Copy(bib, br); err != nil {
d.IOPanic(err, "bitio.Copy")
d.IOPanic(err, "value", "bitio.Copy")
}
}
if unusedBitsCount > 0 {
@ -292,7 +292,7 @@ func decodeASN1BERValue(d *decode.D, bib *bitio.Buffer, sb *strings.Builder, par
if bib != nil {
// TODO: helper?
if _, err := bitio.Copy(bib, br); err != nil {
d.IOPanic(err, "bitio.Copy")
d.IOPanic(err, "value", "bitio.Copy")
}
}
case class == classUniversal && tag == universalTypeNull:

View File

@ -123,6 +123,7 @@ var (
ID3v2 = &decode.Group{Name: "id3v2"}
IPv4Packet = &decode.Group{Name: "ipv4_packet"}
IPv6Packet = &decode.Group{Name: "ipv6_packet"}
JP2C = &decode.Group{Name: "jp2c"}
JPEG = &decode.Group{Name: "jpeg"}
JSON = &decode.Group{Name: "json"}
JSONL = &decode.Group{Name: "jsonl"}

View File

@ -114,7 +114,7 @@ func gzipDecodeMember(d *decode.D) bitio.ReaderAtSeeker {
readCompressedSize, uncompressedBR, err =
d.FieldReaderRange("uncompressed", d.Pos(), d.BitsLeft(), rFn)
if err != nil {
d.IOPanic(err, "TryFieldReaderRange")
d.IOPanic(err, "uncompressed", "FieldReaderRange")
}
d.FieldRawLen("compressed", readCompressedSize)
crc32W := crc32.NewIEEE()
@ -149,7 +149,7 @@ func gzipDecode(d *decode.D) any {
cbr, err := bitio.NewMultiReader(brs...)
if err != nil {
d.IOPanic(err, "NewMultiReader")
d.IOPanic(err, "members", "NewMultiReader")
}
dv, _, _ := d.TryFieldFormatBitBuf("uncompressed", cbr, &probeGroup, format.Probe_In{})
if dv == nil {

View File

@ -302,7 +302,7 @@ func textNullLenFn(encoding int, notFoundFixedBytes int) func(d *decode.D) strin
func(v uint64) bool { return v == 0 },
)
if err != nil {
d.IOPanic(err, "textNullLenFn")
d.IOPanic(err, "", "textNullLenFn")
}
if offset < 0 {
if notFoundFixedBytes < 0 {

144
format/jpeg/jp2c.go Normal file
View File

@ -0,0 +1,144 @@
package jpeg
import (
"github.com/wader/fq/format"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
"github.com/wader/fq/pkg/scalar"
)
func init() {
interp.RegisterFormat(
format.JP2C,
&decode.Format{
Description: "JPEG 2000 codestream",
Groups: []*decode.Group{format.Probe, format.Image},
DecodeFn: jp2cDecode,
RootName: "segments",
RootArray: true,
})
}
const (
JP2_SOC = 0xff_4f /* start of codestream */
JP2_SOT = 0xff_90 /* start of tile */
JP2_SOD = 0xff_93 /* start of data */
JP2_EOC = 0xff_d9 /* end of codestream */
/* fixed information segment */
JP2_SIZ = 0xff_51 /* image and tile size */
/* functional segments */
JP2_COD = 0xff_52 /* coding style default */
JP2_COC = 0xff_53 /* coding style component */
JP2_RGN = 0xff_5e /* region of interest */
JP2_QCD = 0xff_5c /* quantization default */
JP2_QCC = 0xff_5d /* quantization component */
JP2_POC = 0xff_5f /* progression order change */
/* pointer segments */
JP2_TLM = 0xff_55 /* tile-part lengths */
JP2_PLM = 0xff_57 /* packet length (main header) */
JP2_PLT = 0xff_58 /* packet length (tile-part header) */
JP2_PPM = 0xff_60 /* packed packet headers (main header) */
JP2_PPT = 0xff_61 /* packet packet headers (tile-part header) */
/* bitstream internal markers and segments */
JP2_SOP = 0xff_91 /* start of packet */
JP2_EPH = 0xff_92 /* end of packet header */
/* informational segments */
JP2_CRG = 0xff_63 /* component registration */
JP2_COM = 0xff_64 /* comment */
)
var jp2Markers = scalar.UintMap{
JP2_SOC: {Sym: "soc", Description: "Start of codestream"},
JP2_SOT: {Sym: "sot", Description: "Start of tile"},
JP2_SOD: {Sym: "sod", Description: "Start of data"},
JP2_EOC: {Sym: "eoc", Description: "End of codestream"},
JP2_SIZ: {Sym: "siz", Description: "Image and tile size"},
JP2_COD: {Sym: "cod", Description: "Coding style default"},
JP2_COC: {Sym: "coc", Description: "Coding style component"},
JP2_RGN: {Sym: "rgn", Description: "Region of interest"},
JP2_QCD: {Sym: "qcd", Description: "Quantization default"},
JP2_QCC: {Sym: "qcc", Description: "Quantization component"},
JP2_POC: {Sym: "poc", Description: "Progression order change"},
JP2_TLM: {Sym: "tlm", Description: "Tile-part lengths"},
JP2_PLM: {Sym: "plm", Description: "Packet length (main header)"},
JP2_PLT: {Sym: "plt", Description: "Packet length (tile-part header)"},
JP2_PPM: {Sym: "ppm", Description: "Packed packet headers (main header)"},
JP2_PPT: {Sym: "ppt", Description: "Packet packet headers (tile-part header)"},
JP2_SOP: {Sym: "sop", Description: "Start of packet"},
JP2_EPH: {Sym: "eph", Description: "End of packet header"},
JP2_CRG: {Sym: "crg", Description: "Component registration"},
JP2_COM: {Sym: "com", Description: "Comment"},
}
func jp2cDecode(d *decode.D) any {
if d.PeekUintBits(16) != JP2_SOC {
d.Fatalf("no SOC marker")
}
seenSOC := false
seenSIZ := false
seenEOC := false
for !seenEOC && !d.End() {
d.FieldStruct("segment", func(d *decode.D) {
marker := d.FieldU16("marker", jp2Markers, scalar.UintHex)
switch marker {
case JP2_SOC:
// zero length
seenSOC = true
return
case JP2_SOD:
l, _ := d.PeekFind(16, 8, d.BitsLeft(), func(v uint64) bool {
return v == JP2_SOT || v == JP2_EOC
})
d.FieldRawLen("data", l)
case JP2_SOT:
d.FieldU16("l_sot")
d.FieldU16("i_sot")
d.FieldU32("p_sot")
d.FieldU8("tp_sot")
d.FieldU8("tn_sot")
case JP2_SIZ:
seenSIZ = true
d.FieldU16("l_siz")
d.FieldU16("r_siz")
d.FieldU32("x_siz")
d.FieldU32("y_siz")
d.FieldU32("xo_siz")
d.FieldU32("yo_siz")
d.FieldU32("xt_siz")
d.FieldU32("yt_siz")
d.FieldU32("xto_siz")
d.FieldU32("yto_siz")
cSiz := d.FieldU16("c_siz")
d.FieldArray("components", func(d *decode.D) {
for i := 0; i < int(cSiz); i++ {
d.FieldStruct("component", func(d *decode.D) {
d.FieldU8("s_sizi")
d.FieldU8("xr_sizi")
d.FieldU8("yr_sizi")
})
}
})
case JP2_COM:
length := d.FieldU16("length")
d.FieldU16("r_cme")
d.FieldUTF8("data", int(length-4))
case JP2_EOC:
// zero length
seenEOC = true
return
default:
length := d.FieldU16("length")
d.FieldRawLen("data", int64(length-2)*8)
}
})
}
if !(seenSOC && seenSIZ && seenEOC) {
d.Fatalf("SOC, SIZ or EOC marker not found")
}
return nil
}

View File

@ -124,7 +124,7 @@ var markers = scalar.UintMap{
RST6: {Sym: "rst6", Description: "Restart with modulo 8 count 6"},
RST7: {Sym: "rst7", Description: "Restart with modulo 8 count 7"},
SOI: {Sym: "soi", Description: "Start of image"},
EOI: {Sym: "eoi", Description: "End of image true"},
EOI: {Sym: "eoi", Description: "End of image"},
SOS: {Sym: "sos", Description: "Start of scan"},
DQT: {Sym: "dqt", Description: "Define quantization table(s)"},
DNL: {Sym: "dnl", Description: "Define number of lines"},
@ -262,6 +262,29 @@ func jpegDecode(d *decode.D) any {
}
})
})
case DHT:
lH := int64(d.FieldU16("lh"))
d.FramedFn(lH*8-16, func(d *decode.D) {
d.FieldArray("hs", func(d *decode.D) {
for d.NotEnd() {
d.FieldStruct("h", func(d *decode.D) {
d.FieldU4("tc")
d.FieldU4("th")
hK := uint64(0)
hV := uint64(0)
d.FieldArrayLoop("l", func() bool { return hK < 16 }, func(d *decode.D) {
hV += d.FieldU8("l")
hK++
})
hK = 0
d.FieldArrayLoop("v", func() bool { return hK < hV }, func(d *decode.D) {
d.FieldU8("v")
hK++
})
})
}
})
})
case RST0, RST1, RST2, RST3, RST4, RST5, RST6, RST7:
inECD = true
case TEM:

View File

@ -109,15 +109,57 @@ $ fq -d jpeg dv 4x4.jpg
| | | [4]{}: marker 0x66-0x7c (22)
0x60| ff | . | prefix: raw bits (valid) 0x66-0x67 (1)
0x60| c4 | . | code: "dht" (196) (Define Huffman table(s)) 0x67-0x68 (1)
0x60| 00 14 | .. | length: 20 0x68-0x6a (2)
0x60| 00 01 00 00 00 00| ......| data: raw bits 0x6a-0x7c (18)
0x70|00 00 00 00 00 00 00 00 00 00 00 08 |............ |
0x60| 00 14 | .. | lh: 20 0x68-0x6a (2)
| | | hs[0:1]: 0x6a-0x7c (18)
| | | [0]{}: h 0x6a-0x7c (18)
0x60| 00 | . | tc: 0 0x6a-0x6a.4 (0.4)
0x60| 00 | . | th: 0 0x6a.4-0x6b (0.4)
| | | l[0:16]: 0x6b-0x7b (16)
0x60| 01 | . | [0]: 1 l 0x6b-0x6c (1)
0x60| 00 | . | [1]: 0 l 0x6c-0x6d (1)
0x60| 00 | . | [2]: 0 l 0x6d-0x6e (1)
0x60| 00 | . | [3]: 0 l 0x6e-0x6f (1)
0x60| 00| .| [4]: 0 l 0x6f-0x70 (1)
0x70|00 |. | [5]: 0 l 0x70-0x71 (1)
0x70| 00 | . | [6]: 0 l 0x71-0x72 (1)
0x70| 00 | . | [7]: 0 l 0x72-0x73 (1)
0x70| 00 | . | [8]: 0 l 0x73-0x74 (1)
0x70| 00 | . | [9]: 0 l 0x74-0x75 (1)
0x70| 00 | . | [10]: 0 l 0x75-0x76 (1)
0x70| 00 | . | [11]: 0 l 0x76-0x77 (1)
0x70| 00 | . | [12]: 0 l 0x77-0x78 (1)
0x70| 00 | . | [13]: 0 l 0x78-0x79 (1)
0x70| 00 | . | [14]: 0 l 0x79-0x7a (1)
0x70| 00 | . | [15]: 0 l 0x7a-0x7b (1)
| | | v[0:1]: 0x7b-0x7c (1)
0x70| 08 | . | [0]: 8 v 0x7b-0x7c (1)
| | | [5]{}: marker 0x7c-0x92 (22)
0x70| ff | . | prefix: raw bits (valid) 0x7c-0x7d (1)
0x70| c4 | . | code: "dht" (196) (Define Huffman table(s)) 0x7d-0x7e (1)
0x70| 00 14| ..| length: 20 0x7e-0x80 (2)
0x80|10 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| data: raw bits 0x80-0x92 (18)
0x90|00 00 |.. |
0x70| 00 14| ..| lh: 20 0x7e-0x80 (2)
| | | hs[0:1]: 0x80-0x92 (18)
| | | [0]{}: h 0x80-0x92 (18)
0x80|10 |. | tc: 1 0x80-0x80.4 (0.4)
0x80|10 |. | th: 0 0x80.4-0x81 (0.4)
| | | l[0:16]: 0x81-0x91 (16)
0x80| 01 | . | [0]: 1 l 0x81-0x82 (1)
0x80| 00 | . | [1]: 0 l 0x82-0x83 (1)
0x80| 00 | . | [2]: 0 l 0x83-0x84 (1)
0x80| 00 | . | [3]: 0 l 0x84-0x85 (1)
0x80| 00 | . | [4]: 0 l 0x85-0x86 (1)
0x80| 00 | . | [5]: 0 l 0x86-0x87 (1)
0x80| 00 | . | [6]: 0 l 0x87-0x88 (1)
0x80| 00 | . | [7]: 0 l 0x88-0x89 (1)
0x80| 00 | . | [8]: 0 l 0x89-0x8a (1)
0x80| 00 | . | [9]: 0 l 0x8a-0x8b (1)
0x80| 00 | . | [10]: 0 l 0x8b-0x8c (1)
0x80| 00 | . | [11]: 0 l 0x8c-0x8d (1)
0x80| 00 | . | [12]: 0 l 0x8d-0x8e (1)
0x80| 00 | . | [13]: 0 l 0x8e-0x8f (1)
0x80| 00| .| [14]: 0 l 0x8f-0x90 (1)
0x90|00 |. | [15]: 0 l 0x90-0x91 (1)
| | | v[0:1]: 0x91-0x92 (1)
0x90| 00 | . | [0]: 0 v 0x91-0x92 (1)
| | | [6]{}: marker 0x92-0x9c (10)
0x90| ff | . | prefix: raw bits (valid) 0x92-0x93 (1)
0x90| da | . | code: "sos" (218) (Start of scan) 0x93-0x94 (1)
@ -135,4 +177,4 @@ $ fq -d jpeg dv 4x4.jpg
0x90| 3f bf | ?. | [7]: raw bits entropy_coded_data 0x9c-0x9e (2)
| | | [8]{}: marker 0x9e-0xa0 (2)
0x90| ff | . | prefix: raw bits (valid) 0x9e-0x9f (1)
0x90| d9| .| code: "eoi" (217) (End of image true) 0x9f-0xa0 (1)
0x90| d9| .| code: "eoi" (217) (End of image) 0x9f-0xa0 (1)

BIN
format/jpeg/testdata/jp2c_3c.jp2 vendored Normal file

Binary file not shown.

129
format/jpeg/testdata/jp2c_3c.jp2.fqtest vendored Normal file
View File

@ -0,0 +1,129 @@
# jp2c with 3 components
# gm convert -size 4x4 'gradient:#ff00ff-#00ff00' -colors 254 jp2c_3c.jp2
$ fq dv jp2c_3c.jp2
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: jp2c_3c.jp2 (mp4) 0x0-0x18a (394)
| | | boxes[0:4]: 0x0-0x18a (394)
| | | [0]{}: box 0x0-0xc (12)
0x000|00 00 00 0c |.... | size: 12 0x0-0x4 (4)
0x000| 6a 50 20 20 | jP | type: "jP" (JPEG 2000 Signature) 0x4-0x8 (4)
0x000| 0d 0a 87 0a | .... | signature: raw bits (valid) 0x8-0xc (4)
| | | [1]{}: box 0xc-0x20 (20)
0x000| 00 00 00 14| ....| size: 20 0xc-0x10 (4)
0x010|66 74 79 70 |ftyp | type: "ftyp" (File type and compatibility) 0x10-0x14 (4)
0x010| 6a 70 32 20 | jp2 | major_brand: "jp2" 0x14-0x18 (4)
0x010| 00 00 00 00 | .... | minor_version: 0 0x18-0x1c (4)
| | | brands[0:1]: 0x1c-0x20 (4)
0x010| 6a 70 32 20| jp2 | [0]: "jp2" brand 0x1c-0x20 (4)
| | | [2]{}: box 0x20-0x4d (45)
0x020|00 00 00 2d |...- | size: 45 0x20-0x24 (4)
0x020| 6a 70 32 68 | jp2h | type: "jp2h" (Header) 0x24-0x28 (4)
| | | boxes[0:2]: 0x28-0x4d (37)
| | | [0]{}: box 0x28-0x3e (22)
0x020| 00 00 00 16 | .... | size: 22 0x28-0x2c (4)
0x020| 69 68 64 72| ihdr| type: "ihdr" (Image Header) 0x2c-0x30 (4)
0x030|00 00 00 04 |.... | width: 4 0x30-0x34 (4)
0x030| 00 00 00 04 | .... | height: 4 0x34-0x38 (4)
0x030| 00 03 | .. | nc: 3 0x38-0x3a (2)
0x030| 0f | . | bpc: 16 0x3a-0x3b (1)
0x030| 07 | . | c: "jpeg_2000" (7) 0x3b-0x3c (1)
0x030| 00 | . | unk_c: 0 0x3c-0x3d (1)
0x030| 00 | . | ipr: 0 0x3d-0x3e (1)
| | | [1]{}: box 0x3e-0x4d (15)
0x030| 00 00| ..| size: 15 0x3e-0x42 (4)
0x040|00 0f |.. |
0x040| 63 6f 6c 72 | colr | type: "colr" (Specifies the colourspace of the image) 0x42-0x46 (4)
0x040| 01 00 00 00 | .... | parameter_type: "\x01\x00\x00\x00" 0x46-0x4a (4)
0x040| 00 00 10 | ... | data: raw bits 0x4a-0x4d (3)
| | | [3]{}: box 0x4d-0x18a (317)
0x040| 00 00 00| ...| size: 0 (Rest of file) 0x4d-0x51 (4)
0x050|00 |. |
0x050| 6a 70 32 63 | jp2c | type: "jp2c" (JPEG 2000 contiguous codestream) 0x51-0x55 (4)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| segments[0:13]: (jp2c) 0x55-0x18a (309)
| | | [0]{}: segment 0x55-0x57 (2)
0x050| ff 4f | .O | marker: "soc" (0xff4f) (Start of codestream) 0x55-0x57 (2)
| | | [1]{}: segment 0x57-0x88 (49)
0x050| ff 51 | .Q | marker: "siz" (0xff51) (Image and tile size) 0x57-0x59 (2)
0x050| 00 2f | ./ | l_siz: 47 0x59-0x5b (2)
0x050| 00 00 | .. | r_siz: 0 0x5b-0x5d (2)
0x050| 00 00 00| ...| x_siz: 4 0x5d-0x61 (4)
0x060|04 |. |
0x060| 00 00 00 04 | .... | y_siz: 4 0x61-0x65 (4)
0x060| 00 00 00 00 | .... | xo_siz: 0 0x65-0x69 (4)
0x060| 00 00 00 00 | .... | yo_siz: 0 0x69-0x6d (4)
0x060| 00 00 00| ...| xt_siz: 4 0x6d-0x71 (4)
0x070|04 |. |
0x070| 00 00 00 04 | .... | yt_siz: 4 0x71-0x75 (4)
0x070| 00 00 00 00 | .... | xto_siz: 0 0x75-0x79 (4)
0x070| 00 00 00 00 | .... | yto_siz: 0 0x79-0x7d (4)
0x070| 00 03 | .. | c_siz: 3 0x7d-0x7f (2)
| | | components[0:3]: 0x7f-0x88 (9)
| | | [0]{}: component 0x7f-0x82 (3)
0x070| 0f| .| s_sizi: 15 0x7f-0x80 (1)
0x080|01 |. | xr_sizi: 1 0x80-0x81 (1)
0x080| 01 | . | yr_sizi: 1 0x81-0x82 (1)
| | | [1]{}: component 0x82-0x85 (3)
0x080| 0f | . | s_sizi: 15 0x82-0x83 (1)
0x080| 01 | . | xr_sizi: 1 0x83-0x84 (1)
0x080| 01 | . | yr_sizi: 1 0x84-0x85 (1)
| | | [2]{}: component 0x85-0x88 (3)
0x080| 0f | . | s_sizi: 15 0x85-0x86 (1)
0x080| 01 | . | xr_sizi: 1 0x86-0x87 (1)
0x080| 01 | . | yr_sizi: 1 0x87-0x88 (1)
| | | [2]{}: segment 0x88-0xac (36)
0x080| ff 64 | .d | marker: "com" (0xff64) (Comment) 0x88-0x8a (2)
0x080| 00 22 | ." | length: 34 0x8a-0x8c (2)
0x080| 00 01 | .. | r_cme: 1 0x8c-0x8e (2)
0x080| 43 72| Cr| data: "Creator: JasPer Version 2.0.33" 0x8e-0xac (30)
0x090|65 61 74 6f 72 3a 20 4a 61 73 50 65 72 20 56 65|eator: JasPer Ve|
0x0a0|72 73 69 6f 6e 20 32 2e 30 2e 33 33 |rsion 2.0.33 |
| | | [3]{}: segment 0xac-0xba (14)
0x0a0| ff 52 | .R | marker: "cod" (0xff52) (Coding style default) 0xac-0xae (2)
0x0a0| 00 0c| ..| length: 12 0xae-0xb0 (2)
0x0b0|00 00 00 01 01 05 04 04 00 01 |.......... | data: raw bits 0xb0-0xba (10)
| | | [4]{}: segment 0xba-0xcf (21)
0x0b0| ff 5c | .\ | marker: "qcd" (0xff5c) (Quantization default) 0xba-0xbc (2)
0x0b0| 00 13 | .. | length: 19 0xbc-0xbe (2)
0x0b0| 40 80| @.| data: raw bits 0xbe-0xcf (17)
0x0c0|88 88 90 88 88 90 88 88 90 88 88 90 88 88 90 |............... |
| | | [5]{}: segment 0xcf-0xe5 (22)
0x0c0| ff| .| marker: "qcc" (0xff5d) (Quantization component) 0xcf-0xd1 (2)
0x0d0|5d |] |
0x0d0| 00 14 | .. | length: 20 0xd1-0xd3 (2)
0x0d0| 01 40 80 88 88 90 88 88 90 88 88 90 88| .@...........| data: raw bits 0xd3-0xe5 (18)
0x0e0|88 90 88 88 90 |..... |
| | | [6]{}: segment 0xe5-0xfb (22)
0x0e0| ff 5d | .] | marker: "qcc" (0xff5d) (Quantization component) 0xe5-0xe7 (2)
0x0e0| 00 14 | .. | length: 20 0xe7-0xe9 (2)
0x0e0| 02 40 80 88 88 90 88| .@.....| data: raw bits 0xe9-0xfb (18)
0x0f0|88 90 88 88 90 88 88 90 88 88 90 |........... |
| | | [7]{}: segment 0xfb-0x107 (12)
0x0f0| ff 90 | .. | marker: "sot" (0xff90) (Start of tile) 0xfb-0xfd (2)
0x0f0| 00 0a | .. | l_sot: 10 0xfd-0xff (2)
0x0f0| 00| .| i_sot: 0 0xff-0x101 (2)
0x100|00 |. |
0x100| 00 00 00 8d | .... | p_sot: 141 0x101-0x105 (4)
0x100| 00 | . | tp_sot: 0 0x105-0x106 (1)
0x100| 01 | . | tn_sot: 1 0x106-0x107 (1)
| | | [8]{}: segment 0x107-0x11d (22)
0x100| ff 5d | .] | marker: "qcc" (0xff5d) (Quantization component) 0x107-0x109 (2)
0x100| 00 14 | .. | length: 20 0x109-0x10b (2)
0x100| 00 40 80 00 00| .@...| data: raw bits 0x10b-0x11d (18)
0x110|00 00 00 00 00 00 00 88 88 90 88 88 90 |............. |
| | | [9]{}: segment 0x11d-0x133 (22)
0x110| ff 5d | .] | marker: "qcc" (0xff5d) (Quantization component) 0x11d-0x11f (2)
0x110| 00| .| length: 20 0x11f-0x121 (2)
0x120|14 |. |
0x120| 01 40 80 00 00 00 00 00 00 00 00 00 88 88 90| .@.............| data: raw bits 0x121-0x133 (18)
0x130|88 88 90 |... |
| | | [10]{}: segment 0x133-0x149 (22)
0x130| ff 5d | .] | marker: "qcc" (0xff5d) (Quantization component) 0x133-0x135 (2)
0x130| 00 14 | .. | length: 20 0x135-0x137 (2)
0x130| 02 40 80 00 00 00 00 00 00| .@.......| data: raw bits 0x137-0x149 (18)
0x140|00 00 00 88 88 90 88 88 90 |......... |
| | | [11]{}: segment 0x149-0x188 (63)
0x140| ff 93 | .. | marker: "sod" (0xff93) (Start of data) 0x149-0x14b (2)
0x140| c0 00 21 07 cf| ..!..| data: raw bits 0x14b-0x188 (61)
0x150|fc 30 08 05 3f cf fc 30 08 05 3f 80 80 80 80 80|.0..?..0..?.....|
* |until 0x187.7 (61) | |
| | | [12]{}: segment 0x188-0x18a (2)
0x180| ff d9| | ..| | marker: "eoc" (0xffd9) (End of codestream) 0x188-0x18a (2)

BIN
format/jpeg/testdata/test.j2c vendored Normal file

Binary file not shown.

59
format/jpeg/testdata/test.j2c.fqtest vendored Normal file
View File

@ -0,0 +1,59 @@
# test probe
$ fq dv test.j2c
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.[0:9]: test.j2c (jp2c) 0x0-0xa4 (164)
| | | [0]{}: segment 0x0-0x2 (2)
0x00|ff 4f |.O | marker: "soc" (0xff4f) (Start of codestream) 0x0-0x2 (2)
| | | [1]{}: segment 0x2-0x2d (43)
0x00| ff 51 | .Q | marker: "siz" (0xff51) (Image and tile size) 0x2-0x4 (2)
0x00| 00 29 | .) | l_siz: 41 0x4-0x6 (2)
0x00| 00 00 | .. | r_siz: 0 0x6-0x8 (2)
0x00| 00 00 00 01 | .... | x_siz: 1 0x8-0xc (4)
0x00| 00 00 00 01| ....| y_siz: 1 0xc-0x10 (4)
0x10|00 00 00 00 |.... | xo_siz: 0 0x10-0x14 (4)
0x10| 00 00 00 00 | .... | yo_siz: 0 0x14-0x18 (4)
0x10| 00 00 00 01 | .... | xt_siz: 1 0x18-0x1c (4)
0x10| 00 00 00 01| ....| yt_siz: 1 0x1c-0x20 (4)
0x20|00 00 00 00 |.... | xto_siz: 0 0x20-0x24 (4)
0x20| 00 00 00 00 | .... | yto_siz: 0 0x24-0x28 (4)
0x20| 00 01 | .. | c_siz: 1 0x28-0x2a (2)
| | | components[0:1]: 0x2a-0x2d (3)
| | | [0]{}: component 0x2a-0x2d (3)
0x20| 0f | . | s_sizi: 15 0x2a-0x2b (1)
0x20| 01 | . | xr_sizi: 1 0x2b-0x2c (1)
0x20| 01 | . | yr_sizi: 1 0x2c-0x2d (1)
| | | [2]{}: segment 0x2d-0x51 (36)
0x20| ff 64 | .d | marker: "com" (0xff64) (Comment) 0x2d-0x2f (2)
0x20| 00| .| length: 34 0x2f-0x31 (2)
0x30|22 |" |
0x30| 00 01 | .. | r_cme: 1 0x31-0x33 (2)
0x30| 43 72 65 61 74 6f 72 3a 20 4a 61 73 50| Creator: JasP| data: "Creator: JasPer Version 2.0.33" 0x33-0x51 (30)
0x40|65 72 20 56 65 72 73 69 6f 6e 20 32 2e 30 2e 33|er Version 2.0.3|
0x50|33 |3 |
| | | [3]{}: segment 0x51-0x5f (14)
0x50| ff 52 | .R | marker: "cod" (0xff52) (Coding style default) 0x51-0x53 (2)
0x50| 00 0c | .. | length: 12 0x53-0x55 (2)
0x50| 00 00 00 01 00 05 04 04 00 01 | .......... | data: raw bits 0x55-0x5f (10)
| | | [4]{}: segment 0x5f-0x74 (21)
0x50| ff| .| marker: "qcd" (0xff5c) (Quantization default) 0x5f-0x61 (2)
0x60|5c |\ |
0x60| 00 13 | .. | length: 19 0x61-0x63 (2)
0x60| 40 80 88 88 90 88 88 90 88 88 90 88 88| @............| data: raw bits 0x63-0x74 (17)
0x70|90 88 88 90 |.... |
| | | [5]{}: segment 0x74-0x80 (12)
0x70| ff 90 | .. | marker: "sot" (0xff90) (Start of tile) 0x74-0x76 (2)
0x70| 00 0a | .. | l_sot: 10 0x76-0x78 (2)
0x70| 00 00 | .. | i_sot: 0 0x78-0x7a (2)
0x70| 00 00 00 2e | .... | p_sot: 46 0x7a-0x7e (4)
0x70| 00 | . | tp_sot: 0 0x7e-0x7f (1)
0x70| 01| .| tn_sot: 1 0x7f-0x80 (1)
| | | [6]{}: segment 0x80-0x96 (22)
0x80|ff 5d |.] | marker: "qcc" (0xff5d) (Quantization component) 0x80-0x82 (2)
0x80| 00 14 | .. | length: 20 0x82-0x84 (2)
0x80| 00 40 80 00 00 00 00 00 00 00 00 00| .@..........| data: raw bits 0x84-0x96 (18)
0x90|00 00 00 00 00 00 |...... |
| | | [7]{}: segment 0x96-0xa2 (12)
0x90| ff 93 | .. | marker: "sod" (0xff93) (Start of data) 0x96-0x98 (2)
0x90| df f8 90 08 07 80 80 80| ........| data: raw bits 0x98-0xa2 (10)
0xa0|80 80 |.. |
| | | [8]{}: segment 0xa2-0xa4 (2)
0xa0| ff d9| | ..| | marker: "eoc" (0xffd9) (End of codestream) 0xa2-0xa4 (2)

BIN
format/jpeg/testdata/test.jp2 vendored Normal file

Binary file not shown.

98
format/jpeg/testdata/test.jp2.fqtest vendored Normal file
View File

@ -0,0 +1,98 @@
# gm convert "xc:#000000" test.jp2
$ fq dv test.jp2
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: test.jp2 (mp4) 0x0-0xf9 (249)
| | | boxes[0:4]: 0x0-0xf9 (249)
| | | [0]{}: box 0x0-0xc (12)
0x00|00 00 00 0c |.... | size: 12 0x0-0x4 (4)
0x00| 6a 50 20 20 | jP | type: "jP" (JPEG 2000 Signature) 0x4-0x8 (4)
0x00| 0d 0a 87 0a | .... | signature: raw bits (valid) 0x8-0xc (4)
| | | [1]{}: box 0xc-0x20 (20)
0x00| 00 00 00 14| ....| size: 20 0xc-0x10 (4)
0x10|66 74 79 70 |ftyp | type: "ftyp" (File type and compatibility) 0x10-0x14 (4)
0x10| 6a 70 32 20 | jp2 | major_brand: "jp2" 0x14-0x18 (4)
0x10| 00 00 00 00 | .... | minor_version: 0 0x18-0x1c (4)
| | | brands[0:1]: 0x1c-0x20 (4)
0x10| 6a 70 32 20| jp2 | [0]: "jp2" brand 0x1c-0x20 (4)
| | | [2]{}: box 0x20-0x4d (45)
0x20|00 00 00 2d |...- | size: 45 0x20-0x24 (4)
0x20| 6a 70 32 68 | jp2h | type: "jp2h" (Header) 0x24-0x28 (4)
| | | boxes[0:2]: 0x28-0x4d (37)
| | | [0]{}: box 0x28-0x3e (22)
0x20| 00 00 00 16 | .... | size: 22 0x28-0x2c (4)
0x20| 69 68 64 72| ihdr| type: "ihdr" (Image Header) 0x2c-0x30 (4)
0x30|00 00 00 01 |.... | width: 1 0x30-0x34 (4)
0x30| 00 00 00 01 | .... | height: 1 0x34-0x38 (4)
0x30| 00 01 | .. | nc: 1 0x38-0x3a (2)
0x30| 0f | . | bpc: 16 0x3a-0x3b (1)
0x30| 07 | . | c: "jpeg_2000" (7) 0x3b-0x3c (1)
0x30| 00 | . | unk_c: 0 0x3c-0x3d (1)
0x30| 00 | . | ipr: 0 0x3d-0x3e (1)
| | | [1]{}: box 0x3e-0x4d (15)
0x30| 00 00| ..| size: 15 0x3e-0x42 (4)
0x40|00 0f |.. |
0x40| 63 6f 6c 72 | colr | type: "colr" (Specifies the colourspace of the image) 0x42-0x46 (4)
0x40| 01 00 00 00 | .... | parameter_type: "\x01\x00\x00\x00" 0x46-0x4a (4)
0x40| 00 00 11 | ... | data: raw bits 0x4a-0x4d (3)
| | | [3]{}: box 0x4d-0xf9 (172)
0x40| 00 00 00| ...| size: 0 (Rest of file) 0x4d-0x51 (4)
0x50|00 |. |
0x50| 6a 70 32 63 | jp2c | type: "jp2c" (JPEG 2000 contiguous codestream) 0x51-0x55 (4)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| segments[0:9]: (jp2c) 0x55-0xf9 (164)
| | | [0]{}: segment 0x55-0x57 (2)
0x50| ff 4f | .O | marker: "soc" (0xff4f) (Start of codestream) 0x55-0x57 (2)
| | | [1]{}: segment 0x57-0x82 (43)
0x50| ff 51 | .Q | marker: "siz" (0xff51) (Image and tile size) 0x57-0x59 (2)
0x50| 00 29 | .) | l_siz: 41 0x59-0x5b (2)
0x50| 00 00 | .. | r_siz: 0 0x5b-0x5d (2)
0x50| 00 00 00| ...| x_siz: 1 0x5d-0x61 (4)
0x60|01 |. |
0x60| 00 00 00 01 | .... | y_siz: 1 0x61-0x65 (4)
0x60| 00 00 00 00 | .... | xo_siz: 0 0x65-0x69 (4)
0x60| 00 00 00 00 | .... | yo_siz: 0 0x69-0x6d (4)
0x60| 00 00 00| ...| xt_siz: 1 0x6d-0x71 (4)
0x70|01 |. |
0x70| 00 00 00 01 | .... | yt_siz: 1 0x71-0x75 (4)
0x70| 00 00 00 00 | .... | xto_siz: 0 0x75-0x79 (4)
0x70| 00 00 00 00 | .... | yto_siz: 0 0x79-0x7d (4)
0x70| 00 01 | .. | c_siz: 1 0x7d-0x7f (2)
| | | components[0:1]: 0x7f-0x82 (3)
| | | [0]{}: component 0x7f-0x82 (3)
0x70| 0f| .| s_sizi: 15 0x7f-0x80 (1)
0x80|01 |. | xr_sizi: 1 0x80-0x81 (1)
0x80| 01 | . | yr_sizi: 1 0x81-0x82 (1)
| | | [2]{}: segment 0x82-0xa6 (36)
0x80| ff 64 | .d | marker: "com" (0xff64) (Comment) 0x82-0x84 (2)
0x80| 00 22 | ." | length: 34 0x84-0x86 (2)
0x80| 00 01 | .. | r_cme: 1 0x86-0x88 (2)
0x80| 43 72 65 61 74 6f 72 3a| Creator:| data: "Creator: JasPer Version 2.0.33" 0x88-0xa6 (30)
0x90|20 4a 61 73 50 65 72 20 56 65 72 73 69 6f 6e 20| JasPer Version |
0xa0|32 2e 30 2e 33 33 |2.0.33 |
| | | [3]{}: segment 0xa6-0xb4 (14)
0xa0| ff 52 | .R | marker: "cod" (0xff52) (Coding style default) 0xa6-0xa8 (2)
0xa0| 00 0c | .. | length: 12 0xa8-0xaa (2)
0xa0| 00 00 00 01 00 05| ......| data: raw bits 0xaa-0xb4 (10)
0xb0|04 04 00 01 |.... |
| | | [4]{}: segment 0xb4-0xc9 (21)
0xb0| ff 5c | .\ | marker: "qcd" (0xff5c) (Quantization default) 0xb4-0xb6 (2)
0xb0| 00 13 | .. | length: 19 0xb6-0xb8 (2)
0xb0| 40 80 88 88 90 88 88 90| @.......| data: raw bits 0xb8-0xc9 (17)
0xc0|88 88 90 88 88 90 88 88 90 |......... |
| | | [5]{}: segment 0xc9-0xd5 (12)
0xc0| ff 90 | .. | marker: "sot" (0xff90) (Start of tile) 0xc9-0xcb (2)
0xc0| 00 0a | .. | l_sot: 10 0xcb-0xcd (2)
0xc0| 00 00 | .. | i_sot: 0 0xcd-0xcf (2)
0xc0| 00| .| p_sot: 46 0xcf-0xd3 (4)
0xd0|00 00 2e |... |
0xd0| 00 | . | tp_sot: 0 0xd3-0xd4 (1)
0xd0| 01 | . | tn_sot: 1 0xd4-0xd5 (1)
| | | [6]{}: segment 0xd5-0xeb (22)
0xd0| ff 5d | .] | marker: "qcc" (0xff5d) (Quantization component) 0xd5-0xd7 (2)
0xd0| 00 14 | .. | length: 20 0xd7-0xd9 (2)
0xd0| 00 40 80 00 00 00 00| .@.....| data: raw bits 0xd9-0xeb (18)
0xe0|00 00 00 00 00 00 00 00 00 00 00 |........... |
| | | [7]{}: segment 0xeb-0xf7 (12)
0xe0| ff 93 | .. | marker: "sod" (0xff93) (Start of data) 0xeb-0xed (2)
0xe0| df f8 90| ...| data: raw bits 0xed-0xf7 (10)
0xf0|08 07 80 80 80 80 80 |....... |
| | | [8]{}: segment 0xf7-0xf9 (2)
0xf0| ff d9| | ..| | marker: "eoc" (0xffd9) (End of codestream) 0xf7-0xf9 (2)

View File

@ -53,7 +53,7 @@ var subTypeNames = scalar.StrMapDescription{
"subp": "Subpicture",
"text": "Text",
"tmcd": "Time Code",
"url ": "URL",
"url": "URL",
"vide": "Video Track",
}
@ -101,7 +101,7 @@ var dataFormatNames = scalar.StrMapDescription{
"encs": "Encrypted Systems stream",
"enct": "Encrypted Text",
"encv": "Encrypted/protected video",
"fdp ": "File delivery hints",
"fdp": "File delivery hints",
"fLaC": "Fres Lossless Audio Codec",
"g719": "ITU-T Recommendation G.719 (2008)",
"g726": "ITU-T Recommendation G.726 (1990)",
@ -138,13 +138,13 @@ var dataFormatNames = scalar.StrMapDescription{
"Opus": "Opus audio coding",
"pm2t": "Protected MPEG-2 Transport",
"prtp": "Protected RTP Reception",
"raw ": "Uncompressed audio",
"raw": "Uncompressed audio",
"resv": "Restricted Video",
"rm2t": "MPEG-2 Transport Reception",
"rrtp": "RTP reception",
"rsrp": "SRTP Reception",
"rtmd": "Real Time Metadata Sample Entry(XAVC Format)",
"rtp ": "RTP Hints",
"rtp": "RTP Hints",
"s263": "ITU H.263 video (3GPP format)",
"samr": "Narrowband AMR voice",
"sawb": "Wideband AMR voice",
@ -248,7 +248,7 @@ func decodeSampleFlags(d *decode.D) {
func decodeBoxWithParentData(ctx *decodeContext, d *decode.D, parentData any, extraTypeMappers ...scalar.StrMapper) {
var dataSize uint64
typeMappers := []scalar.StrMapper{boxDescriptions}
typeMappers := []scalar.StrMapper{scalar.ActualTrimSpace, boxDescriptions}
if len(extraTypeMappers) > 0 {
typeMappers = append(typeMappers, extraTypeMappers...)
}
@ -519,7 +519,7 @@ func decodeBox(ctx *decodeContext, d *decode.D, typ string) {
d.FieldU8("version")
d.FieldU24("flags")
d.FieldUTF8NullFixedLen("component_type", 4)
subType := d.FieldUTF8("component_subtype", 4, subTypeNames, scalar.ActualTrimSpace)
subType := d.FieldUTF8("component_subtype", 4, scalar.ActualTrimSpace, subTypeNames)
d.FieldUTF8NullFixedLen("component_manufacturer", 4)
d.FieldU32("component_flags")
d.FieldU32("component_flags_mask")
@ -559,12 +559,12 @@ func decodeBox(ctx *decodeContext, d *decode.D, typ string) {
var drefURL string
d.FieldStructArrayLoop("boxes", "box", func() bool { return i < entryCount }, func(d *decode.D) {
size := d.FieldU32("size")
typ := d.FieldUTF8("type", 4)
typ := d.FieldUTF8("type", 4, scalar.ActualTrimSpace)
d.FieldU8("version")
d.FieldU24("flags")
dataSize := size - 12
switch typ {
case "url ":
case "url":
drefURL = d.FieldUTF8("data", int(dataSize))
default:
d.FieldRawLen("data", int64(dataSize*8))
@ -589,7 +589,7 @@ func decodeBox(ctx *decodeContext, d *decode.D, typ string) {
d.FieldArrayLoop("boxes", func() bool { return i < entryCount }, func(d *decode.D) {
d.FieldStruct("box", func(d *decode.D) {
size := d.FieldU32("size")
dataFormat := d.FieldUTF8("type", 4, dataFormatNames)
dataFormat := d.FieldUTF8("type", 4, dataFormatNames, scalar.ActualTrimSpace)
subType := ""
if t := ctx.currentTrack(); t != nil {
t.sampleDescriptions = append(t.sampleDescriptions, sampleDescription{
@ -1242,7 +1242,7 @@ func decodeBox(ctx *decodeContext, d *decode.D, typ string) {
if !d.End() {
d.FieldUTF8Null("content_encoding")
}
case "uri ":
case "uri":
d.FieldUTF8Null("item_uri_type")
}
}
@ -1798,6 +1798,40 @@ func decodeBox(ctx *decodeContext, d *decode.D, typ string) {
default:
d.FieldRawLen("message_data", d.BitsLeft())
}
case "jp2h":
decodeBoxes(ctx, d)
case "ihdr":
d.FieldU32("width")
d.FieldU32("height")
d.FieldU16("nc")
d.FieldU8("bpc", scalar.UintActualAdd(1))
d.FieldU8("c", scalar.UintMapSymStr{
0: "uncompressed",
1: "modified_huffman",
2: "modified_read",
3: "modified_modified_read",
4: "jbig_bi_level",
5: "jpeg",
6: "jpeg_ls",
7: "jpeg_2000",
8: "jbig2",
9: "jbig",
})
d.FieldU8("unk_c")
d.FieldU8("ipr")
case "jP":
d.FieldRawLen("signature", 4*8, d.AssertBitBuf([]byte{0x0d, 0x0a, 0x87, 0x0a}))
case "jp2c":
d.FieldFormat("segments", &jp2cGroup, nil)
case "uinf":
decodeBoxes(ctx, d)
case "ulst":
nu := d.FieldU16("nu")
d.FieldArray("uids", func(d *decode.D) {
for i := 0; i < int(nu); i++ {
d.FieldRawLen("uid", 128)
}
})
default:
// there are at least 4 ways to encode udta metadata in mov/mp4 files.

View File

@ -85,7 +85,7 @@ var brandDescriptions = scalar.StrMapDescription{
"dv3b": "DMB AF extending dv3a with 3GPP timed text, DID, TVA, REL, IPMP",
"dvr1": "DVB RTP",
"dvt1": "DVB Transport Stream",
"dxo ": "DxO ONE camera",
"dxo": "DxO ONE camera",
"emsg": "Event message box present",
"F4A": "Audio for Adobe Flash Player 9+ (.F4A)",
"F4B": "Audio Book for Adobe Flash Player 9+ (.F4B)",
@ -160,7 +160,7 @@ var brandDescriptions = scalar.StrMapDescription{
"pana": "Panasonic Digital Camera",
"piff": "Protected Interoperable File Format",
"pnvi": "Panasonic Video Intercom Video Intercom",
"qt ": "QuickTime",
"qt": "QuickTime",
"risx": "Representation Index Segment used to index MPEG-2 TS based Media Segments",
"ROSS": "Ross Video Ross",
"sdv": "SD Video",

View File

@ -104,7 +104,7 @@ var boxDescriptions = scalar.StrMapDescription{
"irot": "Image rotation imir image mirroring",
"ispe": "Image spatial extents",
"j2kH": "JPEG 2000 header item property",
"jP ": "JPEG 2000 Signature",
"jP": "JPEG 2000 Signature",
"jp2c": "JPEG 2000 contiguous codestream",
"jp2h": "Header",
"jp2i": "Intellectual property information",
@ -174,7 +174,7 @@ var boxDescriptions = scalar.StrMapDescription{
"pssh": "Protection system specific header",
"ptle": "Partial Top Level Entry",
"reel": "Name of the tape reel",
"res ": "Grid resolution",
"res": "Grid resolution",
"resc": "Grid resolution at which the image was captured",
"resd": "Default grid resolution at which the image should be displayed",
"rinf": "Restricted scheme information box",
@ -255,12 +255,12 @@ var boxDescriptions = scalar.StrMapDescription{
"UITS": "Unique Identifier Technology Solution",
"ulst": "A list of UUIDs",
"urat": "User 'star' rating of the media",
"url ": "A URL",
"url": "A URL",
"uuid": "User-extension box",
"vmhd": "Video media header, overall information (video track only)",
"vwdi": "Multiview Scene Information",
"wide": "Expansion space reservation",
"xml ": "XML container",
"xml": "XML container",
"yrrc": "Year when media was recorded",
// from https://wiki.multimedia.cx/index.php/FFmpeg_Metadata

View File

@ -19,6 +19,7 @@ import (
"cmp"
"embed"
"fmt"
"strings"
"github.com/wader/fq/format"
"github.com/wader/fq/pkg/decode"
@ -42,6 +43,7 @@ var hevcCDCRGroup decode.Group
var iccProfileGroup decode.Group
var id3v2Group decode.Group
var imageGroup decode.Group
var jp2cGroup decode.Group
var jpegGroup decode.Group
var mp3FrameGroup decode.Group
var mpegESGroup decode.Group
@ -82,6 +84,7 @@ func init() {
{Groups: []*decode.Group{format.ICC_Profile}, Out: &iccProfileGroup},
{Groups: []*decode.Group{format.ID3v2}, Out: &id3v2Group},
{Groups: []*decode.Group{format.Image}, Out: &imageGroup},
{Groups: []*decode.Group{format.JP2C}, Out: &jp2cGroup},
{Groups: []*decode.Group{format.JPEG}, Out: &jpegGroup},
{Groups: []*decode.Group{format.MP3_Frame}, Out: &mp3FrameGroup},
{Groups: []*decode.Group{format.MPEG_ES}, Out: &mpegESGroup},
@ -465,14 +468,14 @@ func mp4Decode(d *decode.D) any {
if size < 8 {
d.Fatalf("first box size too small < 8")
}
firstType := d.UTF8(4)
firstType := strings.TrimSpace(d.UTF8(4))
switch firstType {
case "styp", // mp4 segment
"ftyp", // mp4 file
"free", // seems to happen
"moov", // seems to happen
"pnot", // video preview file
"jP ": // JPEG 2000
"jP": // JPEG 2000
default:
d.Errorf("no styp, ftyp, free or moov box found")
}

View File

@ -179,7 +179,7 @@ $ fq -d mp4 dv aac.mp4
| | | boxes[0:1]: 0x413-0x41f (12)
| | | [0]{}: box 0x413-0x41f (12)
0x410| 00 00 00 0c | .... | size: 12 0x413-0x417 (4)
0x410| 75 72 6c 20 | url | type: "url " 0x417-0x41b (4)
0x410| 75 72 6c 20 | url | type: "url" 0x417-0x41b (4)
0x410| 00 | . | version: 0 0x41b-0x41c (1)
0x410| 00 00 01 | ... | flags: 1 0x41c-0x41f (3)
| | | data: "" 0x41f-0x41f (0)

View File

@ -164,7 +164,7 @@ $ fq -d mp4 dv av1.mp4
| | | boxes[0:1]: 0x1345-0x1351 (12)
| | | [0]{}: box 0x1345-0x1351 (12)
0x1340| 00 00 00 0c | .... | size: 12 0x1345-0x1349 (4)
0x1340| 75 72 6c 20 | url | type: "url " 0x1349-0x134d (4)
0x1340| 75 72 6c 20 | url | type: "url" 0x1349-0x134d (4)
0x1340| 00 | . | version: 0 0x134d-0x134e (1)
0x1340| 00 00| ..| flags: 1 0x134e-0x1351 (3)
0x1350|01 |. |

View File

@ -182,7 +182,7 @@ $ fq -d mp4 dv avc.mp4
| | | boxes[0:1]: 0xf06-0xf12 (12)
| | | [0]{}: box 0xf06-0xf12 (12)
0x00f00| 00 00 00 0c | .... | size: 12 0xf06-0xf0a (4)
0x00f00| 75 72 6c 20 | url | type: "url " 0xf0a-0xf0e (4)
0x00f00| 75 72 6c 20 | url | type: "url" 0xf0a-0xf0e (4)
0x00f00| 00 | . | version: 0 0xf0e-0xf0f (1)
0x00f00| 00| .| flags: 1 0xf0f-0xf12 (3)
0x00f10|00 01 |.. |

View File

@ -197,7 +197,7 @@ $ fq -d mp4 dv dash_audio_init.mp4
| | | boxes[0:1]: 0x1fa-0x206 (12)
| | | [0]{}: box 0x1fa-0x206 (12)
0x1f0| 00 00 00 0c | .... | size: 12 0x1fa-0x1fe (4)
0x1f0| 75 72| ur| type: "url " 0x1fe-0x202 (4)
0x1f0| 75 72| ur| type: "url" 0x1fe-0x202 (4)
0x200|6c 20 |l |
0x200| 00 | . | version: 0 0x202-0x203 (1)
0x200| 00 00 01 | ... | flags: 1 0x203-0x206 (3)

View File

@ -178,7 +178,7 @@ $ fq -d mp4 dv flac.mp4
| | | boxes[0:1]: 0x40b-0x417 (12)
| | | [0]{}: box 0x40b-0x417 (12)
0x400| 00 00 00 0c | .... | size: 12 0x40b-0x40f (4)
0x400| 75| u| type: "url " 0x40f-0x413 (4)
0x400| 75| u| type: "url" 0x40f-0x413 (4)
0x410|72 6c 20 |rl |
0x410| 00 | . | version: 0 0x413-0x414 (1)
0x410| 00 00 01 | ... | flags: 1 0x414-0x417 (3)

View File

@ -140,7 +140,7 @@ $ fq -d mp4 dv fragmented.mp4
| | | boxes[0:1]: 0x185-0x191 (12)
| | | [0]{}: box 0x185-0x191 (12)
0x00180| 00 00 00 0c | .... | size: 12 0x185-0x189 (4)
0x00180| 75 72 6c 20 | url | type: "url " 0x189-0x18d (4)
0x00180| 75 72 6c 20 | url | type: "url" 0x189-0x18d (4)
0x00180| 00 | . | version: 0 0x18d-0x18e (1)
0x00180| 00 00| ..| flags: 1 0x18e-0x191 (3)
0x00190|01 |. |
@ -402,7 +402,7 @@ $ fq -d mp4 dv fragmented.mp4
| | | [0]{}: box 0x36d-0x379 (12)
0x00360| 00 00 00| ...| size: 12 0x36d-0x371 (4)
0x00370|0c |. |
0x00370| 75 72 6c 20 | url | type: "url " 0x371-0x375 (4)
0x00370| 75 72 6c 20 | url | type: "url" 0x371-0x375 (4)
0x00370| 00 | . | version: 0 0x375-0x376 (1)
0x00370| 00 00 01 | ... | flags: 1 0x376-0x379 (3)
| | | data: "" 0x379-0x379 (0)

View File

@ -182,7 +182,7 @@ $ fq -d mp4 dv hevc.mp4
| | | boxes[0:1]: 0xa06-0xa12 (12)
| | | [0]{}: box 0xa06-0xa12 (12)
0x0a00| 00 00 00 0c | .... | size: 12 0xa06-0xa0a (4)
0x0a00| 75 72 6c 20 | url | type: "url " 0xa0a-0xa0e (4)
0x0a00| 75 72 6c 20 | url | type: "url" 0xa0a-0xa0e (4)
0x0a00| 00 | . | version: 0 0xa0e-0xa0f (1)
0x0a00| 00| .| flags: 1 0xa0f-0xa12 (3)
0x0a10|00 01 |.. |

View File

@ -170,7 +170,7 @@ $ fq dv in24.mp4
| | | boxes[0:1]: 0x2d9-0x2e5 (12)
| | | [0]{}: box 0x2d9-0x2e5 (12)
0x2d0| 00 00 00 0c | .... | size: 12 0x2d9-0x2dd (4)
0x2d0| 75 72 6c| url| type: "url " 0x2dd-0x2e1 (4)
0x2d0| 75 72 6c| url| type: "url" 0x2dd-0x2e1 (4)
0x2e0|20 | |
0x2e0| 00 | . | version: 0 0x2e1-0x2e2 (1)
0x2e0| 00 00 01 | ... | flags: 1 0x2e2-0x2e5 (3)

View File

@ -172,7 +172,7 @@ $ fq dv lpcm.mp4
| | | boxes[0:1]: 0x411-0x41d (12)
| | | [0]{}: box 0x411-0x41d (12)
0x410| 00 00 00 0c | .... | size: 12 0x411-0x415 (4)
0x410| 75 72 6c 20 | url | type: "url " 0x415-0x419 (4)
0x410| 75 72 6c 20 | url | type: "url" 0x415-0x419 (4)
0x410| 00 | . | version: 0 0x419-0x41a (1)
0x410| 00 00 01 | ... | flags: 1 0x41a-0x41d (3)
| | | data: "" 0x41d-0x41d (0)

View File

@ -178,7 +178,7 @@ $ fq -d mp4 dv mp3.mp4
| | | [0]{}: box 0x41f-0x42b (12)
0x410| 00| .| size: 12 0x41f-0x423 (4)
0x420|00 00 0c |... |
0x420| 75 72 6c 20 | url | type: "url " 0x423-0x427 (4)
0x420| 75 72 6c 20 | url | type: "url" 0x423-0x427 (4)
0x420| 00 | . | version: 0 0x427-0x428 (1)
0x420| 00 00 01 | ... | flags: 1 0x428-0x42b (3)
| | | data: "" 0x42b-0x42b (0)

View File

@ -180,7 +180,7 @@ $ fq -d mp4 dv mpeg2.mp4
| | | boxes[0:1]: 0x212b-0x2137 (12)
| | | [0]{}: box 0x212b-0x2137 (12)
0x2120| 00 00 00 0c | .... | size: 12 0x212b-0x212f (4)
0x2120| 75| u| type: "url " 0x212f-0x2133 (4)
0x2120| 75| u| type: "url" 0x212f-0x2133 (4)
0x2130|72 6c 20 |rl |
0x2130| 00 | . | version: 0 0x2133-0x2134 (1)
0x2130| 00 00 01 | ... | flags: 1 0x2134-0x2137 (3)

View File

@ -175,7 +175,7 @@ $ fq -d mp4 dv opus.mp4
| | | boxes[0:1]: 0x318-0x324 (12)
| | | [0]{}: box 0x318-0x324 (12)
0x310| 00 00 00 0c | .... | size: 12 0x318-0x31c (4)
0x310| 75 72 6c 20| url | type: "url " 0x31c-0x320 (4)
0x310| 75 72 6c 20| url | type: "url" 0x31c-0x320 (4)
0x320|00 |. | version: 0 0x320-0x321 (1)
0x320| 00 00 01 | ... | flags: 1 0x321-0x324 (3)
| | | data: "" 0x324-0x324 (0)

View File

@ -182,7 +182,7 @@ $ fq dv png.mp4
| | | [0]{}: box 0x20e-0x21a (12)
0x200| 00 00| ..| size: 12 0x20e-0x212 (4)
0x210|00 0c |.. |
0x210| 75 72 6c 20 | url | type: "url " 0x212-0x216 (4)
0x210| 75 72 6c 20 | url | type: "url" 0x212-0x216 (4)
0x210| 00 | . | version: 0 0x216-0x217 (1)
0x210| 00 00 01 | ... | flags: 1 0x217-0x21a (3)
| | | data: "" 0x21a-0x21a (0)

View File

@ -144,7 +144,7 @@ $ fq dv png_no_hdlr.mp4
| | | boxes[0:1]: 0x17c-0x188 (12)
| | | [0]{}: box 0x17c-0x188 (12)
0x170| 00 00 00 0c| ....| size: 12 0x17c-0x180 (4)
0x180|75 72 6c 20 |url | type: "url " 0x180-0x184 (4)
0x180|75 72 6c 20 |url | type: "url" 0x180-0x184 (4)
0x180| 00 | . | version: 0 0x184-0x185 (1)
0x180| 00 00 01 | ... | flags: 1 0x185-0x188 (3)
| | | data: "" 0x188-0x188 (0)

View File

@ -174,7 +174,7 @@ $ fq -d mp4 'dv' stz2.mp4
| | | boxes[0:1]: 0x1aa-0x1b6 (12)
| | | [0]{}: box 0x1aa-0x1b6 (12)
0x1a0| 00 00 00 0c | .... | size: 12 0x1aa-0x1ae (4)
0x1a0| 75 72| ur| type: "url " 0x1ae-0x1b2 (4)
0x1a0| 75 72| ur| type: "url" 0x1ae-0x1b2 (4)
0x1b0|6c 20 |l |
0x1b0| 00 | . | version: 0 0x1b2-0x1b3 (1)
0x1b0| 00 00 01 | ... | flags: 1 0x1b3-0x1b6 (3)

BIN
format/mp4/testdata/uinf.mp4 vendored Normal file

Binary file not shown.

22
format/mp4/testdata/uinf.mp4.fqtest vendored Normal file
View File

@ -0,0 +1,22 @@
$ fq -d mp4 -o force=true dv uinf.mp4
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: uinf.mp4 (mp4) 0x0-0x64 (100)
| | | boxes[0:1]: 0x0-0x64 (100)
| | | [0]{}: box 0x0-0x64 (100)
0x00|00 00 00 64 |...d | size: 100 0x0-0x4 (4)
0x00| 75 69 6e 66 | uinf | type: "uinf" (A tool by which a vendor may provide access to additional information associated with a UUID) 0x4-0x8 (4)
| | | boxes[0:2]: 0x8-0x64 (92)
| | | [0]{}: box 0x8-0x32 (42)
0x00| 00 00 00 2a | ...* | size: 42 0x8-0xc (4)
0x00| 75 6c 73 74| ulst| type: "ulst" (A list of UUIDs) 0xc-0x10 (4)
0x10|00 02 |.. | nu: 2 0x10-0x12 (2)
| | | uids[0:2]: 0x12-0x32 (32)
0x10| 6a 70 6a 70 6a 70 6a 70 6a 70 6a 70 6a 70| jpjpjpjpjpjpjp| [0]: raw bits uid 0x12-0x22 (16)
0x20|6a 70 |jp |
0x20| 61 62 61 62 61 62 61 62 61 62 61 62 61 62| ababababababab| [1]: raw bits uid 0x22-0x32 (16)
0x30|61 62 |ab |
| | | [1]{}: box 0x32-0x64 (50)
0x30| 00 00 00 32 | ...2 | size: 50 0x32-0x36 (4)
0x30| 75 72 6c 20 | url | type: "url" (A URL) 0x36-0x3a (4)
0x30| 00 00 00 00 68 74| ....ht| data: raw bits 0x3a-0x64 (42)
0x40|74 70 3a 2f 2f 77 77 77 2e 6f 70 65 6e 70 6c 61|tp://www.openpla|
* |until 0x63.7 (end) (42) | |

View File

@ -180,7 +180,7 @@ $ fq -d mp4 dv vorbis.mp4
| | | [0]{}: box 0x35e-0x36a (12)
0x0350| 00 00| ..| size: 12 0x35e-0x362 (4)
0x0360|00 0c |.. |
0x0360| 75 72 6c 20 | url | type: "url " 0x362-0x366 (4)
0x0360| 75 72 6c 20 | url | type: "url" 0x362-0x366 (4)
0x0360| 00 | . | version: 0 0x366-0x367 (1)
0x0360| 00 00 01 | ... | flags: 1 0x367-0x36a (3)
| | | data: "" 0x36a-0x36a (0)

View File

@ -163,7 +163,7 @@ $ fq -d mp4 dv vp9.mp4
| | | boxes[0:1]: 0x16e9-0x16f5 (12)
| | | [0]{}: box 0x16e9-0x16f5 (12)
0x16e0| 00 00 00 0c | .... | size: 12 0x16e9-0x16ed (4)
0x16e0| 75 72 6c| url| type: "url " 0x16ed-0x16f1 (4)
0x16e0| 75 72 6c| url| type: "url" 0x16ed-0x16f1 (4)
0x16f0|20 | |
0x16f0| 00 | . | version: 0 0x16f1-0x16f2 (1)
0x16f0| 00 00 01 | ... | flags: 1 0x16f2-0x16f5 (3)

View File

@ -195,7 +195,7 @@ $ fq -d mp4 dv prores_frame.mov
| | | [0]{}: box 0x6dee-0x6dfa (12)
0x6de0| 00 00| ..| size: 12 0x6dee-0x6df2 (4)
0x6df0|00 0c |.. |
0x6df0| 75 72 6c 20 | url | type: "url " 0x6df2-0x6df6 (4)
0x6df0| 75 72 6c 20 | url | type: "url" 0x6df2-0x6df6 (4)
0x6df0| 00 | . | version: 0 0x6df6-0x6df7 (1)
0x6df0| 00 00 01 | ... | flags: 1 0x6df7-0x6dfa (3)
| | | data: "" 0x6dfa-0x6dfa (0)

View File

@ -59,7 +59,7 @@ var aviListTypeDescriptions = scalar.StrMapDescription{
"hdrl": "AVI main list",
"strl": "Stream list",
"movi": "Stream Data",
"rec ": "Chunk group",
"rec": "Chunk group",
}
const (
@ -245,7 +245,7 @@ func aviDecodeEx(d *decode.D, ai format.AVI_In, extendedChunk bool) {
return true, nil
case "LIST":
typ := d.FieldUTF8("type", 4, aviListTypeDescriptions)
typ := d.FieldUTF8("type", 4, scalar.ActualTrimSpace, aviListTypeDescriptions)
switch typ {
case "strl":
return true, &aviStream{}

2
fq.go
View File

@ -6,7 +6,7 @@ import (
"github.com/wader/fq/pkg/interp"
)
const version = "0.10.0"
const version = "0.11.0"
func main() {
cli.Main(interp.DefaultRegistry, version)

12
go.mod
View File

@ -19,7 +19,7 @@ require (
// bump: gomod-ergochat-readline /github\.com\/ergochat\/readline v(.*)/ https://github.com/ergochat/readline.git|*
// bump: gomod-ergochat-readline command go get -d github.com/ergochat/readline@v$LATEST && go mod tidy
// bump: gomod-ergochat-readline link "Release notes" https://github.com/ergochat/readline/releases/tag/v$LATEST
github.com/ergochat/readline v0.1.0
github.com/ergochat/readline v0.1.1
// bump: gomod-golang-snappy /github\.com\/golang\/snappy v(.*)/ https://github.com/golang/snappy.git|^0
// bump: gomod-golang-snappy command go get -d github.com/golang/snappy@v$LATEST && go mod tidy
@ -48,7 +48,7 @@ require (
// bump: gomod-golang-x-crypto /golang\.org\/x\/crypto v(.*)/ https://github.com/golang/crypto.git|^0
// bump: gomod-golang-x-crypto command go get -d golang.org/x/crypto@v$LATEST && go mod tidy
// bump: gomod-golang-x-crypto link "Tags" https://github.com/golang/crypto/tags
golang.org/x/crypto v0.22.0
golang.org/x/crypto v0.23.0
// has no tags
// go get -d golang.org/x/exp@master && go mod tidy
@ -57,17 +57,17 @@ require (
// bump: gomod-golang-x-net /golang\.org\/x\/net v(.*)/ https://github.com/golang/net.git|^0
// bump: gomod-golang-x-net command go get -d golang.org/x/net@v$LATEST && go mod tidy
// bump: gomod-golang-x-net link "Tags" https://github.com/golang/net/tags
golang.org/x/net v0.24.0
golang.org/x/net v0.25.0
// bump: gomod-golang-x-term /golang\.org\/x\/term v(.*)/ https://github.com/golang/term.git|^0
// bump: gomod-golang-x-term command go get -d golang.org/x/term@v$LATEST && go mod tidy
// bump: gomod-golang-x-term link "Tags" https://github.com/golang/term/tags
golang.org/x/term v0.19.0
golang.org/x/term v0.20.0
// bump: gomod-golang/text /golang\.org\/x\/text v(.*)/ https://github.com/golang/text.git|^0
// bump: gomod-golang/text command go get -d golang.org/x/text@v$LATEST && go mod tidy
// bump: gomod-golang/text link "Source diff $CURRENT..$LATEST" https://github.com/golang/text/compare/v$CURRENT..v$LATEST
golang.org/x/text v0.14.0
golang.org/x/text v0.15.0
// bump: gomod-gopkg.in/yaml.v3 /gopkg\.in\/yaml\.v3 v(.*)/ https://github.com/go-yaml/yaml.git|^3
// bump: gomod-gopkg.in/yaml.v3 command go get -d gopkg.in/yaml.v3@v$LATEST && go mod tidy
@ -79,6 +79,6 @@ require (
github.com/itchyny/timefmt-go v0.1.5 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
golang.org/x/sys v0.19.0 // indirect
golang.org/x/sys v0.20.0 // indirect
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
)

24
go.sum
View File

@ -2,8 +2,8 @@ github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA=
github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
github.com/ergochat/readline v0.1.0 h1:KEIiAnyH9qGZB4K8oq5mgDcExlEKwmZDcyyocgJiABc=
github.com/ergochat/readline v0.1.0/go.mod h1:o3ux9QLHLm77bq7hDB21UTm6HlV2++IPDMfIfKDuOgY=
github.com/ergochat/readline v0.1.1 h1:C8Uuo3ybB23GWOt0uxmHbGzKM9owmtXary6Clrj84s0=
github.com/ergochat/readline v0.1.1/go.mod h1:o3ux9QLHLm77bq7hDB21UTm6HlV2++IPDMfIfKDuOgY=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomarkdown/markdown v0.0.0-20240328165702-4d01890c35c0 h1:4gjrh/PN2MuWCCElk8/I4OCKRKWCCo2zEct3VKCbibU=
@ -25,18 +25,18 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/wader/gojq v0.12.1-0.20240401131232-6c6bc364201a h1:P881Oecjt9FEXrwkGJ6UObJksxejJaF/fKq1ZfXpiVE=
github.com/wader/gojq v0.12.1-0.20240401131232-6c6bc364201a/go.mod h1:qVrzkUdnBtJvM4twyRQ6xdziPSnSp35dLm4s/DN2iP4=
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw=
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ=
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

View File

@ -47,7 +47,7 @@ func utf8Bytes(b byte) int {
}
}
// RuneReadSeeker reads rune from a io.ReadSeeker
// ReadRune reads rune from a io.ReadSeeker
func (brr RuneReadSeeker) ReadRune() (r rune, size int, err error) {
var b [utf8.UTFMax]byte

View File

@ -263,7 +263,7 @@ func (d *D) TryCopyBits(w io.Writer, r bitio.Reader) (int64, error) {
func (d *D) CopyBits(w io.Writer, r bitio.Reader) int64 {
n, err := d.TryCopyBits(w, r)
if err != nil {
d.IOPanic(err, "CopyBits: Copy")
d.IOPanic(err, "", "CopyBits")
}
return n
}
@ -277,7 +277,7 @@ func (d *D) TryCopy(w io.Writer, r io.Reader) (int64, error) {
func (d *D) Copy(w io.Writer, r io.Reader) int64 {
n, err := d.TryCopy(w, r)
if err != nil {
d.IOPanic(err, "Copy")
d.IOPanic(err, "", "Copy")
}
return n
}
@ -285,7 +285,7 @@ func (d *D) Copy(w io.Writer, r io.Reader) int64 {
func (d *D) CloneReadSeeker(br bitio.ReadSeeker) bitio.ReadSeeker {
br, err := bitio.CloneReadSeeker(br)
if err != nil {
d.IOPanic(err, "CloneReadSeeker")
d.IOPanic(err, "", "CloneReadSeeker")
}
return br
}
@ -308,7 +308,7 @@ func (d *D) TryReadAllBits(r bitio.Reader) ([]byte, error) {
func (d *D) ReadAllBits(r bitio.Reader) []byte {
buf, err := d.TryReadAllBits(r)
if err != nil {
d.IOPanic(err, "Bytes ReadAllBytes")
d.IOPanic(err, "", "ReadAllBytes")
}
return buf
}
@ -350,7 +350,7 @@ func (d *D) FillGaps(r ranges.Range, namePrefix string) {
for i, gap := range gaps {
br, err := bitiox.Range(d.bitBuf, gap.Start, gap.Len)
if err != nil {
d.IOPanic(err, "FillGaps: Range")
d.IOPanic(err, namePrefix, "bitiox.Range")
}
v := &Value{
@ -380,8 +380,8 @@ func (d *D) Fatalf(format string, a ...any) {
panic(DecoderError{Reason: fmt.Sprintf(format, a...), Pos: d.Pos()})
}
func (d *D) IOPanic(err error, op string) {
panic(IOError{Err: err, Pos: d.Pos(), Op: op})
func (d *D) IOPanic(err error, name string, op string) {
panic(IOError{Err: err, Name: name, Pos: d.Pos(), Op: op})
}
// TryBits reads nBits bits from buffer
@ -543,7 +543,7 @@ func (d *D) TryPeekFind(nBits int, seekBits int64, maxLen int64, fn func(v uint6
func (d *D) PeekFind(nBits int, seekBits int64, maxLen int64, fn func(v uint64) bool) (int64, uint64) {
peekBits, v, err := d.TryPeekFind(nBits, seekBits, maxLen, fn)
if err != nil {
d.IOPanic(err, "PeekFind: TryPeekFind")
d.IOPanic(err, "", "PeekFind")
}
if peekBits == -1 {
d.Errorf("peek not found")
@ -771,7 +771,7 @@ func (d *D) TrySeekRel(delta int64, fns ...func(d *D)) (int64, error) {
func (d *D) SeekRel(delta int64, fns ...func(d *D)) int64 {
n, err := d.trySeekAbs(d.Pos()+delta, fns...)
if err != nil {
d.IOPanic(err, "SeekRel")
d.IOPanic(err, "", "SeekRel")
}
return n
}
@ -785,7 +785,7 @@ func (d *D) TrySeekAbs(pos int64, fns ...func(d *D)) (int64, error) {
func (d *D) SeekAbs(pos int64, fns ...func(d *D)) int64 {
n, err := d.trySeekAbs(pos, fns...)
if err != nil {
d.IOPanic(err, "SeekAbs")
d.IOPanic(err, "", "SeekAbs")
}
return n
}
@ -955,7 +955,7 @@ func (d *D) RangeFn(firstBit int64, nBits int64, fn func(d *D)) int64 {
// TODO: do some kind of DecodeLimitedLen/RangeFn?
br := d.BitBufRange(0, firstBit+nBits)
if _, err := br.SeekBits(firstBit, io.SeekStart); err != nil {
d.IOPanic(err, "RangeFn: SeekAbs")
d.IOPanic(err, "", "RangeFn: SeekAbs")
}
nd := *d
@ -979,7 +979,7 @@ func (d *D) Format(group *Group, inArg any) any {
ReadBuf: d.readBuf,
})
if dv == nil || dv.Errors() != nil {
d.IOPanic(err, "Format: decode")
d.IOPanic(err, "", "Format: decode")
}
switch vv := dv.V.(type) {
@ -992,7 +992,7 @@ func (d *D) Format(group *Group, inArg any) any {
}
if _, err := d.bitBuf.SeekBits(dv.Range.Len, io.SeekCurrent); err != nil {
d.IOPanic(err, "Format: SeekRel")
d.IOPanic(err, "", "Format: SeekRel")
}
return v
@ -1015,7 +1015,7 @@ func (d *D) TryFieldFormat(name string, group *Group, inArg any) (*Value, any, e
d.AddChild(dv)
if _, err := d.bitBuf.SeekBits(dv.Range.Len, io.SeekCurrent); err != nil {
d.IOPanic(err, "TryFieldFormat: SeekRel")
d.IOPanic(err, name, "TryFieldFormat: SeekRel")
}
return dv, v, err
@ -1024,7 +1024,7 @@ func (d *D) TryFieldFormat(name string, group *Group, inArg any) (*Value, any, e
func (d *D) FieldFormat(name string, group *Group, inArg any) (*Value, any) {
dv, v, err := d.TryFieldFormat(name, group, inArg)
if dv == nil || dv.Errors() != nil {
d.IOPanic(err, "FieldFormat: TryFieldFormat")
d.IOPanic(err, name, "FieldFormat: TryFieldFormat")
}
return dv, v
}
@ -1054,7 +1054,7 @@ func (d *D) TryFieldFormatLen(name string, nBits int64, group *Group, inArg any)
d.AddChild(dv)
if _, err := d.bitBuf.SeekBits(nBits, io.SeekCurrent); err != nil {
d.IOPanic(err, "TryFieldFormatLen: SeekRel")
d.IOPanic(err, name, "TryFieldFormatLen: SeekRel")
}
return dv, v, err
@ -1063,7 +1063,7 @@ func (d *D) TryFieldFormatLen(name string, nBits int64, group *Group, inArg any)
func (d *D) FieldFormatLen(name string, nBits int64, group *Group, inArg any) (*Value, any) {
dv, v, err := d.TryFieldFormatLen(name, nBits, group, inArg)
if dv == nil || dv.Errors() != nil {
d.IOPanic(err, "FieldFormatLen: TryFieldFormatLen")
d.IOPanic(err, name, "FieldFormatLen: TryFieldFormatLen")
}
return dv, v
}
@ -1100,7 +1100,7 @@ func (d *D) TryFieldFormatRange(name string, firstBit int64, nBits int64, group
func (d *D) FieldFormatRange(name string, firstBit int64, nBits int64, group *Group, inArg any) (*Value, any) {
dv, v, err := d.TryFieldFormatRange(name, firstBit, nBits, group, inArg)
if dv == nil || dv.Errors() != nil {
d.IOPanic(err, "FieldFormatRange: TryFieldFormatRange")
d.IOPanic(err, name, "FieldFormatRange: TryFieldFormatRange")
}
return dv, v
@ -1130,7 +1130,7 @@ func (d *D) TryFieldFormatBitBuf(name string, br bitio.ReaderAtSeeker, group *Gr
func (d *D) FieldFormatBitBuf(name string, br bitio.ReaderAtSeeker, group *Group, inArg any) (*Value, any) {
dv, v, err := d.TryFieldFormatBitBuf(name, br, group, inArg)
if dv == nil || dv.Errors() != nil {
d.IOPanic(err, "FieldFormatBitBuf: TryFieldFormatBitBuf")
d.IOPanic(err, name, "FieldFormatBitBuf: TryFieldFormatBitBuf")
}
return dv, v
@ -1141,7 +1141,7 @@ func (d *D) FieldFormatBitBuf(name string, br bitio.ReaderAtSeeker, group *Group
func (d *D) FieldRootBitBuf(name string, br bitio.ReaderAtSeeker, sms ...scalar.BitBufMapper) *Value {
brLen, err := bitiox.Len(br)
if err != nil {
d.IOPanic(err, "br Len")
d.IOPanic(err, name, "br Len")
}
v := &Value{}
@ -1188,17 +1188,17 @@ func (d *D) FieldStructRootBitBufFn(name string, br bitio.ReaderAtSeeker, fn fun
func (d *D) FieldFormatReaderLen(name string, nBits int64, fn func(r io.Reader) (io.ReadCloser, error), group *Group) (*Value, any) {
br, err := d.TryBitBufLen(nBits)
if err != nil {
d.IOPanic(err, "FieldFormatReaderLen: BitBufLen")
d.IOPanic(err, name, "FieldFormatReaderLen: BitBufLen")
}
bbBR := bitio.NewIOReader(br)
r, err := fn(bbBR)
if err != nil {
d.IOPanic(err, "FieldFormatReaderLen: fn")
d.IOPanic(err, name, "FieldFormatReaderLen: fn")
}
rBuf, err := io.ReadAll(r)
if err != nil {
d.IOPanic(err, "FieldFormatReaderLen: ReadAll")
d.IOPanic(err, name, "FieldFormatReaderLen: ReadAll")
}
rBR := bitio.NewBitReader(rBuf, -1)
@ -1245,7 +1245,7 @@ func (d *D) FieldReaderRange(name string, startBit int64, nBits int64, fn func(r
func (d *D) FieldReaderRangeFormat(name string, startBit int64, nBits int64, fn func(r io.Reader) io.Reader, group *Group, inArg any) (int64, bitio.ReaderAtSeeker, *Value, any) {
cz, rBR, dv, v, err := d.TryFieldReaderRangeFormat(name, startBit, nBits, fn, group, inArg)
if err != nil {
d.IOPanic(err, "TryFieldReaderRangeFormat")
d.IOPanic(err, name, "TryFieldReaderRangeFormat")
}
return cz, rBR, dv, v
}
@ -1268,7 +1268,7 @@ func (d *D) TryFieldValue(name string, fn func() (*Value, error)) (*Value, error
func (d *D) FieldValue(name string, fn func() *Value) *Value {
v, err := d.TryFieldValue(name, func() (*Value, error) { return fn(), nil })
if err != nil {
d.IOPanic(err, "FieldValue: TryFieldValue")
d.IOPanic(err, name, "FieldValue: TryFieldValue")
}
return v
}

File diff suppressed because it is too large Load Diff

View File

@ -26,7 +26,7 @@ import (
func (d *D) Field{{$name}}ScalarFn(name string, fn func(d *D) scalar.{{$name}}, sms ...scalar.{{$name}}Mapper) {{$t.go_type}} {
v, err := d.TryFieldScalar{{$name}}Fn(name, func(d *D) (scalar.{{$name}}, error) { return fn(d), nil }, sms...)
if err != nil {
panic(IOError{Err: err, Name: name, Op: "{{$name}}", Pos: d.Pos()})
d.IOPanic(err, name, "{{$name}}")
}
return v.Actual
}
@ -48,7 +48,7 @@ import (
func (d *D) FieldScalar{{$name}}Fn(name string, fn func(d *D) scalar.{{$name}}, sms ...scalar.{{$name}}Mapper) *scalar.{{$name}} {
v, err := d.TryFieldScalar{{$name}}Fn(name, func(d *D) (scalar.{{$name}}, error) { return fn(d), nil }, sms...)
if err != nil {
panic(IOError{Err: err, Name: name, Op: "{{$name}}", Pos: d.Pos()})
d.IOPanic(err, name, "{{$name}}")
}
return v
}
@ -181,7 +181,7 @@ import (
func (d *D) {{$r.name}}{{replace $v.name "$n" $n}}({{$v.params}}) {{$t.go_type}} {
v, err := {{replace $v.call "$n" $n}}
if err != nil {
panic(IOError{Err: err, Op: "{{$r.name}}{{replace $v.name "$n" $n}}", Pos: d.Pos()})
d.IOPanic(err, "", "{{$r.name}}{{replace $v.name "$n" $n}}")
}
return v
}
@ -202,7 +202,7 @@ import (
func (d *D) FieldScalar{{$r.name}}{{replace $v.name "$n" $n}}(name string{{if $v.params}}, {{$v.params}}{{end}}, sms ...scalar.{{$r.type}}Mapper) *scalar.{{$r.type}} {
s, err := d.TryFieldScalar{{$r.name}}{{replace $v.name "$n" $n}}(name{{if $v.args}}, {{$v.args}}{{end}}, sms...)
if err != nil {
panic(IOError{Err: err, Name: name, Op: "{{$r.name}}{{replace $v.name "$n" $n}}", Pos: d.Pos()})
d.IOPanic(err, name, "{{$r.name}}{{replace $v.name "$n" $n}}")
}
return s
}

View File

@ -5,8 +5,6 @@ include "decode";
def intdiv(a; b): _intdiv(a; b);
def trim: capture("^\\s*(?<str>.*?)\\s*$"; "m").str;
# does +1 and [:1] as " "*0 is null
def rpad($s; $w): . + ($s * ($w+1-length))[1:];

View File

@ -54,9 +54,6 @@ Same as recurse without argument.
}
};
def help($_): error("help must be alone or last in pipeline. ex: help(length) or ... | help");
def help: help(null);
def _help_format_enrich($arg0; $f; $include_basic):
( if $include_basic then
.examples +=
@ -76,6 +73,9 @@ def _help_format_enrich($arg0; $f; $include_basic):
end
);
# trailing help gets rewritten to _help_slurp, these are here to catch other variants
def help($_): error("help must be alone or last in pipeline. ex: help(length) or ... | help");
def help: help(null);
def _help($arg0; $topic):
( $topic
| if . == "usage" then
@ -236,7 +236,7 @@ def _help($arg0; $topic):
| ($args | length) as $argc
| if $args == null then
# help
( "Type expression to evaluate"
( "Type jq expression to evaluate"
, "help(...) Help for topic. Ex: help(mp4), help(\"mp4\")"
, "\\t Completion"
, "Up/Down History"

View File

@ -1,6 +1,6 @@
$ fq -ni
null> help
Type expression to evaluate
Type jq expression to evaluate
help(...) Help for topic. Ex: help(mp4), help("mp4")
\t Completion
Up/Down History