diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74d66eaf..04753803 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: pull_request: env: - GOLANGCILINT_VERSION: 1.50.0 + GOLANGCILINT_VERSION: 1.50.1 jobs: lint: diff --git a/Makefile b/Makefile index 99126dfe..bec6cca9 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ gogenerate: lint: # bump: make-golangci-lint /golangci-lint@v([\d.]+)/ git:https://github.com/golangci/golangci-lint.git|^1 # bump: make-golangci-lint link "Release notes" https://github.com/golangci/golangci-lint/releases/tag/v$LATEST - go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.0 run + go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1 run .PHONY: depgraph.svg depgraph.svg: @@ -95,16 +95,19 @@ update-gomod: GOPROXY=direct go get -d github.com/wader/gojq@fq go mod tidy +# Usage: make fuzz # fuzz all foramts +# Usage: make fuzz GROUP=mp4 # fuzz a group (each format is a group also) # TODO: as decode recovers panic and "repanics" unrecoverable errors this is a bit hacky at the moment -# fuzz code is not suppose to print to stderr so log to file # Retrigger: +# try to decode crsash with all formats: # cat format/testdata/fuzz/FuzzFormats/... | go run dev/fuzzbytes.go | go run . -d raw '. as $b | formats | keys[] as $f | $b | decode($f)' +# convert crash into raw bytes: # cat format/testdata/fuzz/FuzzFormats/... | go run dev/fuzzbytes.go | fq -d raw 'tobytes | tobase64' # fq -n '"..." | frombase64 | ...' .PHONY: fuzz fuzz: # in other terminal: tail -f /tmp/repanic - FUZZTEST=1 REPANIC_LOG=/tmp/repanic gotip test -v -run Fuzz -fuzz=Fuzz ./format/ + FUZZTEST=1 REPANIC_LOG=/tmp/repanic go test -v -run Fuzz -fuzz=Fuzz ./format/ # usage: make release VERSION=0.0.1 # tag forked dependeces for history and to make then stay around diff --git a/README.md b/README.md index 0cbd4785..51bd2941 100644 --- a/README.md +++ b/README.md @@ -155,22 +155,22 @@ For details see [usage.md](doc/usage.md) ## Presentations +- "fq - jq for binary formats" at [No time to wait 6](https://mediaarea.net/NoTimeToWait6) - [video](https://www.youtube.com/watch?v=-Pwt5KL-xRs&t=1450s) - [slides](doc/presentations/nttw6/fq-nttw6-slides.pdf) - "fq - jq for binary formats" at [Binary Tools Summit 2022](https://binary-tools.net/summit.html) - [video](https://www.youtube.com/watch?v=GJOq_b0eb-s&list=PLTj8twuHdQz-JcX7k6eOwyVPDB8CyfZc8&index=1) - [slides](doc/presentations/bts2022/fq-bts2022-v1.pdf) ## Install -Use one of the methods listed below or download [release](https://github.com/wader/fq/releases) for your platform. Unarchive it and move the executable to `PATH` etc. +Use one of the methods listed below or download a pre-built [release](https://github.com/wader/fq/releases) for macOS, Linux or Windows. Unarchive it and move the executable to `PATH` etc. -On macOS if you don't install using a method below you might have to manually allow the binary to run. This can be done by trying to run the binary, ignore the warning and then go into security preference and allow it. Or you can run this command: +On macOS if you don't install using a method below you might have to manually allow the binary to run. This can be done by trying to run the binary, ignore the warning and then go into security preference and allow it. Same can be done with this command: ```sh xattr -d com.apple.quarantine fq && spctl --add fq ``` -### Homebrew +### Homebrew (macOS) ```sh -# install latest release brew install wader/tap/fq ``` @@ -224,26 +224,30 @@ apk add -X http://dl-cdn.alpinelinux.org/alpine/edge/testing fq ### Build from source -Make sure you have go 1.18 or later installed. +Make sure you have [go](https://go.dev) 1.18 or later installed. -To install directly from git repository do: +To install directly from git repository (no clone needed) do: ```sh # build and install latest release go install github.com/wader/fq@latest -# or build and install latest master +# build and install latest master go install github.com/wader/fq@master # copy binary to $PATH if needed cp "$(go env GOPATH)/bin/fq" /usr/local/bin ``` -To run and run tests from source directory: +To build, run and test from source: ```sh +# build an run +go run fq.go +# or +go run . +# just build +go build -o fq . # run all tests and build binary make test fq -# it's also possible to use go run -go run fq.go ``` ## TODO and ideas @@ -285,12 +289,13 @@ Licenses of direct dependencies: - Forked version of gojq https://github.com/itchyny/gojq/blob/main/LICENSE (MIT) - Forked version of readline https://github.com/chzyer/readline/blob/master/LICENSE (MIT) -- gopacket https://github.com/gopacket/gopacket/blob/master/LICENSE (BSD) -- mapstructure https://github.com/mitchellh/mapstructure/blob/master/LICENSE (MIT) -- copystructure https://github.com/mitchellh/copystructure/blob/master/LICENSE (MIT) -- go-difflib https://github.com/pmezard/go-difflib/blob/master/LICENSE (BSD) -- golang/x/* https://github.com/golang/text/blob/master/LICENSE (BSD) -- golang/snappy https://github.com/golang/snappy/blob/master/LICENSE (BSD) - github.com/BurntSushi/toml https://github.com/BurntSushi/toml/blob/master/COPYING (MIT) -- gopkg.in/yaml.v3 https://github.com/go-yaml/yaml/blob/v3/LICENSE (MIT) - github.com/creasty/defaults https://github.com/creasty/defaults/blob/master/LICENSE (MIT) +- github.com/gomarkdown/markdown https://github.com/gomarkdown/markdown/blob/master/LICENSE.txt (BSD) +- github.com/gopacket/gopacket https://github.com/gopacket/gopacket/blob/master/LICENSE (BSD) +- github.com/mitchellh/copystructure https://github.com/mitchellh/copystructure/blob/master/LICENSE (MIT) +- github.com/mitchellh/mapstructure https://github.com/mitchellh/mapstructure/blob/master/LICENSE (MIT) +- github.com/pmezard/go-difflib https://github.com/pmezard/go-difflib/blob/master/LICENSE (BSD) +- golang/snappy https://github.com/golang/snappy/blob/master/LICENSE (BSD) +- golang/x/* https://github.com/golang/text/blob/master/LICENSE (BSD) +- gopkg.in/yaml.v3 https://github.com/go-yaml/yaml/blob/v3/LICENSE (MIT) diff --git a/doc/presentations/nttw6/fq-nttw6-slides.pdf b/doc/presentations/nttw6/fq-nttw6-slides.pdf new file mode 100644 index 00000000..d4d477b4 Binary files /dev/null and b/doc/presentations/nttw6/fq-nttw6-slides.pdf differ diff --git a/format/flac/flac_frame.go b/format/flac/flac_frame.go index 8b485b7b..afe10dc9 100644 --- a/format/flac/flac_frame.go +++ b/format/flac/flac_frame.go @@ -452,15 +452,25 @@ func frameDecode(d *decode.D, in any) any { if samplesLen < n+count { d.Fatalf("decodeResiduals outside block size") } + if count < 0 { + d.Fatalf("negative sample count %d", count) + } if riceParameter == riceEscape { escapeSampleSize := int(d.FieldU5("escape_sample_size")) - d.RangeFn(d.Pos(), int64(count*escapeSampleSize), func(d *decode.D) { - d.FieldRawLen("samples", int64(count*escapeSampleSize)) - }) - for j := 0; j < count; j++ { - samples[n] = d.S(escapeSampleSize) - n++ + if escapeSampleSize == 0 { + // Zero sample size, we can just skip ahead count samples as they are already zero. From spec: + // Note that it is possible that the number of bits is 0, which means all residual samples in that partition have + // a value of 0, and no bits code for the partition itself. + n += count + } else { + d.RangeFn(d.Pos(), int64(count*escapeSampleSize), func(d *decode.D) { + d.FieldRawLen("samples", int64(count*escapeSampleSize)) + }) + for j := 0; j < count; j++ { + samples[n] = d.S(escapeSampleSize) + n++ + } } } else { samplesStart := d.Pos() @@ -493,17 +503,15 @@ func frameDecode(d *decode.D, in any) any { } } - var samples []int64 + samples := make([]int64, blockSize) switch subframeTypeS.SymStr() { case SubframeConstant: - samples = make([]int64, blockSize) // Unencoded constant value of the subblock, n = frame's bits-per-sample. v := d.FieldS("value", subframeSampleSize) for i := 0; i < blockSize; i++ { samples[i] = v } case SubframeVerbatim: - samples = make([]int64, blockSize) // Unencoded subblock; n = frame's bits-per-sample, i = frame's blocksize. // TODO: refactor into some kind of FieldBitBufLenFn? d.RangeFn(d.Pos(), int64(blockSize*subframeSampleSize), func(d *decode.D) { @@ -514,8 +522,6 @@ func frameDecode(d *decode.D, in any) any { samples[i] = d.S(subframeSampleSize) } case SubframeFixed: - samples = make([]int64, blockSize) - // Unencoded warm-up samples (n = frame's bits-per-sample * predictor order). decodeWarmupSamples(samples, lpcOrder, subframeSampleSize) // Encoded residual @@ -531,8 +537,6 @@ func frameDecode(d *decode.D, in any) any { coeffs := fixedCoeffs[lpcOrder] decodeLPC(lpcOrder, samples, coeffs, 0) case SubframeLPC: - samples = make([]int64, blockSize) - // Unencoded warm-up samples (n = frame's bits-per-sample * lpc order). decodeWarmupSamples(samples, lpcOrder, subframeSampleSize) // <4> (Quantized linear predictor coefficients' precision in bits)-1 (1111 = invalid). diff --git a/format/flac/testdata/README.md b/format/flac/testdata/README.md new file mode 100644 index 00000000..465f28bc --- /dev/null +++ b/format/flac/testdata/README.md @@ -0,0 +1 @@ +More test files can by found at https://github.com/ietf-wg-cellar/flac-test-files/ at the time of writing the decoder passes tests 01-64. diff --git a/format/flac/testdata/frame b/format/flac/testdata/frame deleted file mode 100644 index 063e2aaa..00000000 Binary files a/format/flac/testdata/frame and /dev/null differ diff --git a/format/flac/testdata/frame.fqtest b/format/flac/testdata/frame.fqtest index 4bb3b504..bce15f29 100644 --- a/format/flac/testdata/frame.fqtest +++ b/format/flac/testdata/frame.fqtest @@ -1,58 +1,42 @@ # test decode separate frame -# ffmpeg -f lavfi -i sine -t 0.01s -f flac pipe:1 | fq - '.frames[0]._bytes' > frame -$ fq -d flac_frame dv frame - |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: frame (flac_frame) 0x0-0x1ff.7 (512) - | | | header{}: 0x0-0x7.7 (8) -0x000|ff f8 |.. | sync: 0b11111111111110 (valid) 0x0-0x1.5 (1.6) -0x000| f8 | . | reserved0: 0 (valid) 0x1.6-0x1.6 (0.1) -0x000| f8 | . | blocking_strategy: "fixed" (0) 0x1.7-0x1.7 (0.1) -0x000| 79 | y | block_size: 0b111 (end of header (16 bit)) 0x2-0x2.3 (0.4) -0x000| 79 | y | sample_rate: 44100 (0b1001) 0x2.4-0x2.7 (0.4) -0x000| 08 | . | channel_assignment: 1 (0) (mono) 0x3-0x3.3 (0.4) -0x000| 08 | . | sample_size: 16 (0b100) 0x3.4-0x3.6 (0.3) -0x000| 08 | . | reserved1: 0 (valid) 0x3.7-0x3.7 (0.1) - | | | end_of_header{}: 0x4-0x6.7 (3) -0x000| 00 | . | frame_number: 0 0x4-0x4.7 (1) -0x000| 01 b8 | .. | block_size: 441 0x5-0x6.7 (2) -0x000| 55 | U | crc: 0x55 (valid) 0x7-0x7.7 (1) - | | | subframes[0:1]: 0x8-0x1fd.2 (501.3) - | | | [0]{}: subframe 0x8-0x1fd.2 (501.3) -0x000| 4e | N | zero_bit: 0 (valid) 0x8-0x8 (0.1) -0x000| 4e | N | subframe_type: "lpc" (0b100111) 0x8.1-0x8.6 (0.6) - | | | lpc_order: 8 0x8.7-NA (0) -0x000| 4e | N | wasted_bits_flag: 0 0x8.7-0x8.7 (0.1) - | | | subframe_sample_size: 16 0x9-NA (0) - | | | warmup_samples[0:8]: 0x9-0x18.7 (16) -0x000| 00 00 | .. | [0]: 0 value 0x9-0xa.7 (2) -0x000| 01 00 | .. | [1]: 256 value 0xb-0xc.7 (2) -0x000| 01 ff | .. | [2]: 511 value 0xd-0xe.7 (2) -0x000| 02| .| [3]: 765 value 0xf-0x10.7 (2) -0x010|fd |. | -0x010| 03 f8 | .. | [4]: 1016 value 0x11-0x12.7 (2) -0x010| 04 ee | .. | [5]: 1262 value 0x13-0x14.7 (2) -0x010| 05 e0 | .. | [6]: 1504 value 0x15-0x16.7 (2) -0x010| 06 cc | .. | [7]: 1740 value 0x17-0x18.7 (2) -0x010| e7 | . | precision: 15 0x19-0x19.3 (0.4) -0x010| e7 b7 | .. | shift: 15 0x19.4-0x1a (0.5) - | | | coefficients[0:8]: 0x1a.1-0x29 (15) -0x010| b7 52 | .R | [0]: 14162 value 0x1a.1-0x1b.7 (1.7) -0x010| 5c 9c | \. | [1]: 11854 value 0x1c-0x1d.6 (1.7) -0x010| 9c 8d e8| ...| [2]: 9082 value 0x1d.7-0x1f.5 (1.7) -0x010| e8| .| [3]: 5975 value 0x1f.6-0x21.4 (1.7) -0x020|ba b8 |.. | -0x020| b8 a5 df | ... | [4]: 2653 value 0x21.5-0x23.3 (1.7) -0x020| df a3 fc | ... | [5]: -737 value 0x23.4-0x25.2 (1.7) -0x020| fc 03 71 | ..q | [6]: -4083 value 0x25.3-0x27.1 (1.7) -0x020| 71 e5 80 | q.. | [7]: -7221 value 0x27.2-0x29 (1.7) -0x020| 80 | . | residual_coding_method: 4 (0) (rice) 0x29.1-0x29.2 (0.2) -0x020| 80 | . | partition_order: 0 0x29.3-0x29.6 (0.4) - | | | rice_partitions: 1 0x29.7-NA (0) - | | | partitions[0:1]: 0x29.7-0x1fd.2 (467.4) - | | | [0]{}: partition 0x29.7-0x1fd.2 (467.4) - | | | count: 433 0x29.7-NA (0) -0x020| 80 ee | .. | rice_parameter: 7 0x29.7-0x2a.2 (0.4) -0x020| ee e7 63 a9 cc e0| ..c...| samples: raw bits 0x2a.3-0x1fd.2 (467) -0x030|6d 35 19 8c 25 c2 c9 4c 9a 47 20 bd ba 36 b3 2f|m5..%..L.G ..6./| -* |until 0x1fd.2 (467) | | -0x1f0| 20 | | byte_align: 0 (valid) 0x1fd.3-0x1fd.7 (0.5) -0x1f0| 7f ab| ..| footer_crc: "7fab" (raw bits) (valid) 0x1fe-0x1ff.7 (2) +$ fq '.frames[1] | tobytes | flac_frame | d' mono8.flac + |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: (flac_frame) + | | | header{}: +0x000|ff f8 |.. | sync: 0b11111111111110 (valid) +0x000| f8 | . | reserved0: 0 (valid) +0x000| f8 | . | blocking_strategy: "fixed" (0) +0x000| c9 | . | block_size: 4096 (0b1100) +0x000| c9 | . | sample_rate: 44100 (0b1001) +0x000| 02 | . | channel_assignment: 1 (0) (mono) +0x000| 02 | . | sample_size: 8 (0b1) +0x000| 02 | . | reserved1: 0 (valid) + | | | end_of_header{}: +0x000| 01 | . | frame_number: 1 +0x000| 10 | . | crc: 0x10 (valid) + | | | subframes[0:1]: + | | | [0]{}: subframe +0x000| 12 | . | zero_bit: 0 (valid) +0x000| 12 | . | subframe_type: "fixed" (0b1001) + | | | lpc_order: 1 +0x000| 12 | . | wasted_bits_flag: 0 + | | | subframe_sample_size: 8 + | | | warmup_samples[0:1]: +0x000| 00 | . | [0]: 0 +0x000| 04 | . | residual_coding_method: 4 (0) (rice) +0x000| 04 | . | partition_order: 1 + | | | rice_partitions: 2 + | | | partitions[0:2]: + | | | [0]{}: partition + | | | count: 2047 +0x000| 04 3f | .? | rice_parameter: 0 +0x000| 3f ff ff ff ff ff ff| ?......| samples: raw bits +0x010|ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff|................| +* |until 0x196.1 (397) | | + | | | [1]{}: partition + | | | count: 2048 +0x190| c1 | . | rice_parameter: 0 +0x190| c1 bb ff 39 99 32 4c 92 64 99| ...9.2L.d.| samples: raw bits +0x1a0|33 39 ff bb 6d 6d b6 df fe 66 66 4c 93 26 4c 99|39..mm...ffL.&L.| +* |until 0x33d.7 (424) | | + | | | byte_align: 0 (valid) +0x330| 5c 65| \e| footer_crc: "5c65" (raw bits) (valid) diff --git a/format/fuzz_test.go b/format/fuzz_test.go index e51952e3..783b3226 100644 --- a/format/fuzz_test.go +++ b/format/fuzz_test.go @@ -66,7 +66,7 @@ func (ft *fuzzTest) Readline(opts interp.ReadlineOpts) (string, error) { func FuzzFormats(f *testing.F) { if os.Getenv("FUZZTEST") == "" { - f.Skip("run with FUZZTEST=1 do fuzz") + f.Skip("run with FUZZTEST=1 to fuzz") } i := 0 @@ -103,7 +103,18 @@ func FuzzFormats(f *testing.F) { } gi := 0 - g := interp.DefaultRegistry.MustAll() + var g decode.Group + + if n := os.Getenv("GROUP"); n != "" { + var err error + g, err = interp.DefaultRegistry.FormatGroup(n) + if err != nil { + f.Fatal(err) + } + f.Logf("GROUP=%s", n) + } else { + g = interp.DefaultRegistry.MustAll() + } f.Fuzz(func(t *testing.T, b []byte) { fz := &fuzzTest{b: b, f: g[gi]} diff --git a/format/id3/id3v2.go b/format/id3/id3v2.go index 52a015ed..9d32d99f 100644 --- a/format/id3/id3v2.go +++ b/format/id3/id3v2.go @@ -287,24 +287,34 @@ func textFn(encoding int, nBytes int) func(d *decode.D) string { } } -func textNullFn(encoding int) func(d *decode.D) string { +func textNullLenFn(encoding int, notFoundFixedBytes int) func(d *decode.D) string { return func(d *decode.D) string { nullLen := encodingLen[encodingUTF8] if n, ok := encodingLen[uint64(encoding)]; ok { nullLen = n } - offset, _ := d.PeekFind( + offset, _, err := d.TryPeekFind( int(nullLen)*8, nullLen*8, - -1, + int64(notFoundFixedBytes)*8, func(v uint64) bool { return v == 0 }, ) + if err != nil { + d.IOPanic(err, "textNullLenFn") + } + if offset < 0 { + if notFoundFixedBytes < 0 { + d.Fatalf("textNullLenFn: null not found") + } + return textFn(encoding, notFoundFixedBytes)(d) + } + offsetBytes := offset / 8 text := textFn(encoding, int(offsetBytes))(d) d.SeekRel(nullLen * 8) - // seems sometimes utf16 etc has en exta null byte + // seems sometimes utf16 etc has one exta null byte if nullLen > 1 && d.PeekBits(8) == 0 { d.SeekRel(8) } @@ -313,6 +323,10 @@ func textNullFn(encoding int) func(d *decode.D) string { } } +func textNullFn(encoding int) func(d *decode.D) string { + return textNullLenFn(encoding, -1) +} + func decodeFrame(d *decode.D, version int) uint64 { var id string var size uint64 @@ -493,7 +507,8 @@ func decodeFrame(d *decode.D, version int) uint64 { "COMM": func(d *decode.D) { encoding := d.FieldU8("text_encoding", encodingNames) d.FieldUTF8("language", 3) - d.FieldStrFn("description", textNullFn(int(encoding))) + // there are COMM frames with no null termination + d.FieldStrFn("description", textNullLenFn(int(encoding), int(d.BitsLeft()/8))) d.FieldStrFn("value", textFn(int(encoding), int(d.BitsLeft()/8))) }, // Text information identifier "T00" - "TZZ" , excluding "TXX", @@ -522,7 +537,8 @@ func decodeFrame(d *decode.D, version int) uint64 { // Value "TXXX": func(d *decode.D) { encoding := d.FieldU8("text_encoding", encodingNames) - d.FieldStrFn("description", textNullFn(int(encoding))) + // there are TXXX frames with no null termination + d.FieldStrFn("description", textNullLenFn(int(encoding), int(d.BitsLeft()/8))) d.FieldStrFn("value", textFn(int(encoding), int(d.BitsLeft()/8))) }, //
diff --git a/format/id3/testdata/txxx-nonullterm b/format/id3/testdata/txxx-nonullterm new file mode 100644 index 00000000..c3cd36c9 Binary files /dev/null and b/format/id3/testdata/txxx-nonullterm differ diff --git a/format/id3/testdata/txxx-nonullterm.fqtest b/format/id3/testdata/txxx-nonullterm.fqtest new file mode 100644 index 00000000..f12bc2b9 --- /dev/null +++ b/format/id3/testdata/txxx-nonullterm.fqtest @@ -0,0 +1,29 @@ +$ fq -d id3v2 dv txxx-nonullterm + |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: txxx-nonullterm (id3v2) 0x0-0x26.7 (39) +0x00|49 44 33 |ID3 | magic: "ID3" (valid) 0x0-0x2.7 (3) +0x00| 03 | . | version: 3 0x3-0x3.7 (1) +0x00| 00 | . | revision: 0 0x4-0x4.7 (1) + | | | flags{}: 0x5-0x5.7 (1) +0x00| 00 | . | unsynchronisation: false 0x5-0x5 (0.1) +0x00| 00 | . | extended_header: false 0x5.1-0x5.1 (0.1) +0x00| 00 | . | experimental_indicator: false 0x5.2-0x5.2 (0.1) +0x00| 00 | . | unused: 0 0x5.3-0x5.7 (0.5) +0x00| 00 00 00 1d | .... | size: 29 0x6-0x9.7 (4) + | | | frames[0:1]: 0xa-0x26.7 (29) + | | | [0]{}: frame 0xa-0x26.7 (29) +0x00| 54 58 58 58 | TXXX | id: "TXXX" (User defined text information frame) 0xa-0xd.7 (4) +0x00| 00 00| ..| size: 19 0xe-0x11.7 (4) +0x10|00 13 |.. | + | | | flags{}: 0x12-0x13.7 (2) +0x10| 00 | . | tag_alter_preservation: false 0x12-0x12 (0.1) +0x10| 00 | . | file_alter_preservation: false 0x12.1-0x12.1 (0.1) +0x10| 00 | . | read_only: false 0x12.2-0x12.2 (0.1) +0x10| 00 | . | unused0: 0 0x12.3-0x12.7 (0.5) +0x10| 00 | . | compression: false 0x13-0x13 (0.1) +0x10| 00 | . | encryption: false 0x13.1-0x13.1 (0.1) +0x10| 00 | . | grouping_identity: false 0x13.2-0x13.2 (0.1) +0x10| 00 | . | unused1: 0 0x13.3-0x13.7 (0.5) +0x10| 01 | . | text_encoding: "utf16" (1) 0x14-0x14.7 (1) +0x10| ff fe 53 00 6f 00 66 00 74 00 77| ..S.o.f.t.w| description: "Software" 0x15-0x26.7 (18) +0x20|00 61 00 72 00 65 00| |.a.r.e.| | + | | | value: "" 0x27-NA (0) diff --git a/format/yaml/testdata/normalize.fqtest b/format/yaml/testdata/normalize.fqtest new file mode 100644 index 00000000..05faf214 --- /dev/null +++ b/format/yaml/testdata/normalize.fqtest @@ -0,0 +1,5 @@ +# normalize uint64 to bigint +$ fq -n '"[18446744073709551615]" | yaml' +[ + 18446744073709551615 +] diff --git a/format/yaml/yaml.go b/format/yaml/yaml.go index df9ea60b..a32704fd 100644 --- a/format/yaml/yaml.go +++ b/format/yaml/yaml.go @@ -45,7 +45,7 @@ func decodeYAML(d *decode.D, _ any) any { } var s scalar.S - s.Actual = r + s.Actual = gojqex.Normalize(r) switch s.Actual.(type) { case map[string]any, diff --git a/go.mod b/go.mod index 495ccc22..300c0510 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( // fork of github.com/itchyny/gojq, see github.com/wader/gojq fq branch - github.com/wader/gojq v0.12.1-0.20220929141349-8874f5c7907c + github.com/wader/gojq v0.12.1-0.20221030123557-e1510a9e80c3 // fork of github.com/chzyer/readline, see github.com/wader/readline fq branch github.com/wader/readline v0.0.0-20220928125628-732951d41240 ) @@ -13,7 +13,7 @@ require ( // bump: gomod-BurntSushi/toml /github\.com\/BurntSushi\/toml v(.*)/ https://github.com/BurntSushi/toml.git|^1 // bump: gomod-BurntSushi/toml command go get -d github.com/BurntSushi/toml@v$LATEST && go mod tidy // bump: gomod-BurntSushi/toml link "Source diff $CURRENT..$LATEST" https://github.com/BurntSushi/toml/compare/v$CURRENT..v$LATEST - github.com/BurntSushi/toml v1.2.0 + github.com/BurntSushi/toml v1.2.1 // bump: gomod-creasty-defaults /github\.com\/creasty\/defaults v(.*)/ https://github.com/creasty/defaults.git|^1 // bump: gomod-creasty-defaults command go get -d github.com/creasty/defaults@v$LATEST && go mod tidy @@ -27,13 +27,13 @@ require ( // has no tags // go get -d github.com/gomarkdown/markdown@master && go mod tidy - github.com/gomarkdown/markdown v0.0.0-20220905174103-7b278df48cfb + github.com/gomarkdown/markdown v0.0.0-20221013030248-663e2500819c // has no tags yet // bump-disabled: gomod-gopacket /github\.com\/gopacket\/gopacket v(.*)/ https://github.com/gopacket/gopacket.git|^1 // bump-disabled: gomod-gopacket command go get -d github.com/gopacket/gopacket@v$LATEST && go mod tidy // bump-disabled: gomod-gopacket link "Release notes" https://github.com/gopacket/gopacket/releases/tag/v$LATEST - github.com/gopacket/gopacket v0.0.0-20220819214934-ee81b8c880da + github.com/gopacket/gopacket v0.0.0-20221006103438-9e6d99b9b443 // bump: gomod-copystructure /github\.com\/mitchellh\/copystructure v(.*)/ https://github.com/mitchellh/copystructure.git|^1 // bump: gomod-copystructure command go get -d github.com/mitchellh/copystructure@v$LATEST && go mod tidy @@ -52,20 +52,20 @@ require ( // has no tags // go get -d golang.org/x/crypto@master && go mod tidy - golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be + golang.org/x/crypto v0.1.1-0.20221024173537-a3485e174077 // has no tags // go get -d golang.org/x/exp@master && go mod tidy - golang.org/x/exp v0.0.0-20220927162542-c76eaa363f9d + golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f // has no tags // go get -d golang.org/x/net@master && go mod tidy - golang.org/x/net v0.0.0-20220927171203-f486391704dc + golang.org/x/net v0.1.1-0.20221027164007-c63010009c80 // bump: gomod-golang/text /golang\.org\/x\/text v(.*)/ https://github.com/golang/text.git|^0 // bump: gomod-golang/text command go get -d golang.org/x/text@v$LATEST && go mod tidy // bump: gomod-golang/text link "Source diff $CURRENT..$LATEST" https://github.com/golang/text/compare/v$CURRENT..v$LATEST - golang.org/x/text v0.3.7 + golang.org/x/text v0.4.0 // bump: gomod-gopkg.in/yaml.v3 /gopkg\.in\/yaml\.v3 v(.*)/ https://github.com/go-yaml/yaml.git|^3 // bump: gomod-gopkg.in/yaml.v3 command go get -d gopkg.in/yaml.v3@v$LATEST && go mod tidy @@ -77,6 +77,6 @@ require ( github.com/itchyny/timefmt-go v0.1.4 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect - golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 // indirect + golang.org/x/sys v0.1.0 // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect ) diff --git a/go.sum b/go.sum index bc81fe8f..603383fa 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,13 @@ -github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= -github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/creasty/defaults v1.6.0 h1:ltuE9cfphUtlrBeomuu8PEyISTXnxqkBIoQfXgv7BSc= github.com/creasty/defaults v1.6.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomarkdown/markdown v0.0.0-20220905174103-7b278df48cfb h1:7h+tPfwoUE+qLvWYmsvKSiRlXv6WGorb6PUKaZUclwc= -github.com/gomarkdown/markdown v0.0.0-20220905174103-7b278df48cfb/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= -github.com/gopacket/gopacket v0.0.0-20220819214934-ee81b8c880da h1:AAwDU9N39fQNYUtg270aiU6N7U2ZVsGZKiRwsCMsWEo= -github.com/gopacket/gopacket v0.0.0-20220819214934-ee81b8c880da/go.mod h1:DlRRfaM/QjAu2ADqraIure1Eif0HpNL8hmyVQ+qci5Y= +github.com/gomarkdown/markdown v0.0.0-20221013030248-663e2500819c h1:iyaGYbCmcYK0Ja9a3OUa2Fo+EaN0cbLu0eKpBwPFzc8= +github.com/gomarkdown/markdown v0.0.0-20221013030248-663e2500819c/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= +github.com/gopacket/gopacket v0.0.0-20221006103438-9e6d99b9b443 h1:nZcBvHjudAMfouyqauDctCqWgl0wsphtiCu4yiVCs6g= +github.com/gopacket/gopacket v0.0.0-20221006103438-9e6d99b9b443/go.mod h1:DlRRfaM/QjAu2ADqraIure1Eif0HpNL8hmyVQ+qci5Y= github.com/itchyny/timefmt-go v0.1.4 h1:hFEfWVdwsEi+CY8xY2FtgWHGQaBaC3JeHd+cve0ynVM= github.com/itchyny/timefmt-go v0.1.4/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -23,21 +23,22 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/wader/gojq v0.12.1-0.20220929141349-8874f5c7907c h1:aQz+gw5Xe2x++st4HBIaMHPPTIyPtESG2HMZJmOaRkc= -github.com/wader/gojq v0.12.1-0.20220929141349-8874f5c7907c/go.mod h1:/+WKHSfP8sgyJDXeD9VjxlGjExGfjPUy0zm/Zy3Ind0= +github.com/wader/gojq v0.12.1-0.20221030123557-e1510a9e80c3 h1:O+cdsjdniOlgdSdruQCNnLTuVvubmvywif6c7J8P3fU= +github.com/wader/gojq v0.12.1-0.20221030123557-e1510a9e80c3/go.mod h1:/+WKHSfP8sgyJDXeD9VjxlGjExGfjPUy0zm/Zy3Ind0= github.com/wader/readline v0.0.0-20220928125628-732951d41240 h1:fqNaldd6kVsMNGXLXH4TtB1kJtaPQOzGwqNYgswIM94= github.com/wader/readline v0.0.0-20220928125628-732951d41240/go.mod h1:Zgz8IJWvJoe7NK23CCPpC109XMCqJCpUhpHcnnA4XaM= -golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be h1:fmw3UbQh+nxngCAHrDCCztao/kbYFnWjoqop8dHx05A= -golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/exp v0.0.0-20220927162542-c76eaa363f9d h1:3wgmvnqHUJ8SxiNWwea5NCzTwAVfhTtuV+0ClVFlClc= -golang.org/x/exp v0.0.0-20220927162542-c76eaa363f9d/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/net v0.0.0-20220927171203-f486391704dc h1:FxpXZdoBqT8RjqTy6i1E8nXHhW21wK7ptQ/EPIGxzPQ= -golang.org/x/net v0.0.0-20220927171203-f486391704dc/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/crypto v0.1.1-0.20221024173537-a3485e174077 h1:t5bjOfJPQfaG9NV1imLZM5E2uzaLGs5/NtyMtRNVjQ4= +golang.org/x/crypto v0.1.1-0.20221024173537-a3485e174077/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f h1:Al51T6tzvuh3oiwX11vex3QgJ2XTedFPGmbEVh8cdoc= +golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/net v0.1.1-0.20221027164007-c63010009c80 h1:CtRWmqbiPSOXwJV1JoY7pWiTx2xzVKQ813bvU+Y/9jI= +golang.org/x/net v0.1.1-0.20221027164007-c63010009c80/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 h1:v6hYoSR9T5oet+pMXwUWkbiVqx/63mlHjefrHmxwfeY= -golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= diff --git a/pkg/decode/decode.go b/pkg/decode/decode.go index 2504bf2a..56a5418e 100644 --- a/pkg/decode/decode.go +++ b/pkg/decode/decode.go @@ -487,6 +487,9 @@ func (d *D) PeekFind(nBits int, seekBits int64, maxLen int64, fn func(v uint64) // Does not update current position. // TODO: nBytes -1? func (d *D) TryBytesRange(bitOffset int64, nBytes int) ([]byte, error) { + if nBytes < 0 { + return nil, fmt.Errorf("negative nBytes %d", nBytes) + } buf := make([]byte, nBytes) n, err := bitio.ReadAtFull(d.bitBuf, buf, int64(nBytes)*8, bitOffset) if n == int64(nBytes)*8 { diff --git a/pkg/decode/read.go b/pkg/decode/read.go index ba2be715..13fb5804 100644 --- a/pkg/decode/read.go +++ b/pkg/decode/read.go @@ -32,8 +32,8 @@ func (d *D) tryUEndian(nBits int, endian Endian) (uint64, error) { } func (d *D) trySEndian(nBits int, endian Endian) (int64, error) { - if nBits < 0 { - return 0, fmt.Errorf("trySEndian nBits must be >= 0 (%d)", nBits) + if nBits < 1 { + return 0, fmt.Errorf("trySEndian nBits must be >= 1 (%d)", nBits) } n, err := d.tryUEndian(nBits, endian) if err != nil { diff --git a/pkg/interp/interp.go b/pkg/interp/interp.go index 683627e3..d4f1423e 100644 --- a/pkg/interp/interp.go +++ b/pkg/interp/interp.go @@ -1126,7 +1126,7 @@ func (i *Interp) NewColorJSON(opts Options) (*colorjson.Encoder, error) { if v, ok := toValue(func() Options { return opts }, v); ok { return v } - panic(fmt.Sprintf("toValue not a JQValue value: %#v", v)) + panic(fmt.Sprintf("toValue not a JQValue value: %#v (%T)", v, v)) }, colorjson.Colors{ Reset: []byte(ansi.Reset.SetString),