wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-11-26 10:33:53 +03:00
Code Issues Projects Releases Wiki Activity

Merge branch 'postgres' into postgres_wal_sync_with_postgres

Browse Source
This commit is contained in:
Pavel Safonov 2022-11-01 08:26:17 +03:00
commit 9c7f20b6a0
19 changed files with 191 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/ci.yml vendored
View File

@ -7,7 +7,7 @@ on:
pull_request:
env:
GOLANGCILINT_VERSION: 1.50.0
GOLANGCILINT_VERSION: 1.50.1
jobs:
lint:

9
Makefile
View File

@ -69,7 +69,7 @@ gogenerate:
lint:
# bump: make-golangci-lint /golangci-lint@v([\d.]+)/ git:https://github.com/golangci/golangci-lint.git|^1
# bump: make-golangci-lint link "Release notes" https://github.com/golangci/golangci-lint/releases/tag/v$LATEST
go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.0 run
go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1 run
.PHONY: depgraph.svg
depgraph.svg:
@ -95,16 +95,19 @@ update-gomod:
GOPROXY=direct go get -d github.com/wader/gojq@fq
go mod tidy
# Usage: make fuzz # fuzz all foramts
# Usage: make fuzz GROUP=mp4 # fuzz a group (each format is a group also)
# TODO: as decode recovers panic and "repanics" unrecoverable errors this is a bit hacky at the moment
# fuzz code is not suppose to print to stderr so log to file
# Retrigger:
# try to decode crsash with all formats:
# cat format/testdata/fuzz/FuzzFormats/... | go run dev/fuzzbytes.go | go run . -d raw '. as $b | formats | keys[] as $f | $b | decode($f)'
# convert crash into raw bytes:
# cat format/testdata/fuzz/FuzzFormats/... | go run dev/fuzzbytes.go | fq -d raw 'tobytes | tobase64'
# fq -n '"..." | frombase64 | ...'
.PHONY: fuzz
fuzz:
# in other terminal: tail -f /tmp/repanic
FUZZTEST=1 REPANIC_LOG=/tmp/repanic gotip test -v -run Fuzz -fuzz=Fuzz ./format/
FUZZTEST=1 REPANIC_LOG=/tmp/repanic go test -v -run Fuzz -fuzz=Fuzz ./format/
# usage: make release VERSION=0.0.1
# tag forked dependeces for history and to make then stay around

39
README.md
View File

@ -155,22 +155,22 @@ For details see [usage.md](doc/usage.md)
## Presentations
- "fq - jq for binary formats" at [No time to wait 6](https://mediaarea.net/NoTimeToWait6) - [video](https://www.youtube.com/watch?v=-Pwt5KL-xRs&t=1450s) - [slides](doc/presentations/nttw6/fq-nttw6-slides.pdf)
- "fq - jq for binary formats" at [Binary Tools Summit 2022](https://binary-tools.net/summit.html) - [video](https://www.youtube.com/watch?v=GJOq_b0eb-s&list=PLTj8twuHdQz-JcX7k6eOwyVPDB8CyfZc8&index=1) - [slides](doc/presentations/bts2022/fq-bts2022-v1.pdf)
## Install
Use one of the methods listed below or download [release](https://github.com/wader/fq/releases) for your platform. Unarchive it and move the executable to `PATH` etc.
Use one of the methods listed below or download a pre-built [release](https://github.com/wader/fq/releases) for macOS, Linux or Windows. Unarchive it and move the executable to `PATH` etc.
On macOS if you don't install using a method below you might have to manually allow the binary to run. This can be done by trying to run the binary, ignore the warning and then go into security preference and allow it. Or you can run this command:
On macOS if you don't install using a method below you might have to manually allow the binary to run. This can be done by trying to run the binary, ignore the warning and then go into security preference and allow it. Same can be done with this command:
```sh
xattr -d com.apple.quarantine fq && spctl --add fq
```
### Homebrew
### Homebrew (macOS)
```sh
# install latest release
brew install wader/tap/fq
```
@ -224,26 +224,30 @@ apk add -X http://dl-cdn.alpinelinux.org/alpine/edge/testing fq
### Build from source
Make sure you have go 1.18 or later installed.
Make sure you have [go](https://go.dev) 1.18 or later installed.
To install directly from git repository do:
To install directly from git repository (no clone needed) do:
```sh
# build and install latest release
go install github.com/wader/fq@latest
# or build and install latest master
# build and install latest master
go install github.com/wader/fq@master
# copy binary to $PATH if needed
cp "$(go env GOPATH)/bin/fq" /usr/local/bin
```
To run and run tests from source directory:
To build, run and test from source:
```sh
# build an run
go run fq.go
# or
go run .
# just build
go build -o fq .
# run all tests and build binary
make test fq
# it's also possible to use go run
go run fq.go
```
## TODO and ideas
@ -285,12 +289,13 @@ Licenses of direct dependencies:
- Forked version of gojq https://github.com/itchyny/gojq/blob/main/LICENSE (MIT)
- Forked version of readline https://github.com/chzyer/readline/blob/master/LICENSE (MIT)
- gopacket https://github.com/gopacket/gopacket/blob/master/LICENSE (BSD)
- mapstructure https://github.com/mitchellh/mapstructure/blob/master/LICENSE (MIT)
- copystructure https://github.com/mitchellh/copystructure/blob/master/LICENSE (MIT)
- go-difflib https://github.com/pmezard/go-difflib/blob/master/LICENSE (BSD)
- golang/x/* https://github.com/golang/text/blob/master/LICENSE (BSD)
- golang/snappy https://github.com/golang/snappy/blob/master/LICENSE (BSD)
- github.com/BurntSushi/toml https://github.com/BurntSushi/toml/blob/master/COPYING (MIT)
- gopkg.in/yaml.v3 https://github.com/go-yaml/yaml/blob/v3/LICENSE (MIT)
- github.com/creasty/defaults https://github.com/creasty/defaults/blob/master/LICENSE (MIT)
- github.com/gomarkdown/markdown https://github.com/gomarkdown/markdown/blob/master/LICENSE.txt (BSD)
- github.com/gopacket/gopacket https://github.com/gopacket/gopacket/blob/master/LICENSE (BSD)
- github.com/mitchellh/copystructure https://github.com/mitchellh/copystructure/blob/master/LICENSE (MIT)
- github.com/mitchellh/mapstructure https://github.com/mitchellh/mapstructure/blob/master/LICENSE (MIT)
- github.com/pmezard/go-difflib https://github.com/pmezard/go-difflib/blob/master/LICENSE (BSD)
- golang/snappy https://github.com/golang/snappy/blob/master/LICENSE (BSD)
- golang/x/* https://github.com/golang/text/blob/master/LICENSE (BSD)
- gopkg.in/yaml.v3 https://github.com/go-yaml/yaml/blob/v3/LICENSE (MIT)

BIN
doc/presentations/nttw6/fq-nttw6-slides.pdf Normal file
View File

Binary file not shown.

30
format/flac/flac_frame.go
View File

@ -452,15 +452,25 @@ func frameDecode(d *decode.D, in any) any {
if samplesLen < n+count {
d.Fatalf("decodeResiduals outside block size")
}
if count < 0 {
d.Fatalf("negative sample count %d", count)
}
if riceParameter == riceEscape {
escapeSampleSize := int(d.FieldU5("escape_sample_size"))
d.RangeFn(d.Pos(), int64(count*escapeSampleSize), func(d *decode.D) {
d.FieldRawLen("samples", int64(count*escapeSampleSize))
})
for j := 0; j < count; j++ {
samples[n] = d.S(escapeSampleSize)
n++
if escapeSampleSize == 0 {
// Zero sample size, we can just skip ahead count samples as they are already zero. From spec:
// Note that it is possible that the number of bits is 0, which means all residual samples in that partition have
// a value of 0, and no bits code for the partition itself.
n += count
} else {
d.RangeFn(d.Pos(), int64(count*escapeSampleSize), func(d *decode.D) {
d.FieldRawLen("samples", int64(count*escapeSampleSize))
})
for j := 0; j < count; j++ {
samples[n] = d.S(escapeSampleSize)
n++
}
}
} else {
samplesStart := d.Pos()
@ -493,17 +503,15 @@ func frameDecode(d *decode.D, in any) any {
}
}
var samples []int64
samples := make([]int64, blockSize)
switch subframeTypeS.SymStr() {
case SubframeConstant:
samples = make([]int64, blockSize)
// <n> Unencoded constant value of the subblock, n = frame's bits-per-sample.
v := d.FieldS("value", subframeSampleSize)
for i := 0; i < blockSize; i++ {
samples[i] = v
}
case SubframeVerbatim:
samples = make([]int64, blockSize)
// <n*i> Unencoded subblock; n = frame's bits-per-sample, i = frame's blocksize.
// TODO: refactor into some kind of FieldBitBufLenFn?
d.RangeFn(d.Pos(), int64(blockSize*subframeSampleSize), func(d *decode.D) {
@ -514,8 +522,6 @@ func frameDecode(d *decode.D, in any) any {
samples[i] = d.S(subframeSampleSize)
}
case SubframeFixed:
samples = make([]int64, blockSize)
// <n> Unencoded warm-up samples (n = frame's bits-per-sample * predictor order).
decodeWarmupSamples(samples, lpcOrder, subframeSampleSize)
// Encoded residual
@ -531,8 +537,6 @@ func frameDecode(d *decode.D, in any) any {
coeffs := fixedCoeffs[lpcOrder]
decodeLPC(lpcOrder, samples, coeffs, 0)
case SubframeLPC:
samples = make([]int64, blockSize)
// <n> Unencoded warm-up samples (n = frame's bits-per-sample * lpc order).
decodeWarmupSamples(samples, lpcOrder, subframeSampleSize)
// <4> (Quantized linear predictor coefficients' precision in bits)-1 (1111 = invalid).

1
format/flac/testdata/README.md vendored Normal file
View File

@ -0,0 +1 @@
More test files can by found at https://github.com/ietf-wg-cellar/flac-test-files/ at the time of writing the decoder passes tests 01-64.

BIN
format/flac/testdata/frame vendored
View File

Binary file not shown.

98
format/flac/testdata/frame.fqtest vendored
View File

@ -1,58 +1,42 @@
# test decode separate frame
# ffmpeg -f lavfi -i sine -t 0.01s -f flac pipe:1 | fq - '.frames[0]._bytes' > frame
$ fq -d flac_frame dv frame
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: frame (flac_frame) 0x0-0x1ff.7 (512)
| | | header{}: 0x0-0x7.7 (8)
0x000|ff f8 |.. | sync: 0b11111111111110 (valid) 0x0-0x1.5 (1.6)
0x000| f8 | . | reserved0: 0 (valid) 0x1.6-0x1.6 (0.1)
0x000| f8 | . | blocking_strategy: "fixed" (0) 0x1.7-0x1.7 (0.1)
0x000| 79 | y | block_size: 0b111 (end of header (16 bit)) 0x2-0x2.3 (0.4)
0x000| 79 | y | sample_rate: 44100 (0b1001) 0x2.4-0x2.7 (0.4)
0x000| 08 | . | channel_assignment: 1 (0) (mono) 0x3-0x3.3 (0.4)
0x000| 08 | . | sample_size: 16 (0b100) 0x3.4-0x3.6 (0.3)
0x000| 08 | . | reserved1: 0 (valid) 0x3.7-0x3.7 (0.1)
| | | end_of_header{}: 0x4-0x6.7 (3)
0x000| 00 | . | frame_number: 0 0x4-0x4.7 (1)
0x000| 01 b8 | .. | block_size: 441 0x5-0x6.7 (2)
0x000| 55 | U | crc: 0x55 (valid) 0x7-0x7.7 (1)
| | | subframes[0:1]: 0x8-0x1fd.2 (501.3)
| | | [0]{}: subframe 0x8-0x1fd.2 (501.3)
0x000| 4e | N | zero_bit: 0 (valid) 0x8-0x8 (0.1)
0x000| 4e | N | subframe_type: "lpc" (0b100111) 0x8.1-0x8.6 (0.6)
| | | lpc_order: 8 0x8.7-NA (0)
0x000| 4e | N | wasted_bits_flag: 0 0x8.7-0x8.7 (0.1)
| | | subframe_sample_size: 16 0x9-NA (0)
| | | warmup_samples[0:8]: 0x9-0x18.7 (16)
0x000| 00 00 | .. | [0]: 0 value 0x9-0xa.7 (2)
0x000| 01 00 | .. | [1]: 256 value 0xb-0xc.7 (2)
0x000| 01 ff | .. | [2]: 511 value 0xd-0xe.7 (2)
0x000| 02| .| [3]: 765 value 0xf-0x10.7 (2)
0x010|fd |. |
0x010| 03 f8 | .. | [4]: 1016 value 0x11-0x12.7 (2)
0x010| 04 ee | .. | [5]: 1262 value 0x13-0x14.7 (2)
0x010| 05 e0 | .. | [6]: 1504 value 0x15-0x16.7 (2)
0x010| 06 cc | .. | [7]: 1740 value 0x17-0x18.7 (2)
0x010| e7 | . | precision: 15 0x19-0x19.3 (0.4)
0x010| e7 b7 | .. | shift: 15 0x19.4-0x1a (0.5)
| | | coefficients[0:8]: 0x1a.1-0x29 (15)
0x010| b7 52 | .R | [0]: 14162 value 0x1a.1-0x1b.7 (1.7)
0x010| 5c 9c | \. | [1]: 11854 value 0x1c-0x1d.6 (1.7)
0x010| 9c 8d e8| ...| [2]: 9082 value 0x1d.7-0x1f.5 (1.7)
0x010| e8| .| [3]: 5975 value 0x1f.6-0x21.4 (1.7)
0x020|ba b8 |.. |
0x020| b8 a5 df | ... | [4]: 2653 value 0x21.5-0x23.3 (1.7)
0x020| df a3 fc | ... | [5]: -737 value 0x23.4-0x25.2 (1.7)
0x020| fc 03 71 | ..q | [6]: -4083 value 0x25.3-0x27.1 (1.7)
0x020| 71 e5 80 | q.. | [7]: -7221 value 0x27.2-0x29 (1.7)
0x020| 80 | . | residual_coding_method: 4 (0) (rice) 0x29.1-0x29.2 (0.2)
0x020| 80 | . | partition_order: 0 0x29.3-0x29.6 (0.4)
| | | rice_partitions: 1 0x29.7-NA (0)
| | | partitions[0:1]: 0x29.7-0x1fd.2 (467.4)
| | | [0]{}: partition 0x29.7-0x1fd.2 (467.4)
| | | count: 433 0x29.7-NA (0)
0x020| 80 ee | .. | rice_parameter: 7 0x29.7-0x2a.2 (0.4)
0x020| ee e7 63 a9 cc e0| ..c...| samples: raw bits 0x2a.3-0x1fd.2 (467)
0x030|6d 35 19 8c 25 c2 c9 4c 9a 47 20 bd ba 36 b3 2f|m5..%..L.G ..6./|
* |until 0x1fd.2 (467) | |
0x1f0| 20 | | byte_align: 0 (valid) 0x1fd.3-0x1fd.7 (0.5)
0x1f0| 7f ab| ..| footer_crc: "7fab" (raw bits) (valid) 0x1fe-0x1ff.7 (2)
$ fq '.frames[1] | tobytes | flac_frame | d' mono8.flac
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: (flac_frame)
| | | header{}:
0x000|ff f8 |.. | sync: 0b11111111111110 (valid)
0x000| f8 | . | reserved0: 0 (valid)
0x000| f8 | . | blocking_strategy: "fixed" (0)
0x000| c9 | . | block_size: 4096 (0b1100)
0x000| c9 | . | sample_rate: 44100 (0b1001)
0x000| 02 | . | channel_assignment: 1 (0) (mono)
0x000| 02 | . | sample_size: 8 (0b1)
0x000| 02 | . | reserved1: 0 (valid)
| | | end_of_header{}:
0x000| 01 | . | frame_number: 1
0x000| 10 | . | crc: 0x10 (valid)
| | | subframes[0:1]:
| | | [0]{}: subframe
0x000| 12 | . | zero_bit: 0 (valid)
0x000| 12 | . | subframe_type: "fixed" (0b1001)
| | | lpc_order: 1
0x000| 12 | . | wasted_bits_flag: 0
| | | subframe_sample_size: 8
| | | warmup_samples[0:1]:
0x000| 00 | . | [0]: 0
0x000| 04 | . | residual_coding_method: 4 (0) (rice)
0x000| 04 | . | partition_order: 1
| | | rice_partitions: 2
| | | partitions[0:2]:
| | | [0]{}: partition
| | | count: 2047
0x000| 04 3f | .? | rice_parameter: 0
0x000| 3f ff ff ff ff ff ff| ?......| samples: raw bits
0x010|ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff|................|
* |until 0x196.1 (397) | |
| | | [1]{}: partition
| | | count: 2048
0x190| c1 | . | rice_parameter: 0
0x190| c1 bb ff 39 99 32 4c 92 64 99| ...9.2L.d.| samples: raw bits
0x1a0|33 39 ff bb 6d 6d b6 df fe 66 66 4c 93 26 4c 99|39..mm...ffL.&L.|
* |until 0x33d.7 (424) | |
| | | byte_align: 0 (valid)
0x330| 5c 65| \e| footer_crc: "5c65" (raw bits) (valid)

15
format/fuzz_test.go
View File

@ -66,7 +66,7 @@ func (ft *fuzzTest) Readline(opts interp.ReadlineOpts) (string, error) {
func FuzzFormats(f *testing.F) {
if os.Getenv("FUZZTEST") == "" {
f.Skip("run with FUZZTEST=1 do fuzz")
f.Skip("run with FUZZTEST=1 to fuzz")
}
i := 0
@ -103,7 +103,18 @@ func FuzzFormats(f *testing.F) {
}
gi := 0
g := interp.DefaultRegistry.MustAll()
var g decode.Group
if n := os.Getenv("GROUP"); n != "" {
var err error
g, err = interp.DefaultRegistry.FormatGroup(n)
if err != nil {
f.Fatal(err)
}
f.Logf("GROUP=%s", n)
} else {
g = interp.DefaultRegistry.MustAll()
}
f.Fuzz(func(t *testing.T, b []byte) {
fz := &fuzzTest{b: b, f: g[gi]}

28
format/id3/id3v2.go
View File

@ -287,24 +287,34 @@ func textFn(encoding int, nBytes int) func(d *decode.D) string {
}
}
func textNullFn(encoding int) func(d *decode.D) string {
func textNullLenFn(encoding int, notFoundFixedBytes int) func(d *decode.D) string {
return func(d *decode.D) string {
nullLen := encodingLen[encodingUTF8]
if n, ok := encodingLen[uint64(encoding)]; ok {
nullLen = n
}
offset, _ := d.PeekFind(
offset, _, err := d.TryPeekFind(
int(nullLen)*8,
nullLen*8,
-1,
int64(notFoundFixedBytes)*8,
func(v uint64) bool { return v == 0 },
)
if err != nil {
d.IOPanic(err, "textNullLenFn")
}
if offset < 0 {
if notFoundFixedBytes < 0 {
d.Fatalf("textNullLenFn: null not found")
}
return textFn(encoding, notFoundFixedBytes)(d)
}
offsetBytes := offset / 8
text := textFn(encoding, int(offsetBytes))(d)
d.SeekRel(nullLen * 8)
// seems sometimes utf16 etc has en exta null byte
// seems sometimes utf16 etc has one exta null byte
if nullLen > 1 && d.PeekBits(8) == 0 {
d.SeekRel(8)
}
@ -313,6 +323,10 @@ func textNullFn(encoding int) func(d *decode.D) string {
}
}
func textNullFn(encoding int) func(d *decode.D) string {
return textNullLenFn(encoding, -1)
}
func decodeFrame(d *decode.D, version int) uint64 {
var id string
var size uint64
@ -493,7 +507,8 @@ func decodeFrame(d *decode.D, version int) uint64 {
"COMM": func(d *decode.D) {
encoding := d.FieldU8("text_encoding", encodingNames)
d.FieldUTF8("language", 3)
d.FieldStrFn("description", textNullFn(int(encoding)))
// there are COMM frames with no null termination
d.FieldStrFn("description", textNullLenFn(int(encoding), int(d.BitsLeft()/8)))
d.FieldStrFn("value", textFn(int(encoding), int(d.BitsLeft()/8)))
},
// Text information identifier "T00" - "TZZ" , excluding "TXX",
@ -522,7 +537,8 @@ func decodeFrame(d *decode.D, version int) uint64 {
// Value <text string according to encoding>
"TXXX": func(d *decode.D) {
encoding := d.FieldU8("text_encoding", encodingNames)
d.FieldStrFn("description", textNullFn(int(encoding)))
// there are TXXX frames with no null termination
d.FieldStrFn("description", textNullLenFn(int(encoding), int(d.BitsLeft()/8)))
d.FieldStrFn("value", textFn(int(encoding), int(d.BitsLeft()/8)))
},
// <Header for 'Private frame', ID: "PRIV">

BIN
format/id3/testdata/txxx-nonullterm vendored Normal file
View File

Binary file not shown.

29
format/id3/testdata/txxx-nonullterm.fqtest vendored Normal file
View File

@ -0,0 +1,29 @@
$ fq -d id3v2 dv txxx-nonullterm
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: txxx-nonullterm (id3v2) 0x0-0x26.7 (39)
0x00|49 44 33 |ID3 | magic: "ID3" (valid) 0x0-0x2.7 (3)
0x00| 03 | . | version: 3 0x3-0x3.7 (1)
0x00| 00 | . | revision: 0 0x4-0x4.7 (1)
| | | flags{}: 0x5-0x5.7 (1)
0x00| 00 | . | unsynchronisation: false 0x5-0x5 (0.1)
0x00| 00 | . | extended_header: false 0x5.1-0x5.1 (0.1)
0x00| 00 | . | experimental_indicator: false 0x5.2-0x5.2 (0.1)
0x00| 00 | . | unused: 0 0x5.3-0x5.7 (0.5)
0x00| 00 00 00 1d | .... | size: 29 0x6-0x9.7 (4)
| | | frames[0:1]: 0xa-0x26.7 (29)
| | | [0]{}: frame 0xa-0x26.7 (29)
0x00| 54 58 58 58 | TXXX | id: "TXXX" (User defined text information frame) 0xa-0xd.7 (4)
0x00| 00 00| ..| size: 19 0xe-0x11.7 (4)
0x10|00 13 |.. |
| | | flags{}: 0x12-0x13.7 (2)
0x10| 00 | . | tag_alter_preservation: false 0x12-0x12 (0.1)
0x10| 00 | . | file_alter_preservation: false 0x12.1-0x12.1 (0.1)
0x10| 00 | . | read_only: false 0x12.2-0x12.2 (0.1)
0x10| 00 | . | unused0: 0 0x12.3-0x12.7 (0.5)
0x10| 00 | . | compression: false 0x13-0x13 (0.1)
0x10| 00 | . | encryption: false 0x13.1-0x13.1 (0.1)
0x10| 00 | . | grouping_identity: false 0x13.2-0x13.2 (0.1)
0x10| 00 | . | unused1: 0 0x13.3-0x13.7 (0.5)
0x10| 01 | . | text_encoding: "utf16" (1) 0x14-0x14.7 (1)
0x10| ff fe 53 00 6f 00 66 00 74 00 77| ..S.o.f.t.w| description: "Software" 0x15-0x26.7 (18)
0x20|00 61 00 72 00 65 00| |.a.r.e.| |
| | | value: "" 0x27-NA (0)

5
format/yaml/testdata/normalize.fqtest vendored Normal file
View File

@ -0,0 +1,5 @@
# normalize uint64 to bigint
$ fq -n '"[18446744073709551615]" | yaml'
[
18446744073709551615
]

2
format/yaml/yaml.go
View File

@ -45,7 +45,7 @@ func decodeYAML(d *decode.D, _ any) any {
}
var s scalar.S
s.Actual = r
s.Actual = gojqex.Normalize(r)
switch s.Actual.(type) {
case map[string]any,

18
go.mod
View File

@ -4,7 +4,7 @@ go 1.18
require (
// fork of github.com/itchyny/gojq, see github.com/wader/gojq fq branch
github.com/wader/gojq v0.12.1-0.20220929141349-8874f5c7907c
github.com/wader/gojq v0.12.1-0.20221030123557-e1510a9e80c3
// fork of github.com/chzyer/readline, see github.com/wader/readline fq branch
github.com/wader/readline v0.0.0-20220928125628-732951d41240
)
@ -13,7 +13,7 @@ require (
// bump: gomod-BurntSushi/toml /github\.com\/BurntSushi\/toml v(.*)/ https://github.com/BurntSushi/toml.git|^1
// bump: gomod-BurntSushi/toml command go get -d github.com/BurntSushi/toml@v$LATEST && go mod tidy
// bump: gomod-BurntSushi/toml link "Source diff $CURRENT..$LATEST" https://github.com/BurntSushi/toml/compare/v$CURRENT..v$LATEST
github.com/BurntSushi/toml v1.2.0
github.com/BurntSushi/toml v1.2.1
// bump: gomod-creasty-defaults /github\.com\/creasty\/defaults v(.*)/ https://github.com/creasty/defaults.git|^1
// bump: gomod-creasty-defaults command go get -d github.com/creasty/defaults@v$LATEST && go mod tidy
@ -27,13 +27,13 @@ require (
// has no tags
// go get -d github.com/gomarkdown/markdown@master && go mod tidy
github.com/gomarkdown/markdown v0.0.0-20220905174103-7b278df48cfb
github.com/gomarkdown/markdown v0.0.0-20221013030248-663e2500819c
// has no tags yet
// bump-disabled: gomod-gopacket /github\.com\/gopacket\/gopacket v(.*)/ https://github.com/gopacket/gopacket.git|^1
// bump-disabled: gomod-gopacket command go get -d github.com/gopacket/gopacket@v$LATEST && go mod tidy
// bump-disabled: gomod-gopacket link "Release notes" https://github.com/gopacket/gopacket/releases/tag/v$LATEST
github.com/gopacket/gopacket v0.0.0-20220819214934-ee81b8c880da
github.com/gopacket/gopacket v0.0.0-20221006103438-9e6d99b9b443
// bump: gomod-copystructure /github\.com\/mitchellh\/copystructure v(.*)/ https://github.com/mitchellh/copystructure.git|^1
// bump: gomod-copystructure command go get -d github.com/mitchellh/copystructure@v$LATEST && go mod tidy
@ -52,20 +52,20 @@ require (
// has no tags
// go get -d golang.org/x/crypto@master && go mod tidy
golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be
golang.org/x/crypto v0.1.1-0.20221024173537-a3485e174077
// has no tags
// go get -d golang.org/x/exp@master && go mod tidy
golang.org/x/exp v0.0.0-20220927162542-c76eaa363f9d
golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f
// has no tags
// go get -d golang.org/x/net@master && go mod tidy
golang.org/x/net v0.0.0-20220927171203-f486391704dc
golang.org/x/net v0.1.1-0.20221027164007-c63010009c80
// bump: gomod-golang/text /golang\.org\/x\/text v(.*)/ https://github.com/golang/text.git|^0
// bump: gomod-golang/text command go get -d golang.org/x/text@v$LATEST && go mod tidy
// bump: gomod-golang/text link "Source diff $CURRENT..$LATEST" https://github.com/golang/text/compare/v$CURRENT..v$LATEST
golang.org/x/text v0.3.7
golang.org/x/text v0.4.0
// bump: gomod-gopkg.in/yaml.v3 /gopkg\.in\/yaml\.v3 v(.*)/ https://github.com/go-yaml/yaml.git|^3
// bump: gomod-gopkg.in/yaml.v3 command go get -d gopkg.in/yaml.v3@v$LATEST && go mod tidy
@ -77,6 +77,6 @@ require (
github.com/itchyny/timefmt-go v0.1.4 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 // indirect
golang.org/x/sys v0.1.0 // indirect
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
)

35
go.sum
View File

@ -1,13 +1,13 @@
github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/creasty/defaults v1.6.0 h1:ltuE9cfphUtlrBeomuu8PEyISTXnxqkBIoQfXgv7BSc=
github.com/creasty/defaults v1.6.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomarkdown/markdown v0.0.0-20220905174103-7b278df48cfb h1:7h+tPfwoUE+qLvWYmsvKSiRlXv6WGorb6PUKaZUclwc=
github.com/gomarkdown/markdown v0.0.0-20220905174103-7b278df48cfb/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
github.com/gopacket/gopacket v0.0.0-20220819214934-ee81b8c880da h1:AAwDU9N39fQNYUtg270aiU6N7U2ZVsGZKiRwsCMsWEo=
github.com/gopacket/gopacket v0.0.0-20220819214934-ee81b8c880da/go.mod h1:DlRRfaM/QjAu2ADqraIure1Eif0HpNL8hmyVQ+qci5Y=
github.com/gomarkdown/markdown v0.0.0-20221013030248-663e2500819c h1:iyaGYbCmcYK0Ja9a3OUa2Fo+EaN0cbLu0eKpBwPFzc8=
github.com/gomarkdown/markdown v0.0.0-20221013030248-663e2500819c/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
github.com/gopacket/gopacket v0.0.0-20221006103438-9e6d99b9b443 h1:nZcBvHjudAMfouyqauDctCqWgl0wsphtiCu4yiVCs6g=
github.com/gopacket/gopacket v0.0.0-20221006103438-9e6d99b9b443/go.mod h1:DlRRfaM/QjAu2ADqraIure1Eif0HpNL8hmyVQ+qci5Y=
github.com/itchyny/timefmt-go v0.1.4 h1:hFEfWVdwsEi+CY8xY2FtgWHGQaBaC3JeHd+cve0ynVM=
github.com/itchyny/timefmt-go v0.1.4/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@ -23,21 +23,22 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/wader/gojq v0.12.1-0.20220929141349-8874f5c7907c h1:aQz+gw5Xe2x++st4HBIaMHPPTIyPtESG2HMZJmOaRkc=
github.com/wader/gojq v0.12.1-0.20220929141349-8874f5c7907c/go.mod h1:/+WKHSfP8sgyJDXeD9VjxlGjExGfjPUy0zm/Zy3Ind0=
github.com/wader/gojq v0.12.1-0.20221030123557-e1510a9e80c3 h1:O+cdsjdniOlgdSdruQCNnLTuVvubmvywif6c7J8P3fU=
github.com/wader/gojq v0.12.1-0.20221030123557-e1510a9e80c3/go.mod h1:/+WKHSfP8sgyJDXeD9VjxlGjExGfjPUy0zm/Zy3Ind0=
github.com/wader/readline v0.0.0-20220928125628-732951d41240 h1:fqNaldd6kVsMNGXLXH4TtB1kJtaPQOzGwqNYgswIM94=
github.com/wader/readline v0.0.0-20220928125628-732951d41240/go.mod h1:Zgz8IJWvJoe7NK23CCPpC109XMCqJCpUhpHcnnA4XaM=
golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be h1:fmw3UbQh+nxngCAHrDCCztao/kbYFnWjoqop8dHx05A=
golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20220927162542-c76eaa363f9d h1:3wgmvnqHUJ8SxiNWwea5NCzTwAVfhTtuV+0ClVFlClc=
golang.org/x/exp v0.0.0-20220927162542-c76eaa363f9d/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
golang.org/x/net v0.0.0-20220927171203-f486391704dc h1:FxpXZdoBqT8RjqTy6i1E8nXHhW21wK7ptQ/EPIGxzPQ=
golang.org/x/net v0.0.0-20220927171203-f486391704dc/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/crypto v0.1.1-0.20221024173537-a3485e174077 h1:t5bjOfJPQfaG9NV1imLZM5E2uzaLGs5/NtyMtRNVjQ4=
golang.org/x/crypto v0.1.1-0.20221024173537-a3485e174077/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f h1:Al51T6tzvuh3oiwX11vex3QgJ2XTedFPGmbEVh8cdoc=
golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/net v0.1.1-0.20221027164007-c63010009c80 h1:CtRWmqbiPSOXwJV1JoY7pWiTx2xzVKQ813bvU+Y/9jI=
golang.org/x/net v0.1.1-0.20221027164007-c63010009c80/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 h1:v6hYoSR9T5oet+pMXwUWkbiVqx/63mlHjefrHmxwfeY=
golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=

3
pkg/decode/decode.go
View File

@ -487,6 +487,9 @@ func (d *D) PeekFind(nBits int, seekBits int64, maxLen int64, fn func(v uint64)
// Does not update current position.
// TODO: nBytes -1?
func (d *D) TryBytesRange(bitOffset int64, nBytes int) ([]byte, error) {
if nBytes < 0 {
return nil, fmt.Errorf("negative nBytes %d", nBytes)
}
buf := make([]byte, nBytes)
n, err := bitio.ReadAtFull(d.bitBuf, buf, int64(nBytes)*8, bitOffset)
if n == int64(nBytes)*8 {

4
pkg/decode/read.go
View File

@ -32,8 +32,8 @@ func (d *D) tryUEndian(nBits int, endian Endian) (uint64, error) {
}
func (d *D) trySEndian(nBits int, endian Endian) (int64, error) {
if nBits < 0 {
return 0, fmt.Errorf("trySEndian nBits must be >= 0 (%d)", nBits)
if nBits < 1 {
return 0, fmt.Errorf("trySEndian nBits must be >= 1 (%d)", nBits)
}
n, err := d.tryUEndian(nBits, endian)
if err != nil {

2
pkg/interp/interp.go
View File

@ -1126,7 +1126,7 @@ func (i *Interp) NewColorJSON(opts Options) (*colorjson.Encoder, error) {
if v, ok := toValue(func() Options { return opts }, v); ok {
return v
}
panic(fmt.Sprintf("toValue not a JQValue value: %#v", v))
panic(fmt.Sprintf("toValue not a JQValue value: %#v (%T)", v, v))
},
colorjson.Colors{
Reset: []byte(ansi.Reset.SetString),