Pull latest from wader

2024-11-22 07:16:49 +03:00 · 2022-01-19 19:39:12 -06:00 · 2022-01-19 19:39:12 -06:00 · 6e1f338ac3
commit 6e1f338ac3
parent de64a99e72 2173d9e1f9
49 changed files with 2993 additions and 2215 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,2 +1,3 @@
 .DS_Store
 testfiles/
+fq
--- a/.goreleaser.yml
+++ b/.goreleaser.yml
@ -22,7 +22,6 @@ builds:
    ldflags:
      # omit symbol table and debug information
      - -s -w
-      - -X main.version={{.Version}}
      - -X main.commit={{.Commit}}
      - -X main.date={{.CommitDate}}
      - -X main.builtBy=goreleaser
--- a/10
+++ b/10
@ -5,9 +5,8 @@ GO_TEST_RACE_FLAGS ?=-race
 all: test fq

 .PHONY: fq
-fq: VERSION=$(shell git describe --all --long --dirty 2>/dev/null || echo nogit)
 fq:
-	CGO_ENABLED=0 go build -o fq -ldflags "${GO_BUILD_LDFLAGS} -X main.version=${VERSION}" ${GO_BUILD_FLAGS} .
+	CGO_ENABLED=0 go build -o fq -ldflags "${GO_BUILD_LDFLAGS}" ${GO_BUILD_FLAGS} .

 .PHONY: test
 test: testgo testjq testcli
@ -112,6 +111,12 @@ release:
 	@echo make lint test doc
 	@echo go mod tidy
 	@echo git diff
+	@echo
+	@echo "sed 's/version = "\\\(.*\\\)"/version = \"${VERSION}\"/' fq.go > fq.go.new && mv fq.go.new fq.go"
+	@echo git add fq.go
+	@echo git commit -m "fq: Update version to ${VERSION}"
+	@echo git push wader master
+	@echo
 	@echo "# make sure head master commit CI was successful"
 	@echo open https://github.com/wader/fq/commit/master
 	@echo git tag v${VERSION}
@ -127,4 +132,3 @@ release:
 	@echo "# wader/fq":
 	@echo git push wader v${VERSION}:v${VERSION}
 	@echo "# edit draft release notes and publish"
-	@echo open https://github.com/wader/fq/releases/tag/v${VERSION}
--- a/README.md
+++ b/README.md
@ -1,27 +1,29 @@
 # fq

-Tool, language and decoders for inspecting binary data.
+Tool, language and decoders for working with binary data.

 ![fq demo](doc/demo.svg)

 In most cases fq works the same way as jq but instead of reading JSON it reads binary data.
-The result is a JSON compatible structures where each value has a bit range, symbolic
-interpretations and know how to be presented in a useful way.
+The result is a JSON compatible structure where values have a bit range, can have symbolic
+values and know how to be presented in a useful ways.

-You can pronounce the name as you wish, i pronounce jq /‘dʒei’kju:/ so I usually prefer /‘ef’kju:/.
+It was initially developed to debug, inspect and query media files but has since been extended
+to handle a variety of binary formats.

-**NOTE:** fq is early in development and many things are missing, broken or do not make sense.
-That also means there is a great opportunity to help out!
+You can pronounce the name as you wish, I pronounce jq /‘dʒei’kju:/ so I usually pronounce fq /‘ef’kju:/.

-## Goals
+**NOTE:** fq is still early in development so some things are broken or do not make sense.
+That also means that there is a great opportunity to help out!
+
+### Goals

 - Make binary formats accessible and queryable.
 - Nested formats and bit-oriented decoding.
 - Quick and comfortable CLI tool.
- Bit and byte transformations and conversions.
- Programmer's calculator.
+- Bits and bytes transformations.

-## Hopes
+### Hopes
 - Make it useful enough that people want to help improve it.
 - Inspire people to create similar tools.

@ -29,7 +31,7 @@ That also means there is a great opportunity to help out!

 Basic usage is `fq . file`.

-For details see [usage.md](doc/usage.md)
+For more details see [usage.md](doc/usage.md)

 ## Install

--- a/doc/TODO.md
+++ b/doc/TODO.md
@ -1,5 +1,9 @@
 ### Known bugs to fix

+- `fq -n '"aabbccdd" | hex | tobytes[1:] | raw | tobytes'` create buffer `aabbcc` should be `bbccdd`. I think decode (raw in this case) is confused by root value buffer.
+- Buffers/string duality is confusing, most string functions should be wrapped to understand buffers.
+- `fq -n '[([0xab] | tobits[:4]), ([0xdc] | tobits[:4]), 0] | tobytes'` should create a `ad` buffer, now `a0`. Probably because use of `io.Copy` that will ends up padding on byte boundaries. Should use `bitio.Copy` and create a `bitio.Writer` that can transform to a `io.Writer`.
+- REPL cancel seems to sometimes exit a sub-REPl without properly cleanup options.
 - Value errors, can only be accessed with `._error`.
 - Framed (add unknown in gaps) decode should be on struct level not format?
 - `tovalue({bits_format: "base64"})` only affect root value.
@ -51,7 +55,6 @@
 - `open` leak, file and ctxreadseeker
 - Summary tree with format specific summaries for each format, sample count etc etc?
 - List all unique paths in some compact form?
- Make buffer work with `test` and `capture`?

 ### Tests

@ -86,6 +89,9 @@

 #### Formats

+- `asn1_ber` `asn1_der`, `asn1_cer` decoder
+- `flatbuffer` decoder
+- `capnproto` decoder
 - Pass argument to format
 - Value decoder in jq `u(32)`, `u32`?
 - Warnings and errors
--- a/doc/formats.svg
+++ b/doc/formats.svg
--- a/doc/formats_diagram.jq
+++ b/doc/formats_diagram.jq
@ -11,23 +11,19 @@ def _formats_dot:
    ] | flatten | join("");
  ( "# ... | dot -Tsvg -o formats.svg"
  , "digraph formats {"
-  , "  concentrate=True"
  , "  rankdir=TB"
-  , "  graph ["
-  , "  ]"
-  , "  node [shape=\"none\"style=\"\"]"
-  , "  edge [arrowsize=\"0.7\"]"
+  , "  node [shape=\"none\" style=\"\"]"
  , ( .[]
    | . as $f
    | .dependencies
    | flatten?
    | .[]
-    | "  \"\($f.name)\":\(.) -> \(.)"
+    | "  \"\($f.name)\":\(.):e -> \(.):n"
    )
  , ( .[]
    | .name as $name
    | .groups[]?
-    | "  \(.) -> \"\($name)\":\($name)"
+    | "  \(.) -> \"\($name)\":\($name):n"
    )
  , ( to_entries[]
    | "  \(.key) [color=\"paleturquoise\", label=\(_record(.key; (.value.dependencies // [])))]"
--- a/doc/usage.md
+++ b/doc/usage.md
@ -103,16 +103,19 @@ Use Ctrl-D to exit and Ctrl-C to interrupt current evaluation.
 ## Example usages

 #### Second mp3 frame header as JSON
+
 ```sh
 fq '.frames[1].header | tovalue' file.mp3
 ```

 #### Byte start position for the first 10 mp3 frames in an array
+
 ```sh
 fq '.frames[0:10] | map(tobytesrange.start)' file.mp3
 ```

 #### Decode at range
+
 ```sh
 # decode byte range 100 to end
 fq -d raw 'tobytes[100:] | mp3_frame | d' file.mp3
@ -128,6 +131,7 @@ decoded value for `a.mp4` and `b.mp4` filtered thru `f`.
 ```sh
 fq -n 'def f: .. | select(format=="avc_sps"); diff(input|f; input|f)' a.mp4 b.mp4
 ```
+
 #### Extract first JPEG found in file

 Recursively look for first value that is a `jpeg` decode value root. Use `tobytes` to get bytes buffer for value. Redirect bytes to a file.
@ -152,7 +156,19 @@ Use `grep` to recursively find strings matching a regexp.
 fq '.tcp_connections | grep("GET /.* HTTP/1.?")' file.pcap
 ```

-###
+#### Use representation of a format
+
+Some formats like `msgpack`, `bson` etc are used to represent some data structure. In those cases the `torepr`
+function can be used to get the representation.
+
+```sh
+# whole represented value
+fq -d msgpack torepr file.msgpack
+# value of the key "field" from the represented value
+fq -d msgpack `torepr.field` file.msgpack
+# query or transform represented value
+fq -d msgpack 'torepr | ...' file.msgpack
+```

 #### Widest PNG in a directory
 ```sh
--- a/format/bson/bson.go
+++ b/format/bson/bson.go
@ -82,6 +82,7 @@ func decodeBSONDocument(d *decode.D) {
 						d.FieldU8("subtype")
 						d.FieldRawLen("value", int64(length)*8)
 					case elementTypeUndefined:
+						//deprecated
 					case elementTypeObjectID:
 						d.FieldRawLen("value", 12*8)
 					case elementTypeBoolean:
@ -89,6 +90,7 @@ func decodeBSONDocument(d *decode.D) {
 					case elementTypeDatatime:
 						d.FieldS32("value")
 					case elementTypeNull:
+						d.FieldValueNil("value")
 					case elementTypeRegexp:
 						d.FieldUTF8Null("value")
 						d.FieldUTF8Null("options")
--- a/format/bson/testdata/test.fqtest
+++ b/format/bson/testdata/test.fqtest
@ -59,6 +59,7 @@ $ fq -d bson verbose /test.bson
 0x60|                                    0a         |            .   |      type: "null" (10) (Null value) 0x6c-0x6c.7 (1)
 0x60|                                       6e 75 6c|             nul|      name: "null" 0x6d-0x71.7 (5)
 0x70|6c 00                                          |l.              |
+    |                                               |                |      value: null 0x72-NA (0)
 0x70|      00|                                      |  .|            |  terminator: 0 (valid) 0x72-0x72.7 (1)
 $ fq -d bson torepr /test.bson
 {
--- a/format/cbor/cbor.go
+++ b/format/cbor/cbor.go
@ -9,10 +9,12 @@ package cbor
 import (
 	"bytes"
 	"embed"
+	"math/big"
 	"strings"

 	"github.com/wader/fq/format"
 	"github.com/wader/fq/format/registry"
+	"github.com/wader/fq/internal/mathextra"
 	"github.com/wader/fq/pkg/bitio"
 	"github.com/wader/fq/pkg/decode"
 	"github.com/wader/fq/pkg/scalar"
@ -109,11 +111,13 @@ func decodeCBORValue(d *decode.D) interface{} {
 	majorTypeMap := majorTypeEntries{
 		majorTypePositiveInt: {s: scalar.S{Sym: "positive_int"}, d: func(d *decode.D, shortCount uint64, count uint64) interface{} {
 			d.FieldValueU("value", count)
-			return count
+			return nil
 		}},
 		majorTypeNegativeInt: {s: scalar.S{Sym: "negative_int"}, d: func(d *decode.D, shortCount uint64, count uint64) interface{} {
-			d.FieldValueS("value", int64(^count))
-			return count
+			n := new(big.Int)
+			n.SetUint64(count).Neg(n).Sub(n, mathextra.BigIntOne)
+			d.FieldValueBigInt("value", n)
+			return nil
 		}},
 		majorTypeBytes: {s: scalar.S{Sym: "bytes"}, d: func(d *decode.D, shortCount uint64, count uint64) interface{} {
 			if shortCount == shortCountIndefinite {
@ -204,7 +208,7 @@ func decodeCBORValue(d *decode.D) interface{} {
 		majorTypeSematic: {s: scalar.S{Sym: "semantic"}, d: func(d *decode.D, shortCount uint64, count uint64) interface{} {
 			d.FieldValueU("tag", count, tagMap)
 			d.FieldStruct("value", func(d *decode.D) { decodeCBORValue(d) })
-			return count
+			return nil
 		}},
 		majorTypeSpecialFloat: {s: scalar.S{Sym: "special_float"}, d: func(d *decode.D, shortCount uint64, count uint64) interface{} {
 			switch shortCount {
@ -214,7 +218,7 @@ func decodeCBORValue(d *decode.D) interface{} {
 			case shortCountSpecialTrue:
 				d.FieldValueBool("value", true)
 			case shortCountSpecialNull:
-				// TODO: null
+				d.FieldValueNil("value")
 			case shortCountSpecialUndefined:
 				// TODO: undefined
 			case 24:
--- a/format/cbor/testdata/appendix_a.fqtest
+++ b/format/cbor/testdata/appendix_a.fqtest
@ -1,6 +1,8 @@
 # appendix_a.json from https://github.com/cbor/test-vectors
-# TODO: "O///////////" fails due lack of bigint support (is smaller thax mmin int64), also the test JSON has issues truncating decoded value
 # TODO: "w0kBAAAAAAAAAAA=" "wkkBAAAAAAAAAAA=" semantic bigint
+# NOTE: "O///////////" test uses bigint and is correct but test success currently relay on -18446744073709551616
+# in input json being turned into a float as it can't be represented in json and cbor decoded bigint will also be
+# converted to a float when comparing.
 $ fq -i -d json . appendix_a.json
 json> length
 82
@ -19,15 +21,6 @@ json> map(select(.decoded) | (.cbor | base64 | cbor | torepr) as $a | select( .d
      "roundtrip": true
    }
  },
-  {
-    "actual": 0,
-    "test": {
-      "cbor": "O///////////",
-      "decoded": -18446744073709552000,
-      "hex": "3bffffffffffffffff",
-      "roundtrip": true
-    }
-  },
  {
    "actual": {
      "major_type": "bytes",
@ -106,7 +99,7 @@ json> .[] | select(.decoded) | .cbor | base64 | cbor | v
 0x0|3b                                             |;               |  major_type: "negative_int" (1) 0x0-0x0.2 (0.3)
 0x0|3b                                             |;               |  short_count: "64bit" (27) 0x0.3-0x0.7 (0.5)
 0x0|   ff ff ff ff ff ff ff ff|                    | ........|      |  variable_count: 18446744073709551615 0x1-0x8.7 (8)
-   |                                               |                |  value: 0 0x9-NA (0)
+   |                                               |                |  value: -18446744073709551616 0x9-NA (0)
   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: (cbor) 0x0-0xa.7 (11)
 0x0|c3                                             |.               |  major_type: "semantic" (6) 0x0-0x0.2 (0.3)
 0x0|c3                                             |.               |  short_count: 3 0x0.3-0x0.7 (0.5)
--- a/format/flac/flac.go
+++ b/format/flac/flac.go
@ -13,7 +13,7 @@ import (

 	"github.com/wader/fq/format"
 	"github.com/wader/fq/format/registry"
-	"github.com/wader/fq/internal/num"
+	"github.com/wader/fq/internal/mathextra"
 	"github.com/wader/fq/pkg/bitio"
 	"github.com/wader/fq/pkg/decode"
 	"github.com/wader/fq/pkg/scalar"
@ -67,7 +67,7 @@ func flacDecode(d *decode.D, in interface{}) interface{} {

 			samplesInFrame := ffo.Samples
 			if streamTotalSamples > 0 {
-				samplesInFrame = num.MinUInt64(streamTotalSamples-streamDecodedSamples, ffo.Samples)
+				samplesInFrame = mathextra.MinUInt64(streamTotalSamples-streamDecodedSamples, ffo.Samples)
 			}
 			frameStreamSamplesBuf := ffo.SamplesBuf[0 : samplesInFrame*uint64(ffo.Channels*ffo.BitsPerSample/8)]
 			framesNDecodedSamples += ffo.Samples
--- a/format/flac/flac_frame.go
+++ b/format/flac/flac_frame.go
@ -6,7 +6,7 @@ import (

 	"github.com/wader/fq/format"
 	"github.com/wader/fq/format/registry"
-	"github.com/wader/fq/internal/num"
+	"github.com/wader/fq/internal/mathextra"
 	"github.com/wader/fq/pkg/checksum"
 	"github.com/wader/fq/pkg/decode"
 	"github.com/wader/fq/pkg/scalar"
@ -378,8 +378,8 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
 				}

 				subframeSampleSize := sampleSize - wastedBitsK
-				if subframeSampleSize < 0 {
-					d.Fatalf("negative subframeSampleSize %d", subframeSampleSize)
+				if subframeSampleSize < 1 {
+					d.Fatalf("subframeSampleSize %d < 1", subframeSampleSize)
 				}
 				// if channel is side, add en extra sample bit
 				// https://github.com/xiph/flac/blob/37e675b777d4e0de53ac9ff69e2aea10d92e729c/src/libFLAC/stream_decoder.c#L2040
@ -471,7 +471,7 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
 										_ = high
 										low := d.U(riceParameter)
 										_ = low
-										samples[n] = num.ZigZag(high<<riceParameter | low)
+										samples[n] = mathextra.ZigZag(high<<riceParameter | low)
 										n++
 									}
 									samplesStop := d.Pos()
--- a/format/icc/icc_profile.go
+++ b/format/icc/icc_profile.go
@ -6,7 +6,7 @@ package icc
 import (
 	"github.com/wader/fq/format"
 	"github.com/wader/fq/format/registry"
-	"github.com/wader/fq/internal/num"
+	"github.com/wader/fq/internal/mathextra"
 	"github.com/wader/fq/pkg/decode"
 )

@ -165,7 +165,7 @@ func iccProfileDecode(d *decode.D, in interface{}) interface{} {
 						// was. instead add alignment after if offset+size does not align and to be sure clamp it if outside buffer.
 						paddingStart := int64(offset) + int64(size)
 						paddingBytes := (4 - (int64(offset)+int64(size))%4) % 4
-						paddingBytes = num.MinInt64(paddingBytes, d.Len()-(paddingStart+paddingBytes))
+						paddingBytes = mathextra.MinInt64(paddingBytes, d.Len()-(paddingStart+paddingBytes))
 						if paddingBytes != 0 {
 							d.RangeFn(paddingStart*8, paddingBytes*8, func(d *decode.D) {
 								d.FieldRawLen("alignment", d.BitsLeft())
--- a/format/mp4/boxes.go
+++ b/format/mp4/boxes.go
@ -257,7 +257,7 @@ func init() {

 			// TODO: dup track id?
 			if _, ok := ctx.tracks[trackID]; !ok {
-				t := &track{id: trackID}
+				t := &track{id: int(trackID)}
 				ctx.tracks[trackID] = t
 				ctx.currentTrack = t
 			}
@ -518,8 +518,8 @@ func init() {

 				if ctx.currentTrack != nil {
 					ctx.currentTrack.stsc = append(ctx.currentTrack.stsc, stsc{
-						firstChunk:      firstChunk,
-						samplesPerChunk: samplesPerChunk,
+						firstChunk:      int(firstChunk),
+						samplesPerChunk: int(samplesPerChunk),
 					})
 				}
 				i++
@ -536,18 +536,19 @@ func init() {
 				d.FieldArrayLoop("entries", func() bool { return i < entryCount }, func(d *decode.D) {
 					size := uint32(d.FieldU32("size"))
 					if ctx.currentTrack != nil {
-						ctx.currentTrack.stsz = append(ctx.currentTrack.stsz, size)
+						ctx.currentTrack.stsz = append(ctx.currentTrack.stsz, stsz{
+							size:  int64(size),
+							count: 1,
+						})
 					}
 					i++
 				})
 			} else {
 				if ctx.currentTrack != nil {
-					if entryCount > maxSampleEntryCount {
-						d.Errorf("too many constant stsz entries %d > %d", entryCount, maxSampleEntryCount)
-					}
-					for i := uint64(0); i < entryCount; i++ {
-						ctx.currentTrack.stsz = append(ctx.currentTrack.stsz, uint32(sampleSize))
-					}
+					ctx.currentTrack.stsz = append(ctx.currentTrack.stsz, stsz{
+						size:  int64(sampleSize),
+						count: int(entryCount),
+					})
 				}
 			}
 		},
@ -559,7 +560,7 @@ func init() {
 			d.FieldArrayLoop("entries", func() bool { return i < entryCount }, func(d *decode.D) {
 				chunkOffset := d.FieldU32("chunk_offset")
 				if ctx.currentTrack != nil {
-					ctx.currentTrack.stco = append(ctx.currentTrack.stco, chunkOffset)
+					ctx.currentTrack.stco = append(ctx.currentTrack.stco, int64(chunkOffset))
 				}
 				i++
 			})
@ -615,7 +616,7 @@ func init() {
 			d.FieldArrayLoop("entries", func() bool { return i < entryCount }, func(d *decode.D) {
 				offset := d.FieldU64("offset")
 				if ctx.currentTrack != nil {
-					ctx.currentTrack.stco = append(ctx.currentTrack.stco, offset)
+					ctx.currentTrack.stco = append(ctx.currentTrack.stco, int64(offset))
 				}
 				i++
 			})
@ -720,13 +721,13 @@ func init() {
 				d.FieldU64("base_data_offset")
 			}
 			if sampleDescriptionIndexPresent {
-				m.defaultSampleDescriptionIndex = uint32(d.FieldU32("sample_description_index"))
+				m.defaultSampleDescriptionIndex = int(d.FieldU32("sample_description_index"))
 			}
 			if defaultSampleDurationPresent {
 				d.FieldU32("default_sample_duration")
 			}
 			if defaultSampleSizePresent {
-				m.defaultSampleSize = uint32(d.FieldU32("default_sample_size"))
+				m.defaultSampleSize = int64(d.FieldU32("default_sample_size"))
 			}
 			if defaultSampleFlagsPresent {
 				d.FieldU32("default_sample_flags")
@ -760,14 +761,14 @@ func init() {
 			})
 			sampleCount := d.FieldU32("sample_count")
 			if dataOffsetPresent {
-				m.dataOffset = uint32(d.FieldS32("data_offset"))
+				m.dataOffset = d.FieldS32("data_offset")
 			}
 			if firstSampleFlagsPresent {
 				d.FieldU32("first_sample_flags")
 			}

 			if sampleCount > maxSampleEntryCount {
-				d.Errorf("too many constant trun entries %d > %d", sampleCount, maxSampleEntryCount)
+				d.Errorf("too many sample trun entries %d > %d", sampleCount, maxSampleEntryCount)
 			}

 			d.FieldArray("samples", func(d *decode.D) {
@ -778,7 +779,7 @@ func init() {
 							d.FieldU32("sample_duration")
 						}
 						if sampleSizePresent {
-							sampleSize = uint32(d.FieldU32("sample_size"))
+							sampleSize = int64(d.FieldU32("sample_size"))
 						}
 						if sampleFlagsPresent {
 							d.FieldU32("sample_flags")
--- a/format/mp4/mp4.go
+++ b/format/mp4/mp4.go
@ -17,6 +17,7 @@ package mp4

 import (
 	"embed"
+	"fmt"
 	"sort"

 	"github.com/wader/fq/format"
@ -86,16 +87,16 @@ func init() {
 }

 type stsc struct {
-	firstChunk      uint32
-	samplesPerChunk uint32
+	firstChunk      int
+	samplesPerChunk int
 }

 type moof struct {
 	offset                        int64
-	defaultSampleSize             uint32
-	defaultSampleDescriptionIndex uint32
-	dataOffset                    uint32
-	samplesSizes                  []uint32
+	defaultSampleSize             int64
+	defaultSampleDescriptionIndex int
+	dataOffset                    int64
+	samplesSizes                  []int64
 }

 type sampleDescription struct {
@ -103,13 +104,18 @@ type sampleDescription struct {
 	originalFormat string
 }

+type stsz struct {
+	size  int64
+	count int
+}
+
 type track struct {
-	id                 uint32
+	id                 int
 	sampleDescriptions []sampleDescription
 	subType            string
-	stco               []uint64 //
+	stco               []int64
 	stsc               []stsc
-	stsz               []uint32
+	stsz               []stsz
 	formatInArg        interface{}
 	objectType         int // if data format is "mp4a"

@ -196,68 +202,86 @@ func mp4Decode(d *decode.D, in interface{}) interface{} {
 			d.FieldStruct("track", func(d *decode.D) {
 				// TODO: handle progressive/fragmented mp4 differently somehow?

-				trackSdDataFormat := "unknown"
+				trackSDDataFormat := "unknown"
 				if len(t.sampleDescriptions) > 0 {
 					sd := t.sampleDescriptions[0]
-					trackSdDataFormat = sd.dataFormat
+					trackSDDataFormat = sd.dataFormat
 					if sd.originalFormat != "" {
-						trackSdDataFormat = sd.originalFormat
+						trackSDDataFormat = sd.originalFormat
 					}
 				}

 				d.FieldArray("samples", func(d *decode.D) {
-					stscIndex := 0
-					chunkNr := uint32(0)
-					sampleNr := uint64(0)
+					// TODO: warning? could also be init fragment etc

-					for sampleNr < uint64(len(t.stsz)) {
-						if stscIndex >= len(t.stsc) {
-							// TODO: add warning
-							break
-						}
+					if len(t.stsz) > 0 && len(t.stsc) > 0 && len(t.stco) > 0 {
+						stszIndex := 0
+						stszEntryNr := 0
+						sampleNr := 0
+						stscIndex := 0
+						stscEntryNr := 0
+						stcoIndex := 0
+
+						stszEntry := t.stsz[stszIndex]
 						stscEntry := t.stsc[stscIndex]
-						if int(chunkNr) >= len(t.stco) {
-							// TODO: add warning
-							break
-						}
-						sampleOffset := t.stco[chunkNr]
+						sampleOffset := t.stco[stcoIndex]

-						for i := uint32(0); i < stscEntry.samplesPerChunk; i++ {
-							if int(sampleNr) >= len(t.stsz) {
-								// TODO: add warning
-								break
+						logStrFn := func() string {
+							return fmt.Sprintf("%d: %s: nr=%d: stsz[%d/%d] nr=%d %#v stsc[%d/%d] nr=%d %#v stco[%d/%d]=%d \n",
+								t.id,
+								trackSDDataFormat,
+								sampleNr,
+								stszIndex, len(t.stsz), stszEntryNr, stszEntry,
+								stscIndex, len(t.stsc), stscEntryNr, stscEntry,
+								stcoIndex, len(t.stco), sampleOffset,
+							)
+						}
+
+						for stszIndex < len(t.stsz) {
+							if stszEntryNr >= stszEntry.count {
+								stszIndex++
+								if stszIndex >= len(t.stsz) {
+									// TODO: warning if unused stsc/stco entries?
+									break
+								}
+
+								stszEntry = t.stsz[stszIndex]
+								stszEntryNr = 0
 							}

-							sampleSize := t.stsz[sampleNr]
-							decodeSampleRange(d, t, trackSdDataFormat, "sample", int64(sampleOffset)*8, int64(sampleSize)*8, t.formatInArg)
+							if stscEntryNr >= stscEntry.samplesPerChunk {
+								stscEntryNr = 0
+								stcoIndex++
+								if stcoIndex >= len(t.stco) {
+									d.Fatalf("outside stco: %s", logStrFn())
+								}
+								sampleOffset = t.stco[stcoIndex]

-							// log.Printf("%s %d/%d %d/%d sample=%d/%d chunk=%d size=%d %d-%d\n",
-							// 	trackSdDataFormat, stscIndex, len(t.stsc),
-							// 	i, stscEntry.samplesPerChunk,
-							// 	sampleNr, len(t.stsz),
-							// 	chunkNr,
-							// 	sampleSize,
-							// 	sampleOffset,
-							// 	sampleOffset+uint64(sampleSize))
+								if stscIndex < len(t.stsc)-1 && stcoIndex >= t.stsc[stscIndex+1].firstChunk-1 {
+									stscIndex++
+									if stscIndex >= len(t.stsc) {
+										d.Fatalf("outside stsc: %s", logStrFn())
+									}
+									stscEntry = t.stsc[stscIndex]
+								}
+							}

-							sampleOffset += uint64(sampleSize)
+							// log.Println(logStrFn())
+
+							decodeSampleRange(d, t, trackSDDataFormat, "sample", sampleOffset*8, stszEntry.size*8, t.formatInArg)
+
+							sampleOffset += stszEntry.size
+							stscEntryNr++
+							stszEntryNr++
 							sampleNr++
-
-						}
-
-						chunkNr++
-						if stscIndex < len(t.stsc)-1 && chunkNr >= t.stsc[stscIndex+1].firstChunk-1 {
-							stscIndex++
 						}
 					}

 					for _, m := range t.moofs {
-						sampleOffset := m.offset + int64(m.dataOffset)
+						sampleOffset := m.offset + m.dataOffset
 						for _, sz := range m.samplesSizes {
-							// log.Printf("moof sample %s %d-%d\n", t.dataFormat, sampleOffset, int64(sz))
-
-							dataFormat := trackSdDataFormat
-							if m.defaultSampleDescriptionIndex != 0 && int(m.defaultSampleDescriptionIndex-1) < len(t.sampleDescriptions) {
+							dataFormat := trackSDDataFormat
+							if m.defaultSampleDescriptionIndex != 0 && m.defaultSampleDescriptionIndex-1 < len(t.sampleDescriptions) {
 								sd := t.sampleDescriptions[m.defaultSampleDescriptionIndex-1]
 								dataFormat = sd.dataFormat
 								if sd.originalFormat != "" {
@ -265,10 +289,8 @@ func mp4Decode(d *decode.D, in interface{}) interface{} {
 								}
 							}

-							// log.Printf("moof %#+v dataFormat: %#+v\n", m, dataFormat)
-
-							decodeSampleRange(d, t, dataFormat, "sample", sampleOffset*8, int64(sz)*8, t.formatInArg)
-							sampleOffset += int64(sz)
+							decodeSampleRange(d, t, dataFormat, "sample", sampleOffset*8, sz*8, t.formatInArg)
+							sampleOffset += sz
 						}
 					}
 				})
--- a/format/mpeg/avc_nalu.go
+++ b/format/mpeg/avc_nalu.go
@ -5,7 +5,7 @@ package mpeg
 import (
 	"github.com/wader/fq/format"
 	"github.com/wader/fq/format/registry"
-	"github.com/wader/fq/internal/num"
+	"github.com/wader/fq/internal/mathextra"
 	"github.com/wader/fq/pkg/decode"
 	"github.com/wader/fq/pkg/scalar"
 )
@ -48,7 +48,7 @@ func uEV(d *decode.D) uint64 { return expGolomb(d) }

 func sEV(d *decode.D) int64 {
 	v := expGolomb(d) + 1
-	return num.ZigZag(v) - -int64(v&1)
+	return mathextra.ZigZag(v) - -int64(v&1)
 }

 const (
--- a/format/msgpack/msgpack.go
+++ b/format/msgpack/msgpack.go
@ -100,7 +100,7 @@ func decodeMsgPackValue(d *decode.D) {
 			d.FieldUTF8("value", int(length))
 		}},
 		{r: [2]byte{0xc0, 0xc0}, s: scalar.S{Sym: "nil"}, d: func(d *decode.D) {
-			// TODO: fq has no good null type atm
+			d.FieldValueNil("value")
 		}},
 		{r: [2]byte{0xc1, 0xc1}, s: scalar.S{Sym: "never_used"}, d: func(d *decode.D) {
 			d.Fatalf("0xc1 never used")
--- a/format/msgpack/testdata/test.fqtest
+++ b/format/msgpack/testdata/test.fqtest
@ -84,6 +84,7 @@ $ fq -d msgpack v test.msgpack
 0x40|6c 6c                                          |ll              |
    |                                               |                |      value{}: 0x42-0x42.7 (1)
 0x40|      c0|                                      |  .|            |        type: "nil" (0xc0) 0x42-0x42.7 (1)
+    |                                               |                |        value: null 0x43-NA (0)
 $ fq -d msgpack torepr test.msgpack
 {
  "array": [
--- a/format/protobuf/protobuf.go
+++ b/format/protobuf/protobuf.go
@ -3,7 +3,7 @@ package protobuf
 import (
 	"github.com/wader/fq/format"
 	"github.com/wader/fq/format/registry"
-	"github.com/wader/fq/internal/num"
+	"github.com/wader/fq/internal/mathextra"
 	"github.com/wader/fq/pkg/decode"
 	"github.com/wader/fq/pkg/scalar"
 )
@ -74,7 +74,7 @@ func protobufDecodeField(d *decode.D, pbm *format.ProtoBufMessage) {

 				switch pbf.Type {
 				case format.ProtoBufTypeInt32, format.ProtoBufTypeInt64:
-					v := num.ZigZag(value)
+					v := mathextra.ZigZag(value)
 					d.FieldValueS("value", v)
 					if len(pbf.Enums) > 0 {
 						d.FieldValueStr("enum", pbf.Enums[uint64(v)])
@ -86,7 +86,7 @@ func protobufDecodeField(d *decode.D, pbm *format.ProtoBufMessage) {
 					}
 				case format.ProtoBufTypeSInt32, format.ProtoBufTypeSInt64:
 					// TODO: correct? 32 different?
-					v := num.TwosComplement(64, value)
+					v := mathextra.TwosComplement(64, value)
 					d.FieldValueS("value", v)
 					if len(pbf.Enums) > 0 {
 						d.FieldValueStr("enum", pbf.Enums[uint64(v)])
--- a/format/tar/tar.go
+++ b/format/tar/tar.go
@ -50,7 +50,9 @@ func tarDecode(d *decode.D, in interface{}) interface{} {

 	// end marker is 512*2 zero bytes
 	endMarker := [blockBytes * 2]byte{}
-	foundEndMarker := false
+	var endMarkerStart int64
+	var endMarkerEnd int64
+	filesCount := 0

 	d.FieldArray("files", func(d *decode.D) {
 		for !d.End() {
@ -68,10 +70,7 @@ func tarDecode(d *decode.D, in interface{}) interface{} {
 				d.FieldUTF8NullFixedLen("chksum", 8, mapOctStrToSymU)
 				d.FieldUTF8("typeflag", 1, mapTrimSpaceNull)
 				d.FieldUTF8("linkname", 100, mapTrimSpaceNull)
-				magic := d.FieldUTF8("magic", 6, mapTrimSpaceNull)
-				if magic != "ustar" {
-					d.Errorf("invalid magic %s", magic)
-				}
+				d.FieldUTF8("magic", 6, mapTrimSpaceNull, d.AssertStr("ustar"))
 				d.FieldUTF8NullFixedLen("version", 2, mapOctStrToSymU)
 				d.FieldUTF8("uname", 32, mapTrimSpaceNull)
 				d.FieldUTF8("gname", 32, mapTrimSpaceNull)
@ -87,17 +86,29 @@ func tarDecode(d *decode.D, in interface{}) interface{} {

 				d.FieldRawLen("data_block_padding", blockPadding(d), d.BitBufIsZero())
 			})
+			filesCount++

-			bs := d.PeekBytes(blockBytes * 2)
-			if bytes.Equal(bs, endMarker[:]) {
-				foundEndMarker = true
+			if d.BitsLeft() >= int64(len(endMarker))*8 && bytes.Equal(d.PeekBytes(len(endMarker)), endMarker[:]) {
+				endMarkerStart = d.Pos()
+				// consensus seems to be to allow more than 2 zero blocks at end
+				d.SeekRel(int64(len(endMarker)) * 8)
+				zeroBlock := [blockBytes]byte{}
+				for d.BitsLeft() >= blockBytes*8 && bytes.Equal(d.PeekBytes(blockBytes), zeroBlock[:]) {
+					d.SeekRel(int64(len(zeroBlock)) * 8)
+				}
+				endMarkerEnd = d.Pos()
 				break
 			}
 		}
 	})
-	d.FieldRawLen("end_marker", int64(len(endMarker))*8)
+	endMarkerSize := endMarkerEnd - endMarkerStart
+	if endMarkerSize > 0 {
+		d.RangeFn(endMarkerStart, endMarkerSize, func(d *decode.D) {
+			d.FieldRawLen("end_marker", d.BitsLeft())
+		})
+	}

-	if !foundEndMarker {
+	if filesCount == 0 {
 		d.Errorf("no files found")
 	}

--- a/format/tar/testdata/no_end_marker.fqtest
+++ b/format/tar/testdata/no_end_marker.fqtest
@ -0,0 +1,39 @@
+# fq 'tobytes[0:.end_marker | tobytesrange.start]' test.tar > no_end_marker.tar
+$ fq v no_end_marker.tar
+     |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: no_end_marker.tar (tar) 0x0-0x3ff.7 (1024)
+     |                                               |                |  files[0:1]: 0x0-0x3ff.7 (1024)
+     |                                               |                |    [0]{}: file 0x0-0x3ff.7 (1024)
+0x000|74 65 73 74 00 00 00 00 00 00 00 00 00 00 00 00|test............|      name: "test" 0x0-0x63.7 (100)
+*    |until 0x63.7 (100)                             |                |
+0x060|            30 30 30 36 34 34 20 00            |    000644 .    |      mode: 420 ("000644 ") 0x64-0x6b.7 (8)
+0x060|                                    30 30 30 37|            0007|      uid: 501 ("000765 ") 0x6c-0x73.7 (8)
+0x070|36 35 20 00                                    |65 .            |
+0x070|            30 30 30 30 32 34 20 00            |    000024 .    |      gid: 20 ("000024 ") 0x74-0x7b.7 (8)
+0x070|                                    30 30 30 30|            0000|      size: 6 ("00000000006 ") 0x7c-0x87.7 (12)
+0x080|30 30 30 30 30 30 36 20                        |0000006         |
+0x080|                        31 34 31 33 33 36 32 35|        14133625|      mtime: 1634675538 ("14133625522 ") 0x88-0x93.7 (12)
+0x090|35 32 32 20                                    |522             |
+0x090|            30 31 32 32 32 34 00 20            |    012224.     |      chksum: 5268 ("012224") 0x94-0x9b.7 (8)
+0x090|                                    30         |            0   |      typeflag: "0" 0x9c-0x9c.7 (1)
+0x090|                                       00 00 00|             ...|      linkname: "" 0x9d-0x100.7 (100)
+0x0a0|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
+*    |until 0x100.7 (100)                            |                |
+0x100|   75 73 74 61 72 00                           | ustar.         |      magic: "ustar" (valid) 0x101-0x106.7 (6)
+0x100|                     30 30                     |       00       |      version: 0 ("00") 0x107-0x108.7 (2)
+0x100|                           77 61 64 65 72 00 00|         wader..|      uname: "wader" 0x109-0x128.7 (32)
+0x110|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
+0x120|00 00 00 00 00 00 00 00 00                     |.........       |
+0x120|                           73 74 61 66 66 00 00|         staff..|      gname: "staff" 0x129-0x148.7 (32)
+0x130|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
+0x140|00 00 00 00 00 00 00 00 00                     |.........       |
+0x140|                           30 30 30 30 30 30 20|         000000 |      devmajor: 0 ("000000 ") 0x149-0x150.7 (8)
+0x150|00                                             |.               |
+0x150|   30 30 30 30 30 30 20 00                     | 000000 .       |      devminor: 0 ("000000 ") 0x151-0x158.7 (8)
+0x150|                           00 00 00 00 00 00 00|         .......|      prefix: "" 0x159-0x1f3.7 (155)
+0x160|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
+*    |until 0x1f3.7 (155)                            |                |
+0x1f0|            00 00 00 00 00 00 00 00 00 00 00 00|    ............|      header_block_padding: raw bits (all zero) 0x1f4-0x1ff.7 (12)
+0x200|68 65 6c 6c 6f 0a                              |hello.          |      data: raw bits 0x200-0x205.7 (6)
+0x200|                  00 00 00 00 00 00 00 00 00 00|      ..........|      data_block_padding: raw bits (all zero) 0x206-0x3ff.7 (506)
+0x210|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
+*    |until 0x3ff.7 (end) (506)                      |                |
--- a/format/tar/testdata/no_end_marker.tar
+++ b/format/tar/testdata/no_end_marker.tar
--- a/format/tar/testdata/tar.fqtest
+++ b/format/tar/testdata/tar.fqtest
@ -19,7 +19,7 @@ $ fq -d tar v /test.tar
 0x0090|                                       00 00 00|             ...|      linkname: "" 0x9d-0x100.7 (100)
 0x00a0|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
 *     |until 0x100.7 (100)                            |                |
-0x0100|   75 73 74 61 72 00                           | ustar.         |      magic: "ustar" 0x101-0x106.7 (6)
+0x0100|   75 73 74 61 72 00                           | ustar.         |      magic: "ustar" (valid) 0x101-0x106.7 (6)
 0x0100|                     30 30                     |       00       |      version: 0 ("00") 0x107-0x108.7 (2)
 0x0100|                           77 61 64 65 72 00 00|         wader..|      uname: "wader" 0x109-0x128.7 (32)
 0x0110|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
@ -38,7 +38,5 @@ $ fq -d tar v /test.tar
 0x0200|                  00 00 00 00 00 00 00 00 00 00|      ..........|      data_block_padding: raw bits (all zero) 0x206-0x3ff.7 (506)
 0x0210|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
 *     |until 0x3ff.7 (506)                            |                |
-0x0400|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|  end_marker: raw bits 0x400-0x7ff.7 (1024)
-*     |until 0x7ff.7 (1024)                           |                |
-0x0800|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|  unknown0: raw bits 0x800-0x27ff.7 (8192)
-*     |until 0x27ff.7 (end) (8192)                    |                |
+0x0400|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|  end_marker: raw bits 0x400-0x27ff.7 (9216)
+*     |until 0x27ff.7 (end) (9216)                    |                |
--- a/BIN
+++ b/BIN
--- a/fq.go
+++ b/fq.go
@ -6,7 +6,7 @@ import (
 	"github.com/wader/fq/pkg/cli"
 )

-var version = "dev"
+const version = "0.0.3"

 func main() {
 	cli.Main(registry.Default, version)
--- a/go.mod
+++ b/go.mod
@ -25,7 +25,7 @@ require (
 	// fork of github.com/itchyny/gojq, see github.com/wader/gojq fq branch
 	github.com/wader/gojq v0.12.1-0.20220108235115-6a05b6c59ace
 	// fork of github.com/chzyer/readline, see github.com/wader/readline fq branch
-	github.com/wader/readline v0.0.0-20210920124728-5a81f7707bac
+	github.com/wader/readline v0.0.0-20220117233529-692d84ca36e2
 )

 require (
--- a/go.sum
+++ b/go.sum
@ -1,8 +1,3 @@
-github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
@ -12,14 +7,12 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs=
 github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354/go.mod h1:KSVJerMDfblTH7p5MZaTt+8zaT2iEk3AkVb9PQdZuE8=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/wader/gojq v0.12.1-0.20220108235115-6a05b6c59ace h1:pt07NaC7OhePrQVRKRxZy9umeWkjr28AmbtQC9CrtVQ=
 github.com/wader/gojq v0.12.1-0.20220108235115-6a05b6c59ace/go.mod h1:tdC5h6dXdwAJs7eJUw4681AzsgfOSBrAV+cZzEbCZs4=
-github.com/wader/readline v0.0.0-20210920124728-5a81f7707bac h1:F5x54dwg6vGyf+8XhujiyXr651E3tKpcL1mqGmS7/MU=
-github.com/wader/readline v0.0.0-20210920124728-5a81f7707bac/go.mod h1:jYXyt9wQg3DifxQ8FM5M/ZoskO23GIwmo05QLHtO9CQ=
+github.com/wader/readline v0.0.0-20220117233529-692d84ca36e2 h1:AK4wt6mSypGEVAzUcCfrJqVD5hju+w81b9J/k0swV/8=
+github.com/wader/readline v0.0.0-20220117233529-692d84ca36e2/go.mod h1:TJUJCkylZhI0Z07t2Nw6l6Ck7NiZqUpnMlkjEzN7+yM=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
--- a/internal/gojqextra/totype.go
+++ b/internal/gojqextra/totype.go
@ -87,6 +87,7 @@ func ToInt(x interface{}) (int, bool) {
 	case gojq.JQValue:
 		return ToInt(x.JQValueToGoJQ())
 	default:
+		// nil and other should fail, "null | tonumber" in jq is an error
 		return 0, false
 	}
 }
@ -112,6 +113,7 @@ func ToFloat(x interface{}) (float64, bool) {
 	case gojq.JQValue:
 		return ToFloat(x.JQValueToGoJQ())
 	default:
+		// nil and other should fail, "null | tonumber" in jq is an error
 		return 0.0, false
 	}
 }
--- a/internal/hexdump/hexdump.go
+++ b/internal/hexdump/hexdump.go
@ -5,7 +5,7 @@ import (
 	"strings"

 	"github.com/wader/fq/internal/columnwriter"
-	"github.com/wader/fq/internal/num"
+	"github.com/wader/fq/internal/mathextra"
 	"github.com/wader/fq/pkg/bitio"
 )

@ -60,7 +60,7 @@ func New(w io.Writer, startOffset int64, addrLen int, addrBase int, lineBytes in

 func (d *Dumper) flush() error {
 	if _, err := d.columnW.Columns[0].Write([]byte(
-		d.dumpAddrFn(num.PadFormatInt(((d.offset-1)/d.lineBytes)*d.lineBytes, d.addrBase, true, d.addrLen)))); err != nil {
+		d.dumpAddrFn(mathextra.PadFormatInt(((d.offset-1)/d.lineBytes)*d.lineBytes, d.addrBase, true, d.addrLen)))); err != nil {
 		return err
 	}
 	if _, err := d.separatorsW.Write([]byte(d.column)); err != nil {
@ -76,7 +76,7 @@ func (d *Dumper) WriteBits(p []byte, nBits int) (n int, err error) {
 	pos := 0
 	rBits := nBits
 	if d.bitsBufN > 0 {
-		r := num.MinInt(8-d.bitsBufN, nBits)
+		r := mathextra.MinInt(8-d.bitsBufN, nBits)
 		v := bitio.Read64(p, 0, r)
 		bitio.Write64(v, r, d.bitsBuf, d.bitsBufN)

@ -122,7 +122,7 @@ func (d *Dumper) Write(p []byte) (n int, err error) {
 		}
 		for i := int64(0); i < d.lineBytes; i++ {
 			headerSB := &strings.Builder{}
-			if _, err := headerSB.Write([]byte(num.PadFormatInt(i, d.addrBase, false, 2))); err != nil {
+			if _, err := headerSB.Write([]byte(mathextra.PadFormatInt(i, d.addrBase, false, 2))); err != nil {
 				return 0, err
 			}
 			if i < d.lineBytes-1 {
--- a/internal/hexpairwriter/hexpairwriter.go
+++ b/internal/hexpairwriter/hexpairwriter.go
@ -5,7 +5,7 @@ package hexpairwriter
 import (
 	"io"

-	"github.com/wader/fq/internal/num"
+	"github.com/wader/fq/internal/mathextra"
 	"github.com/wader/fq/pkg/bitio"
 )

@ -115,7 +115,7 @@ func (h *Writer) WriteBits(p []byte, nBits int) (n int, err error) {
 	pos := 0
 	rBits := nBits
 	if h.bitsBufN > 0 {
-		r := num.MinInt(8-h.bitsBufN, nBits)
+		r := mathextra.MinInt(8-h.bitsBufN, nBits)
 		v := bitio.Read64(p, 0, r)
 		bitio.Write64(v, r, h.bitsBuf, h.bitsBufN)

--- a/internal/mathextra/big.go
+++ b/internal/mathextra/big.go
@ -0,0 +1,13 @@
+package mathextra
+
+import "math/big"
+
+var BigIntOne = big.NewInt(1)
+
+func BigIntSetBytesSigned(n *big.Int, buf []byte) *big.Int {
+	n.SetBytes(buf)
+	if len(buf) > 0 && buf[0]&0x80 > 0 {
+		n.Sub(n, new(big.Int).Lsh(BigIntOne, uint(len(buf))*8))
+	}
+	return n
+}
--- a/internal/mathextra/big_test.go
+++ b/internal/mathextra/big_test.go
@ -0,0 +1,37 @@
+package mathextra_test
+
+import (
+	"fmt"
+	"math/big"
+	"testing"
+
+	"github.com/wader/fq/internal/mathextra"
+)
+
+func TestBigIntSetBytesSigned(t *testing.T) {
+	testCases := []struct {
+		buf []byte
+		s   string
+	}{
+		{[]byte{1}, "1"},
+		{[]byte{0b1111_1111}, "-1"},
+		{[]byte{0b1000_0000}, "-128"},
+		{[]byte{0b0111_1111}, "127"},
+		{[]byte{0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, "9223372036854775807"},
+		{[]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, "-1"},
+		{[]byte{0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, "-9223372036854775808"},
+		{[]byte{0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, "2361183241434822606847"},
+		{[]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, "-1"},
+		{[]byte{0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, "-2361183241434822606848"},
+	}
+	for _, tC := range testCases {
+		t.Run(fmt.Sprintf("%v %s", tC.buf, tC.s), func(t *testing.T) {
+			var n big.Int
+			expected := tC.s
+			actual := mathextra.BigIntSetBytesSigned(&n, tC.buf).String()
+			if expected != actual {
+				t.Errorf("expected %s, got %s", expected, actual)
+			}
+		})
+	}
+}
--- a/internal/mathextra/float16.go
+++ b/internal/mathextra/float16.go
@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 //nolint:revive
-package num
+package mathextra

 import "unsafe"

--- a/internal/mathextra/num.go
+++ b/internal/mathextra/num.go
@ -1,8 +1,9 @@
-package num
+package mathextra

 import (
 	"fmt"
 	"math"
+	"math/big"
 	"strconv"
 	"strings"

@ -46,8 +47,12 @@ func PadFormatInt(i int64, base int, basePrefix bool, width int) string {

 func PadFormatUint(i uint64, base int, basePrefix bool, width int) string {
 	return padFormatNumber(strconv.FormatUint(i, base), base, basePrefix, width)
-
 }
+
+func PadFormatBigInt(i *big.Int, base int, basePrefix bool, width int) string {
+	return padFormatNumber(i.Text(base), base, basePrefix, width)
+}
+
 func MaxUInt64(a, b uint64) uint64 {
 	if a < b {
 		return b
--- a/pkg/decode/decode.go
+++ b/pkg/decode/decode.go
@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/big"

 	"github.com/wader/fq/internal/recoverfn"
 	"github.com/wader/fq/pkg/bitio"
@ -636,6 +637,10 @@ func (d *D) FieldValueS(name string, a int64, sms ...scalar.Mapper) {
 	d.FieldScalarFn(name, func(_ scalar.S) (scalar.S, error) { return scalar.S{Actual: a}, nil }, sms...)
 }

+func (d *D) FieldValueBigInt(name string, a *big.Int, sms ...scalar.Mapper) {
+	d.FieldScalarFn(name, func(_ scalar.S) (scalar.S, error) { return scalar.S{Actual: a}, nil }, sms...)
+}
+
 func (d *D) FieldValueBool(name string, a bool, sms ...scalar.Mapper) {
 	d.FieldScalarFn(name, func(_ scalar.S) (scalar.S, error) { return scalar.S{Actual: a}, nil }, sms...)
 }
@ -648,6 +653,10 @@ func (d *D) FieldValueStr(name string, a string, sms ...scalar.Mapper) {
 	d.FieldScalarFn(name, func(_ scalar.S) (scalar.S, error) { return scalar.S{Actual: a}, nil }, sms...)
 }

+func (d *D) FieldValueNil(name string, sms ...scalar.Mapper) {
+	d.FieldScalarFn(name, func(_ scalar.S) (scalar.S, error) { return scalar.S{Actual: nil}, nil }, sms...)
+}
+
 func (d *D) FieldValueRaw(name string, a []byte, sms ...scalar.Mapper) {
 	d.FieldScalarFn(name, func(_ scalar.S) (scalar.S, error) {
 		return scalar.S{Actual: bitio.NewBufferFromBytes(a, -1)}, nil
--- a/pkg/decode/decode_gen.go
+++ b/pkg/decode/decode_gen.go
--- a/pkg/decode/decode_gen.go.tmpl
+++ b/pkg/decode/decode_gen.go.tmpl
@ -3,6 +3,7 @@ package decode

 import (
 	"fmt"
+	"math/big"

 	"github.com/wader/fq/pkg/bitio"
 	"github.com/wader/fq/pkg/scalar"
--- a/pkg/decode/errors.go
+++ b/pkg/decode/errors.go
@ -4,7 +4,7 @@ import (
 	"fmt"
 	"strings"

-	"github.com/wader/fq/internal/num"
+	"github.com/wader/fq/internal/mathextra"
 	"github.com/wader/fq/internal/recoverfn"
 )

@ -72,7 +72,7 @@ func (e IOError) Error() string {
 	}

 	return fmt.Sprintf("%s: failed at position %s (read size %s seek pos %s): %s",
-		prefix, num.Bits(e.Pos).StringByteBits(10), num.Bits(e.ReadSize).StringByteBits(10), num.Bits(e.SeekPos).StringByteBits(10), e.Err)
+		prefix, mathextra.Bits(e.Pos).StringByteBits(10), mathextra.Bits(e.ReadSize).StringByteBits(10), mathextra.Bits(e.SeekPos).StringByteBits(10), e.Err)
 }
 func (e IOError) Unwrap() error { return e.Err }

@ -84,7 +84,7 @@ type DecoderError struct {
 }

 func (e DecoderError) Error() string {
-	return fmt.Sprintf("error at position %s: %s", num.Bits(e.Pos).StringByteBits(16), e.Reason)
+	return fmt.Sprintf("error at position %s: %s", mathextra.Bits(e.Pos).StringByteBits(16), e.Reason)
 }

 func (DecoderError) IsRecoverableError() bool { return true }
--- a/pkg/decode/read.go
+++ b/pkg/decode/read.go
@ -4,14 +4,22 @@ import (
 	"bytes"
 	"fmt"
 	"math"
+	"math/big"

-	"github.com/wader/fq/internal/num"
+	"github.com/wader/fq/internal/mathextra"
 	"github.com/wader/fq/pkg/bitio"
 	"golang.org/x/text/encoding"
 	"golang.org/x/text/encoding/unicode"
 )

-func (d *D) tryUE(nBits int, endian Endian) (uint64, error) {
+func (d *D) tryBitBuf(nBits int64) (*bitio.Buffer, error) {
+	return d.bitBuf.BitBufLen(nBits)
+}
+
+func (d *D) tryUEndian(nBits int, endian Endian) (uint64, error) {
+	if nBits < 0 {
+		return 0, fmt.Errorf("tryUEndian nBits must be >= 0 (%d)", nBits)
+	}
 	n, err := d.bits(nBits)
 	if err != nil {
 		return 0, err
@ -23,21 +31,14 @@ func (d *D) tryUE(nBits int, endian Endian) (uint64, error) {
 	return n, nil
 }

-func (d *D) tryBitBuf(nBits int64) (*bitio.Buffer, error) {
-	return d.bitBuf.BitBufLen(nBits)
-}
-
-func (d *D) trySE(nBits int, endian Endian) (int64, error) {
-	n, err := d.bits(nBits)
+func (d *D) trySEndian(nBits int, endian Endian) (int64, error) {
+	if nBits < 0 {
+		return 0, fmt.Errorf("trySEndian nBits must be >= 0 (%d)", nBits)
+	}
+	n, err := d.tryUEndian(nBits, endian)
 	if err != nil {
 		return 0, err
 	}
-	if nBits == 0 {
-		return 0, nil
-	}
-	if endian == LittleEndian {
-		n = bitio.Uint64ReverseBytes(nBits, n)
-	}
 	var s int64
 	if n&(1<<(nBits-1)) > 0 {
 		// two's complement
@ -49,7 +50,44 @@ func (d *D) trySE(nBits int, endian Endian) (int64, error) {
 	return s, nil
 }

-func (d *D) tryFE(nBits int, endian Endian) (float64, error) {
+// from https://github.com/golang/go/wiki/SliceTricks#reversing
+func reverseBytes(a []byte) {
+	for i := len(a)/2 - 1; i >= 0; i-- {
+		opp := len(a) - 1 - i
+		a[i], a[opp] = a[opp], a[i]
+	}
+}
+
+func (d *D) tryBigIntEndianSign(nBits int, endian Endian, sign bool) (*big.Int, error) {
+	if nBits < 0 {
+		return nil, fmt.Errorf("tryBigIntEndianSign nBits must be >= 0 (%d)", nBits)
+	}
+	b := int(bitio.BitsByteCount(int64(nBits)))
+	buf := d.SharedReadBuf(b)[0:b]
+	_, err := bitio.ReadFull(d.bitBuf, buf, nBits)
+	if err != nil {
+		return nil, err
+	}
+
+	if endian == LittleEndian {
+		reverseBytes(buf)
+	}
+
+	n := new(big.Int)
+	if sign {
+		mathextra.BigIntSetBytesSigned(n, buf)
+	} else {
+		n.SetBytes(buf)
+	}
+	n.Rsh(n, uint((8-nBits%8)%8))
+
+	return n, nil
+}
+
+func (d *D) tryFEndian(nBits int, endian Endian) (float64, error) {
+	if nBits < 0 {
+		return 0, fmt.Errorf("tryFEndian nBits must be >= 0 (%d)", nBits)
+	}
 	n, err := d.bits(nBits)
 	if err != nil {
 		return 0, err
@ -59,7 +97,7 @@ func (d *D) tryFE(nBits int, endian Endian) (float64, error) {
 	}
 	switch nBits {
 	case 16:
-		return float64(num.Float16(uint16(n)).Float32()), nil
+		return float64(mathextra.Float16(uint16(n)).Float32()), nil
 	case 32:
 		return float64(math.Float32frombits(uint32(n))), nil
 	case 64:
@ -69,7 +107,10 @@ func (d *D) tryFE(nBits int, endian Endian) (float64, error) {
 	}
 }

-func (d *D) tryFPE(nBits int, fBits int, endian Endian) (float64, error) {
+func (d *D) tryFPEndian(nBits int, fBits int, endian Endian) (float64, error) {
+	if nBits < 0 {
+		return 0, fmt.Errorf("tryFPEndian nBits must be >= 0 (%d)", nBits)
+	}
 	n, err := d.bits(nBits)
 	if err != nil {
 		return 0, err
--- a/pkg/decode/types.json
+++ b/pkg/decode/types.json
@ -5,7 +5,8 @@
        "S": {"go_type": "int64", "zero": "0", "compare": "a == b", "range": "a >= start && a <= end"},
        "F": {"go_type": "float64", "map_from": false, "zero": "0", "compare": "a == b", "range": "a >= start && a <= end"},
        "Bool": {"go_type": "bool", "zero": "false", "compare": "a == b"},
-        "BitBuf": {"go_type": "*bitio.Buffer", "zero": "nil", "map_from": false, "map_to": false}
+        "BitBuf": {"go_type": "*bitio.Buffer", "zero": "nil", "map_from": false, "map_to": false},
+        "BigInt": {"go_type": "*big.Int", "zero": "nil", "map_from": false, "map_to": false, "compare": "a.Cmp(b) == 0", "range": "a.Cmp(start) >= 0 && a.Cmp(end) <= 0"}
    },
    "readers": [
        {
@ -22,51 +23,83 @@
            "name": "U",
            "type": "U",
            "variants": [
-                {"name": "", "args": "nBits", "params": "nBits int", "call": "d.tryUE(nBits, d.Endian)", "doc": "nBits bits unsigned integer in current endian"},
+                {"name": "", "args": "nBits", "params": "nBits int", "call": "d.tryUEndian(nBits, d.Endian)", "doc": "nBits bits unsigned integer in current endian"},
                {
                    "name": "E",
                    "args": "nBits, endian",
                    "params": "nBits int, endian Endian",
-                    "call": "d.tryUE(nBits, endian)",
+                    "call": "d.tryUEndian(nBits, endian)",
                    "doc": "nBits unsigned integer in specified endian"
                },
-                { "name": "$n", "range": [1, 64], "args": "", "params": "", "call": "d.tryUE($n, d.Endian)", "doc": "$n bit unsigned integer in current endian" },
-                { "name": "$nLE", "range": [8, 64], "args": "", "params": "", "call": "d.tryUE($n, LittleEndian)", "doc": "$n bit unsigned integer in little-endian" },
-                { "name": "$nBE", "range": [8, 64], "args": "", "params": "", "call": "d.tryUE($n, BigEndian)", "doc": "$n bit unsigned integer in big-endian" }
+                { "name": "$n", "range": [1, 64], "args": "", "params": "", "call": "d.tryUEndian($n, d.Endian)", "doc": "$n bit unsigned integer in current endian" },
+                { "name": "$nLE", "range": [8, 64], "args": "", "params": "", "call": "d.tryUEndian($n, LittleEndian)", "doc": "$n bit unsigned integer in little-endian" },
+                { "name": "$nBE", "range": [8, 64], "args": "", "params": "", "call": "d.tryUEndian($n, BigEndian)", "doc": "$n bit unsigned integer in big-endian" }
            ]
        },
        {
            "name": "S",
            "type": "S",
            "variants": [
-                {"name": "", "args": "nBits", "params": "nBits int", "call": "d.trySE(nBits, d.Endian)", "doc": "nBits bits signed integer in current endian"},
+                {"name": "", "args": "nBits", "params": "nBits int", "call": "d.trySEndian(nBits, d.Endian)", "doc": "nBits bits signed integer in current endian"},
                {
                    "name": "E",
                    "args": "nBits, endian",
                    "params": "nBits int, endian Endian",
-                    "call": "d.trySE(nBits, endian)",
+                    "call": "d.trySEndian(nBits, endian)",
                    "doc": "nBits signed integer in specified endian"
                },
-                { "name": "$n", "range": [1, 64], "args": "", "params": "", "call": "d.trySE($n, d.Endian)", "doc": "$n bit signed integer in current endian" },
-                { "name": "$nLE", "range": [8, 64], "args": "", "params": "", "call": "d.trySE($n, LittleEndian)", "doc": "$n bit signed integer in little-endian" },
-                { "name": "$nBE", "range": [8, 64], "args": "", "params": "", "call": "d.trySE($n, BigEndian)", "doc": "$n bit signed integer in big-endian" }
+                { "name": "$n", "range": [1, 64], "args": "", "params": "", "call": "d.trySEndian($n, d.Endian)", "doc": "$n bit signed integer in current endian" },
+                { "name": "$nLE", "range": [8, 64], "args": "", "params": "", "call": "d.trySEndian($n, LittleEndian)", "doc": "$n bit signed integer in little-endian" },
+                { "name": "$nBE", "range": [8, 64], "args": "", "params": "", "call": "d.trySEndian($n, BigEndian)", "doc": "$n bit signed integer in big-endian" }
+            ]
+        },
+        {
+            "name": "UBigInt",
+            "type": "BigInt",
+            "variants": [
+                {"name": "", "args": "nBits", "params": "nBits int", "call": "d.tryBigIntEndianSign(nBits, d.Endian, false)", "doc": "nBits bits signed integer in current endian"},
+                {
+                    "name": "E",
+                    "args": "nBits, endian",
+                    "params": "nBits int, endian Endian",
+                    "call": "d.tryBigIntEndianSign(nBits, endian, false)",
+                    "doc": "nBits signed integer in specified endian"
+                },
+                {"name": "LE", "args": "nBits", "params": "nBits int", "call": "d.tryBigIntEndianSign(nBits, LittleEndian, false)", "doc": "nBits bit signed integer in little-endian"},
+                {"name": "BE", "args": "nBits", "params": "nBits int", "call": "d.tryBigIntEndianSign(nBits, BigEndian, false)", "doc": "nBits bit signed integer in big-endian"}
+            ]
+        },
+        {
+            "name": "SBigInt",
+            "type": "BigInt",
+            "variants": [
+                {"name": "", "args": "nBits", "params": "nBits int", "call": "d.tryBigIntEndianSign(nBits, d.Endian, true)", "doc": "nBits bits signed integer in current endian"},
+                {
+                    "name": "E",
+                    "args": "nBits, endian",
+                    "params": "nBits int, endian Endian",
+                    "call": "d.tryBigIntEndianSign(nBits, endian, true)",
+                    "doc": "nBits signed integer in specified endian"
+                },
+                {"name": "LE", "args": "nBits", "params": "nBits int", "call": "d.tryBigIntEndianSign(nBits, LittleEndian, true)", "doc": "nBits bit signed integer in little-endian"},
+                {"name": "BE", "args": "nBits", "params": "nBits int", "call": "d.tryBigIntEndianSign(nBits, BigEndian, true)", "doc": "nBits bit signed integer in big-endian"}
            ]
        },
        {
            "name": "F",
            "type": "F",
            "variants": [
-                {"name": "", "args": "nBits", "params": "nBits int", "call": "d.tryFE(nBits, d.Endian)", "doc": "nBit IEEE 754 float in current endian"},
-                {"name": "E", "args": "nBits, endian", "params": "nBits int, endian Endian", "call": "d.tryFE(nBits, endian)", "doc": "nBit IEEE 754 float in specified endian"},
-                {"name": "16", "args": "", "params": "", "call": "d.tryFE(16, d.Endian)", "doc": "16 bit IEEE 754 float in current endian"},
-                {"name": "32", "args": "", "params": "", "call": "d.tryFE(32, d.Endian)", "doc": "32 bit IEEE 754 float in current endian"},
-                {"name": "64", "args": "", "params": "", "call": "d.tryFE(64, d.Endian)", "doc": "64 bit IEEE 754 float in current endian"},
-                {"name": "16LE", "args": "", "params": "", "call": "d.tryFE(16, LittleEndian)", "doc": "16 bit IEEE 754 float in little-endian"},
-                {"name": "32LE", "args": "", "params": "", "call": "d.tryFE(32, LittleEndian)", "doc": "32 bit IEEE 754 float in little-endian"},
-                {"name": "64LE", "args": "", "params": "", "call": "d.tryFE(64, LittleEndian)", "doc": "64 bit IEEE 754 float in little-endian"},
-                {"name": "16BE", "args": "", "params": "", "call": "d.tryFE(16, BigEndian)", "doc": "16 bit IEEE 754 float in big-endian"},
-                {"name": "32BE", "args": "", "params": "", "call": "d.tryFE(32, BigEndian)", "doc": "32 bit IEEE 754 float in big-endian"},
-                {"name": "64BE", "args": "", "params": "", "call": "d.tryFE(64, BigEndian)", "doc": "64 bit IEEE 754 float in big-endian"}
+                {"name": "", "args": "nBits", "params": "nBits int", "call": "d.tryFEndian(nBits, d.Endian)", "doc": "nBit IEEE 754 float in current endian"},
+                {"name": "E", "args": "nBits, endian", "params": "nBits int, endian Endian", "call": "d.tryFEndian(nBits, endian)", "doc": "nBit IEEE 754 float in specified endian"},
+                {"name": "16", "args": "", "params": "", "call": "d.tryFEndian(16, d.Endian)", "doc": "16 bit IEEE 754 float in current endian"},
+                {"name": "32", "args": "", "params": "", "call": "d.tryFEndian(32, d.Endian)", "doc": "32 bit IEEE 754 float in current endian"},
+                {"name": "64", "args": "", "params": "", "call": "d.tryFEndian(64, d.Endian)", "doc": "64 bit IEEE 754 float in current endian"},
+                {"name": "16LE", "args": "", "params": "", "call": "d.tryFEndian(16, LittleEndian)", "doc": "16 bit IEEE 754 float in little-endian"},
+                {"name": "32LE", "args": "", "params": "", "call": "d.tryFEndian(32, LittleEndian)", "doc": "32 bit IEEE 754 float in little-endian"},
+                {"name": "64LE", "args": "", "params": "", "call": "d.tryFEndian(64, LittleEndian)", "doc": "64 bit IEEE 754 float in little-endian"},
+                {"name": "16BE", "args": "", "params": "", "call": "d.tryFEndian(16, BigEndian)", "doc": "16 bit IEEE 754 float in big-endian"},
+                {"name": "32BE", "args": "", "params": "", "call": "d.tryFEndian(32, BigEndian)", "doc": "32 bit IEEE 754 float in big-endian"},
+                {"name": "64BE", "args": "", "params": "", "call": "d.tryFEndian(64, BigEndian)", "doc": "64 bit IEEE 754 float in big-endian"}
            ]
        },
        {
@ -77,25 +110,25 @@
                    "name": "",
                    "args": "nBits, fBits",
                    "params": "nBits int, fBits int",
-                    "call": "d.tryFPE(nBits, fBits, d.Endian)",
+                    "call": "d.tryFPEndian(nBits, fBits, d.Endian)",
                    "doc": "nBits fixed-point number in current endian"
                },
                {
                    "name": "E",
                    "args": "nBits, fBits, endian",
                    "params": "nBits int, fBits int, endian Endian",
-                    "call": "d.tryFPE(nBits, fBits, endian)",
+                    "call": "d.tryFPEndian(nBits, fBits, endian)",
                    "doc": "nBits fixed-point number in specified endian"
                },
-                {"name": "16", "args": "", "params": "", "call": "d.tryFPE(16, 8, d.Endian)", "doc": "16 bit fixed-point number in current endian"},
-                {"name": "32", "args": "", "params": "", "call": "d.tryFPE(32, 16, d.Endian)", "doc": "32 bit fixed-point number in current endian"},
-                {"name": "64", "args": "", "params": "", "call": "d.tryFPE(64, 32, d.Endian)", "doc": "64 bit fixed-point number in current endian"},
-                {"name": "16LE", "args": "", "params": "", "call": "d.tryFPE(16, 8, LittleEndian)", "doc": "16 bit fixed-point number in little-endian"},
-                {"name": "32LE", "args": "", "params": "", "call": "d.tryFPE(32, 16, LittleEndian)", "doc": "32 bit fixed-point number in little-endian"},
-                {"name": "64LE", "args": "", "params": "", "call": "d.tryFPE(64, 32, LittleEndian)", "doc": "64 bit fixed-point number in little-endian"},
-                {"name": "16BE", "args": "", "params": "", "call": "d.tryFPE(16, 8, BigEndian)", "doc": "16 bit fixed-point number in big-endian"},
-                {"name": "32BE", "args": "", "params": "", "call": "d.tryFPE(32, 16, BigEndian)", "doc": "32 bit fixed-point number in big-endian"},
-                {"name": "64BE", "args": "", "params": "", "call": "d.tryFPE(64, 32, BigEndian)", "doc": "64 bit fixed-point number in big-endian"}
+                {"name": "16", "args": "", "params": "", "call": "d.tryFPEndian(16, 8, d.Endian)", "doc": "16 bit fixed-point number in current endian"},
+                {"name": "32", "args": "", "params": "", "call": "d.tryFPEndian(32, 16, d.Endian)", "doc": "32 bit fixed-point number in current endian"},
+                {"name": "64", "args": "", "params": "", "call": "d.tryFPEndian(64, 32, d.Endian)", "doc": "64 bit fixed-point number in current endian"},
+                {"name": "16LE", "args": "", "params": "", "call": "d.tryFPEndian(16, 8, LittleEndian)", "doc": "16 bit fixed-point number in little-endian"},
+                {"name": "32LE", "args": "", "params": "", "call": "d.tryFPEndian(32, 16, LittleEndian)", "doc": "32 bit fixed-point number in little-endian"},
+                {"name": "64LE", "args": "", "params": "", "call": "d.tryFPEndian(64, 32, LittleEndian)", "doc": "64 bit fixed-point number in little-endian"},
+                {"name": "16BE", "args": "", "params": "", "call": "d.tryFPEndian(16, 8, BigEndian)", "doc": "16 bit fixed-point number in big-endian"},
+                {"name": "32BE", "args": "", "params": "", "call": "d.tryFPEndian(32, 16, BigEndian)", "doc": "32 bit fixed-point number in big-endian"},
+                {"name": "64BE", "args": "", "params": "", "call": "d.tryFPEndian(64, 32, BigEndian)", "doc": "64 bit fixed-point number in big-endian"}
            ]
        },
        {
--- a/pkg/interp/decode.go
+++ b/pkg/interp/decode.go
@ -337,6 +337,11 @@ func makeDecodeValue(dv *decode.Value) interface{} {
 				JQValue:         gojqextra.Null{},
 				decodeValueBase: decodeValueBase{dv},
 			}
+		case *big.Int:
+			return decodeValue{
+				JQValue:         gojqextra.Number{V: vv},
+				decodeValueBase: decodeValueBase{dv},
+			}
 		default:
 			panic(fmt.Sprintf("unreachable vv %#+v", vv))
 		}
--- a/pkg/interp/decorator.go
+++ b/pkg/interp/decorator.go
@ -2,6 +2,7 @@ package interp

 import (
 	"log"
+	"math/big"
 	"strconv"
 	"strings"

@ -120,6 +121,8 @@ func decoratorFromOptions(opts Options) Decorator {
 			case int, float64, int64, uint64:
 				// TODO: clean up number types
 				return d.Number
+			case *big.Int:
+				return d.Number
 			default:
 				log.Printf("v: %#+v\n", v)
 				panic("unreachable")
--- a/pkg/interp/dump.go
+++ b/pkg/interp/dump.go
@ -10,7 +10,7 @@ import (
 	"github.com/wader/fq/internal/asciiwriter"
 	"github.com/wader/fq/internal/columnwriter"
 	"github.com/wader/fq/internal/hexpairwriter"
-	"github.com/wader/fq/internal/num"
+	"github.com/wader/fq/internal/mathextra"
 	"github.com/wader/fq/pkg/bitio"
 	"github.com/wader/fq/pkg/decode"
 	"github.com/wader/fq/pkg/scalar"
@ -55,7 +55,7 @@ func dumpEx(v *decode.Value, buf []byte, cw *columnwriter.Writer, depth int, roo
 	var asciiHeader string
 	if depth == 0 {
 		for i := 0; i < opts.LineBytes; i++ {
-			s := num.PadFormatInt(int64(i), opts.AddrBase, false, 2)
+			s := mathextra.PadFormatInt(int64(i), opts.AddrBase, false, 2)
 			hexHeader += s
 			if i < opts.LineBytes-1 {
 				hexHeader += " "
@ -173,7 +173,7 @@ func dumpEx(v *decode.Value, buf []byte, cw *columnwriter.Writer, depth int, roo

 	if opts.Verbose {
 		cfmt(colField, " %s (%s)",
-			num.BitRange(innerRange).StringByteBits(opts.AddrBase), num.Bits(innerRange.Len).StringByteBits(opts.SizeBase))
+			mathextra.BitRange(innerRange).StringByteBits(opts.AddrBase), mathextra.Bits(innerRange.Len).StringByteBits(opts.SizeBase))
 	}

 	cprint(colField, "\n")
@ -258,7 +258,7 @@ func dumpEx(v *decode.Value, buf []byte, cw *columnwriter.Writer, depth int, roo
 	// has length and is not compound or a collapsed struct/array (max depth)
 	if innerRange.Len > 0 && (!isCompound(v) || (opts.Depth != 0 && opts.Depth == depth)) {
 		cfmt(colAddr, "%s%s\n",
-			rootIndent, deco.DumpAddr.F(num.PadFormatInt(startLineByte, opts.AddrBase, true, addrWidth)))
+			rootIndent, deco.DumpAddr.F(mathextra.PadFormatInt(startLineByte, opts.AddrBase, true, addrWidth)))

 		vBitBuf, err := rootV.RootBitBuf.BitBufRange(startByte*8, displaySizeBits)
 		if err != nil {
@ -287,7 +287,7 @@ func dumpEx(v *decode.Value, buf []byte, cw *columnwriter.Writer, depth int, roo
 		for i := int64(1); i < addrLines; i++ {
 			lineStartByte := startLineByte + i*int64(opts.LineBytes)
 			columns()
-			cfmt(colAddr, "%s%s\n", rootIndent, deco.DumpAddr.F(num.PadFormatInt(lineStartByte, opts.AddrBase, true, addrWidth)))
+			cfmt(colAddr, "%s%s\n", rootIndent, deco.DumpAddr.F(mathextra.PadFormatInt(lineStartByte, opts.AddrBase, true, addrWidth)))
 		}
 		// TODO: correct? should rethink columnwriter api maybe?
 		lastLineStopByte := startLineByte + addrLines*int64(opts.LineBytes) - 1
@ -308,9 +308,9 @@ func dumpEx(v *decode.Value, buf []byte, cw *columnwriter.Writer, depth int, roo
 			cprint(colHex, "\n")
 			// TODO: truncate if display_bytes is small?
 			cfmt(colHex, "until %s%s (%s)",
-				num.Bits(stopBit).StringByteBits(opts.AddrBase),
+				mathextra.Bits(stopBit).StringByteBits(opts.AddrBase),
 				isEnd,
-				num.PadFormatInt(bitio.BitsByteCount(sizeBits), opts.SizeBase, true, 0))
+				mathextra.PadFormatInt(bitio.BitsByteCount(sizeBits), opts.SizeBase, true, 0))
 			// TODO: dump last line?
 		}
 	}
@ -335,9 +335,9 @@ func dump(v *decode.Value, w io.Writer, opts Options) error {
 	}

 	_ = v.WalkPreOrder(makeWalkFn(func(v *decode.Value, rootV *decode.Value, depth int, rootDepth int) error {
-		maxAddrIndentWidth = num.MaxInt(
+		maxAddrIndentWidth = mathextra.MaxInt(
 			maxAddrIndentWidth,
-			rootDepth+num.DigitsInBase(bitio.BitsByteCount(v.InnerRange().Stop()), true, opts.AddrBase),
+			rootDepth+mathextra.DigitsInBase(bitio.BitsByteCount(v.InnerRange().Stop()), true, opts.AddrBase),
 		)
 		return nil
 	}))
--- a/pkg/interp/interp.go
+++ b/pkg/interp/interp.go
@ -22,7 +22,7 @@ import (
 	"github.com/wader/fq/internal/colorjson"
 	"github.com/wader/fq/internal/ctxstack"
 	"github.com/wader/fq/internal/ioextra"
-	"github.com/wader/fq/internal/num"
+	"github.com/wader/fq/internal/mathextra"
 	"github.com/wader/fq/internal/pos"
 	"github.com/wader/fq/pkg/bitio"
 	"github.com/wader/fq/pkg/decode"
@ -948,7 +948,7 @@ func bitsFormatFnFromOptions(opts Options) func(bb *bitio.Buffer) (interface{},
 				return "", err
 			}
 			e.Close()
-			return fmt.Sprintf("<%s>%s", num.Bits(bb.Len()).StringByteBits(opts.SizeBase), b.String()), nil
+			return fmt.Sprintf("<%s>%s", mathextra.Bits(bb.Len()).StringByteBits(opts.SizeBase), b.String()), nil
 		}
 	}
 }
@ -981,12 +981,12 @@ func (i *Interp) variables() map[string]interface{} {
 func (i *Interp) Options(v interface{}) Options {
 	var opts Options
 	_ = mapstructure.Decode(v, &opts)
-	opts.ArrayTruncate = num.MaxInt(0, opts.ArrayTruncate)
-	opts.Depth = num.MaxInt(0, opts.Depth)
-	opts.AddrBase = num.ClampInt(2, 36, opts.AddrBase)
-	opts.SizeBase = num.ClampInt(2, 36, opts.SizeBase)
-	opts.LineBytes = num.MaxInt(0, opts.LineBytes)
-	opts.DisplayBytes = num.MaxInt(0, opts.DisplayBytes)
+	opts.ArrayTruncate = mathextra.MaxInt(0, opts.ArrayTruncate)
+	opts.Depth = mathextra.MaxInt(0, opts.Depth)
+	opts.AddrBase = mathextra.ClampInt(2, 36, opts.AddrBase)
+	opts.SizeBase = mathextra.ClampInt(2, 36, opts.SizeBase)
+	opts.LineBytes = mathextra.MaxInt(0, opts.LineBytes)
+	opts.DisplayBytes = mathextra.MaxInt(0, opts.DisplayBytes)
 	opts.Decorator = decoratorFromOptions(opts)
 	opts.BitsFormatFn = bitsFormatFnFromOptions(opts)

--- a/pkg/interp/preview.go
+++ b/pkg/interp/preview.go
@ -2,9 +2,10 @@ package interp

 import (
 	"fmt"
+	"math/big"
 	"strconv"

-	"github.com/wader/fq/internal/num"
+	"github.com/wader/fq/internal/mathextra"
 	"github.com/wader/fq/pkg/bitio"
 	"github.com/wader/fq/pkg/scalar"
 )
@ -18,12 +19,12 @@ func previewValue(v interface{}, df scalar.DisplayFormat) string {
 		return "false"
 	case int:
 		// TODO: DisplayFormat is weird
-		return num.PadFormatInt(int64(vv), df.FormatBase(), true, 0)
+		return mathextra.PadFormatInt(int64(vv), df.FormatBase(), true, 0)
 	case int64:
 		// TODO: DisplayFormat is weird
-		return num.PadFormatInt(vv, df.FormatBase(), true, 0)
+		return mathextra.PadFormatInt(vv, df.FormatBase(), true, 0)
 	case uint64:
-		return num.PadFormatUint(vv, df.FormatBase(), true, 0)
+		return mathextra.PadFormatUint(vv, df.FormatBase(), true, 0)
 	case float64:
 		// TODO: float32? better truncated to significant digits?
 		return strconv.FormatFloat(vv, 'g', -1, 64)
@ -32,8 +33,12 @@ func previewValue(v interface{}, df scalar.DisplayFormat) string {
 			return fmt.Sprintf("%q", vv[0:50]) + "..."
 		}
 		return fmt.Sprintf("%q", vv)
+	case nil:
+		return "null"
 	case *bitio.Buffer:
 		return "raw bits"
+	case *big.Int:
+		return mathextra.PadFormatBigInt(vv, df.FormatBase(), true, 0)
 	default:
 		panic("unreachable")
 	}
--- a/pkg/scalar/scalar_gen.go
+++ b/pkg/scalar/scalar_gen.go
@ -3,10 +3,31 @@ package scalar

 import (
 	"fmt"
+	"math/big"

 	"github.com/wader/fq/pkg/bitio"
 )

+// Type BigInt
+
+// ActualBigInt asserts actual value is a BigInt and returns it
+func (s S) ActualBigInt() *big.Int {
+	v, ok := s.Actual.(*big.Int)
+	if !ok {
+		panic(fmt.Sprintf("failed to type assert s.Actual %v as *big.Int", s.Actual))
+	}
+	return v
+}
+
+// SymBigInt asserts symbolic value is a BigInt and returns it
+func (s S) SymBigInt() *big.Int {
+	v, ok := s.Sym.(*big.Int)
+	if !ok {
+		panic(fmt.Sprintf("failed to type assert s.Sym %v as *big.Int", s.Sym))
+	}
+	return v
+}
+
 // Type BitBuf

 // ActualBitBuf asserts actual value is a BitBuf and returns it
--- a/pkg/scalar/scalar_gen.go.tmpl
+++ b/pkg/scalar/scalar_gen.go.tmpl
@ -3,6 +3,7 @@ package scalar

 import (
 	"fmt"
+	"math/big"

 	"github.com/wader/fq/pkg/bitio"
 )