interp: Rework buffer, still confusing

pkg/bitio/bitio.go

@@ -372,7 +372,10 @@ func NewMultiBitReader(rs []BitReadAtSeeker) (*MultiBitReader, error) {
 }
 
 func (m *MultiBitReader) ReadBitsAt(p []byte, nBits int, bitOff int64) (n int, err error) {
-	end := m.readerEnds[len(m.readers)-1]
+	var end int64
+	if len(m.readers) > 0 {
+		end = m.readerEnds[len(m.readers)-1]
+	}
 	if end <= bitOff {
 		return 0, io.EOF
 	}
@@ -406,7 +409,10 @@ func (m *MultiBitReader) ReadBits(p []byte, nBits int) (n int, err error) {
 
 func (m *MultiBitReader) SeekBits(bitOff int64, whence int) (int64, error) {
 	var p int64
-	end := m.readerEnds[len(m.readerEnds)-1]
+	var end int64
+	if len(m.readers) > 0 {
+		end = m.readerEnds[len(m.readers)-1]
+	}
 
 	switch whence {
 	case io.SeekStart:
@@ -431,9 +437,6 @@ func (m *MultiBitReader) Read(p []byte) (n int, err error) {
 	n, err = m.ReadBitsAt(p, len(p)*8, m.pos)
 	m.pos += int64(n)
 
-	// log.Printf("n: %#+v\n", n)
-	// log.Printf("err: %#+v\n", err)
-
 	if err != nil {
 		return int(BitsByteCount(int64(n))), err
 	}

pkg/interp/buffer.go

@@ -8,12 +8,11 @@ import (
 
 	"github.com/wader/fq/internal/gojqextra"
 	"github.com/wader/fq/pkg/bitio"
-	"github.com/wader/fq/pkg/decode"
 	"github.com/wader/fq/pkg/ranges"
 )
 
-var _ Value = (*BufferView)(nil)
-var _ ToBuffer = (*BufferView)(nil)
+var _ Value = BufferView{}
+var _ ToBufferView = BufferView{}
 
 type BufferView struct {
 	bb   *bitio.Buffer
@@ -21,9 +20,7 @@ type BufferView struct {
 	unit int
 }
 
-// TODO: JQArray
-
-func newBifBufObject(bb *bitio.Buffer, unit int) BufferView {
+func bufferViewFromBuffer(bb *bitio.Buffer, unit int) BufferView {
 	return BufferView{
 		bb:   bb,
 		r:    ranges.Range{Start: 0, Len: bb.Len()},
@@ -31,8 +28,20 @@ func newBifBufObject(bb *bitio.Buffer, unit int) BufferView {
 	}
 }
 
-func (*BufferView) DisplayName() string { return "buffer" }
-func (*BufferView) ExtKeys() []string {
+func (bv BufferView) toBytesBuffer(r ranges.Range) (*bytes.Buffer, error) {
+	bb, err := bv.bb.BitBufRange(r.Start, r.Len)
+	if err != nil {
+		return nil, err
+	}
+	buf := &bytes.Buffer{}
+	if _, err := io.Copy(buf, bb.Copy()); err != nil {
+		return nil, err
+	}
+	return buf, nil
+}
+
+func (BufferView) DisplayName() string { return "buffer" }
+func (BufferView) ExtKeys() []string {
 	return []string{
 		"size",
 		"start",
@@ -42,116 +51,100 @@ func (*BufferView) ExtKeys() []string {
 	}
 }
 
-func (bo BufferView) JQValueLength() interface{} {
-	return int(bo.r.Len / int64(bo.unit))
+func (bv BufferView) ToBufferView() (BufferView, error) {
+	return bv, nil
 }
-func (bo BufferView) JQValueSliceLen() interface{} {
-	return bo.JQValueLength()
+
+func (bv BufferView) JQValueLength() interface{} {
+	return int(bv.r.Len / int64(bv.unit))
 }
-func (bo BufferView) JQValueIndex(index int) interface{} {
-	// TODO: use bitio
-	/*
-		pos, err := bo.bbr.bb.Pos()
-		if err != nil {
-			return err
-		}
-		if _, err := bo.bbr.bb.SeekAbs(int64(index) * int64(bo.unit)); err != nil {
-			return err
-		}
-		v, err := bo.bbr.bb.U(bo.unit)
-		if err != nil {
-			return err
-		}
-		if _, err := bo.bbr.bb.SeekAbs(pos); err != nil {
-			return err
-		}
-		return int(v)
-	*/
-	return nil
+func (bv BufferView) JQValueSliceLen() interface{} {
+	return bv.JQValueLength()
 }
-func (bo BufferView) JQValueSlice(start int, end int) interface{} {
-	rStart := int64(start * bo.unit)
-	rLen := int64((end - start) * bo.unit)
+func (bv BufferView) JQValueIndex(index int) interface{} {
+	if index < 0 {
+		return ""
+	}
+	buf, err := bv.toBytesBuffer(ranges.Range{Start: bv.r.Start + int64(index*bv.unit), Len: int64(bv.unit)})
+	if err != nil {
+		return err
+	}
+	s := buf.String()
+	return s[0:1]
+}
+func (bv BufferView) JQValueSlice(start int, end int) interface{} {
+	rStart := int64(start * bv.unit)
+	rLen := int64((end - start) * bv.unit)
 
 	return BufferView{
-		bb:   bo.bb,
-		r:    ranges.Range{Start: bo.r.Start + rStart, Len: rLen},
-		unit: bo.unit,
+		bb:   bv.bb,
+		r:    ranges.Range{Start: bv.r.Start + rStart, Len: rLen},
+		unit: bv.unit,
 	}
 }
-func (bo BufferView) JQValueKey(name string) interface{} {
+func (bv BufferView) JQValueKey(name string) interface{} {
 	switch name {
 	case "size":
-		return new(big.Int).SetInt64(bo.r.Len / int64(bo.unit))
+		return new(big.Int).SetInt64(bv.r.Len / int64(bv.unit))
 	case "start":
-		return new(big.Int).SetInt64(bo.r.Start / int64(bo.unit))
+		return new(big.Int).SetInt64(bv.r.Start / int64(bv.unit))
 	case "stop":
-		stop := bo.r.Stop()
-		stopUnits := stop / int64(bo.unit)
-		if stop%int64(bo.unit) != 0 {
+		stop := bv.r.Stop()
+		stopUnits := stop / int64(bv.unit)
+		if stop%int64(bv.unit) != 0 {
 			stopUnits++
 		}
 		return new(big.Int).SetInt64(stopUnits)
 	case "bits":
-		if bo.unit == 1 {
-			return bo
+		if bv.unit == 1 {
+			return bv
 		}
-		return BufferView{bb: bo.bb, r: bo.r, unit: 1}
+		return BufferView{bb: bv.bb, r: bv.r, unit: 1}
 	case "bytes":
-		if bo.unit == 8 {
-			return bo
+		if bv.unit == 8 {
+			return bv
 		}
-		return BufferView{bb: bo.bb, r: bo.r, unit: 8}
+		return BufferView{bb: bv.bb, r: bv.r, unit: 8}
 	}
 	return nil
 }
-func (bo BufferView) JQValueEach() interface{} {
+func (bv BufferView) JQValueEach() interface{} {
 	return nil
 }
-func (bo BufferView) JQValueType() string {
+func (bv BufferView) JQValueType() string {
 	return "buffer"
 }
-func (bo BufferView) JQValueKeys() interface{} {
+func (bv BufferView) JQValueKeys() interface{} {
 	return gojqextra.FuncTypeError{Name: "keys", Typ: "buffer"}
 }
-func (bo BufferView) JQValueHas(key interface{}) interface{} {
+func (bv BufferView) JQValueHas(key interface{}) interface{} {
 	return gojqextra.HasKeyTypeError{L: "buffer", R: fmt.Sprintf("%v", key)}
 }
-func (bo BufferView) JQValueToNumber() interface{} {
-	bb, err := bo.ToBuffer()
+func (bv BufferView) JQValueToNumber() interface{} {
+	buf, err := bv.toBytesBuffer(bv.r)
 	if err != nil {
 		return err
 	}
-	buf := &bytes.Buffer{}
-	if _, err := io.Copy(buf, bb.Copy()); err != nil {
-		return err
-	}
-	extraBits := uint((8 - bo.r.Len%8) % 8)
+	extraBits := uint((8 - bv.r.Len%8) % 8)
 	return new(big.Int).Rsh(new(big.Int).SetBytes(buf.Bytes()), extraBits)
 }
-func (bo BufferView) JQValueToString() interface{} {
-	return bo.JQValueToGoJQ()
+func (bv BufferView) JQValueToString() interface{} {
+	return bv.JQValueToGoJQ()
 }
-
-func (bo BufferView) JQValueToGoJQ() interface{} {
-	bb, err := bo.ToBuffer()
+func (bv BufferView) JQValueToGoJQ() interface{} {
+	buf, err := bv.toBytesBuffer(bv.r)
 	if err != nil {
 		return err
 	}
-	buf := &bytes.Buffer{}
-	if _, err := io.Copy(buf, bb.Copy()); err != nil {
-		return err
-	}
 	return buf.String()
 }
-
-func (bo BufferView) JQValueUpdate(key interface{}, u interface{}, delpath bool) interface{} {
+func (bv BufferView) JQValueUpdate(key interface{}, u interface{}, delpath bool) interface{} {
 	return notUpdateableError{Key: fmt.Sprintf("%v", key), Typ: "buffer"}
 }
 
-func (bo BufferView) Display(w io.Writer, opts Options) error {
+func (bv BufferView) Display(w io.Writer, opts Options) error {
 	if opts.RawOutput {
-		bb, err := bo.ToBuffer()
+		bb, err := bv.toBuffer()
 		if err != nil {
 			return err
 		}
@@ -161,27 +154,9 @@ func (bo BufferView) Display(w io.Writer, opts Options) error {
 		return nil
 	}
 
-	unitNames := map[int]string{
-		1: "bits",
-		8: "bytes",
-	}
-	unitName := unitNames[bo.unit]
-	if unitName == "" {
-		unitName = "units"
-	}
-
-	// TODO: hack
-	return dump(
-		&decode.Value{
-			Range:       bo.r,
-			RootBitBuf:  bo.bb.Copy(),
-			Description: fmt.Sprintf("%d %s", bo.r.Len/int64(bo.unit), unitName),
-		},
-		w,
-		opts,
-	)
+	return hexdump(w, bv, opts)
 }
 
-func (bo BufferView) ToBuffer() (*bitio.Buffer, error) {
-	return bo.bb.BitBufRange(bo.r.Start, bo.r.Len)
+func (bv BufferView) toBuffer() (*bitio.Buffer, error) {
+	return bv.bb.BitBufRange(bv.r.Start, bv.r.Len)
 }

pkg/interp/dump.go

@@ -349,3 +349,16 @@ func dump(v *decode.Value, w io.Writer, opts Options) error {
 		return dumpEx(v, buf, cw, depth, rootV, rootDepth, maxAddrIndentWidth-rootDepth, opts)
 	}))
 }
+
+func hexdump(w io.Writer, bv BufferView, opts Options) error {
+	// TODO: hack
+	opts.Verbose = true
+	return dump(
+		&decode.Value{
+			Range:      bv.r,
+			RootBitBuf: bv.bb.Copy(),
+		},
+		w,
+		opts,
+	)
+}

pkg/interp/funcs.go

@@ -56,6 +56,7 @@ func (i *Interp) makeFunctions() []Function {
 
 		{[]string{"format"}, 0, 0, i.format, nil},
 		{[]string{"_display"}, 1, 1, nil, i._display},
+		{[]string{"_hexdump"}, 1, 1, nil, i._hexdump},
 
 		{[]string{"tobytes"}, 0, 0, i.toBytes, nil},
 		{[]string{"tobits"}, 0, 0, i.toBits, nil},
@@ -122,7 +123,7 @@ func makeStringBitBufTransformFn(
 			}
 			outBB := bitio.NewBufferFromBytes(buf.Bytes(), -1)
 
-			return newBifBufObject(outBB, 8)
+			return bufferViewFromBuffer(outBB, 8)
 		default:
 			bb, err := toBuffer(c)
 			if err != nil {
@@ -168,7 +169,7 @@ func makeBitBufTransformFn(fn func(r io.Reader) (io.Reader, error)) func(c inter
 
 		outBB := bitio.NewBufferFromBytes(outBuf.Bytes(), -1)
 
-		return newBifBufObject(outBB, 8)
+		return bufferViewFromBuffer(outBB, 8)
 	}
 }
 
@@ -190,7 +191,7 @@ func makeHashFn(fn func() (hash.Hash, error)) func(c interface{}, a []interface{
 
 		outBB := bitio.NewBufferFromBytes(h.Sum(nil), -1)
 
-		return newBifBufObject(outBB, 8)
+		return bufferViewFromBuffer(outBB, 8)
 	}
 }
 
@@ -488,15 +489,15 @@ type bitBufFile struct {
 	progressFn progressreadseeker.ProgressFn
 }
 
-var _ ToBuffer = (*bitBufFile)(nil)
+var _ ToBufferView = (*bitBufFile)(nil)
 
 func (bbf *bitBufFile) Display(w io.Writer, opts Options) error {
 	_, err := fmt.Fprintln(w, bbf.JQValue.JQValueToString())
 	return err
 }
 
-func (bbf *bitBufFile) ToBuffer() (*bitio.Buffer, error) {
-	return bbf.bb.Copy(), nil
+func (bbf *bitBufFile) ToBufferView() (BufferView, error) {
+	return bufferViewFromBuffer(bbf.bb, 8), nil
 }
 
 // def open: #:: string| => buffer
@@ -700,7 +701,7 @@ func (i *Interp) toBytes(c interface{}, a []interface{}) interface{} {
 	if err != nil {
 		return err
 	}
-	return newBifBufObject(bb, 8)
+	return bufferViewFromBuffer(bb, 8)
 }
 
 func (i *Interp) toBits(c interface{}, a []interface{}) interface{} {
@@ -708,7 +709,7 @@ func (i *Interp) toBits(c interface{}, a []interface{}) interface{} {
 	if err != nil {
 		return err
 	}
-	return newBifBufObject(bb, 1)
+	return bufferViewFromBuffer(bb, 1)
 }
 
 func (i *Interp) toValue(c interface{}, a []interface{}) interface{} {
@@ -833,8 +834,24 @@ func (i *Interp) find(c interface{}, a []interface{}) gojq.Iter {
 		return gojq.NewIter()
 	}
 
-	bbo := newBifBufObject(bb, 8)
+	bbo := bufferViewFromBuffer(bb, 8)
 	// log.Printf("bbo: %#+v\n", bbo)
 
 	return gojq.NewIter(bbo)
 }
+
+func (i *Interp) _hexdump(c interface{}, a []interface{}) gojq.Iter {
+	opts, err := i.Options(a...)
+	if err != nil {
+		return gojq.NewIter(err)
+	}
+	bv, err := toBufferView(c)
+	if err != nil {
+		return gojq.NewIter(err)
+	}
+	if err := hexdump(i.evalContext.output, bv, opts); err != nil {
+		return gojq.NewIter(err)
+	}
+
+	return gojq.NewIter()
+}

pkg/interp/funcs.jq

@@ -28,8 +28,8 @@ def verbose($opts): _display({verbose: true, array_truncate: 0} + $opts);
 def verbose: verbose({});
 def v($opts): verbose($opts);
 def v: verbose;
-def hexdump($opts): _display({display_bytes: 0} + $opts);
-def hexdump: _display({display_bytes: 0});
+def hexdump($opts): _hexdump({display_bytes: 0} + $opts);
+def hexdump: _hexdump({display_bytes: 0});
 def hd($opts): hexdump($opts);
 def hd: hexdump;
 

pkg/interp/interp.go

@@ -179,10 +179,6 @@ type Display interface {
 	Display(w io.Writer, opts Options) error
 }
 
-type ToBuffer interface {
-	ToBuffer() (*bitio.Buffer, error)
-}
-
 type ToBufferView interface {
 	ToBufferView() (BufferView, error)
 }
@@ -293,8 +289,12 @@ func toBuffer(v interface{}) (*bitio.Buffer, error) {
 // TODO: refactor to return struct?
 func toBufferEx(v interface{}, inArray bool) (*bitio.Buffer, error) {
 	switch vv := v.(type) {
-	case ToBuffer:
-		return vv.ToBuffer()
+	case ToBufferView:
+		bv, err := vv.ToBufferView()
+		if err != nil {
+			return nil, err
+		}
+		return bv.bb.BitBufRange(bv.r.Start, bv.r.Len)
 	case string:
 		return bitio.NewBufferFromBytes([]byte(vv), -1), nil
 	case []byte:
@@ -343,6 +343,19 @@ func toBufferEx(v interface{}, inArray bool) (*bitio.Buffer, error) {
 	}
 }
 
+func toBufferView(v interface{}) (BufferView, error) {
+	switch vv := v.(type) {
+	case ToBufferView:
+		return vv.ToBufferView()
+	default:
+		bb, err := toBuffer(v)
+		if err != nil {
+			return BufferView{}, err
+		}
+		return bufferViewFromBuffer(bb, 8), nil
+	}
+}
+
 func toValue(opts Options, v interface{}) (interface{}, bool) {
 	switch v := v.(type) {
 	case JQValueEx:

pkg/interp/testdata/buffer.fqtest

@@ -1,37 +1,35 @@
 /test.mp3:
 $ fq -d mp3 '.headers[0].magic._bits[8:16] | hd' /test.mp3
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|   44                                          | D              |.: none (8 bits)
+0x0|   44                                          | D              |.: none 0x1-0x1.7 (1)
 $ fq -d mp3 '.headers[0].magic._bits | [.[8:16], .[0:8]] | hd' /test.mp3
-[
-  "D",
-  "I"
-]
+   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x0|44 49|                                         |DI|             |.: none 0x0-0x1.7 (2)
 $ fq -d mp3 '.headers[0].magic._bits | [.[8:16], .[0:8]] | tobits | hd' /test.mp3
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|44 49|                                         |DI|             |.: none (16 bits)
+0x0|44 49|                                         |DI|             |.: none 0x0-0x1.7 (2)
 $ fq -d mp3 '.headers[0].magic._bits | [.[8:16], .[0:8]] | tobytes | hd' /test.mp3
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|44 49|                                         |DI|             |.: none (2 bytes)
+0x0|44 49|                                         |DI|             |.: none 0x0-0x1.7 (2)
 $ fq -n '"12" | tobytes | hd'
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|31 32|                                         |12|             |.: none (2 bytes)
+0x0|31 32|                                         |12|             |.: none 0x0-0x1.7 (2)
 $ fq -n '"12" | tobits | hd'
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|31 32|                                         |12|             |.: none (16 bits)
+0x0|31 32|                                         |12|             |.: none 0x0-0x1.7 (2)
 $ fq -n '["12", "3"] | tobytes | hd'
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|31 32 33|                                      |123|            |.: none (3 bytes)
+0x0|31 32 33|                                      |123|            |.: none 0x0-0x2.7 (3)
 $ fq -n '["12", "3"] | tobits  | hd'
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|31 32 33|                                      |123|            |.: none (24 bits)
+0x0|31 32 33|                                      |123|            |.: none 0x0-0x2.7 (3)
 $ fq -n '[("11" | hex), ("22" | hex)] | tobits  | hd'
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|11 22|                                         |."|             |.: none (16 bits)
+0x0|11 22|                                         |."|             |.: none 0x0-0x1.7 (2)
 # TODO: bug, hexdump uses io.Copy which is byte oritneted
 $ fq -n '[("12" | hex | .bits[4:]), ("34" | hex | .bits[0:4])] | tobits  | hd'
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|20|                                            | |              |.: none (8 bits)
+0x0|20|                                            | |              |.: none 0x0-0x0.7 (1)
 $ fq -d mp3 '.frames[]._bits[0:12] | tonumber' /test.mp3
 4095
 4095

pkg/interp/testdata/hexdump.fqtest

@@ -2,11 +2,11 @@
 /test.mp3:
 $ fq -d mp3 '._bits[0:(16+8)*8] | hexdump' /test.mp3
     |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x00|49 44 33 04 00 00 00 00 00 23 54 53 53 45 00 00|ID3......#TSSE..|.: none (192 bits)
+0x00|49 44 33 04 00 00 00 00 00 23 54 53 53 45 00 00|ID3......#TSSE..|.: none 0x0-0x17.7 (24)
 0x10|00 0f 00 00 03 4c 61 76                        |.....Lav        |
 $ fq -d mp3 '.frames[1].header.layer | hexdump' /test.mp3
     |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0xe0|            fb                                 |    .           |.frames[1].header.layer: Layer 3 (3)
+0xe0|            fb                                 |    .           |.: none 0xe4.5-0xe4.6 (0.2)
 $ fq -d mp3 '.frames[1].header.layer._bytes | hexdump' /test.mp3
-   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|40|                                            |@|              |.: none (0 bytes)
+    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0xe0|            fb                                 |    .           |.: none 0xe4.5-0xe4.6 (0.2)

pkg/interp/testdata/raw.fqtest

@@ -1,5 +1,5 @@
 $ fq -n '"test" | tobytes'
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|74 65 73 74|                                   |test|           |.: none (4 bytes)
+0x0|74 65 73 74|                                   |test|           |.: none 0x0-0x3.7 (4)
 $ fq -n -o raw_output=true '"test" | tobytes'
 test\

pkg/interp/testdata/value.fqtest

@@ -138,11 +138,11 @@ mp3> ._path
 []
 mp3> ._bits
      |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x000|49 44 33 04 00 00 00 00 00 23 54 53 53 45 00 00|ID3......#TSSE..|.: none (5152 bits)
+0x000|49 44 33 04 00 00 00 00 00 23 54 53 53 45 00 00|ID3......#TSSE..|.: none 0x0-0x283.7 (644)
 *    |until 0x283.7 (end) (644)                      |                |
 mp3> ._bytes
      |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x000|49 44 33 04 00 00 00 00 00 23 54 53 53 45 00 00|ID3......#TSSE..|.: none (644 bytes)
+0x000|49 44 33 04 00 00 00 00 00 23 54 53 53 45 00 00|ID3......#TSSE..|.: none 0x0-0x283.7 (644)
 *    |until 0x283.7 (end) (644)                      |                |
 mp3> ._error
 null
@@ -152,4 +152,4 @@ mp3> ._format
 "mp3"
 mp3> ._abc
 error: expected a extkey but got: _abc
-mp3> ^D
+mp3> ^D

pkg/interp/testdata/value_array.fqtest

@@ -165,15 +165,15 @@ mp3> .headers._path | ., type, length?
 1
 mp3> .headers._bits | ., type, length?
     |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x00|49 44 33 04 00 00 00 00 00 23 54 53 53 45 00 00|ID3......#TSSE..|.: none (360 bits)
-*   |until 0x2c.7 (end) (45)                        |                |
+0x00|49 44 33 04 00 00 00 00 00 23 54 53 53 45 00 00|ID3......#TSSE..|.: none 0x0-0x2c.7 (45)
+*   |until 0x2c.7 (45)                              |                |
 "buffer"
 360
 mp3> 
 mp3> .headers._bytes | ., type, length?
     |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x00|49 44 33 04 00 00 00 00 00 23 54 53 53 45 00 00|ID3......#TSSE..|.: none (45 bytes)
-*   |until 0x2c.7 (end) (45)                        |                |
+0x00|49 44 33 04 00 00 00 00 00 23 54 53 53 45 00 00|ID3......#TSSE..|.: none 0x0-0x2c.7 (45)
+*   |until 0x2c.7 (45)                              |                |
 "buffer"
 45
 mp3> 

pkg/interp/testdata/value_boolean.fqtest

@@ -72,13 +72,13 @@ mp3> .headers[0].flags.unsynchronisation._path | ., type, length?
 4
 mp3> .headers[0].flags.unsynchronisation._bits | ., type, length?
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|00|                                            |.|              |.: none (1 bits)
+0x0|               00                              |     .          |.: none 0x5-0x5 (0.1)
 "buffer"
 1
 mp3> 
 mp3> .headers[0].flags.unsynchronisation._bytes | ., type, length?
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|00|                                            |.|              |.: none (0 bytes)
+0x0|               00                              |     .          |.: none 0x5-0x5 (0.1)
 "buffer"
 0
 mp3> 

pkg/interp/testdata/value_json_array.fqtest

@@ -81,13 +81,13 @@ json> (.)._path | ., type, length?
 0
 json> (.)._bits | ., type, length?
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|5b 5d|                                         |[]|             |.: none (16 bits)
+0x0|5b 5d|                                         |[]|             |.: none 0x0-0x1.7 (2)
 "buffer"
 16
 json> 
 json> (.)._bytes | ., type, length?
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|5b 5d|                                         |[]|             |.: none (2 bytes)
+0x0|5b 5d|                                         |[]|             |.: none 0x0-0x1.7 (2)
 "buffer"
 2
 json> 

pkg/interp/testdata/value_json_object.fqtest

@@ -71,13 +71,13 @@ json> (.)._path | ., type, length?
 0
 json> (.)._bits | ., type, length?
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|7b 7d|                                         |{}|             |.: none (16 bits)
+0x0|7b 7d|                                         |{}|             |.: none 0x0-0x1.7 (2)
 "buffer"
 16
 json> 
 json> (.)._bytes | ., type, length?
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|7b 7d|                                         |{}|             |.: none (2 bytes)
+0x0|7b 7d|                                         |{}|             |.: none 0x0-0x1.7 (2)
 "buffer"
 2
 json> 

pkg/interp/testdata/value_null.fqtest

@@ -71,14 +71,14 @@ mp3> .headers[0].padding._path | ., type, length?
 "array"
 3
 mp3> .headers[0].padding._bits | ., type, length?
-   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|00 00 00 00 00 00 00 00 00 00|                 |..........|     |.: none (80 bits)
+    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x20|         00 00 00 00 00 00 00 00 00 00         |   ..........   |.: none 0x23-0x2c.7 (10)
 "buffer"
 80
 mp3> 
 mp3> .headers[0].padding._bytes | ., type, length?
-   |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|00 00 00 00 00 00 00 00 00 00|                 |..........|     |.: none (10 bytes)
+    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
+0x20|         00 00 00 00 00 00 00 00 00 00         |   ..........   |.: none 0x23-0x2c.7 (10)
 "buffer"
 10
 mp3> 

pkg/interp/testdata/value_number.fqtest

@@ -72,13 +72,13 @@ mp3> .headers[0].version._path | ., type, length?
 3
 mp3> .headers[0].version._bits | ., type, length?
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|04|                                            |.|              |.: none (8 bits)
+0x0|         04                                    |   .            |.: none 0x3-0x3.7 (1)
 "buffer"
 8
 mp3> 
 mp3> .headers[0].version._bytes | ., type, length?
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|04|                                            |.|              |.: none (1 bytes)
+0x0|         04                                    |   .            |.: none 0x3-0x3.7 (1)
 "buffer"
 1
 mp3> 

pkg/interp/testdata/value_object.fqtest

@@ -88,13 +88,13 @@ mp3> .headers[0].flags._path | ., type, length?
 3
 mp3> .headers[0].flags._bits | ., type, length?
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|00|                                            |.|              |.: none (8 bits)
+0x0|               00                              |     .          |.: none 0x5-0x5.7 (1)
 "buffer"
 8
 mp3> 
 mp3> .headers[0].flags._bytes | ., type, length?
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|00|                                            |.|              |.: none (1 bytes)
+0x0|               00                              |     .          |.: none 0x5-0x5.7 (1)
 "buffer"
 1
 mp3> 

pkg/interp/testdata/value_string.fqtest

@@ -84,13 +84,13 @@ mp3> .headers[0].magic._path | ., type, length?
 3
 mp3> .headers[0].magic._bits | ., type, length?
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|49 44 33|                                      |ID3|            |.: none (24 bits)
+0x0|49 44 33                                       |ID3             |.: none 0x0-0x2.7 (3)
 "buffer"
 24
 mp3> 
 mp3> .headers[0].magic._bytes | ., type, length?
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
-0x0|49 44 33|                                      |ID3|            |.: none (3 bytes)
+0x0|49 44 33                                       |ID3             |.: none 0x0-0x2.7 (3)
 "buffer"
 3
 mp3> 

pkg/interp/value.go

@@ -1,7 +1,6 @@
 package interp
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"io"
@@ -11,6 +10,7 @@ import (
 	"github.com/wader/fq/internal/gojqextra"
 	"github.com/wader/fq/pkg/bitio"
 	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/ranges"
 
 	"github.com/wader/gojq"
 )
@@ -35,7 +35,7 @@ func (err notUpdateableError) Error() string {
 // TODO: rename
 type valueIf interface {
 	Value
-	ToBuffer
+	ToBufferView
 }
 
 func valueKey(name string, a, b func(name string) interface{}) interface{} {
@@ -134,11 +134,8 @@ func (dvb decodeValueBase) DisplayName() string {
 }
 
 func (dvb decodeValueBase) Display(w io.Writer, opts Options) error { return dump(dvb.dv, w, opts) }
-func (dvb decodeValueBase) ToBuffer() (*bitio.Buffer, error) {
-	return dvb.dv.RootBitBuf.Copy().BitBufRange(dvb.dv.Range.Start, dvb.dv.Range.Len)
-}
 func (dvb decodeValueBase) ToBufferView() (BufferView, error) {
-	return BufferView{bb: dvb.dv.RootBitBuf.Copy(), r: dvb.dv.Range, unit: 8}, nil
+	return BufferView{bb: dvb.dv.RootBitBuf, r: dvb.dv.Range, unit: 8}, nil
 }
 func (dvb decodeValueBase) ExtKeys() []string {
 	kv := []string{
@@ -206,17 +203,17 @@ func (dvb decodeValueBase) JQValueKey(name string) interface{} {
 
 		return dv.Err
 	case "_bits":
-		bb, err := dv.RootBitBuf.BitBufRange(dv.Range.Start, dv.Range.Len)
-		if err != nil {
-			return err
+		return BufferView{
+			bb:   dv.RootBitBuf,
+			r:    dv.Range,
+			unit: 1,
 		}
-		return newBifBufObject(bb, 1)
 	case "_bytes":
-		bb, err := dv.RootBitBuf.BitBufRange(dv.Range.Start, dv.Range.Len)
-		if err != nil {
-			return err
+		return BufferView{
+			bb:   dv.RootBitBuf,
+			r:    dv.Range,
+			unit: 8,
 		}
-		return newBifBufObject(bb, 8)
 	case "_format":
 		if dvb.dv.Format == nil {
 			return nil
@@ -278,17 +275,14 @@ func (v BufferDecodeValue) JQValueIndex(index int) interface{} {
 	return v.JQValueSlice(index, index+1)
 }
 func (v BufferDecodeValue) JQValueSlice(start int, end int) interface{} {
-	bb := v.Buffer.Copy()
-	if start != 0 {
-		if _, err := bb.SeekAbs(int64(start) * 8); err != nil {
-			return err
-		}
+	rStart := int64(start * 8)
+	rLen := int64((end - start) * 8)
+
+	return BufferView{
+		bb:   v.Buffer,
+		r:    ranges.Range{Start: rStart, Len: rLen},
+		unit: 8,
 	}
-	b := &bytes.Buffer{}
-	if _, err := io.CopyN(b, bb, int64(end-start)); err != nil {
-		return err
-	}
-	return b.String()
 }
 func (v BufferDecodeValue) JQValueUpdate(key interface{}, u interface{}, delpath bool) interface{} {
 	return notUpdateableError{Key: fmt.Sprintf("%v", key), Typ: "string"}