wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-09-11 12:05:39 +03:00
Code Issues Projects Releases Wiki Activity

bitio,decode: Refactor bitio usage and make buffer slicing more correct

Browse Source 
Remove bitio.Buffer layer. bitio.Buffer was a kitchen sink layer with helpers
now it's just a buffer and most functions have been moved to decode instead.

bitio package now only have primitive types and functions simialar to standard
library io and bytes packages.

Make nearly eveything internally use bitio.Bit* interfaces so that slicing work
correctly this will also make it possible to start experimenting with more
complicated silcing helpers, ex things like:
breplace(.header.bitrate; 123) to get a new buffer with bitrate changed.
This commit is contained in:
Mattias Wadman 2022-01-24 21:21:48 +01:00
parent 9ed08ec108
commit 7c5215347d
69 changed files with 1696 additions and 1491 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
doc/dev.md
View File

@ -150,11 +150,7 @@ aheadreadseeker.Reader does readahead caching
^
| (io.ReadSeeker interface)
|
bitio.Reader (implements bitio.Bit* interfaces)
^
| (bitio.Bit* interfaces)
|
bitio.Buffer convenience wrapper to read bytes from bit reader, create section readers etc
bitio.IOBitReader (implements bitio.Bit* interfaces)
SectionBitReader
MultiBitReader
```

9
format/bzip2/bzip2.go
View File

@ -14,6 +14,7 @@ import (
"github.com/wader/fq/format"
"github.com/wader/fq/format/registry"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/scalar"
)
@ -108,14 +109,14 @@ func bzip2Decode(d *decode.D, in interface{}) interface{} {
compressedStart := d.Pos()
readCompressedSize, uncompressedBB, dv, _, _ := d.TryFieldReaderRangeFormat("uncompressed", 0, d.Len(), bzip2.NewReader, probeGroup, nil)
if uncompressedBB != nil {
readCompressedSize, uncompressedBR, dv, _, _ := d.TryFieldReaderRangeFormat("uncompressed", 0, d.Len(), bzip2.NewReader, probeGroup, nil)
if uncompressedBR != nil {
if dv == nil {
d.FieldRootBitBuf("uncompressed", uncompressedBB)
d.FieldRootBitBuf("uncompressed", uncompressedBR)
}
blockCRC32W := crc32.NewIEEE()
d.MustCopy(blockCRC32W, bitFlipReader{uncompressedBB.Clone()})
d.MustCopy(blockCRC32W, bitFlipReader{bitio.NewIOReader(uncompressedBR)})
blockCRC32N := bits.Reverse32(binary.BigEndian.Uint32(blockCRC32W.Sum(nil)))
_ = blockCRCValue.TryScalarFn(d.ValidateU(uint64(blockCRC32N)))
streamCRCN = blockCRC32N ^ ((streamCRCN << 1) | (streamCRCN >> 31))

11
format/cbor/cbor.go
View File

@ -135,17 +135,14 @@ func decodeCBORValue(d *decode.D) interface{} {
})
}
})
d.FieldRootBitBuf("value", bitio.NewBufferFromBytes(bb.Bytes(), -1))
d.FieldRootBitBuf("value", bitio.NewBitReader(bb.Bytes(), -1))
// nil, nested indefinite bytes is not allowed
return nil
}
bib := d.FieldRawLen("value", int64(count)*8)
bs, err := bib.Bytes()
if err != nil {
d.IOPanic(err, "bytes bb.Bytes")
}
return bs
buf := d.MustReadAllBits(d.FieldRawLen("value", int64(count)*8))
return buf
}},
majorTypeUTF8: {s: scalar.S{Sym: "utf8"}, d: func(d *decode.D, shortCount uint64, count uint64) interface{} {
if shortCount == shortCountIndefinite {

2
format/flac/flac.go
View File

@ -80,7 +80,7 @@ func flacDecode(d *decode.D, in interface{}) interface{} {
}
})
md5CalcValue := d.FieldRootBitBuf("md5_calculated", bitio.NewBufferFromBytes(md5Samples.Sum(nil), -1))
md5CalcValue := d.FieldRootBitBuf("md5_calculated", bitio.NewBitReader(md5Samples.Sum(nil), -1))
_ = md5CalcValue.TryScalarFn(d.ValidateBitBuf(streamInfo.MD5), scalar.RawHex)
d.FieldValueU("decoded_samples", framesNDecodedSamples)

4
format/flac/flac_frame.go
View File

@ -327,7 +327,7 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
})
headerCRC := &checksum.CRC{Bits: 8, Table: checksum.ATM8Table}
d.MustCopy(headerCRC, d.BitBufRange(frameStart, d.Pos()-frameStart))
d.MustCopyBits(headerCRC, d.BitBufRange(frameStart, d.Pos()-frameStart))
d.FieldU8("crc", d.ValidateUBytes(headerCRC.Sum(nil)), scalar.Hex)
})
@ -573,7 +573,7 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
d.FieldU("byte_align", d.ByteAlignBits(), d.AssertU(0))
// <16> CRC-16 (polynomial = x^16 + x^15 + x^2 + x^0, initialized with 0) of everything before the crc, back to and including the frame header sync code
footerCRC := &checksum.CRC{Bits: 16, Table: checksum.ANSI16Table}
d.MustCopy(footerCRC, d.BitBufRange(frameStart, d.Pos()-frameStart))
d.MustCopyBits(footerCRC, d.BitBufRange(frameStart, d.Pos()-frameStart))
d.FieldRawLen("footer_crc", 16, d.ValidateBitBuf(footerCRC.Sum(nil)), scalar.RawHex)
streamSamples := len(channelSamples[0])

7
format/flac/flac_streaminfo.go
View File

@ -26,11 +26,8 @@ func streaminfoDecode(d *decode.D, in interface{}) interface{} {
// <5> (bits per sample)-1. FLAC supports from 4 to 32 bits per sample. Currently the reference encoder and decoders only support up to 24 bits per sample.
bitPerSample := d.FieldU5("bits_per_sample", scalar.UAdd(1))
totalSamplesInStream := d.FieldU("total_samples_in_stream", 36)
md5BB := d.FieldRawLen("md5", 16*8, scalar.RawHex)
md5b, err := md5BB.Bytes()
if err != nil {
d.IOPanic(err, "md5BB.Bytes")
}
md5BR := d.FieldRawLen("md5", 16*8, scalar.RawHex)
md5b := d.MustReadAllBits(md5BR)
return format.FlacStreaminfoOut{
StreamInfo: format.FlacStreamInfo{

10
format/gif/gif.go
View File

@ -73,9 +73,9 @@ func gifDecode(d *decode.D, in interface{}) interface{} {
blocks:
for {
switch d.PeekBits(8) {
case 0x3b: /* ";" */
case ';':
break blocks
case 0x21: /* "!" */
case '!': /* "!" */
d.FieldStruct("extension_block", func(d *decode.D) {
d.FieldU8("introducer")
functionCode := d.FieldU8("function_code", extensionNames, scalar.Hex)
@ -93,7 +93,7 @@ func gifDecode(d *decode.D, in interface{}) interface{} {
d.FieldU8("terminator")
seenTerminator = true
}
d.MustCopy(dataBytes, b.Clone())
d.MustCopyBits(dataBytes, d.MustClone(b))
})
}
})
@ -105,12 +105,12 @@ func gifDecode(d *decode.D, in interface{}) interface{} {
// case extensionGraphicalControl:
// d.FieldFormatBitBuf(
// "graphics_control",
// bitio.NewBufferFromBytes(dataBytes.Bytes(), -1),
// bitio.NewReader(dataBytes.Bytes(), -1),
// )
// }
})
case 0x2c: /* "," */
case ',':
d.FieldStruct("image", func(d *decode.D) {
d.FieldU8("separator_character")
d.FieldU16("left")

11
format/gzip/gzip.go
View File

@ -101,20 +101,21 @@ func gzDecode(d *decode.D, in interface{}) interface{} {
var rFn func(r io.Reader) io.Reader
switch compressionMethod {
case delfateMethod:
// *bitio.Buffer implements io.ByteReader so hat deflate don't do own
// bitio.NewIOReadSeeker implements io.ByteReader so that deflate don't do own
// buffering and might read more than needed messing up knowing compressed size
rFn = func(r io.Reader) io.Reader { return flate.NewReader(r) }
}
if rFn != nil {
readCompressedSize, uncompressedBB, dv, _, _ := d.TryFieldReaderRangeFormat("uncompressed", d.Pos(), d.BitsLeft(), rFn, probeFormat, nil)
if uncompressedBB != nil {
readCompressedSize, uncompressedBR, dv, _, _ := d.TryFieldReaderRangeFormat("uncompressed", d.Pos(), d.BitsLeft(), rFn, probeFormat, nil)
if uncompressedBR != nil {
if dv == nil {
d.FieldRootBitBuf("uncompressed", uncompressedBB)
d.FieldRootBitBuf("uncompressed", uncompressedBR)
}
d.FieldRawLen("compressed", readCompressedSize)
crc32W := crc32.NewIEEE()
d.MustCopy(crc32W, uncompressedBB.Clone())
// TODO: cleanup clone
d.MustCopyBits(crc32W, d.MustClone(uncompressedBR))
d.FieldU32("crc32", d.ValidateUBytes(crc32W.Sum(nil)), scalar.Hex)
d.FieldU32("isize")
}

14
format/gzip/testdata/compressed_size.fqtest vendored Normal file
View File

@ -0,0 +1,14 @@
# this tests compressed size (TryFieldReaderRangeFormat)
$ fq -d gzip 'tobits | chunk(3) | gzip' test.gz
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: (gzip)
0x00|1f 8b |.. | identification: raw bits (valid)
0x00| 08 | . | compression_method: "deflate" (8)
0x00| 00 | . | flags{}:
0x00| 41 02 ea 5f | A.._ | mtime: 1609171521
0x00| 00 | . | extra_flags: 0
0x00| 03 | . | os: "Unix" (3)
0x0|74 65 73 74 0a| |test.| | uncompressed: raw bits
0x00| 2b 49 2d 2e e1 02| +I-...| compressed: raw bits
0x10|00 |. |
0x10| c6 35 b9 3b | .5.; | crc32: 0x3bb935c6 (valid)
0x10| 05 00 00 00| | ....| | isize: 5

5
format/id3/id3v2.go
View File

@ -11,6 +11,7 @@ import (
"github.com/wader/fq/format"
"github.com/wader/fq/format/registry"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/scalar"
"golang.org/x/text/encoding"
@ -533,8 +534,8 @@ func decodeFrame(d *decode.D, version int) uint64 {
if unsyncFlag {
// TODO: DecodeFn
// TODO: unknown after frame decode
unsyncedBb := d.MustNewBitBufFromReader(unsyncReader{Reader: d.BitBufRange(d.Pos(), int64(dataSize)*8)})
d.FieldFormatBitBuf("unsync", unsyncedBb, decode.FormatFn(func(d *decode.D, in interface{}) interface{} {
unsyncedBR := d.MustNewBitBufFromReader(unsyncReader{Reader: bitio.NewIOReader(d.BitBufRange(d.Pos(), int64(dataSize)*8))})
d.FieldFormatBitBuf("unsync", unsyncedBR, decode.FormatFn(func(d *decode.D, in interface{}) interface{} {
if fn, ok := frames[idNormalized]; ok {
fn(d)
} else {

5
format/inet/ipv4_packet.go
View File

@ -6,6 +6,7 @@ import (
"github.com/wader/fq/format"
"github.com/wader/fq/format/registry"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/checksum"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/scalar"
@ -98,8 +99,8 @@ func decodeIPv4(d *decode.D, in interface{}) interface{} {
headerEnd := d.Pos()
ipv4Checksum := &checksum.IPv4{}
d.MustCopy(ipv4Checksum, d.BitBufRange(0, checksumStart))
d.MustCopy(ipv4Checksum, d.BitBufRange(checksumEnd, headerEnd-checksumEnd))
d.MustCopy(ipv4Checksum, bitio.NewIOReader(d.BitBufRange(0, checksumStart)))
d.MustCopy(ipv4Checksum, bitio.NewIOReader(d.BitBufRange(checksumEnd, headerEnd-checksumEnd)))
_ = d.FieldMustGet("header_checksum").TryScalarFn(d.ValidateUBytes(ipv4Checksum.Sum(nil)), scalar.Hex)
dataLen := int64(totalLength-(ihl*4)) * 8

7
format/jpeg/jpeg.go
View File

@ -306,10 +306,7 @@ func jpegDecode(d *decode.D, in interface{}) interface{} {
// TODO: FieldBitsLen? concat bitbuf?
chunk := d.FieldRawLen("data", d.BitsLeft())
// TODO: redo this? multi reader?
chunkBytes, err := chunk.Bytes()
if err != nil {
d.Fatalf("failed to read xmp chunk: %s", err)
}
chunkBytes := d.MustReadAllBits(chunk)
if extendedXMP == nil {
extendedXMP = make([]byte, fullLength)
@ -348,7 +345,7 @@ func jpegDecode(d *decode.D, in interface{}) interface{} {
}
if extendedXMP != nil {
d.FieldRootBitBuf("extended_xmp", bitio.NewBufferFromBytes(extendedXMP, -1))
d.FieldRootBitBuf("extended_xmp", bitio.NewBitReader(extendedXMP, -1))
}
return nil

5
format/json/json.go
View File

@ -5,6 +5,7 @@ import (
"github.com/wader/fq/format"
"github.com/wader/fq/format/registry"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/scalar"
)
@ -25,8 +26,8 @@ func init() {
}
func decodeJSON(d *decode.D, in interface{}) interface{} {
bb := d.RawLen(d.Len())
jd := stdjson.NewDecoder(bb)
br := d.RawLen(d.Len())
jd := stdjson.NewDecoder(bitio.NewIOReader(br))
var s scalar.S
if err := jd.Decode(&s.Actual); err != nil {
d.Fatalf(err.Error())

7
format/mp4/boxes.go
View File

@ -964,12 +964,9 @@ func init() {
version := d.FieldU8("version")
d.FieldU24("flags")
systemIDBB := d.FieldRawLen("system_id", 6*8, systemIDNames)
systemIDBR := d.FieldRawLen("system_id", 6*8, systemIDNames)
// TODO: make nicer
systemID, err := systemIDBB.Bytes()
if err != nil {
d.IOPanic(err, "systemIDBB.Bytes")
}
systemID := d.MustReadAllBits(systemIDBR)
switch version {
case 0:
case 1:

9
format/mpeg/avc_nalu.go
View File

@ -6,6 +6,7 @@ import (
"github.com/wader/fq/format"
"github.com/wader/fq/format/registry"
"github.com/wader/fq/internal/mathextra"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/scalar"
)
@ -101,7 +102,7 @@ func avcNALUDecode(d *decode.D, in interface{}) interface{} {
d.FieldBool("forbidden_zero_bit")
d.FieldU2("nal_ref_idc")
nalType := d.FieldU5("nal_unit_type", avcNALNames)
unescapedBb := d.MustNewBitBufFromReader(decode.NALUnescapeReader{Reader: d.BitBufRange(d.Pos(), d.BitsLeft())})
unescapedBR := d.MustNewBitBufFromReader(decode.NALUnescapeReader{Reader: bitio.NewIOReader(d.BitBufRange(d.Pos(), d.BitsLeft()))})
switch nalType {
case avcNALCodedSliceNonIDR,
@ -118,11 +119,11 @@ func avcNALUDecode(d *decode.D, in interface{}) interface{} {
// TODO: if ( separate_colour_plane_flag from SPS ) colour_plane_id; frame_num
})
case avcNALSupplementalEnhancementInformation:
d.FieldFormatBitBuf("sei", unescapedBb, avcSEIFormat, nil)
d.FieldFormatBitBuf("sei", unescapedBR, avcSEIFormat, nil)
case avcNALSequenceParameterSet:
d.FieldFormatBitBuf("sps", unescapedBb, avcSPSFormat, nil)
d.FieldFormatBitBuf("sps", unescapedBR, avcSPSFormat, nil)
case avcNALPictureParameterSet:
d.FieldFormatBitBuf("pps", unescapedBb, avcPPSFormat, nil)
d.FieldFormatBitBuf("pps", unescapedBR, avcPPSFormat, nil)
}
d.FieldRawLen("data", d.BitsLeft())

5
format/mpeg/hevc_nalu.go
View File

@ -3,6 +3,7 @@ package mpeg
import (
"github.com/wader/fq/format"
"github.com/wader/fq/format/registry"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/scalar"
)
@ -72,9 +73,9 @@ func hevcNALUDecode(d *decode.D, in interface{}) interface{} {
nalType := d.FieldU6("nal_unit_type", hevcNALNames)
d.FieldU6("nuh_layer_id")
d.FieldU3("nuh_temporal_id_plus1")
unescapedBb := d.MustNewBitBufFromReader(decode.NALUnescapeReader{Reader: d.BitBufRange(d.Pos(), d.BitsLeft())})
unescapedBR := d.MustNewBitBufFromReader(decode.NALUnescapeReader{Reader: bitio.NewIOReader(d.BitBufRange(d.Pos(), d.BitsLeft()))})
_ = unescapedBb
_ = unescapedBR
_ = nalType
d.FieldRawLen("data", d.BitsLeft())

4
format/mpeg/mp3_frame.go
View File

@ -386,8 +386,8 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
crcHash := &checksum.CRC{Bits: 16, Current: 0xffff, Table: checksum.ANSI16Table}
// 2 bytes after sync and some other fields + all of side info
d.MustCopy(crcHash, d.BitBufRange(2*8, 2*8))
d.MustCopy(crcHash, d.BitBufRange(6*8, sideInfoBytes*8))
d.MustCopyBits(crcHash, d.BitBufRange(2*8, 2*8))
d.MustCopyBits(crcHash, d.BitBufRange(6*8, sideInfoBytes*8))
if crcValue != nil {
_ = crcValue.TryScalarFn(d.ValidateUBytes(crcHash.Sum(nil)))

9
format/mpeg/mpeg_pes.go
View File

@ -61,17 +61,16 @@ func pesDecode(d *decode.D, in interface{}) interface{} {
s = &subStream{}
substreams[dvv.number] = s
}
b, _ := dvv.bb.BytesRange(0, int(dvv.bb.Len()/8))
s.b = append(s.b, b...)
s.b = append(s.b, dvv.buf...)
if s.l == 0 && len(b) >= 2 {
s.l = int(b[0])<<8 | int(b[1])
if s.l == 0 && len(s.b) >= 2 {
s.l = int(s.b[0])<<8 | int(s.b[1])
// TODO: zero l?
}
// TODO: is this how spu end is signalled?
if s.l == len(s.b) {
spuD.FieldFormatBitBuf("spu", bitio.NewBufferFromBytes(s.b, -1), spuFormat, nil)
spuD.FieldFormatBitBuf("spu", bitio.NewBitReader(s.b, -1), spuFormat, nil)
s.b = nil
s.l = 0
}

7
format/mpeg/mpeg_pes_packet.go
View File

@ -6,7 +6,6 @@ package mpeg
import (
"github.com/wader/fq/format"
"github.com/wader/fq/format/registry"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/scalar"
)
@ -28,7 +27,7 @@ const (
type subStreamPacket struct {
number int
bb *bitio.Buffer
buf []byte
}
var startAndStreamNames = scalar.URangeToScalar{
@ -174,11 +173,11 @@ func pesPacketDecode(d *decode.D, in interface{}) interface{} {
d.FieldStruct("data", func(d *decode.D) {
d.LenFn(dataLen, func(d *decode.D) {
substreamNumber := d.FieldU8("substream")
substreamBB := d.FieldRawLen("data", dataLen-8)
substreamBR := d.FieldRawLen("data", dataLen-8)
v = subStreamPacket{
number: int(substreamNumber),
bb: substreamBB,
buf: d.MustReadAllBits(substreamBR),
}
})
})

33
format/ogg/ogg.go
View File

@ -105,14 +105,14 @@ func decodeOgg(d *decode.D, in interface{}) interface{} {
for _, bs := range oggPageOut.Segments {
s.packetBuf = append(s.packetBuf, bs...)
if len(bs) < 255 {
bb := bitio.NewBufferFromBytes(s.packetBuf, -1)
br := bitio.NewBitReader(s.packetBuf, -1)
if s.codec == codecUnknown {
if b, err := bb.PeekBytes(len(vorbisIdentification)); err == nil && bytes.Equal(b, vorbisIdentification) {
if bytes.HasPrefix(s.packetBuf, vorbisIdentification) {
s.codec = codecVorbis
} else if b, err := bb.PeekBytes(len(opusIdentification)); err == nil && bytes.Equal(b, opusIdentification) {
} else if bytes.HasPrefix(s.packetBuf, opusIdentification) {
s.codec = codecOpus
} else if b, err := bb.PeekBytes(len(flacIdentification)); err == nil && bytes.Equal(b, flacIdentification) {
} else if bytes.HasPrefix(s.packetBuf, flacIdentification) {
s.codec = codecFlac
}
}
@ -120,25 +120,22 @@ func decodeOgg(d *decode.D, in interface{}) interface{} {
switch s.codec {
case codecVorbis:
// TODO: err
if _, _, err := s.packetD.TryFieldFormatBitBuf("packet", bb, vorbisPacketFormat, nil); err != nil {
s.packetD.FieldRootBitBuf("packet", bb)
if _, _, err := s.packetD.TryFieldFormatBitBuf("packet", br, vorbisPacketFormat, nil); err != nil {
s.packetD.FieldRootBitBuf("packet", br)
}
case codecOpus:
// TODO: err
if _, _, err := s.packetD.TryFieldFormatBitBuf("packet", bb, opusPacketFormat, nil); err != nil {
s.packetD.FieldRootBitBuf("packet", bb)
if _, _, err := s.packetD.TryFieldFormatBitBuf("packet", br, opusPacketFormat, nil); err != nil {
s.packetD.FieldRootBitBuf("packet", br)
}
case codecFlac:
var firstByte byte
bs, err := bb.PeekBytes(1)
if err != nil {
if len(s.packetBuf) == 0 {
return
}
firstByte = bs[0]
switch {
case firstByte == 0x7f:
s.packetD.FieldStructRootBitBufFn("packet", bb, func(d *decode.D) {
case s.packetBuf[0] == 0x7f:
s.packetD.FieldStructRootBitBufFn("packet", br, func(d *decode.D) {
d.FieldU8("type")
d.FieldUTF8("signature", 4)
d.FieldU8("major")
@ -152,13 +149,13 @@ func decodeOgg(d *decode.D, in interface{}) interface{} {
}
s.flacStreamInfo = flacMetadatablockOut.StreamInfo
})
case firstByte == 0xff:
s.packetD.FieldFormatBitBuf("packet", bb, flacFrameFormat, nil)
case s.packetBuf[0] == 0xff:
s.packetD.FieldFormatBitBuf("packet", br, flacFrameFormat, nil)
default:
s.packetD.FieldFormatBitBuf("packet", bb, flacMetadatablockFormat, nil)
s.packetD.FieldFormatBitBuf("packet", br, flacMetadatablockFormat, nil)
}
case codecUnknown:
s.packetD.FieldRootBitBuf("packet", bb)
s.packetD.FieldRootBitBuf("packet", br)
}
s.packetBuf = nil

12
format/ogg/ogg_page.go
View File

@ -5,6 +5,7 @@ import (
"github.com/wader/fq/format"
"github.com/wader/fq/format/registry"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/checksum"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/scalar"
@ -43,10 +44,7 @@ func pageDecode(d *decode.D, in interface{}) interface{} {
})
d.FieldArray("segments", func(d *decode.D) {
for _, ss := range segmentTable {
bs, err := d.FieldRawLen("segment", int64(ss)*8).Bytes()
if err != nil {
d.IOPanic(err, "d.BitBufRange segment")
}
bs := d.MustReadAllBits(d.FieldRawLen("segment", int64(ss)*8))
p.Segments = append(p.Segments, bs)
}
})
@ -54,9 +52,9 @@ func pageDecode(d *decode.D, in interface{}) interface{} {
pageChecksumValue := d.FieldGet("crc")
pageCRC := &checksum.CRC{Bits: 32, Table: checksum.Poly04c11db7Table}
d.MustCopy(pageCRC, d.BitBufRange(startPos, pageChecksumValue.Range.Start-startPos)) // header before checksum
d.MustCopy(pageCRC, bytes.NewReader([]byte{0, 0, 0, 0})) // zero checksum bits
d.MustCopy(pageCRC, d.BitBufRange(pageChecksumValue.Range.Stop(), endPos-pageChecksumValue.Range.Stop())) // rest of page
d.MustCopy(pageCRC, bitio.NewIOReader(d.BitBufRange(startPos, pageChecksumValue.Range.Start-startPos))) // header before checksum
d.MustCopy(pageCRC, bytes.NewReader([]byte{0, 0, 0, 0})) // zero checksum bits
d.MustCopy(pageCRC, bitio.NewIOReader(d.BitBufRange(pageChecksumValue.Range.Stop(), endPos-pageChecksumValue.Range.Stop()))) // rest of page
_ = pageChecksumValue.TryScalarFn(d.ValidateUBytes(pageCRC.Sum(nil)))
return p

7
format/pcap/pcap.go
View File

@ -78,12 +78,7 @@ func decodePcap(d *decode.D, in interface{}) interface{} {
d.Errorf("incl_len %d > orig_len %d", inclLen, origLen)
}
bb := d.BitBufRange(d.Pos(), int64(inclLen)*8)
bs, err := bb.Bytes()
if err != nil {
// TODO:
d.IOPanic(err, "BitBufRange: inclLen")
}
bs := d.MustReadAllBits(d.BitBufRange(d.Pos(), int64(inclLen)*8))
if fn, ok := linkToDecodeFn[linkType]; ok {
// TODO: report decode errors

5
format/pcap/pcapng.go
View File

@ -230,10 +230,7 @@ var blockFns = map[uint64]func(d *decode.D, dc *decodeContext){
capturedLength := d.FieldU32("capture_packet_length")
d.FieldU32("original_packet_length")
bs, err := d.BitBufRange(d.Pos(), int64(capturedLength)*8).Bytes()
if err != nil {
d.IOPanic(err, "d.BitBufRange")
}
bs := d.MustReadAllBits(d.BitBufRange(d.Pos(), int64(capturedLength)*8))
linkType := dc.interfaceTypes[int(interfaceID)]

18
format/pcap/shared.go
View File

@ -36,14 +36,14 @@ var linkToDecodeFn = map[int]func(fd *flowsdecoder.Decoder, bs []byte) error{
func fieldFlows(d *decode.D, fd *flowsdecoder.Decoder, tcpStreamFormat decode.Group, ipv4PacketFormat decode.Group) {
d.FieldArray("ipv4_reassembled", func(d *decode.D) {
for _, p := range fd.IPV4Reassembled {
bb := bitio.NewBufferFromBytes(p.Datagram, -1)
br := bitio.NewBitReader(p.Datagram, -1)
if dv, _, _ := d.TryFieldFormatBitBuf(
"ipv4_packet",
bb,
br,
ipv4PacketFormat,
nil,
); dv == nil {
d.FieldRootBitBuf("ipv4_packet", bb)
d.FieldRootBitBuf("ipv4_packet", br)
}
}
})
@ -55,30 +55,30 @@ func fieldFlows(d *decode.D, fd *flowsdecoder.Decoder, tcpStreamFormat decode.Gr
d.FieldValueU("source_port", uint64(s.ClientEndpoint.Port), format.TCPPortMap)
d.FieldValueStr("destination_ip", s.ServerEndpoint.IP.String())
d.FieldValueU("destination_port", uint64(s.ServerEndpoint.Port), format.TCPPortMap)
csBB := bitio.NewBufferFromBytes(s.ClientToServer.Bytes(), -1)
csBR := bitio.NewBitReader(s.ClientToServer.Bytes(), -1)
if dv, _, _ := d.TryFieldFormatBitBuf(
"client_stream",
csBB,
csBR,
tcpStreamFormat,
format.TCPStreamIn{
SourcePort: s.ClientEndpoint.Port,
DestinationPort: s.ServerEndpoint.Port,
},
); dv == nil {
d.FieldRootBitBuf("client_stream", csBB)
d.FieldRootBitBuf("client_stream", csBR)
}
scBB := bitio.NewBufferFromBytes(s.ServerToClient.Bytes(), -1)
scBR := bitio.NewBitReader(s.ServerToClient.Bytes(), -1)
if dv, _, _ := d.TryFieldFormatBitBuf(
"server_stream",
scBB,
scBR,
tcpStreamFormat,
format.TCPStreamIn{
SourcePort: s.ClientEndpoint.Port,
DestinationPort: s.ServerEndpoint.Port,
},
); dv == nil {
d.FieldRootBitBuf("server_stream", scBB)
d.FieldRootBitBuf("server_stream", scBR)
}
})
}

3
format/png/png.go
View File

@ -10,6 +10,7 @@ import (
"github.com/wader/fq/format"
"github.com/wader/fq/format/registry"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/scalar"
)
@ -227,7 +228,7 @@ func pngDecode(d *decode.D, in interface{}) interface{} {
})
chunkCRC := crc32.NewIEEE()
d.MustCopy(chunkCRC, d.BitBufRange(crcStartPos, d.Pos()-crcStartPos))
d.MustCopy(chunkCRC, bitio.NewIOReader(d.BitBufRange(crcStartPos, d.Pos()-crcStartPos)))
d.FieldU32("crc", d.ValidateUBytes(chunkCRC.Sum(nil)), scalar.Hex)
})

6
format/vorbis/vorbis_comment.go
View File

@ -42,14 +42,14 @@ func commentDecode(d *decode.D, in interface{}) interface{} {
base64Offset := int64(len(metadataBlockPicturePreix)) * 8
base64Len := int64(len(userComment))*8 - base64Offset
_, base64BB, dv, _, _ := d.TryFieldReaderRangeFormat(
_, base64Br, dv, _, _ := d.TryFieldReaderRangeFormat(
"picture",
userCommentStart+base64Offset, base64Len,
func(r io.Reader) io.Reader { return base64.NewDecoder(base64.StdEncoding, r) },
flacPicture, nil,
)
if dv == nil && base64BB != nil {
d.FieldRootBitBuf("picture", base64BB)
if dv == nil && base64Br != nil {
d.FieldRootBitBuf("picture", base64Br)
}
}
i++

8
format/zip/zip.go
View File

@ -285,15 +285,15 @@ func zipDecode(d *decode.D, in interface{}) interface{} {
var rFn func(r io.Reader) io.Reader
switch compressionMethod {
case compressionMethodDeflated:
// *bitio.Buffer implements io.ByteReader so hat deflate don't do own
// bitio.NewIOReadSeeker implements io.ByteReader so that deflate don't do own
// buffering and might read more than needed messing up knowing compressed size
rFn = func(r io.Reader) io.Reader { return flate.NewReader(r) }
}
if rFn != nil {
readCompressedSize, uncompressedBB, dv, _, _ := d.TryFieldReaderRangeFormat("uncompressed", d.Pos(), compressedLimit, rFn, probeFormat, nil)
if dv == nil && uncompressedBB != nil {
d.FieldRootBitBuf("uncompressed", uncompressedBB)
readCompressedSize, uncompressedBR, dv, _, _ := d.TryFieldReaderRangeFormat("uncompressed", d.Pos(), compressedLimit, rFn, probeFormat, nil)
if dv == nil && uncompressedBR != nil {
d.FieldRootBitBuf("uncompressed", uncompressedBR)
}
if compressedSize == 0 {
compressedSize = readCompressedSize

56
internal/bitioextra/bitioextra.go Normal file
View File

@ -0,0 +1,56 @@
package bitioextra
// bitio helpers that don't really belong in bitio
import (
"errors"
"io"
"github.com/wader/fq/pkg/bitio"
)
func CopyBitsBuffer(dst io.Writer, src bitio.Reader, buf []byte) (int64, error) {
return io.CopyBuffer(dst, bitio.NewIOReader(src), buf)
}
func CopyBits(dst io.Writer, src bitio.Reader) (int64, error) {
return CopyBitsBuffer(dst, src, nil)
}
func Clone(br bitio.ReaderAtSeeker) (bitio.ReaderAtSeeker, error) {
l, err := Len(br)
if err != nil {
return nil, err
}
return bitio.NewSectionReader(br, 0, l), nil
}
func Len(br bitio.ReadAtSeeker) (int64, error) {
bPos, err := br.SeekBits(0, io.SeekCurrent)
if err != nil {
return 0, err
}
bEnd, err := br.SeekBits(0, io.SeekEnd)
if err != nil {
return 0, err
}
if _, err := br.SeekBits(bPos, io.SeekStart); err != nil {
return 0, err
}
return bEnd, nil
}
func Range(br bitio.ReadAtSeeker, firstBitOffset int64, nBits int64) (bitio.ReaderAtSeeker, error) {
l, err := Len(br)
if err != nil {
return nil, err
}
// TODO: move error check?
if nBits < 0 {
return nil, errors.New("negative nBits")
}
if firstBitOffset+nBits > l {
return nil, errors.New("outside buffer")
}
return bitio.NewSectionReader(br, firstBitOffset, nBits), nil
}

12
internal/hexdump/hexdump.go
View File

@ -25,7 +25,7 @@ type Dumper struct {
hasWrittenHeader bool
bitsBuf []byte
bitsBufN int
bitsBufN int64
}
// TODO: something more generic? bin, octal, arbitrary base?
@ -72,11 +72,11 @@ func (d *Dumper) flush() error {
return nil
}
func (d *Dumper) WriteBits(p []byte, nBits int) (n int, err error) {
pos := 0
func (d *Dumper) WriteBits(p []byte, nBits int64) (n int64, err error) {
pos := int64(0)
rBits := nBits
if d.bitsBufN > 0 {
r := mathextra.MinInt(8-d.bitsBufN, nBits)
r := mathextra.MinInt64(8-d.bitsBufN, nBits)
v := bitio.Read64(p, 0, r)
bitio.Write64(v, r, d.bitsBuf, d.bitsBufN)
@ -86,7 +86,7 @@ func (d *Dumper) WriteBits(p []byte, nBits int) (n int, err error) {
return nBits, nil
}
if n, err := d.Write(d.bitsBuf); err != nil {
return n * 8, err
return int64(n) * 8, err
}
pos = r
rBits -= r
@ -97,7 +97,7 @@ func (d *Dumper) WriteBits(p []byte, nBits int) (n int, err error) {
b[0] = byte(bitio.Read64(p, pos, 8))
if n, err := d.Write(b[:]); err != nil {
return n * 8, err
return int64(n) * 8, err
}
pos += 8

53
internal/hexpairwriter/hexpairwriter.go
View File

@ -4,9 +4,6 @@ package hexpairwriter
import (
"io"
"github.com/wader/fq/internal/mathextra"
"github.com/wader/fq/pkg/bitio"
)
type Writer struct {
@ -17,9 +14,6 @@ type Writer struct {
offset int
buf []byte
bufOffset int
bitsBuf []byte
bitsBufN int
}
// for i in $(seq 0 255) ; do printf "%02x" $i ; done | while read -n 32 s ; do echo "\"$s\""+; done
@ -53,9 +47,7 @@ func New(w io.Writer, width int, startLineOffset int, fn func(b byte) string) *W
fn: fn,
offset: 0,
// TODO: ansi length? nicer reusable buffer?
buf: make([]byte, width*200+1), // worst case " " or "\n" + width*(XX "+ansi) + "\n"
bufOffset: 0,
bitsBuf: make([]byte, 1),
buf: make([]byte, width*200+1), // worst case " " or "\n" + width*(XX "+ansi) + "\n"
}
}
@ -110,46 +102,3 @@ func (h *Writer) Write(p []byte) (n int, err error) {
return len(p), nil
}
func (h *Writer) WriteBits(p []byte, nBits int) (n int, err error) {
pos := 0
rBits := nBits
if h.bitsBufN > 0 {
r := mathextra.MinInt(8-h.bitsBufN, nBits)
v := bitio.Read64(p, 0, r)
bitio.Write64(v, r, h.bitsBuf, h.bitsBufN)
h.bitsBufN += r
if h.bitsBufN < 8 {
return nBits, nil
}
if n, err := h.Write(h.bitsBuf); err != nil {
return n * 8, err
}
pos = r
rBits -= r
}
for rBits >= 8 {
b := [1]byte{0}
b[0] = byte(bitio.Read64(p, pos, 8))
if n, err := h.Write(b[:]); err != nil {
return n * 8, err
}
pos += 8
rBits -= 8
}
if rBits > 0 {
h.bitsBuf[0] = byte(bitio.Read64(p, pos, rBits)) << (8 - rBits)
h.bitsBufN = rBits
} else {
h.bitsBufN = 0
}
return nBits, nil
}

18
pkg/bitio/README.md Normal file
View File

@ -0,0 +1,18 @@
The bitio package tries to mimic the standard library packages io and bytes as much as possible.
- bitio.Buffer same as bytes.Buffer
- bitio.IOBitReadSeeker is a bitio.ReaderAtSeeker that from a io.ReadSeeker
- bitio.IOBitWriter a bitio.BitWriter that write bytes to a io.Writer, use Flush() to write possible unaligned byte
- bitio.IOReader is a io.Reader that reads bytes from a bit reader, will zero pad on unaligned byte eof
- bitio.IOReadSeeker is a io.ReadSeeker that read/seek bytes in a bit stream, will zero pad on unaligned - bitio.NewBitReader same as bytes.NewReader
- bitio.LimitReader same as io.LimitReader
- bitio.MultiReader same as io.MultiReader
- bitio.SectionReader same as io.SectionReader
- bitio.Copy* same as io.Copy*
- bitio.ReadFull same as io.ReadFull
TODO:
- bitio.IOBitReader bitio.Reader that reads from a io.Reader
- bitio.IOBitWriteSeeker bitio.BitWriteSeeker that writes to a io.WriteSeeker
- bitio.CopyN
- speed up read by using a cache somehow ([]byte or just a uint64?)

162
pkg/bitio/bitio.go
View File

@ -1,40 +1,107 @@
package bitio
import (
"bytes"
"errors"
"fmt"
"io"
"strings"
)
var ErrOffset = errors.New("invalid seek offset")
var ErrNegativeNBits = errors.New("negative number of bits")
type BitReaderAt interface {
ReadBitsAt(p []byte, nBits int, bitOff int64) (n int, err error)
type ReaderAt interface {
ReadBitsAt(p []byte, nBits int64, bitOff int64) (n int64, err error)
}
type BitReader interface {
ReadBits(p []byte, nBits int) (n int, err error)
type Reader interface {
ReadBits(p []byte, nBits int64) (n int64, err error)
}
type BitSeeker interface {
type Seeker interface {
SeekBits(bitOffset int64, whence int) (int64, error)
}
type BitReadSeeker interface {
BitReader
BitSeeker
type ReadSeeker interface {
Reader
Seeker
}
type BitReadAtSeeker interface {
BitReaderAt
BitSeeker
type ReadAtSeeker interface {
ReaderAt
Seeker
}
type BitWriter interface {
WriteBits(p []byte, nBits int) (n int, err error)
type ReaderAtSeeker interface {
Reader
ReaderAt
Seeker
}
func CopyBuffer(dst BitWriter, src BitReader, buf []byte) (n int64, err error) {
type Writer interface {
WriteBits(p []byte, nBits int64) (n int64, err error)
}
// NewBitReader reading nBits bits from a []byte
// If nBits is -1 all bits will be used.
// Similar to bytes.NewReader
func NewBitReader(buf []byte, nBits int64) *SectionReader {
if nBits < 0 {
nBits = int64(len(buf)) * 8
}
return NewSectionReader(
NewIOBitReadSeeker(bytes.NewReader(buf)),
0,
nBits,
)
}
// BitsByteCount returns smallest amount of bytes to fit nBits bits
func BitsByteCount(nBits int64) int64 {
n := nBits / 8
if nBits%8 != 0 {
n++
}
return n
}
// BytesFromBitString []byte from bit string, ex: "0101" -> ([]byte{0x50}, 4)
func BytesFromBitString(s string) ([]byte, int64) {
r := len(s) % 8
bufLen := len(s) / 8
if r > 0 {
bufLen++
}
buf := make([]byte, bufLen)
for i := 0; i < len(s); i++ {
d := s[i] - '0'
if d != 0 && d != 1 {
panic(fmt.Sprintf("invalid bit string %q at index %d %q", s, i, s[i]))
}
buf[i/8] |= d << (7 - i%8)
}
return buf, int64(len(s))
}
// BitStringFromBytes string from []byte], ex: ([]byte{0x50}, 4) -> "0101"
func BitStringFromBytes(buf []byte, nBits int64) string {
sb := &strings.Builder{}
for i := int64(0); i < nBits; i++ {
if buf[i/8]&(1<<(7-i%8)) > 0 {
sb.WriteString("1")
} else {
sb.WriteString("0")
}
}
return sb.String()
}
// CopyBuffer bits from src to dst using provided buffer
// Similar to io.CopyBuffer
func CopyBuffer(dst Writer, src Reader, buf []byte) (n int64, err error) {
// same default size as io.Copy
if buf == nil {
buf = make([]byte, 32*1024)
@ -42,10 +109,10 @@ func CopyBuffer(dst BitWriter, src BitReader, buf []byte) (n int64, err error) {
var written int64
for {
rBits, rErr := src.ReadBits(buf, len(buf)*8)
rBits, rErr := src.ReadBits(buf, int64(len(buf))*8)
if rBits > 0 {
wBits, wErr := dst.WriteBits(buf, rBits)
written += int64(wBits)
written += wBits
if wErr != nil {
err = wErr
break
@ -66,25 +133,41 @@ func CopyBuffer(dst BitWriter, src BitReader, buf []byte) (n int64, err error) {
return written, err
}
func Copy(dst BitWriter, src BitReader) (n int64, err error) {
// Copy bits from src to dst
// Similar to io.Copy
func Copy(dst Writer, src Reader) (n int64, err error) {
return CopyBuffer(dst, src, nil)
}
// BitsByteCount returns smallest amount of bytes to fit nBits bits
func BitsByteCount(nBits int64) int64 {
n := nBits / 8
if nBits%8 != 0 {
n++
// TODO: make faster, align and use copy()
func copyBits(dst []byte, dstStart int64, src []byte, srcStart int64, n int64, zero bool) {
l := n
off := int64(0)
for l > 0 {
c := int64(64)
if l < c {
c = l
}
u := Read64(src, srcStart+off, c)
Write64(u, c, dst, dstStart+off)
off += c
l -= c
}
// zero fill last bits if not aligned
e := dstStart + n
if zero && e%8 != 0 {
Write64(0, 8-(e%8), dst, e)
}
return n
}
func readFull(p []byte, nBits int, bitOff int64, fn func(p []byte, nBits int, bitOff int64) (int, error)) (int, error) {
// TODO: redo?
func readFull(p []byte, nBits int64, bitOff int64, fn func(p []byte, nBits int64, bitOff int64) (int64, error)) (int64, error) {
if nBits < 0 {
return 0, ErrNegativeNBits
}
readBitOffset := 0
readBitOffset := int64(0)
for readBitOffset < nBits {
byteOffset := readBitOffset / 8
byteBitsOffset := readBitOffset % 8
@ -98,7 +181,7 @@ func readFull(p []byte, nBits int, bitOff int64, fn func(p []byte, nBits int, bi
}
var pb [1]byte
rBits, err := fn(pb[:], readBits, bitOff+int64(readBitOffset))
rBits, err := fn(pb[:], readBits, bitOff+readBitOffset)
Write64(uint64(pb[0]>>(8-rBits)), rBits, p, readBitOffset)
readBitOffset += rBits
@ -109,7 +192,7 @@ func readFull(p []byte, nBits int, bitOff int64, fn func(p []byte, nBits int, bi
continue
}
rBits, err := fn(p[byteOffset:], nBits-readBitOffset, bitOff+int64(readBitOffset))
rBits, err := fn(p[byteOffset:], nBits-readBitOffset, bitOff+readBitOffset)
readBitOffset += rBits
if err != nil {
@ -120,31 +203,14 @@ func readFull(p []byte, nBits int, bitOff int64, fn func(p []byte, nBits int, bi
return nBits, nil
}
func ReadAtFull(r BitReaderAt, p []byte, nBits int, bitOff int64) (int, error) {
return readFull(p, nBits, bitOff, func(p []byte, nBits int, bitOff int64) (int, error) {
func ReadAtFull(r ReaderAt, p []byte, nBits int64, bitOff int64) (int64, error) {
return readFull(p, nBits, bitOff, func(p []byte, nBits int64, bitOff int64) (int64, error) {
return r.ReadBitsAt(p, nBits, bitOff)
})
}
func ReadFull(r BitReader, p []byte, nBits int) (int, error) {
return readFull(p, nBits, 0, func(p []byte, nBits int, bitOff int64) (int, error) {
func ReadFull(r Reader, p []byte, nBits int64) (int64, error) {
return readFull(p, nBits, 0, func(p []byte, nBits int64, bitOff int64) (int64, error) {
return r.ReadBits(p, nBits)
})
}
// TODO: move?
func EndPos(rs BitSeeker) (int64, error) {
c, err := rs.SeekBits(0, io.SeekCurrent)
if err != nil {
return 0, err
}
e, err := rs.SeekBits(0, io.SeekEnd)
if err != nil {
return 0, err
}
_, err = rs.SeekBits(c, io.SeekStart)
if err != nil {
return 0, err
}
return e, nil
}

163
pkg/bitio/bitio_test.go
View File

@ -1,163 +0,0 @@
package bitio_test
import (
"bytes"
"io"
"log"
"testing"
"github.com/wader/fq/pkg/bitio"
)
type shortBitReader struct {
bitio.BitReaderAt
}
func (b shortBitReader) ReadBitsAt(p []byte, nBits int, bitOff int64) (n int, err error) {
if nBits > 3 {
nBits = 3
}
return b.BitReaderAt.ReadBitsAt(p, nBits, bitOff)
}
func TestReader(t *testing.T) {
bb, bbBits := bitio.BytesFromBitString("011011110")
br := bitio.NewReaderFromReadSeeker(bytes.NewReader(bb))
sbr := bitio.NewSectionBitReader(br, 0, int64(bbBits))
sbr2 := bitio.NewSectionBitReader(sbr, 0, 4)
ob := make([]byte, 2)
obBits, _ := sbr2.ReadBits(ob, 4)
obs := bitio.BitStringFromBytes(ob, obBits)
log.Printf("obs: %#+v\n", obs)
}
func TestIOCopy(t *testing.T) {
br := bitio.NewReaderFromReadSeeker(bytes.NewReader([]byte{0xf0, 0xff, 0xff}))
sbr := bitio.NewSectionBitReader(br, 0, 8*3-1)
b := &bytes.Buffer{}
n, err := io.Copy(b, sbr)
log.Printf("n: %d err: %v b=%v\n", n, err, b.Bytes())
}
func TestReadFull(t *testing.T) {
bb, bbBits := bitio.BytesFromBitString("011011110")
br := bitio.NewReaderFromReadSeeker(bytes.NewReader(bb))
ob := make([]byte, 2)
obBits, _ := bitio.ReadAtFull(shortBitReader{br}, ob, bbBits, 0)
obs := bitio.BitStringFromBytes(ob, obBits)
log.Printf("obs: %#+v\n", obs)
}
func TestMultiBitReader(t *testing.T) {
bb1, bb1Bits := bitio.BytesFromBitString("101")
br1 := bitio.NewSectionBitReader(bitio.NewReaderFromReadSeeker(bytes.NewReader(bb1)), 0, int64(bb1Bits))
bb2, bb2Bits := bitio.BytesFromBitString("0001")
br2 := bitio.NewSectionBitReader(bitio.NewReaderFromReadSeeker(bytes.NewReader(bb2)), 0, int64(bb2Bits))
mb, _ := bitio.NewMultiBitReader([]bitio.BitReadAtSeeker{br1, br2})
ob := make([]byte, 2)
obBits, _ := bitio.ReadAtFull(mb, ob, 7, 0)
obs := bitio.BitStringFromBytes(ob, obBits)
log.Printf("obs: %#+v\n", obs)
}
func TestMultiBitReader11(t *testing.T) {
bb1, bb1Bits := bitio.BytesFromBitString("11111111")
br1 := bitio.NewSectionBitReader(bitio.NewReaderFromReadSeeker(bytes.NewReader(bb1)), 0, int64(bb1Bits))
bb2, bb2Bits := bitio.BytesFromBitString("11111111")
br2 := bitio.NewSectionBitReader(bitio.NewReaderFromReadSeeker(bytes.NewReader(bb2)), 0, int64(bb2Bits))
mb, _ := bitio.NewMultiBitReader([]bitio.BitReadAtSeeker{br1, br2})
ob := make([]byte, 2)
obBits, _ := bitio.ReadAtFull(mb, ob, 11, 0)
obs := bitio.BitStringFromBytes(ob, obBits)
log.Printf("obs: %#+v\n", obs)
}
type testBW struct{}
func (testBW) WriteBits(p []byte, nBits int) (n int, err error) {
log.Printf("WriteBits p: %#+v nBits=%d\n", len(p), nBits)
for i := 0; i < nBits; i++ {
if bitio.Read64(p, i, 1) != 0 {
log.Print("1")
} else {
log.Print("0")
}
log.Println()
}
return nBits, nil
}
func TestCopy(t *testing.T) {
bb1, bb1Bits := bitio.BytesFromBitString("101")
br1 := bitio.NewSectionBitReader(bitio.NewReaderFromReadSeeker(bytes.NewReader(bb1)), 0, int64(bb1Bits))
bb2, bb2Bits := bitio.BytesFromBitString("0001")
br2 := bitio.NewSectionBitReader(bitio.NewReaderFromReadSeeker(bytes.NewReader(bb2)), 0, int64(bb2Bits))
mb, _ := bitio.NewMultiBitReader([]bitio.BitReadAtSeeker{br1, br2})
n, err := bitio.Copy(testBW{}, mb)
log.Printf("n: %#+v\n", n)
log.Printf("err: %#+v\n", err)
}
func TestAlignBitWriter(t *testing.T) {
bb1, bb1Bits := bitio.BytesFromBitString("101")
br1 := bitio.NewSectionBitReader(bitio.NewReaderFromReadSeeker(bytes.NewReader(bb1)), 0, int64(bb1Bits))
bb2, bb2Bits := bitio.BytesFromBitString("0001")
br2 := bitio.NewSectionBitReader(bitio.NewReaderFromReadSeeker(bytes.NewReader(bb2)), 0, int64(bb2Bits))
mb, _ := bitio.NewMultiBitReader([]bitio.BitReadAtSeeker{br1, br2})
mbEnd, _ := bitio.EndPos(mb)
alignN := int64(8)
b := make([]byte, alignN/8+1)
bLeft := (alignN - mbEnd%alignN) % alignN
log.Printf("bLeft: %#+v\n", bLeft)
mb, _ = bitio.NewMultiBitReader([]bitio.BitReadAtSeeker{mb, bitio.NewBufferFromBytes(b, bLeft)})
bw := testBW{}
// ab := &bitio.AlignBitWriter{N: 8, W: bw}
n, err := bitio.Copy(bw, mb)
// ab.Close()
log.Printf("n: %#+v\n", n)
log.Printf("err: %#+v\n", err)
}

39
pkg/bitio/bitstring.go
View File

@ -1,39 +0,0 @@
package bitio
import (
"fmt"
"strings"
)
// BytesFromBitString []byte from bit string, ex: "0101" -> ([]byte{0x50}, 4)
func BytesFromBitString(s string) ([]byte, int) {
r := len(s) % 8
bufLen := len(s) / 8
if r > 0 {
bufLen++
}
buf := make([]byte, bufLen)
for i := 0; i < len(s); i++ {
d := s[i] - '0'
if d != 0 && d != 1 {
panic(fmt.Sprintf("invalid bit string %q at index %d %q", s, i, s[i]))
}
buf[i/8] |= d << (7 - i%8)
}
return buf, len(s)
}
// BitStringFromBytes string from []byte], ex: ([]byte{0x50}, 4) -> "0101"
func BitStringFromBytes(buf []byte, nBits int) string {
sb := &strings.Builder{}
for i := 0; i < nBits; i++ {
if buf[i/8]&(1<<(7-i%8)) > 0 {
sb.WriteString("1")
} else {
sb.WriteString("0")
}
}
return sb.String()
}

36
pkg/bitio/bitstring_test.go
View File

@ -1,36 +0,0 @@
package bitio_test
import (
"testing"
"github.com/wader/fq/pkg/bitio"
)
func TestBitString(t *testing.T) {
testCases := []string{
"",
"1",
"0",
"10",
"01",
"11",
"00",
"1000001",
"0000000",
"10000001",
"00000000",
"100000001",
"000000000",
"101010101",
"111100000",
}
for _, tC := range testCases {
t.Run(tC, func(t *testing.T) {
bb, bbBits := bitio.BytesFromBitString(tC)
actual := bitio.BitStringFromBytes(bb, bbBits)
if tC != actual {
t.Errorf("expected %s, got %s", tC, actual)
}
})
}
}

311
pkg/bitio/buffer.go
View File

@ -1,280 +1,71 @@
package bitio
// TODO: NewBuffer with []byte arg to save on alloc
import (
"bytes"
"errors"
"fmt"
"io"
"strings"
)
type br interface {
io.Reader // both Reader and SectionBitReader implement io.Reader
io.Seeker
BitReadSeeker
BitReader
BitReaderAt
}
// Buffer is a bit buffer
// Buffer is a bitio.Reader and bitio.Writer providing a bit buffer
// Similar to bytes.Buffer
type Buffer struct {
// embed to forward to underlaying reader
br
bitLen int64 // mostly to cache len
buf []byte
bufBits int64
bitsOff int64
}
// NewBufferFromReadSeeker new Buffer from io.ReadSeeker, start at firstBit with bit length lenBits
// buf is not copied.
func NewBufferFromReadSeeker(rs io.ReadSeeker) (*Buffer, error) {
bPos, err := rs.Seek(0, io.SeekCurrent)
if err != nil {
return nil, err
}
bEnd, err := rs.Seek(0, io.SeekEnd)
if err != nil {
return nil, err
}
if _, err := rs.Seek(bPos, io.SeekStart); err != nil {
return nil, err
}
func (b *Buffer) Len() int64 { return b.bufBits - b.bitsOff }
return &Buffer{
br: NewReaderFromReadSeeker(rs),
bitLen: bEnd * 8,
}, nil
func (b *Buffer) Reset() {
b.bufBits = 0
b.bitsOff = 0
}
func NewBufferFromBitReadSeeker(br br) (*Buffer, error) {
bPos, err := br.SeekBits(0, io.SeekCurrent)
if err != nil {
return nil, err
}
bEnd, err := br.SeekBits(0, io.SeekEnd)
if err != nil {
return nil, err
}
if _, err := br.SeekBits(bPos, io.SeekStart); err != nil {
return nil, err
}
return &Buffer{
br: br,
bitLen: bEnd,
}, nil
func (b *Buffer) Bytes() ([]byte, int64) {
l := b.Len()
buf := make([]byte, BitsByteCount(l))
copyBits(buf, 0, b.buf, b.bitsOff, l, true)
return buf, b.bufBits
}
// NewBufferFromBytes new Buffer from bytes
// if nBits is < 0 nBits is all bits in buf
func NewBufferFromBytes(buf []byte, nBits int64) *Buffer {
if nBits < 0 {
nBits = int64(len(buf)) * 8
}
return &Buffer{
br: NewSectionBitReader(NewReaderFromReadSeeker(bytes.NewReader(buf)), 0, nBits),
bitLen: nBits,
}
}
func (b *Buffer) WriteBits(p []byte, nBits int64) (n int64, err error) {
tBytes := BitsByteCount(b.bufBits + nBits)
// NewBufferFromBitString new Buffer from bit string, ex: "0101"
func NewBufferFromBitString(s string) *Buffer {
b, bBits := BytesFromBitString(s)
return NewBufferFromBytes(b, int64(bBits))
}
// BitBufRange reads nBits bits starting from start
// Does not update current position.
// if nBits is < 0 nBits is all bits after firstBitOffset
func (b *Buffer) BitBufRange(firstBitOffset int64, nBits int64) (*Buffer, error) {
// TODO: move error check?
if nBits < 0 {
return nil, errors.New("negative nBits")
}
if firstBitOffset+nBits > b.bitLen {
return nil, errors.New("outside buffer")
}
return &Buffer{
br: NewSectionBitReader(b.br, firstBitOffset, nBits),
bitLen: nBits,
}, nil
}
// Clone buffer and reset position to zero
func (b *Buffer) Clone() *Buffer {
return &Buffer{
br: NewSectionBitReader(b.br, 0, b.bitLen),
bitLen: b.bitLen,
}
}
func (b *Buffer) Pos() (int64, error) {
bPos, err := b.br.SeekBits(0, io.SeekCurrent)
if err != nil {
return 0, err
}
return bPos, nil
}
func (b *Buffer) Len() int64 {
return b.bitLen
}
// BitBufLen reads nBits
func (b *Buffer) BitBufLen(nBits int64) (*Buffer, error) {
bPos, err := b.Pos()
if err != nil {
return nil, err
}
bb, err := b.BitBufRange(bPos, nBits)
if err != nil {
return nil, err
}
if _, err := b.br.SeekBits(nBits, io.SeekCurrent); err != nil {
return nil, err
}
return bb, nil
}
// PeekBytes peek nBytes bytes from buffer
func (b *Buffer) PeekBytes(nBytes int) ([]byte, error) {
start, err := b.br.SeekBits(0, io.SeekCurrent)
if err != nil {
return nil, err
}
bs, err := b.BytesLen(nBytes)
if _, err := b.br.SeekBits(start, io.SeekStart); err != nil {
return nil, err
}
return bs, err
}
// required by some decompressors (like deflate) to not do own buffering
func (b *Buffer) ReadByte() (byte, error) {
var rb [1]byte
_, err := b.br.Read(rb[:])
return rb[0], err
}
// BytesRange reads nBytes bytes starting bit position start
// Does not update current position.
// TODO: swap args
// TODO: nBytes -1?
func (b *Buffer) BytesRange(bitOffset int64, nBytes int) ([]byte, error) {
buf := make([]byte, nBytes)
n, err := ReadAtFull(b.br, buf, nBytes*8, bitOffset)
if n == nBytes*8 {
err = nil
}
return buf, err
}
// BytesLen reads nBytes bytes
func (b *Buffer) BytesLen(nBytes int) ([]byte, error) {
buf := make([]byte, nBytes)
_, err := io.ReadAtLeast(b, buf, nBytes)
return buf, err
}
// Bytes all bytes
func (b *Buffer) Bytes() ([]byte, error) {
nBytes := int(b.bitLen) / 8
if b.bitLen%8 != 0 {
nBytes++
}
buf := make([]byte, nBytes)
_, err := ReadAtFull(b, buf, int(b.bitLen), 0)
return buf, err
}
// End is true if current position is at the end
func (b *Buffer) End() (bool, error) {
bPos, err := b.Pos()
if err != nil {
return false, err
}
return bPos >= b.bitLen, nil
}
// BitsLeft number of bits left until end
func (b *Buffer) BitsLeft() (int64, error) {
bPos, err := b.Pos()
if err != nil {
return 0, err
}
return b.bitLen - bPos, nil
}
// AlignBits number of bits to next nBits align
func (b *Buffer) AlignBits(nBits int) (int, error) {
bPos, err := b.Pos()
if err != nil {
return 0, err
}
return int((int64(nBits) - (bPos % int64(nBits))) % int64(nBits)), nil
}
// ByteAlignBits number of bits to next byte align
func (b *Buffer) ByteAlignBits() (int, error) {
return b.AlignBits(8)
}
// BytePos byte position of current bit position
func (b *Buffer) BytePos() (int64, error) {
bPos, err := b.Pos()
if err != nil {
return 0, err
}
return bPos & 0x7, nil
}
// SeekRel seeks relative to current bit position
// TODO: better name?
func (b *Buffer) SeekRel(delta int64) (int64, error) {
return b.br.SeekBits(delta, io.SeekCurrent)
}
// SeekAbs seeks to absolute position
func (b *Buffer) SeekAbs(pos int64) (int64, error) {
return b.br.SeekBits(pos, io.SeekStart)
}
func (b *Buffer) String() string {
truncLen := b.bitLen
truncS := ""
if truncLen > 64 {
truncLen, truncS = 64, "..."
}
truncBB, err := b.BitBufRange(0, truncLen)
if err != nil {
return err.Error()
}
bitString, err := truncBB.BitString()
if err != nil {
return err.Error()
}
return fmt.Sprintf("0b%s%s /* %d bits */", bitString, truncS, b.bitLen)
}
// BitString return bit string representation
func (b *Buffer) BitString() (string, error) {
var ss []string
for {
var buf [1]byte
_, err := ReadFull(b, buf[:], 1)
if err != nil {
if !errors.Is(err, io.EOF) {
return "", err
}
break
}
if buf[0] != 0b1000_0000 {
ss = append(ss, "0")
if tBytes > int64(len(b.buf)) {
if tBytes <= int64(cap(b.buf)) {
b.buf = b.buf[:tBytes]
} else {
ss = append(ss, "1")
buf := make([]byte, tBytes, tBytes*2)
copy(buf, b.buf)
b.buf = buf
}
}
return strings.Join(ss, ""), nil
copyBits(b.buf, b.bufBits, p, 0, nBits, true)
b.bufBits += nBits
return nBits, nil
}
func (b *Buffer) empty() bool { return b.bufBits <= b.bitsOff }
func (b *Buffer) ReadBits(p []byte, nBits int64) (n int64, err error) {
if b.empty() {
b.Reset()
if nBits == 0 {
return 0, nil
}
return 0, io.EOF
}
c := nBits
left := b.bufBits - b.bitsOff
if c > left {
c = left
}
copyBits(p, 0, b.buf, b.bitsOff, c, true)
b.bitsOff += c
return c, nil
}

96
pkg/bitio/buffer_test.go
View File

@ -1,96 +0,0 @@
package bitio_test
// TODO: unbreak, check err
import (
"fmt"
"math/rand"
"strings"
"testing"
"github.com/wader/fq/pkg/bitio"
)
func TestBufferBitString(t *testing.T) {
testCases := []string{
"",
"1",
"0",
"10",
"01",
"11",
"00",
"1000001",
"0000000",
"10000001",
"00000000",
"100000001",
"000000000",
"101010101",
"111100000",
}
for _, tC := range testCases {
t.Run(tC, func(t *testing.T) {
bb := bitio.NewBufferFromBitString(tC)
actual, err := bb.BitString()
if err != nil {
t.Error(err)
}
if tC != actual {
t.Errorf("expected %s, got %s", tC, actual)
}
for i := int64(0); i < bb.Len(); i++ {
t.Run(fmt.Sprintf("%s_%d", tC, i), func(t *testing.T) {
startBb, _ := bb.BitBufRange(i, bb.Len()-i)
startExpected := tC[i : i+bb.Len()-i]
startActual, _ := startBb.BitString()
if startExpected != startActual {
t.Errorf("startBb expected %s, got %s", startExpected, startActual)
}
lenBb, _ := bb.BitBufRange(0, i)
lenExpected := tC[0:i]
lenActual, _ := lenBb.BitString()
if lenExpected != lenActual {
t.Errorf("lenBb expected %s, got %s", lenExpected, lenActual)
}
})
}
})
}
}
func TestBitStringRandom(t *testing.T) {
r := rand.New(rand.NewSource(0)) //nolint:gosec
for i := 0; i < 10000; i++ {
var ss []string
for j := uint32(0); j < r.Uint32()%1000; j++ {
switch r.Uint32() % 2 {
case 0:
ss = append(ss, "0")
case 1:
ss = append(ss, "1")
}
}
expected := strings.Join(ss, "")
bb := bitio.NewBufferFromBitString(expected)
actual, _ := bb.BitString()
if expected != actual {
t.Errorf("expected %s, got %s", expected, actual)
}
}
}
func TestInvalidBitString(t *testing.T) {
// TODO: check panic?
defer func() {
if err := recover(); err != nil {
// nop
} else {
t.Error("did not panic")
}
}()
bitio.NewBufferFromBitString("01invalid")
}

85
pkg/bitio/iobitreadseeker.go Normal file
View File

@ -0,0 +1,85 @@
package bitio
import (
"errors"
"io"
)
// IOBitReadSeeker is a bitio.BitReadAtSeeker reading from a io.ReadSeeker
type IOBitReadSeeker struct {
bitPos int64
rs io.ReadSeeker
buf []byte
}
func NewIOBitReadSeeker(rs io.ReadSeeker) *IOBitReadSeeker {
return &IOBitReadSeeker{
bitPos: 0,
rs: rs,
}
}
func (r *IOBitReadSeeker) ReadBitsAt(p []byte, nBits int64, bitOffset int64) (int64, error) {
if nBits < 0 {
return 0, ErrNegativeNBits
}
readBytePos := bitOffset / 8
readSkipBits := bitOffset % 8
wantReadBits := readSkipBits + nBits
wantReadBytes := int(BitsByteCount(wantReadBits))
if wantReadBytes > len(r.buf) {
// TODO: use append somehow?
r.buf = make([]byte, wantReadBytes)
}
_, err := r.rs.Seek(readBytePos, io.SeekStart)
if err != nil {
return 0, err
}
// TODO: nBits should be available
readBytes, err := io.ReadFull(r.rs, r.buf[0:wantReadBytes])
if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) {
return 0, err
} else if errors.Is(err, io.ErrUnexpectedEOF) {
nBits = int64(readBytes) * 8
err = io.EOF
}
if readSkipBits == 0 && nBits%8 == 0 {
copy(p[0:readBytes], r.buf[0:readBytes])
return nBits, err
}
nBytes := nBits / 8
restBits := nBits % 8
// TODO: copy smartness if many bytes
for i := int64(0); i < nBytes; i++ {
p[i] = byte(Read64(r.buf, readSkipBits+i*8, 8))
}
if restBits != 0 {
p[nBytes] = byte(Read64(r.buf, readSkipBits+nBytes*8, restBits)) << (8 - restBits)
}
return nBits, err
}
func (r *IOBitReadSeeker) ReadBits(p []byte, nBits int64) (n int64, err error) {
rBits, err := r.ReadBitsAt(p, nBits, r.bitPos)
r.bitPos += rBits
return rBits, err
}
func (r *IOBitReadSeeker) SeekBits(bitOff int64, whence int) (int64, error) {
seekBytesPos, err := r.rs.Seek(bitOff/8, whence)
if err != nil {
return 0, err
}
seekBitPos := seekBytesPos*8 + bitOff%8
r.bitPos = seekBitPos
return seekBitPos, nil
}

63
pkg/bitio/iobitwriter.go Normal file
View File

@ -0,0 +1,63 @@
package bitio
import (
"io"
)
// IOBitWriter is a bitio.BitWriter that writes to a io.Writer
// Use Flush to write possible unaligned byte zero bit padded.
type IOBitWriter struct {
w io.Writer
b Buffer
}
func NewIOBitWriter(w io.Writer) *IOBitWriter {
return &IOBitWriter{w: w}
}
func (w *IOBitWriter) WriteBits(p []byte, nBits int64) (n int64, err error) {
if n, err = w.b.WriteBits(p, nBits); err != nil {
return n, err
}
var sn int64
for {
l := w.b.Len()
if l < 8 {
break
}
var buf [32 * 1024]byte
n, err := w.b.ReadBits(buf[:], l-(l%8))
if err != nil {
return sn, err
}
rn, err := w.w.Write(buf[:n/8])
sn += int64(rn)
if err != nil {
return sn, err
}
sn += n
}
return nBits, nil
}
// Flush write possible unaligned byte zero bit padded.
func (w *IOBitWriter) Flush() error {
if w.b.Len() == 0 {
return nil
}
var buf [1]byte
_, err := w.b.ReadBits(buf[:], w.b.Len())
if err != nil {
return err
}
_, err = w.w.Write(buf[:])
return err
}

76
pkg/bitio/ioreader.go Normal file
View File

@ -0,0 +1,76 @@
package bitio
import (
"errors"
"io"
)
// IOReader is a io.Reader that reads bytes from a bitio.BitReader
// Unaligned byte at EOF will be zero bit padded
type IOReader struct {
r Reader
rErr error
b Buffer
}
func NewIOReader(r Reader) *IOReader {
return &IOReader{r: r}
}
func (r *IOReader) Read(p []byte) (n int, err error) {
var ns int64
for {
var err error
// uses p even if returning nothing, io.Reader docs says:
// "it may use all of p as scratch space during the call"
if r.rErr == nil {
var rn int64
rn, err = r.r.ReadBits(p, int64(len(p))*8)
r.rErr = err
ns += rn
_, err = r.b.WriteBits(p, rn)
if err != nil {
return 0, err
}
}
if r.b.Len() >= 8 {
// read whole bytes
rBits := int64(len(p)) * 8
bBits := r.b.Len()
aBits := bBits - bBits%8
if rBits > aBits {
rBits = aBits
}
rn, rErr := r.b.ReadBits(p, rBits)
if rErr != nil {
return int(rn / 8), rErr
}
return int(rn / 8), nil
} else if r.rErr != nil {
if errors.Is(r.rErr, io.EOF) && r.b.Len() > 0 {
// TODO: hmm io.Buffer does this
if len(p) == 0 {
return 0, nil
}
_, err := r.b.ReadBits(p, r.b.Len())
if err != nil {
return 0, err
}
return 1, nil
}
return 0, r.rErr
}
}
}
// required to make some readers like deflate not do their own buffering
func (r *IOReader) ReadByte() (byte, error) {
var rb [1]byte
_, err := r.Read(rb[:])
return rb[0], err
}

37
pkg/bitio/ioreadseeker.go Normal file
View File

@ -0,0 +1,37 @@
package bitio
// IOReadSeeker is a io.ReaSeeker that reads and seeks bytes from a bitio.BitReadSeeker
// Unaligned byte at EOF will be zero bit padded
type IOReadSeeker struct {
IOReader
s Seeker
sPos int64
}
func NewIOReadSeeker(rs ReadSeeker) *IOReadSeeker {
return &IOReadSeeker{
IOReader: IOReader{r: rs},
s: rs,
}
}
func (r *IOReadSeeker) Read(p []byte) (n int, err error) {
n, err = r.IOReader.Read(p)
r.sPos += int64(n)
return n, err
}
func (r *IOReadSeeker) Seek(offset int64, whence int) (int64, error) {
n, err := r.s.SeekBits(offset*8, whence)
// TODO: reset last error on seek. some nicer way?
r.IOReader.rErr = nil
if n != r.sPos {
r.b.Reset()
r.sPos = n / 8
}
if err != nil {
return n / 8, err
}
return n / 8, err
}

152
pkg/bitio/ioreadseeker_test.go Normal file
View File

@ -0,0 +1,152 @@
package bitio_test
import (
"bytes"
"fmt"
"io"
"strings"
"testing"
"github.com/wader/fq/pkg/bitio"
)
func sb(s string) *bitio.SectionReader {
buf, nBits := bitio.BytesFromBitString(s)
return bitio.NewBitReader(buf, nBits)
}
func bs(br bitio.Reader) string {
bib := &bitio.Buffer{}
_, err := bitio.Copy(bib, br)
if err != nil {
panic(err)
}
buf, nBits := bib.Bytes()
return bitio.BitStringFromBytes(buf, nBits)
}
func Test(t *testing.T) {
testCases := []struct {
bs string
}{
{""},
{"|"},
{"0"},
{"1"},
{"1|"},
{"|1|"},
{"|1|"},
{"0|"},
{"|0|"},
{"0|"},
{"1"},
{"10"},
{"101"},
{"1011"},
{"10110"},
{"101101"},
{"1011011"},
{"10110110"},
{"101101101"},
{"1011011010"},
{"10110110101"},
{"101101101011"},
{"1011011010110"},
{"10110110101101"},
{"101101101011011"},
{"1011011010110110"},
{"10110110|1"},
{"10110110|10"},
{"10110110|101"},
{"10110110|1011"},
{"10110110|10110"},
{"10110110|101101"},
{"10110110|1011011"},
{"10110110|10110110"},
{"1|10110110|1"},
{"10|10110110|10"},
{"101|10110110|101"},
{"1011|10110110|1011"},
{"10110|10110110|10110"},
{"101101|10110110|101101"},
{"1011011|10110110|1011011"},
{"10110110|10110110|10110110"},
{"1|1|0110110|1"},
{"10|10|110110|10"},
{"101|101|10110|101"},
{"1011|1011|0110|1011"},
{"10110|10110|110|10110"},
{"101101|101101|10|101101"},
{"1011011|1011011|0|1011011"},
{"1|10110110|10110110|1"},
{"10|10110110|10110110|10"},
{"101|10110110|10110110|101"},
{"1011|10110110|10110110|1011"},
{"10110|10110110|10110110|10110"},
{"101101|10110110|10110110|101101"},
{"1011011|10110110|10110110|1011011"},
{"10110110|10110110|10110110|10110110"},
{"1|101101101011011010110110101101101011011010110110|1"},
{"10|101101101011011010110110101101101011011010110110|10"},
{"101|101101101011011010110110101101101011011010110110|101"},
{"1011|101101101011011010110110101101101011011010110110|1011"},
{"10110|101101101011011010110110101101101011011010110110|10110"},
{"101101|101101101011011010110110101101101011011010110110|101101"},
{"1011011|101101101011011010110110101101101011011010110110|1011011"},
{"10110110|101101101011011010110110101101101011011010110110|10110110"},
}
for _, tC := range testCases {
bsParts := strings.Split(tC.bs, "|")
var bsBRs []bitio.ReadAtSeeker
for _, p := range bsParts {
bsBRs = append(bsBRs, sb(p))
}
bsBR, err := bitio.NewMultiBitReader(bsBRs...)
if err != nil {
panic(err)
}
bsBitString := strings.ReplaceAll(tC.bs, "|", "")
for i := 0; i < len(bsBitString); i++ {
t.Run(fmt.Sprintf("%s_%d", tC.bs, i), func(t *testing.T) {
_, err = bsBR.SeekBits(int64(i), io.SeekStart)
if err != nil {
t.Fatal(err)
}
expectedBitString := bsBitString[i:]
actualBitString := bs(bsBR)
if expectedBitString != actualBitString {
t.Errorf("expected bits %q, got %q", expectedBitString, actualBitString)
}
_, err = bsBR.SeekBits(int64(i), io.SeekStart)
if err != nil {
t.Fatal(err)
}
r := bitio.NewIOReader(bsBR)
bb := &bytes.Buffer{}
if _, err := io.Copy(bb, r); err != nil {
t.Fatal(err)
}
expecetdByteBitString := expectedBitString + strings.Repeat("0", (8-(len(expectedBitString)%8))%8)
actualByteBitString := bitio.BitStringFromBytes(bb.Bytes(), int64(bb.Len()*8))
if expecetdByteBitString != actualByteBitString {
t.Errorf("expected bytes %q, got %q", expecetdByteBitString, actualByteBitString)
}
})
}
}
}

26
pkg/bitio/limitreader.go Normal file
View File

@ -0,0 +1,26 @@
package bitio
import (
"io"
)
func NewLimitReader(r Reader, n int64) *LimitReader { return &LimitReader{r, n} }
// LimitReader is a bitio.Reader that reads a limited amount of bits from a bitio.Reader
// Similar to bytes.LimitedReader but for bits
type LimitReader struct {
r Reader
n int64
}
func (l *LimitReader) ReadBits(p []byte, nBits int64) (n int64, err error) {
if l.n <= 0 {
return 0, io.EOF
}
if nBits > l.n {
nBits = l.n
}
n, err = l.r.ReadBits(p, nBits)
l.n -= n
return n, err
}

108
pkg/bitio/multireader.go Normal file
View File

@ -0,0 +1,108 @@
package bitio
import (
"errors"
"io"
)
// TODO: smarter, track index?
func endPos(rs Seeker) (int64, error) {
c, err := rs.SeekBits(0, io.SeekCurrent)
if err != nil {
return 0, err
}
e, err := rs.SeekBits(0, io.SeekEnd)
if err != nil {
return 0, err
}
_, err = rs.SeekBits(c, io.SeekStart)
if err != nil {
return 0, err
}
return e, nil
}
// MultiReader is a bitio.ReaderAtSeeker concatinating multiple bitio.ReadAtSeeker:s
// Similar to io.MultiReader but for bits
type MultiReader struct {
pos int64
readers []ReadAtSeeker
readerEnds []int64
}
func NewMultiBitReader(rs ...ReadAtSeeker) (*MultiReader, error) {
readerEnds := make([]int64, len(rs))
var esSum int64
for i, r := range rs {
e, err := endPos(r)
if err != nil {
return nil, err
}
esSum += e
readerEnds[i] = esSum
}
return &MultiReader{readers: rs, readerEnds: readerEnds}, nil
}
func (m *MultiReader) ReadBitsAt(p []byte, nBits int64, bitOff int64) (n int64, err error) {
var end int64
if len(m.readers) > 0 {
end = m.readerEnds[len(m.readers)-1]
}
if end <= bitOff {
return 0, io.EOF
}
prevAtEnd := int64(0)
readerAt := m.readers[0]
for i, end := range m.readerEnds {
if bitOff < end {
readerAt = m.readers[i]
break
}
prevAtEnd = end
}
rBits, err := readerAt.ReadBitsAt(p, nBits, bitOff-prevAtEnd)
if errors.Is(err, io.EOF) {
if bitOff+rBits < end {
err = nil
}
}
return rBits, err
}
func (m *MultiReader) ReadBits(p []byte, nBits int64) (n int64, err error) {
n, err = m.ReadBitsAt(p, nBits, m.pos)
m.pos += n
return n, err
}
func (m *MultiReader) SeekBits(bitOff int64, whence int) (int64, error) {
var p int64
var end int64
if len(m.readers) > 0 {
end = m.readerEnds[len(m.readers)-1]
}
switch whence {
case io.SeekStart:
p = bitOff
case io.SeekCurrent:
p = m.pos + bitOff
case io.SeekEnd:
p = end + bitOff
default:
panic("unknown whence")
}
if p < 0 || p > end {
return 0, ErrOffset
}
m.pos = p
return p, nil
}

269
pkg/bitio/reader.go
View File

@ -1,269 +0,0 @@
package bitio
import (
"errors"
"io"
)
// Reader is a BitReadSeeker and BitReaderAt reading from a io.ReadSeeker
type Reader struct {
bitPos int64
rs io.ReadSeeker
buf []byte
}
func NewReaderFromReadSeeker(rs io.ReadSeeker) *Reader {
return &Reader{
bitPos: 0,
rs: rs,
}
}
func (r *Reader) ReadBitsAt(p []byte, nBits int, bitOffset int64) (int, error) {
if nBits < 0 {
return 0, ErrNegativeNBits
}
readBytePos := bitOffset / 8
readSkipBits := int(bitOffset % 8)
wantReadBits := readSkipBits + nBits
wantReadBytes := int(BitsByteCount(int64(wantReadBits)))
if wantReadBytes > len(r.buf) {
// TODO: use append somehow?
r.buf = make([]byte, wantReadBytes)
}
_, err := r.rs.Seek(readBytePos, io.SeekStart)
if err != nil {
return 0, err
}
// TODO: nBits should be available
readBytes, err := io.ReadFull(r.rs, r.buf[0:wantReadBytes])
if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) {
return 0, err
} else if errors.Is(err, io.ErrUnexpectedEOF) {
nBits = readBytes * 8
err = io.EOF
}
if readSkipBits == 0 && nBits%8 == 0 {
copy(p[0:readBytes], r.buf[0:readBytes])
return nBits, err
}
nBytes := nBits / 8
restBits := nBits % 8
// TODO: copy smartness if many bytes
for i := 0; i < nBytes; i++ {
p[i] = byte(Read64(r.buf, readSkipBits+i*8, 8))
}
if restBits != 0 {
p[nBytes] = byte(Read64(r.buf, readSkipBits+nBytes*8, restBits)) << (8 - restBits)
}
return nBits, err
}
func (r *Reader) ReadBits(p []byte, nBits int) (n int, err error) {
rBits, err := r.ReadBitsAt(p, nBits, r.bitPos)
r.bitPos += int64(rBits)
return rBits, err
}
func (r *Reader) SeekBits(bitOff int64, whence int) (int64, error) {
seekBytesPos, err := r.rs.Seek(bitOff/8, whence)
if err != nil {
return 0, err
}
seekBitPos := seekBytesPos*8 + bitOff%8
r.bitPos = seekBitPos
return seekBitPos, nil
}
func (r *Reader) Read(p []byte) (n int, err error) {
n, err = r.ReadBitsAt(p, len(p)*8, r.bitPos)
r.bitPos += int64(n)
if err != nil {
return int(BitsByteCount(int64(n))), err
}
return int(BitsByteCount(int64(n))), nil
}
func (r *Reader) Seek(offset int64, whence int) (int64, error) {
seekBytesPos, err := r.rs.Seek(offset, whence)
if err != nil {
return 0, err
}
r.bitPos = seekBytesPos * 8
return seekBytesPos, nil
}
// SectionBitReader is a BitReadSeeker reading from a BitReaderAt
// modelled after io.SectionReader
type SectionBitReader struct {
r BitReaderAt
bitBase int64
bitOff int64
bitLimit int64
}
func NewSectionBitReader(r BitReaderAt, bitOff int64, nBits int64) *SectionBitReader {
return &SectionBitReader{
r: r,
bitBase: bitOff,
bitOff: bitOff,
bitLimit: bitOff + nBits,
}
}
func (r *SectionBitReader) ReadBitsAt(p []byte, nBits int, bitOff int64) (int, error) {
if bitOff < 0 || bitOff >= r.bitLimit-r.bitBase {
return 0, io.EOF
}
bitOff += r.bitBase
if maxBits := int(r.bitLimit - bitOff); nBits > maxBits {
nBits = maxBits
rBits, err := r.r.ReadBitsAt(p, nBits, bitOff)
return rBits, err
}
return r.r.ReadBitsAt(p, nBits, bitOff)
}
func (r *SectionBitReader) ReadBits(p []byte, nBits int) (n int, err error) {
rBits, err := r.ReadBitsAt(p, nBits, r.bitOff-r.bitBase)
r.bitOff += int64(rBits)
return rBits, err
}
func (r *SectionBitReader) SeekBits(bitOff int64, whence int) (int64, error) {
switch whence {
case io.SeekStart:
bitOff += r.bitBase
case io.SeekCurrent:
bitOff += r.bitOff
case io.SeekEnd:
bitOff += r.bitLimit
default:
panic("unknown whence")
}
if bitOff < r.bitBase {
return 0, ErrOffset
}
r.bitOff = bitOff
return bitOff - r.bitBase, nil
}
func (r *SectionBitReader) Read(p []byte) (n int, err error) {
n, err = r.ReadBitsAt(p, len(p)*8, r.bitOff-r.bitBase)
r.bitOff += int64(n)
return int(BitsByteCount(int64(n))), err
}
func (r *SectionBitReader) Seek(offset int64, whence int) (int64, error) {
seekBitsPos, err := r.SeekBits(offset*8, whence)
return seekBitsPos / 8, err
}
// TODO: smart, track index?
type MultiBitReader struct {
pos int64
readers []BitReadAtSeeker
readerEnds []int64
}
func NewMultiBitReader(rs []BitReadAtSeeker) (*MultiBitReader, error) {
readerEnds := make([]int64, len(rs))
var esSum int64
for i, r := range rs {
e, err := EndPos(r)
if err != nil {
return nil, err
}
esSum += e
readerEnds[i] = esSum
}
return &MultiBitReader{readers: rs, readerEnds: readerEnds}, nil
}
func (m *MultiBitReader) ReadBitsAt(p []byte, nBits int, bitOff int64) (n int, err error) {
var end int64
if len(m.readers) > 0 {
end = m.readerEnds[len(m.readers)-1]
}
if end <= bitOff {
return 0, io.EOF
}
prevAtEnd := int64(0)
readerAt := m.readers[0]
for i, end := range m.readerEnds {
if bitOff < end {
readerAt = m.readers[i]
break
}
prevAtEnd = end
}
rBits, err := readerAt.ReadBitsAt(p, nBits, bitOff-prevAtEnd)
if errors.Is(err, io.EOF) {
if bitOff+int64(rBits) < end {
err = nil
}
}
return rBits, err
}
func (m *MultiBitReader) ReadBits(p []byte, nBits int) (n int, err error) {
n, err = m.ReadBitsAt(p, nBits, m.pos)
m.pos += int64(n)
return n, err
}
func (m *MultiBitReader) SeekBits(bitOff int64, whence int) (int64, error) {
var p int64
var end int64
if len(m.readers) > 0 {
end = m.readerEnds[len(m.readers)-1]
}
switch whence {
case io.SeekStart:
p = bitOff
case io.SeekCurrent:
p = m.pos + bitOff
case io.SeekEnd:
p = end + bitOff
default:
panic("unknown whence")
}
if p < 0 || p > end {
return 0, ErrOffset
}
m.pos = p
return p, nil
}
func (m *MultiBitReader) Read(p []byte) (n int, err error) {
n, err = m.ReadBitsAt(p, len(p)*8, m.pos)
m.pos += int64(n)
if err != nil {
return int(BitsByteCount(int64(n))), err
}
return int(BitsByteCount(int64(n))), nil
}
func (m *MultiBitReader) Seek(offset int64, whence int) (int64, error) {
seekBitsPos, err := m.SeekBits(offset*8, whence)
return seekBitsPos / 8, err
}

27
pkg/bitio/rw64.go → pkg/bitio/readwrite64.go
View File

@ -7,7 +7,7 @@ import (
// Read64 read nBits bits large unsigned integer from buf starting from firstBit.
// Integer is read most significant bit first.
func Read64(buf []byte, firstBit int, nBits int) uint64 {
func Read64(buf []byte, firstBit int64, nBits int64) uint64 {
if nBits < 0 || nBits > 64 {
panic(fmt.Sprintf("nBits must be 0-64 (%d)", nBits))
}
@ -90,7 +90,7 @@ func Read64(buf []byte, firstBit int, nBits int) uint64 {
return n
}
func Write64(v uint64, nBits int, buf []byte, firstBit int) {
func Write64(v uint64, nBits int64, buf []byte, firstBit int64) {
if nBits < 0 || nBits > 64 {
panic(fmt.Sprintf("nBits must be 0-64 (%d)", nBits))
}
@ -169,26 +169,3 @@ func Write64(v uint64, nBits int, buf []byte, firstBit int) {
}
}
}
func Uint64ReverseBytes(nBits int, n uint64) uint64 {
switch {
case nBits <= 8:
return n
case nBits <= 16:
return n&0xff00>>8 | n&0xff<<8
case nBits <= 24:
return n&0xff<<16 | n&0xff00 | n&0xff0000>>16
case nBits <= 32:
return n&0xff<<24 | n&0xff00<<8 | n&0xff0000>>8 | n&0xff000000>>24
case nBits <= 40:
return n&0xff<<32 | n&0xff00<<16 | n&0xff0000 | n&0xff000000>>16 | n&0xff00000000>>32
case nBits <= 48:
return n&0xff<<40 | n&0xff00<<24 | n&0xff0000<<8 | n&0xff000000>>8 | n&0xff00000000>>24 | n&0xff0000000000>>40
case nBits <= 56:
return n&0xff<<48 | n&0xff00<<32 | n&0xff0000<<16 | n&0xff000000 | n&0xff00000000>>16 | n&0xff0000000000>>32 | n&0xff000000000000>>48
case nBits <= 64:
return n&0xff<<56 | n&0xff00<<40 | n&0xff0000<<24 | n&0xff000000<<8 | n&0xff00000000>>8 | n&0xff0000000000>>24 | n&0xff000000000000>>40 | n&0xff00000000000000>>56
default:
panic(fmt.Sprintf("unsupported bit length %d", nBits))
}
}

49
pkg/bitio/rw64_test.go → pkg/bitio/readwrite64_test.go
View File

@ -12,8 +12,8 @@ import (
func TestRead64(t *testing.T) {
testCases := []struct {
buf []byte
firstBit int
nBits int
firstBit int64
nBits int64
expected uint64
}{
{buf: []byte{0xff}, firstBit: 0, nBits: 8, expected: 0b11111111},
@ -93,9 +93,9 @@ func TestRead64(t *testing.T) {
func TestWrite64(t *testing.T) {
testCases := []struct {
v uint64
nBits int
nBits int64
buf []byte
firstBit int
firstBit int64
expectedBuf []byte
}{
{0x0123456789abcdef, 8, []byte{0, 0, 0, 0, 0, 0, 0, 0}, 0, []byte{0xef, 0x00, 0x00, 0x0, 0x0, 0x00, 0x00, 0x00}},
@ -145,44 +145,3 @@ func TestWrite64(t *testing.T) {
})
}
}
// func TestWrite64(t *testing.T) {
// b := make([]byte, 2)
// bitio.Write64(0xf, 4, b, 0)
// }
func TestUint64Panic(t *testing.T) {
// TODO: check panic string
defer func() { _ = recover() }()
bitio.Read64([]byte{}, 0, 65)
t.Error("should panic")
}
func TestUint64ReverseBytes(t *testing.T) {
testCases := []struct {
nBits int
n uint64
expected uint64
}{
{nBits: 0, n: 0, expected: 0},
{nBits: 8, n: 0x01, expected: 0x01},
{nBits: 16, n: 0x0123, expected: 0x2301},
{nBits: 24, n: 0x012345, expected: 0x452301},
{nBits: 32, n: 0x01234567, expected: 0x67452301},
{nBits: 40, n: 0x0123456789, expected: 0x8967452301},
{nBits: 48, n: 0x0123456789ab, expected: 0xab8967452301},
{nBits: 56, n: 0x0123456789abcd, expected: 0xcdab8967452301},
{nBits: 64, n: 0x0123456789abcdef, expected: 0xefcdab8967452301},
}
for _, tC := range testCases {
t.Run(fmt.Sprintf("%d %x %x", tC.nBits, tC.n, tC.expected), func(t *testing.T) {
actual := bitio.Uint64ReverseBytes(tC.nBits, tC.n)
if tC.expected != actual {
t.Errorf("expected %x, got %x", tC.expected, actual)
}
})
}
}

9
pkg/bitio/reversebytes.go
View File

@ -1,9 +0,0 @@
package bitio
func ReverseBytes(bs []byte) []byte {
l := len(bs)
for i := 0; i < l/2; i++ {
bs[i], bs[l-i-1] = bs[l-i-1], bs[i]
}
return bs
}

28
pkg/bitio/reversebytes64.go Normal file
View File

@ -0,0 +1,28 @@
package bitio
import "fmt"
// ReverseBytes64 reverses the bytes part of the lowest nBits
// Similar to bits.ReverseBytes64 but only rotates to lowest bytes and rest of bytes will be zero
func ReverseBytes64(nBits int, n uint64) uint64 {
switch {
case nBits <= 8:
return n
case nBits <= 16:
return n&0xff00>>8 | n&0xff<<8
case nBits <= 24:
return n&0xff<<16 | n&0xff00 | n&0xff0000>>16
case nBits <= 32:
return n&0xff<<24 | n&0xff00<<8 | n&0xff0000>>8 | n&0xff000000>>24
case nBits <= 40:
return n&0xff<<32 | n&0xff00<<16 | n&0xff0000 | n&0xff000000>>16 | n&0xff00000000>>32
case nBits <= 48:
return n&0xff<<40 | n&0xff00<<24 | n&0xff0000<<8 | n&0xff000000>>8 | n&0xff00000000>>24 | n&0xff0000000000>>40
case nBits <= 56:
return n&0xff<<48 | n&0xff00<<32 | n&0xff0000<<16 | n&0xff000000 | n&0xff00000000>>16 | n&0xff0000000000>>32 | n&0xff000000000000>>48
case nBits <= 64:
return n&0xff<<56 | n&0xff00<<40 | n&0xff0000<<24 | n&0xff000000<<8 | n&0xff00000000>>8 | n&0xff0000000000>>24 | n&0xff000000000000>>40 | n&0xff00000000000000>>56
default:
panic(fmt.Sprintf("unsupported bit length %d", nBits))
}
}

41
pkg/bitio/reversebytes64_test.go Normal file
View File

@ -0,0 +1,41 @@
package bitio_test
import (
"fmt"
"testing"
"github.com/wader/fq/pkg/bitio"
)
func TestReverseBytes64Panic(t *testing.T) {
// TODO: check panic string
defer func() { _ = recover() }()
bitio.Read64([]byte{}, 0, 65)
t.Error("should panic")
}
func TestReverseBytes64(t *testing.T) {
testCases := []struct {
nBits int
n uint64
expected uint64
}{
{nBits: 0, n: 0, expected: 0},
{nBits: 8, n: 0x01, expected: 0x01},
{nBits: 16, n: 0x0123, expected: 0x2301},
{nBits: 24, n: 0x012345, expected: 0x452301},
{nBits: 32, n: 0x01234567, expected: 0x67452301},
{nBits: 40, n: 0x0123456789, expected: 0x8967452301},
{nBits: 48, n: 0x0123456789ab, expected: 0xab8967452301},
{nBits: 56, n: 0x0123456789abcd, expected: 0xcdab8967452301},
{nBits: 64, n: 0x0123456789abcdef, expected: 0xefcdab8967452301},
}
for _, tC := range testCases {
t.Run(fmt.Sprintf("%d %x %x", tC.nBits, tC.n, tC.expected), func(t *testing.T) {
actual := bitio.ReverseBytes64(tC.nBits, tC.n)
if tC.expected != actual {
t.Errorf("expected %x, got %x", tC.expected, actual)
}
})
}
}

30
pkg/bitio/reversebytes_test.go
View File

@ -1,30 +0,0 @@
package bitio_test
import (
"bytes"
"fmt"
"testing"
"github.com/wader/fq/pkg/bitio"
)
func TestReverseBytes(t *testing.T) {
testCases := []struct {
input []byte
expected []byte
}{
{nil, nil},
{[]byte{1}, []byte{1}},
{[]byte{1, 2}, []byte{2, 1}},
{[]byte{1, 2, 3}, []byte{3, 2, 1}},
}
for _, tC := range testCases {
t.Run(fmt.Sprintf("%v", tC.input), func(t *testing.T) {
actual := append([]byte(nil), tC.input...)
bitio.ReverseBytes(actual)
if !bytes.Equal(tC.expected, actual) {
t.Errorf("expected %v, got %v", tC.expected, actual)
}
})
}
}

60
pkg/bitio/sectiontreader.go Normal file
View File

@ -0,0 +1,60 @@
package bitio
import (
"io"
)
// SectionReader is a bitio.BitReaderAtSeeker reading a section of a bitio.ReaderAt
// Similar to io.SectionReader but for bits
type SectionReader struct {
r ReaderAt
bitBase int64
bitOff int64
bitLimit int64
}
func NewSectionReader(r ReaderAt, bitOff int64, nBits int64) *SectionReader {
return &SectionReader{
r: r,
bitBase: bitOff,
bitOff: bitOff,
bitLimit: bitOff + nBits,
}
}
func (r *SectionReader) ReadBitsAt(p []byte, nBits int64, bitOff int64) (int64, error) {
if bitOff < 0 || bitOff >= r.bitLimit-r.bitBase {
return 0, io.EOF
}
bitOff += r.bitBase
if maxBits := r.bitLimit - bitOff; nBits > maxBits {
nBits = maxBits
rBits, err := r.r.ReadBitsAt(p, nBits, bitOff)
return rBits, err
}
return r.r.ReadBitsAt(p, nBits, bitOff)
}
func (r *SectionReader) ReadBits(p []byte, nBits int64) (n int64, err error) {
rBits, err := r.ReadBitsAt(p, nBits, r.bitOff-r.bitBase)
r.bitOff += rBits
return rBits, err
}
func (r *SectionReader) SeekBits(bitOff int64, whence int) (int64, error) {
switch whence {
case io.SeekStart:
bitOff += r.bitBase
case io.SeekCurrent:
bitOff += r.bitOff
case io.SeekEnd:
bitOff += r.bitLimit
default:
panic("unknown whence")
}
if bitOff < r.bitBase {
return 0, ErrOffset
}
r.bitOff = bitOff
return bitOff - r.bitBase, nil
}

352
pkg/decode/decode.go
View File

@ -8,6 +8,7 @@ import (
"io/ioutil"
"math/big"
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/internal/recoverfn"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/ranges"
@ -38,14 +39,19 @@ type Options struct {
}
// Decode try decode group and return first success and all other decoder errors
func Decode(ctx context.Context, bb *bitio.Buffer, group Group, opts Options) (*Value, interface{}, error) {
return decode(ctx, bb, group, opts)
func Decode(ctx context.Context, br bitio.ReaderAtSeeker, group Group, opts Options) (*Value, interface{}, error) {
return decode(ctx, br, group, opts)
}
func decode(ctx context.Context, bb *bitio.Buffer, group Group, opts Options) (*Value, interface{}, error) {
func decode(ctx context.Context, br bitio.ReaderAtSeeker, group Group, opts Options) (*Value, interface{}, error) {
brLen, err := bitioextra.Len(br)
if err != nil {
return nil, nil, err
}
decodeRange := opts.Range
if decodeRange.IsZero() {
decodeRange = ranges.Range{Len: bb.Len()}
decodeRange = ranges.Range{Len: brLen}
}
if group == nil {
@ -55,12 +61,12 @@ func decode(ctx context.Context, bb *bitio.Buffer, group Group, opts Options) (*
formatsErr := FormatsError{}
for _, g := range group {
cbb, err := bb.BitBufRange(decodeRange.Start, decodeRange.Len)
cBR, err := bitioextra.Range(br, decodeRange.Start, decodeRange.Len)
if err != nil {
return nil, nil, IOError{Err: err, Op: "BitBufRange", ReadSize: decodeRange.Len, Pos: decodeRange.Start}
}
d := newDecoder(ctx, g, cbb, opts)
d := newDecoder(ctx, g, cBR, opts)
var decodeV interface{}
r, rOk := recoverfn.Run(func() {
@ -105,7 +111,7 @@ func decode(ctx context.Context, bb *bitio.Buffer, group Group, opts Options) (*
if err := d.Value.WalkRootPreOrder(func(v *Value, rootV *Value, depth int, rootDepth int) error {
minMaxRange = ranges.MinMax(minMaxRange, v.Range)
v.Range.Start += decodeRange.Start
v.RootBitBuf = bb
v.RootBitBuf = br
return nil
}); err != nil {
return nil, nil, err
@ -133,14 +139,14 @@ type D struct {
Value *Value
Options Options
bitBuf *bitio.Buffer
bitBuf bitio.ReaderAtSeeker
readBuf *[]byte
}
// TODO: new struct decoder?
// note bb is assumed to be a non-shared buffer
func newDecoder(ctx context.Context, format Format, bb *bitio.Buffer, opts Options) *D {
// note br is assumed to be a non-shared buffer
func newDecoder(ctx context.Context, format Format, br bitio.ReaderAtSeeker, opts Options) *D {
name := format.RootName
if opts.Name != "" {
name = opts.Name
@ -158,18 +164,18 @@ func newDecoder(ctx context.Context, format Format, bb *bitio.Buffer, opts Optio
Value: &Value{
Name: name,
V: rootV,
RootBitBuf: bb,
RootBitBuf: br,
Range: ranges.Range{Start: 0, Len: 0},
IsRoot: opts.IsRoot,
},
Options: opts,
bitBuf: bb,
bitBuf: br,
readBuf: opts.ReadBuf,
}
}
func (d *D) FieldDecoder(name string, bitBuf *bitio.Buffer, v interface{}) *D {
func (d *D) FieldDecoder(name string, bitBuf bitio.ReaderAtSeeker, v interface{}) *D {
return &D{
Ctx: d.Ctx,
Endian: d.Endian,
@ -186,24 +192,63 @@ func (d *D) FieldDecoder(name string, bitBuf *bitio.Buffer, v interface{}) *D {
}
}
func (d *D) Copy(r io.Writer, w io.Reader) (int64, error) {
func (d *D) CopyBits(w io.Writer, r bitio.Reader) (int64, error) {
// TODO: what size? now same as io.Copy
buf := d.SharedReadBuf(32 * 1024)
return io.CopyBuffer(r, w, buf)
return bitioextra.CopyBitsBuffer(w, r, buf)
}
func (d *D) MustCopy(r io.Writer, w io.Reader) int64 {
n, err := d.Copy(r, w)
func (d *D) MustCopyBits(w io.Writer, r bitio.Reader) int64 {
n, err := d.CopyBits(w, r)
if err != nil {
d.IOPanic(err, "MustCopy: Copy")
}
return n
}
func (d *D) MustNewBitBufFromReader(r io.Reader) *bitio.Buffer {
func (d *D) Copy(w io.Writer, r io.Reader) (int64, error) {
// TODO: what size? now same as io.Copy
buf := d.SharedReadBuf(32 * 1024)
return io.CopyBuffer(w, r, buf)
}
func (d *D) MustCopy(w io.Writer, r io.Reader) int64 {
n, err := d.Copy(w, r)
if err != nil {
d.IOPanic(err, "MustCopy: Copy")
}
return n
}
func (d *D) MustClone(br bitio.ReaderAtSeeker) bitio.ReaderAtSeeker {
br, err := bitioextra.Clone(br)
if err != nil {
d.IOPanic(err, "MustClone")
}
return br
}
func (d *D) MustNewBitBufFromReader(r io.Reader) bitio.ReaderAtSeeker {
b := &bytes.Buffer{}
d.MustCopy(b, r)
return bitio.NewBufferFromBytes(b.Bytes(), -1)
return bitio.NewBitReader(b.Bytes(), -1)
}
func (d *D) ReadAllBits(r bitio.Reader) ([]byte, error) {
bb := &bytes.Buffer{}
buf := d.SharedReadBuf(32 * 1024)
if _, err := bitioextra.CopyBitsBuffer(bb, r, buf); err != nil {
return nil, err
}
return bb.Bytes(), nil
}
func (d *D) MustReadAllBits(r bitio.Reader) []byte {
buf, err := d.ReadAllBits(r)
if err != nil {
d.IOPanic(err, "Bytes ReadAllBytes")
}
return buf
}
func (d *D) SharedReadBuf(n int) []byte {
@ -241,7 +286,7 @@ func (d *D) FillGaps(r ranges.Range, namePrefix string) {
gaps := ranges.Gaps(r, valueRanges)
for i, gap := range gaps {
bb, err := d.bitBuf.BitBufRange(gap.Start, gap.Len)
br, err := bitioextra.Range(d.bitBuf, gap.Start, gap.Len)
if err != nil {
d.IOPanic(err, "FillGaps: BitBufRange")
}
@ -249,7 +294,7 @@ func (d *D) FillGaps(r ranges.Range, namePrefix string) {
v := &Value{
Name: fmt.Sprintf("%s%d", namePrefix, i),
V: &scalar.S{
Actual: bb,
Actual: br,
Unknown: true,
},
RootBitBuf: d.bitBuf,
@ -283,12 +328,12 @@ func (d *D) bits(nBits int) (uint64, error) {
}
// 64 bits max, 9 byte worse case if not byte aligned
buf := d.SharedReadBuf(9)
_, err := bitio.ReadFull(d.bitBuf, buf, nBits)
_, err := bitio.ReadFull(d.bitBuf, buf, int64(nBits)) // TODO: int64?
if err != nil {
return 0, err
}
return bitio.Read64(buf[:], 0, nBits), nil
return bitio.Read64(buf[:], 0, int64(nBits)), nil // TODO: int64
}
// Bits reads nBits bits from buffer
@ -308,8 +353,20 @@ func (d *D) PeekBits(nBits int) uint64 {
return n
}
func (d *D) TryPeekBytes(nBytes int) ([]byte, error) {
start, err := d.bitBuf.SeekBits(0, io.SeekCurrent)
if err != nil {
return nil, err
}
bs, err := d.TryBytesLen(nBytes)
if _, err := d.bitBuf.SeekBits(start, io.SeekStart); err != nil {
return nil, err
}
return bs, err
}
func (d *D) PeekBytes(nBytes int) []byte {
bs, err := d.bitBuf.PeekBytes(nBytes)
bs, err := d.TryPeekBytes(nBytes)
if err != nil {
panic(IOError{Err: err, Op: "PeekBytes", ReadSize: int64(nBytes) * 8, Pos: d.Pos()})
}
@ -410,16 +467,34 @@ func (d *D) TryPeekFind(nBits int, seekBits int64, maxLen int64, fn func(v uint6
return count, v, nil
}
// BytesRange reads nBytes bytes starting bit position start
// Does not update current position.
// TODO: nBytes -1?
func (d *D) TryBytesRange(bitOffset int64, nBytes int) ([]byte, error) {
buf := make([]byte, nBytes)
n, err := bitio.ReadAtFull(d.bitBuf, buf, int64(nBytes)*8, bitOffset)
if n == int64(nBytes)*8 {
err = nil
}
return buf, err
}
func (d *D) BytesRange(firstBit int64, nBytes int) []byte {
bs, err := d.bitBuf.BytesRange(firstBit, nBytes)
bs, err := d.TryBytesRange(firstBit, nBytes)
if err != nil {
panic(IOError{Err: err, Op: "BytesRange", ReadSize: int64(nBytes) * 8, Pos: firstBit})
}
return bs
}
func (d *D) TryBytesLen(nBytes int) ([]byte, error) {
buf := make([]byte, nBytes)
_, err := bitio.ReadFull(d.bitBuf, buf, int64(nBytes)*8)
return buf, err
}
func (d *D) BytesLen(nBytes int) []byte {
bs, err := d.bitBuf.BytesLen(nBytes)
bs, err := d.TryBytesLen(nBytes)
if err != nil {
panic(IOError{Err: err, Op: "BytesLen", ReadSize: int64(nBytes) * 8, Pos: d.Pos()})
}
@ -427,98 +502,202 @@ func (d *D) BytesLen(nBytes int) []byte {
}
// TODO: rename/remove BitBuf name?
func (d *D) BitBufRange(firstBit int64, nBits int64) *bitio.Buffer {
bb, err := d.bitBuf.BitBufRange(firstBit, nBits)
func (d *D) TryBitBufRange(firstBit int64, nBits int64) (bitio.ReaderAtSeeker, error) {
return bitioextra.Range(d.bitBuf, firstBit, nBits)
}
func (d *D) BitBufRange(firstBit int64, nBits int64) bitio.ReaderAtSeeker {
br, err := bitioextra.Range(d.bitBuf, firstBit, nBits)
if err != nil {
panic(IOError{Err: err, Op: "BitBufRange", ReadSize: nBits, Pos: firstBit})
}
return bb
return br
}
func (d *D) TryPos() (int64, error) {
return d.bitBuf.SeekBits(0, io.SeekCurrent)
}
func (d *D) Pos() int64 {
bPos, err := d.bitBuf.Pos()
bPos, err := d.TryPos()
if err != nil {
panic(IOError{Err: err, Op: "Pos", ReadSize: 0, Pos: bPos})
}
return bPos
}
func (d *D) TryLen() (int64, error) {
return bitioextra.Len(d.bitBuf)
}
func (d *D) Len() int64 {
return d.bitBuf.Len()
l, err := d.TryLen()
if err != nil {
panic(IOError{Err: err, Op: "Len"})
}
return l
}
////
// BitBufLen reads nBits
func (d *D) TryBitBufLen(nBits int64) (bitio.ReaderAtSeeker, error) {
bPos, err := d.TryPos()
if err != nil {
return nil, err
}
br, err := d.TryBitBufRange(bPos, nBits)
if err != nil {
return nil, err
}
if _, err := d.TrySeekRel(nBits); err != nil {
return nil, err
}
return br, nil
}
////
// End is true if current position is at the end
func (d *D) TryEnd() (bool, error) {
bPos, err := d.TryPos()
if err != nil {
return false, err
}
bLen, err := d.TryLen()
if err != nil {
return false, err
}
return bPos >= bLen, nil
}
func (d *D) End() bool {
bEnd, err := d.bitBuf.End()
bEnd, err := d.TryEnd()
if err != nil {
panic(IOError{Err: err, Op: "Len", ReadSize: 0, Pos: d.Pos()})
panic(IOError{Err: err, Op: "End", ReadSize: 0, Pos: d.Pos()})
}
return bEnd
}
func (d *D) NotEnd() bool { return !d.End() }
// BitsLeft number of bits left until end
func (d *D) TryBitsLeft() (int64, error) {
bPos, err := d.TryPos()
if err != nil {
return 0, err
}
bLen, err := d.TryLen()
if err != nil {
return 0, err
}
return bLen - bPos, nil
}
func (d *D) BitsLeft() int64 {
bBitsLeft, err := d.bitBuf.BitsLeft()
bBitsLeft, err := d.TryBitsLeft()
if err != nil {
panic(IOError{Err: err, Op: "BitsLeft", ReadSize: 0, Pos: d.Pos()})
}
return bBitsLeft
}
// AlignBits number of bits to next nBits align
func (d *D) TryAlignBits(nBits int) (int, error) {
bPos, err := d.TryPos()
if err != nil {
return 0, err
}
return int((int64(nBits) - (bPos % int64(nBits))) % int64(nBits)), nil
}
func (d *D) AlignBits(nBits int) int {
bByteAlignBits, err := d.bitBuf.AlignBits(nBits)
bByteAlignBits, err := d.TryAlignBits(nBits)
if err != nil {
panic(IOError{Err: err, Op: "AlignBits", ReadSize: 0, Pos: d.Pos()})
}
return bByteAlignBits
}
// ByteAlignBits number of bits to next byte align
func (d *D) TryByteAlignBits() (int, error) {
return d.TryAlignBits(8)
}
func (d *D) ByteAlignBits() int {
bByteAlignBits, err := d.bitBuf.ByteAlignBits()
bByteAlignBits, err := d.TryByteAlignBits()
if err != nil {
panic(IOError{Err: err, Op: "ByteAlignBits", ReadSize: 0, Pos: d.Pos()})
}
return bByteAlignBits
}
// BytePos byte position of current bit position
func (d *D) TryBytePos() (int64, error) {
bPos, err := d.TryPos()
if err != nil {
return 0, err
}
return bPos & 0x7, nil
}
func (d *D) BytePos() int64 {
bBytePos, err := d.bitBuf.BytePos()
bBytePos, err := d.TryBytePos()
if err != nil {
panic(IOError{Err: err, Op: "BytePos", ReadSize: 0, Pos: d.Pos()})
}
return bBytePos
}
func (d *D) seekAbs(pos int64, name string, fns ...func(d *D)) int64 {
func (d *D) trySeekAbs(pos int64, fns ...func(d *D)) (int64, error) {
var oldPos int64
if len(fns) > 0 {
oldPos = d.Pos()
}
pos, err := d.bitBuf.SeekAbs(pos)
pos, err := d.bitBuf.SeekBits(pos, io.SeekStart)
if err != nil {
panic(IOError{Err: err, Op: name, SeekPos: pos, Pos: d.Pos()})
return 0, err
}
if len(fns) > 0 {
for _, fn := range fns {
fn(d)
}
_, err := d.bitBuf.SeekAbs(oldPos)
_, err := d.bitBuf.SeekBits(oldPos, io.SeekStart)
if err != nil {
panic(IOError{Err: err, Op: name, SeekPos: pos, Pos: d.Pos()})
return 0, err
}
}
return pos
return pos, nil
}
func (d *D) SeekRel(deltaPos int64, fns ...func(d *D)) int64 {
return d.seekAbs(d.Pos()+deltaPos, "SeekRel", fns...)
// SeekRel seeks relative to current bit position
func (d *D) TrySeekRel(delta int64, fns ...func(d *D)) (int64, error) {
return d.trySeekAbs(d.Pos()+delta, fns...)
}
func (d *D) SeekRel(delta int64, fns ...func(d *D)) int64 {
n, err := d.trySeekAbs(d.Pos()+delta, fns...)
if err != nil {
d.IOPanic(err, "SeekRel")
}
return n
}
// SeekAbs seeks to absolute position
func (d *D) TrySeekAbs(pos int64, fns ...func(d *D)) (int64, error) {
return d.trySeekAbs(pos, fns...)
}
func (d *D) SeekAbs(pos int64, fns ...func(d *D)) int64 {
return d.seekAbs(pos, "SeekAbs", fns...)
n, err := d.trySeekAbs(pos, fns...)
if err != nil {
d.IOPanic(err, "SeekAbs")
}
return n
}
func (d *D) AddChild(v *Value) {
@ -659,7 +838,7 @@ func (d *D) FieldValueNil(name string, sms ...scalar.Mapper) {
func (d *D) FieldValueRaw(name string, a []byte, sms ...scalar.Mapper) {
d.FieldScalarFn(name, func(_ scalar.S) (scalar.S, error) {
return scalar.S{Actual: bitio.NewBufferFromBytes(a, -1)}, nil
return scalar.S{Actual: bitio.NewBitReader(a, -1)}, nil
}, sms...)
}
@ -685,11 +864,11 @@ func (d *D) RangeFn(firstBit int64, nBits int64, fn func(d *D)) {
}
// TODO: do some kind of DecodeLimitedLen/RangeFn?
bb := d.BitBufRange(0, firstBit+nBits)
if _, err := bb.SeekAbs(firstBit); err != nil {
br := d.BitBufRange(0, firstBit+nBits)
if _, err := br.SeekBits(firstBit, io.SeekStart); err != nil {
d.IOPanic(err, "RangeFn: SeekAbs")
}
sd := d.FieldDecoder("", bb, subV)
sd := d.FieldDecoder("", br, subV)
fn(sd)
@ -735,7 +914,7 @@ func (d *D) Format(group Group, inArg interface{}) interface{} {
panic("unreachable")
}
if _, err := d.bitBuf.SeekRel(dv.Range.Len); err != nil {
if _, err := d.bitBuf.SeekBits(dv.Range.Len, io.SeekCurrent); err != nil {
d.IOPanic(err, "Format: SeekRel")
}
@ -757,7 +936,7 @@ func (d *D) TryFieldFormat(name string, group Group, inArg interface{}) (*Value,
}
d.AddChild(dv)
if _, err := d.bitBuf.SeekRel(dv.Range.Len); err != nil {
if _, err := d.bitBuf.SeekBits(dv.Range.Len, io.SeekCurrent); err != nil {
d.IOPanic(err, "TryFieldFormat: SeekRel")
}
@ -787,7 +966,7 @@ func (d *D) TryFieldFormatLen(name string, nBits int64, group Group, inArg inter
}
d.AddChild(dv)
if _, err := d.bitBuf.SeekRel(nBits); err != nil {
if _, err := d.bitBuf.SeekBits(nBits, io.SeekCurrent); err != nil {
d.IOPanic(err, "TryFieldFormatLen: SeekRel")
}
@ -831,8 +1010,8 @@ func (d *D) FieldFormatRange(name string, firstBit int64, nBits int64, group Gro
return dv, v
}
func (d *D) TryFieldFormatBitBuf(name string, bb *bitio.Buffer, group Group, inArg interface{}) (*Value, interface{}, error) {
dv, v, err := decode(d.Ctx, bb, group, Options{
func (d *D) TryFieldFormatBitBuf(name string, br bitio.ReaderAtSeeker, group Group, inArg interface{}) (*Value, interface{}, error) {
dv, v, err := decode(d.Ctx, br, group, Options{
Name: name,
Force: d.Options.Force,
FillGaps: true,
@ -851,8 +1030,8 @@ func (d *D) TryFieldFormatBitBuf(name string, bb *bitio.Buffer, group Group, inA
return dv, v, err
}
func (d *D) FieldFormatBitBuf(name string, bb *bitio.Buffer, group Group, inArg interface{}) (*Value, interface{}) {
dv, v, err := d.TryFieldFormatBitBuf(name, bb, group, inArg)
func (d *D) FieldFormatBitBuf(name string, br bitio.ReaderAtSeeker, group Group, inArg interface{}) (*Value, interface{}) {
dv, v, err := d.TryFieldFormatBitBuf(name, br, group, inArg)
if dv == nil || dv.Errors() != nil {
d.IOPanic(err, "FieldFormatBitBuf: TryFieldFormatBitBuf")
}
@ -861,23 +1040,31 @@ func (d *D) FieldFormatBitBuf(name string, bb *bitio.Buffer, group Group, inArg
}
// TODO: rethink these
func (d *D) FieldRootBitBuf(name string, bb *bitio.Buffer, sms ...scalar.Mapper) *Value {
func (d *D) FieldRootBitBuf(name string, br bitio.ReaderAtSeeker, sms ...scalar.Mapper) *Value {
brLen, err := bitioextra.Len(br)
if err != nil {
d.IOPanic(err, "br Len")
}
v := &Value{}
v.V = &scalar.S{Actual: bb}
v.V = &scalar.S{Actual: br}
v.Name = name
v.RootBitBuf = bb
v.RootBitBuf = br
v.IsRoot = true
v.Range = ranges.Range{Start: d.Pos(), Len: bb.Len()}
v.Range = ranges.Range{Start: d.Pos(), Len: brLen}
if err := v.TryScalarFn(sms...); err != nil {
d.Fatalf("%v", err)
}
d.AddChild(v)
return v
}
func (d *D) FieldArrayRootBitBufFn(name string, bb *bitio.Buffer, fn func(d *D)) *Value {
cd := d.FieldDecoder(name, bb, &Compound{IsArray: true})
func (d *D) FieldArrayRootBitBufFn(name string, br bitio.ReaderAtSeeker, fn func(d *D)) *Value {
cd := d.FieldDecoder(name, br, &Compound{IsArray: true})
cd.Value.IsRoot = true
d.AddChild(cd.Value)
fn(cd)
@ -887,8 +1074,8 @@ func (d *D) FieldArrayRootBitBufFn(name string, bb *bitio.Buffer, fn func(d *D))
return cd.Value
}
func (d *D) FieldStructRootBitBufFn(name string, bb *bitio.Buffer, fn func(d *D)) *Value {
cd := d.FieldDecoder(name, bb, &Compound{})
func (d *D) FieldStructRootBitBufFn(name string, br bitio.ReaderAtSeeker, fn func(d *D)) *Value {
cd := d.FieldDecoder(name, br, &Compound{})
cd.Value.IsRoot = true
d.AddChild(cd.Value)
fn(cd)
@ -900,55 +1087,56 @@ func (d *D) FieldStructRootBitBufFn(name string, bb *bitio.Buffer, fn func(d *D)
// TODO: range?
func (d *D) FieldFormatReaderLen(name string, nBits int64, fn func(r io.Reader) (io.ReadCloser, error), group Group) (*Value, interface{}) {
bb, err := d.bitBuf.BitBufLen(nBits)
br, err := d.TryBitBufLen(nBits)
if err != nil {
d.IOPanic(err, "FieldFormatReaderLen: BitBufLen")
}
zr, err := fn(bb)
bbBR := bitio.NewIOReader(br)
r, err := fn(bbBR)
if err != nil {
d.IOPanic(err, "FieldFormatReaderLen: fn")
}
zd, err := ioutil.ReadAll(zr)
rBuf, err := ioutil.ReadAll(r)
if err != nil {
d.IOPanic(err, "FieldFormatReaderLen: ReadAll")
}
zbb := bitio.NewBufferFromBytes(zd, -1)
rBR := bitio.NewBitReader(rBuf, -1)
return d.FieldFormatBitBuf(name, zbb, group, nil)
return d.FieldFormatBitBuf(name, rBR, group, nil)
}
// TODO: too mant return values
func (d *D) TryFieldReaderRangeFormat(name string, startBit int64, nBits int64, fn func(r io.Reader) io.Reader, group Group, inArg interface{}) (int64, *bitio.Buffer, *Value, interface{}, error) {
func (d *D) TryFieldReaderRangeFormat(name string, startBit int64, nBits int64, fn func(r io.Reader) io.Reader, group Group, inArg interface{}) (int64, bitio.ReaderAtSeeker, *Value, interface{}, error) {
bitLen := nBits
if bitLen == -1 {
bitLen = d.BitsLeft()
}
bb, err := d.bitBuf.BitBufRange(startBit, bitLen)
br, err := d.TryBitBufRange(startBit, bitLen)
if err != nil {
return 0, nil, nil, nil, err
}
r := fn(bb)
// TODO: check if io.Closer?
rb, err := ioutil.ReadAll(r)
r := bitio.NewIOReadSeeker(br)
rb, err := ioutil.ReadAll(fn(r))
if err != nil {
return 0, nil, nil, nil, err
}
cz, err := bb.Pos()
rbb := bitio.NewBufferFromBytes(rb, -1)
cz, err := r.Seek(0, io.SeekCurrent)
rbr := bitio.NewBitReader(rb, -1)
if err != nil {
return 0, nil, nil, nil, err
}
dv, v, err := d.TryFieldFormatBitBuf(name, rbb, group, inArg)
dv, v, err := d.TryFieldFormatBitBuf(name, rbr, group, inArg)
return cz, rbb, dv, v, err
return cz * 8, rbr, dv, v, err
}
func (d *D) FieldReaderRangeFormat(name string, startBit int64, nBits int64, fn func(r io.Reader) io.Reader, group Group, inArg interface{}) (int64, *bitio.Buffer, *Value, interface{}) {
cz, rbb, dv, v, err := d.TryFieldReaderRangeFormat(name, startBit, nBits, fn, group, inArg)
func (d *D) FieldReaderRangeFormat(name string, startBit int64, nBits int64, fn func(r io.Reader) io.Reader, group Group, inArg interface{}) (int64, bitio.ReaderAtSeeker, *Value, interface{}) {
cz, rBR, dv, v, err := d.TryFieldReaderRangeFormat(name, startBit, nBits, fn, group, inArg)
if err != nil {
d.IOPanic(err, "TryFieldReaderRangeFormat")
}
return cz, rbb, dv, v
return cz, rBR, dv, v
}
func (d *D) TryFieldValue(name string, fn func() (*Value, error)) (*Value, error) {

30
pkg/decode/decode_gen.go
View File

@ -62,7 +62,7 @@ func (d *D) FieldScalarBigIntFn(name string, fn func(d *D) *big.Int, sms ...scal
// Type BitBuf
// TryFieldBitBufScalarFn tries to add a field, calls scalar functions and returns actual value as a BitBuf
func (d *D) TryFieldBitBufScalarFn(name string, fn func(d *D) (scalar.S, error), sms ...scalar.Mapper) (*bitio.Buffer, error) {
func (d *D) TryFieldBitBufScalarFn(name string, fn func(d *D) (scalar.S, error), sms ...scalar.Mapper) (bitio.ReaderAtSeeker, error) {
v, err := d.TryFieldScalarFn(name, func(_ scalar.S) (scalar.S, error) { return fn(d) }, sms...)
if err != nil {
return nil, err
@ -71,7 +71,7 @@ func (d *D) TryFieldBitBufScalarFn(name string, fn func(d *D) (scalar.S, error),
}
// FieldBitBufScalarFn adds a field, calls scalar functions and returns actual value as a BitBuf
func (d *D) FieldBitBufScalarFn(name string, fn func(d *D) scalar.S, sms ...scalar.Mapper) *bitio.Buffer {
func (d *D) FieldBitBufScalarFn(name string, fn func(d *D) scalar.S, sms ...scalar.Mapper) bitio.ReaderAtSeeker {
v, err := d.TryFieldScalarFn(name, func(_ scalar.S) (scalar.S, error) { return fn(d), nil }, sms...)
if err != nil {
panic(IOError{Err: err, Name: name, Op: "BitBuf", Pos: d.Pos()})
@ -79,30 +79,30 @@ func (d *D) FieldBitBufScalarFn(name string, fn func(d *D) scalar.S, sms ...scal
return v.ActualBitBuf()
}
// FieldBitBufFn adds a field, calls *bitio.Buffer decode function and returns actual value as a BitBuf
func (d *D) FieldBitBufFn(name string, fn func(d *D) *bitio.Buffer, sms ...scalar.Mapper) *bitio.Buffer {
// FieldBitBufFn adds a field, calls bitio.ReaderAtSeeker decode function and returns actual value as a BitBuf
func (d *D) FieldBitBufFn(name string, fn func(d *D) bitio.ReaderAtSeeker, sms ...scalar.Mapper) bitio.ReaderAtSeeker {
return d.FieldBitBufScalarFn(name, func(d *D) scalar.S { return scalar.S{Actual: fn(d)} }, sms...)
}
// TryFieldBitBufFn tries to add a field, calls *bitio.Buffer decode function and returns actual value as a BitBuf
func (d *D) TryFieldBitBufFn(name string, fn func(d *D) (*bitio.Buffer, error), sms ...scalar.Mapper) (*bitio.Buffer, error) {
// TryFieldBitBufFn tries to add a field, calls bitio.ReaderAtSeeker decode function and returns actual value as a BitBuf
func (d *D) TryFieldBitBufFn(name string, fn func(d *D) (bitio.ReaderAtSeeker, error), sms ...scalar.Mapper) (bitio.ReaderAtSeeker, error) {
return d.TryFieldBitBufScalarFn(name, func(d *D) (scalar.S, error) {
v, err := fn(d)
return scalar.S{Actual: v}, err
}, sms...)
}
// TryFieldScalarBitBufFn tries to add a field, calls *bitio.Buffer decode function and returns scalar
func (d *D) TryFieldScalarBitBufFn(name string, fn func(d *D) (*bitio.Buffer, error), sms ...scalar.Mapper) (*scalar.S, error) {
// TryFieldScalarBitBufFn tries to add a field, calls bitio.ReaderAtSeeker decode function and returns scalar
func (d *D) TryFieldScalarBitBufFn(name string, fn func(d *D) (bitio.ReaderAtSeeker, error), sms ...scalar.Mapper) (*scalar.S, error) {
return d.TryFieldScalarFn(name, func(_ scalar.S) (scalar.S, error) {
v, err := fn(d)
return scalar.S{Actual: v}, err
}, sms...)
}
// FieldScalarBitBufFn tries to add a field, calls *bitio.Buffer decode function and returns scalar
func (d *D) FieldScalarBitBufFn(name string, fn func(d *D) *bitio.Buffer, sms ...scalar.Mapper) *scalar.S {
v, err := d.TryFieldScalarBitBufFn(name, func(d *D) (*bitio.Buffer, error) { return fn(d), nil }, sms...)
// FieldScalarBitBufFn tries to add a field, calls bitio.ReaderAtSeeker decode function and returns scalar
func (d *D) FieldScalarBitBufFn(name string, fn func(d *D) bitio.ReaderAtSeeker, sms ...scalar.Mapper) *scalar.S {
v, err := d.TryFieldScalarBitBufFn(name, func(d *D) (bitio.ReaderAtSeeker, error) { return fn(d), nil }, sms...)
if err != nil {
panic(IOError{Err: err, Name: name, Op: "BitBuf", Pos: d.Pos()})
}
@ -758,10 +758,10 @@ func (d *D) ValidateURange(start, end uint64) scalar.Mapper {
// Reader RawLen
// TryRawLen tries to read nBits raw bits
func (d *D) TryRawLen(nBits int64) (*bitio.Buffer, error) { return d.tryBitBuf(nBits) }
func (d *D) TryRawLen(nBits int64) (bitio.ReaderAtSeeker, error) { return d.tryBitBuf(nBits) }
// RawLen reads nBits raw bits
func (d *D) RawLen(nBits int64) *bitio.Buffer {
func (d *D) RawLen(nBits int64) bitio.ReaderAtSeeker {
v, err := d.tryBitBuf(nBits)
if err != nil {
panic(IOError{Err: err, Op: "RawLen", Pos: d.Pos()})
@ -792,13 +792,13 @@ func (d *D) FieldScalarRawLen(name string, nBits int64, sms ...scalar.Mapper) *s
}
// TryFieldRawLen tries to add a field and read nBits raw bits
func (d *D) TryFieldRawLen(name string, nBits int64, sms ...scalar.Mapper) (*bitio.Buffer, error) {
func (d *D) TryFieldRawLen(name string, nBits int64, sms ...scalar.Mapper) (bitio.ReaderAtSeeker, error) {
s, err := d.TryFieldScalarRawLen(name, nBits, sms...)
return s.ActualBitBuf(), err
}
// FieldRawLen adds a field and reads nBits raw bits
func (d *D) FieldRawLen(name string, nBits int64, sms ...scalar.Mapper) *bitio.Buffer {
func (d *D) FieldRawLen(name string, nBits int64, sms ...scalar.Mapper) bitio.ReaderAtSeeker {
return d.FieldScalarRawLen(name, nBits, sms...).ActualBitBuf()
}

20
pkg/decode/read.go
View File

@ -12,8 +12,8 @@ import (
"golang.org/x/text/encoding/unicode"
)
func (d *D) tryBitBuf(nBits int64) (*bitio.Buffer, error) {
return d.bitBuf.BitBufLen(nBits)
func (d *D) tryBitBuf(nBits int64) (bitio.ReaderAtSeeker, error) {
return d.TryBitBufLen(nBits)
}
func (d *D) tryUEndian(nBits int, endian Endian) (uint64, error) {
@ -25,7 +25,7 @@ func (d *D) tryUEndian(nBits int, endian Endian) (uint64, error) {
return 0, err
}
if endian == LittleEndian {
n = bitio.Uint64ReverseBytes(nBits, n)
n = bitio.ReverseBytes64(nBits, n)
}
return n, nil
@ -64,7 +64,7 @@ func (d *D) tryBigIntEndianSign(nBits int, endian Endian, sign bool) (*big.Int,
}
b := int(bitio.BitsByteCount(int64(nBits)))
buf := d.SharedReadBuf(b)[0:b]
_, err := bitio.ReadFull(d.bitBuf, buf, nBits)
_, err := bitio.ReadFull(d.bitBuf, buf, int64(nBits))
if err != nil {
return nil, err
}
@ -93,7 +93,7 @@ func (d *D) tryFEndian(nBits int, endian Endian) (float64, error) {
return 0, err
}
if endian == LittleEndian {
n = bitio.Uint64ReverseBytes(nBits, n)
n = bitio.ReverseBytes64(nBits, n)
}
switch nBits {
case 16:
@ -116,7 +116,7 @@ func (d *D) tryFPEndian(nBits int, fBits int, endian Endian) (float64, error) {
return 0, err
}
if endian == LittleEndian {
n = bitio.Uint64ReverseBytes(nBits, n)
n = bitio.ReverseBytes64(nBits, n)
}
return float64(n) / float64(uint64(1<<fBits)), nil
}
@ -135,7 +135,7 @@ func (d *D) tryText(nBytes int, e encoding.Encoding) (string, error) {
return "", fmt.Errorf("tryText nBytes %d outside buffer, %d bytes left", nBytes, bytesLeft)
}
bs, err := d.bitBuf.BytesLen(nBytes)
bs, err := d.TryBytesLen(nBytes)
if err != nil {
return "", err
}
@ -173,7 +173,7 @@ func (d *D) tryTextLenPrefixed(lenBits int, fixedBytes int, e encoding.Encoding)
}
}
bs, err := d.bitBuf.BytesLen(n)
bs, err := d.TryBytesLen(n)
if err != nil {
d.SeekAbs(p)
return "", err
@ -192,7 +192,7 @@ func (d *D) tryTextNull(nullBytes int, e encoding.Encoding) (string, error) {
return "", err
}
n := (int(peekBits) / 8) + nullBytes
bs, err := d.bitBuf.BytesLen(n)
bs, err := d.TryBytesLen(n)
if err != nil {
d.SeekAbs(p)
return "", err
@ -210,7 +210,7 @@ func (d *D) tryTextNullLen(fixedBytes int, e encoding.Encoding) (string, error)
return "", fmt.Errorf("tryTextNullLen fixedBytes %d outside, %d bytes left", fixedBytes, bytesLeft)
}
bs, err := d.bitBuf.BytesLen(fixedBytes)
bs, err := d.TryBytesLen(fixedBytes)
if err != nil {
return "", err
}

31
pkg/decode/scalar.go
View File

@ -5,12 +5,13 @@ import (
"encoding/binary"
"errors"
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/scalar"
)
func bitBufIsZero(s scalar.S, isValidate bool) (scalar.S, error) {
bb, ok := s.Actual.(*bitio.Buffer)
br, ok := s.Actual.(bitio.ReaderAtSeeker)
if !ok {
return s, nil
}
@ -18,13 +19,17 @@ func bitBufIsZero(s scalar.S, isValidate bool) (scalar.S, error) {
isZero := true
// TODO: shared
b := make([]byte, 32*1024)
bLen := len(b) * 8
bbLeft := int(bb.Len())
bbPos := int64(0)
bLen := int64(len(b)) * 8
brLen, err := bitioextra.Len(br)
if err != nil {
return scalar.S{}, err
}
brLeft := brLen
brPos := int64(0)
for bbLeft > 0 {
rl := bbLeft
if bbLeft > bLen {
for brLeft > 0 {
rl := brLeft
if brLeft > bLen {
rl = bLen
}
// zero last byte if uneven read
@ -32,11 +37,11 @@ func bitBufIsZero(s scalar.S, isValidate bool) (scalar.S, error) {
b[rl/8] = 0
}
n, err := bitio.ReadAtFull(bb, b, rl, bbPos)
n, err := bitio.ReadAtFull(br, b, rl, brPos)
if err != nil {
return s, err
}
nb := int(bitio.BitsByteCount(int64(n)))
nb := int(bitio.BitsByteCount(n))
for i := 0; i < nb; i++ {
if b[i] != 0 {
@ -45,7 +50,7 @@ func bitBufIsZero(s scalar.S, isValidate bool) (scalar.S, error) {
}
}
bbLeft -= n
brLeft -= n
}
if isZero {
@ -74,12 +79,12 @@ func (d *D) BitBufValidateIsZero() scalar.Mapper {
// TODO: generate?
func assertBitBuf(s scalar.S, isErr bool, bss ...[]byte) (scalar.S, error) {
ab, err := s.ActualBitBuf().Bytes()
if err != nil {
bb := &bytes.Buffer{}
if _, err := bitioextra.CopyBits(bb, s.ActualBitBuf()); err != nil {
return s, err
}
for _, bs := range bss {
if bytes.Equal(ab, bs) {
if bytes.Equal(bb.Bytes(), bs) {
s.Description = "valid"
return s, nil
}

50
pkg/decode/types.json
View File

@ -5,7 +5,7 @@
"S": {"go_type": "int64", "zero": "0", "compare": "a == b", "range": "a >= start && a <= end"},
"F": {"go_type": "float64", "map_from": false, "zero": "0", "compare": "a == b", "range": "a >= start && a <= end"},
"Bool": {"go_type": "bool", "zero": "false", "compare": "a == b"},
"BitBuf": {"go_type": "*bitio.Buffer", "zero": "nil", "map_from": false, "map_to": false},
"BitBuf": {"go_type": "bitio.ReaderAtSeeker", "zero": "nil", "map_from": false, "map_to": false},
"BigInt": {"go_type": "*big.Int", "zero": "nil", "map_from": false, "map_to": false, "compare": "a.Cmp(b) == 0", "range": "a.Cmp(start) >= 0 && a.Cmp(end) <= 0"}
},
"readers": [
@ -57,7 +57,13 @@
"name": "UBigInt",
"type": "BigInt",
"variants": [
{"name": "", "args": "nBits", "params": "nBits int", "call": "d.tryBigIntEndianSign(nBits, d.Endian, false)", "doc": "nBits bits signed integer in current endian"},
{
"name": "",
"args": "nBits",
"params": "nBits int",
"call": "d.tryBigIntEndianSign(nBits, d.Endian, false)",
"doc": "nBits bits signed integer in current endian"
},
{
"name": "E",
"args": "nBits, endian",
@ -65,15 +71,33 @@
"call": "d.tryBigIntEndianSign(nBits, endian, false)",
"doc": "nBits signed integer in specified endian"
},
{"name": "LE", "args": "nBits", "params": "nBits int", "call": "d.tryBigIntEndianSign(nBits, LittleEndian, false)", "doc": "nBits bit signed integer in little-endian"},
{"name": "BE", "args": "nBits", "params": "nBits int", "call": "d.tryBigIntEndianSign(nBits, BigEndian, false)", "doc": "nBits bit signed integer in big-endian"}
{
"name": "LE",
"args": "nBits",
"params": "nBits int",
"call": "d.tryBigIntEndianSign(nBits, LittleEndian, false)",
"doc": "nBits bit signed integer in little-endian"
},
{
"name": "BE",
"args": "nBits",
"params": "nBits int",
"call": "d.tryBigIntEndianSign(nBits, BigEndian, false)",
"doc": "nBits bit signed integer in big-endian"
}
]
},
{
"name": "SBigInt",
"type": "BigInt",
"variants": [
{"name": "", "args": "nBits", "params": "nBits int", "call": "d.tryBigIntEndianSign(nBits, d.Endian, true)", "doc": "nBits bits signed integer in current endian"},
{
"name": "",
"args": "nBits",
"params": "nBits int",
"call": "d.tryBigIntEndianSign(nBits, d.Endian, true)",
"doc": "nBits bits signed integer in current endian"
},
{
"name": "E",
"args": "nBits, endian",
@ -81,7 +105,13 @@
"call": "d.tryBigIntEndianSign(nBits, endian, true)",
"doc": "nBits signed integer in specified endian"
},
{"name": "LE", "args": "nBits", "params": "nBits int", "call": "d.tryBigIntEndianSign(nBits, LittleEndian, true)", "doc": "nBits bit signed integer in little-endian"},
{
"name": "LE",
"args": "nBits",
"params": "nBits int",
"call": "d.tryBigIntEndianSign(nBits, LittleEndian, true)",
"doc": "nBits bit signed integer in little-endian"
},
{"name": "BE", "args": "nBits", "params": "nBits int", "call": "d.tryBigIntEndianSign(nBits, BigEndian, true)", "doc": "nBits bit signed integer in big-endian"}
]
},
@ -90,7 +120,13 @@
"type": "F",
"variants": [
{"name": "", "args": "nBits", "params": "nBits int", "call": "d.tryFEndian(nBits, d.Endian)", "doc": "nBit IEEE 754 float in current endian"},
{"name": "E", "args": "nBits, endian", "params": "nBits int, endian Endian", "call": "d.tryFEndian(nBits, endian)", "doc": "nBit IEEE 754 float in specified endian"},
{
"name": "E",
"args": "nBits, endian",
"params": "nBits int, endian Endian",
"call": "d.tryFEndian(nBits, endian)",
"doc": "nBit IEEE 754 float in specified endian"
},
{"name": "16", "args": "", "params": "", "call": "d.tryFEndian(16, d.Endian)", "doc": "16 bit IEEE 754 float in current endian"},
{"name": "32", "args": "", "params": "", "call": "d.tryFEndian(32, d.Endian)", "doc": "32 bit IEEE 754 float in current endian"},
{"name": "64", "args": "", "params": "", "call": "d.tryFEndian(64, d.Endian)", "doc": "64 bit IEEE 754 float in current endian"},

2
pkg/decode/value.go
View File

@ -27,7 +27,7 @@ type Value struct {
V interface{} // scalar.S or Compound (array/struct)
Index int // index in parent array/struct
Range ranges.Range
RootBitBuf *bitio.Buffer
RootBitBuf bitio.ReaderAtSeeker
IsRoot bool // TODO: rework?
}

102
pkg/interp/buffer.go
View File

@ -9,6 +9,7 @@ import (
"math/big"
"github.com/wader/fq/internal/aheadreadseeker"
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/internal/ctxreadseeker"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/ioextra"
@ -30,20 +31,20 @@ type ToBuffer interface {
ToBuffer() (Buffer, error)
}
func toBitBuf(v interface{}) (*bitio.Buffer, error) {
func toBitBuf(v interface{}) (bitio.ReaderAtSeeker, error) {
return toBitBufEx(v, false)
}
func toBitBufEx(v interface{}, inArray bool) (*bitio.Buffer, error) {
func toBitBufEx(v interface{}, inArray bool) (bitio.ReaderAtSeeker, error) {
switch vv := v.(type) {
case ToBuffer:
bv, err := vv.ToBuffer()
if err != nil {
return nil, err
}
return bv.bb.BitBufRange(bv.r.Start, bv.r.Len)
return bitioextra.Range(bv.br, bv.r.Start, bv.r.Len)
case string:
return bitio.NewBufferFromBytes([]byte(vv), -1), nil
return bitio.NewBitReader([]byte(vv), -1), nil
case int, float64, *big.Int:
bi, err := toBigInt(v)
if err != nil {
@ -56,39 +57,40 @@ func toBitBufEx(v interface{}, inArray bool) (*bitio.Buffer, error) {
}
n := bi.Uint64()
b := [1]byte{byte(n)}
return bitio.NewBufferFromBytes(b[:], -1), nil
return bitio.NewBitReader(b[:], -1), nil
}
bitLen := int64(bi.BitLen())
// bit.Int "The bit length of 0 is 0."
if bitLen == 0 {
var z [1]byte
return bitio.NewBitReader(z[:], 1), nil
}
// TODO: how should this work? "0xf | tobytes" 4bits or 8bits? now 4
//padBefore := (8 - (bi.BitLen() % 8)) % 8
padBefore := 0
bb, err := bitio.NewBufferFromBytes(bi.Bytes(), -1).BitBufRange(int64(padBefore), int64(bi.BitLen()))
padBefore := (8 - (bitLen % 8)) % 8
// padBefore := 0
br, err := bitioextra.Range(bitio.NewBitReader(bi.Bytes(), -1), padBefore, bitLen)
if err != nil {
return nil, err
}
return bb, nil
return br, nil
case []interface{}:
var rr []bitio.BitReadAtSeeker
rr := make([]bitio.ReadAtSeeker, 0, len(vv))
// TODO: optimize byte array case, flatten into one slice
for _, e := range vv {
eBB, eErr := toBitBufEx(e, true)
eBR, eErr := toBitBufEx(e, true)
if eErr != nil {
return nil, eErr
}
rr = append(rr, eBB)
rr = append(rr, eBR)
}
mb, err := bitio.NewMultiBitReader(rr)
mb, err := bitio.NewMultiBitReader(rr...)
if err != nil {
return nil, err
}
bb, err := bitio.NewBufferFromBitReadSeeker(mb)
if err != nil {
return nil, err
}
return bb, nil
return mb, nil
default:
return nil, fmt.Errorf("value can't be a buffer")
}
@ -99,11 +101,11 @@ func toBuffer(v interface{}) (Buffer, error) {
case ToBuffer:
return vv.ToBuffer()
default:
bb, err := toBitBuf(v)
br, err := toBitBuf(v)
if err != nil {
return Buffer{}, err
}
return newBufferFromBuffer(bb, 8), nil
return newBufferFromBuffer(br, 8)
}
}
@ -140,8 +142,15 @@ func (i *Interp) _toBitsRange(c interface{}, a []interface{}) interface{} {
bv.unit = unit
if !r {
bb, _ := bv.toBuffer()
return newBufferFromBuffer(bb, unit)
br, err := bv.toBuffer()
if err != nil {
return err
}
bb, err := newBufferFromBuffer(br, unit)
if err != nil {
return err
}
return bb
}
return bv
@ -162,12 +171,12 @@ func (of *openFile) Display(w io.Writer, opts Options) error {
}
func (of *openFile) ToBuffer() (Buffer, error) {
return newBufferFromBuffer(of.bb, 8), nil
return newBufferFromBuffer(of.br, 8)
}
// def open: #:: string| => buffer
// opens a file for reading from filesystem
// TODO: when to close? when bb loses all refs? need to use finalizer somehow?
// TODO: when to close? when br loses all refs? need to use finalizer somehow?
func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
var err error
var f fs.File
@ -237,7 +246,7 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
// bitio.Buffer -> (bitio.Reader) -> aheadreadseeker -> progressreadseeker -> ctxreadseeker -> readseeker
bbf.bb, err = bitio.NewBufferFromReadSeeker(aheadRs)
bbf.br = bitio.NewIOBitReadSeeker(aheadRs)
if err != nil {
return err
}
@ -249,28 +258,34 @@ var _ Value = Buffer{}
var _ ToBuffer = Buffer{}
type Buffer struct {
bb *bitio.Buffer
br bitio.ReaderAtSeeker
r ranges.Range
unit int
}
func newBufferFromBuffer(bb *bitio.Buffer, unit int) Buffer {
return Buffer{
bb: bb,
r: ranges.Range{Start: 0, Len: bb.Len()},
unit: unit,
func newBufferFromBuffer(br bitio.ReaderAtSeeker, unit int) (Buffer, error) {
l, err := bitioextra.Len(br)
if err != nil {
return Buffer{}, err
}
return Buffer{
br: br,
r: ranges.Range{Start: 0, Len: l},
unit: unit,
}, nil
}
func (b Buffer) toBytesBuffer(r ranges.Range) (*bytes.Buffer, error) {
bb, err := b.bb.BitBufRange(r.Start, r.Len)
br, err := bitioextra.Range(b.br, r.Start, r.Len)
if err != nil {
return nil, err
}
buf := &bytes.Buffer{}
if _, err := io.Copy(buf, bb.Clone()); err != nil {
if _, err := bitioextra.CopyBits(buf, br); err != nil {
return nil, err
}
return buf, nil
}
@ -307,7 +322,8 @@ func (b Buffer) JQValueIndex(index int) interface{} {
return err
}
extraBits := uint((8 - b.r.Len%8) % 8)
extraBits := uint((8 - b.unit%8) % 8)
return new(big.Int).Rsh(new(big.Int).SetBytes(buf.Bytes()), extraBits)
}
func (b Buffer) JQValueSlice(start int, end int) interface{} {
@ -315,7 +331,7 @@ func (b Buffer) JQValueSlice(start int, end int) interface{} {
rLen := int64((end - start) * b.unit)
return Buffer{
bb: b.bb,
br: b.br,
r: ranges.Range{Start: b.r.Start + rStart, Len: rLen},
unit: b.unit,
}
@ -337,12 +353,12 @@ func (b Buffer) JQValueKey(name string) interface{} {
if b.unit == 1 {
return b
}
return Buffer{bb: b.bb, r: b.r, unit: 1}
return Buffer{br: b.br, r: b.r, unit: 1}
case "bytes":
if b.unit == 8 {
return b
}
return Buffer{bb: b.bb, r: b.r, unit: 8}
return Buffer{br: b.br, r: b.r, unit: 8}
}
return nil
}
@ -382,19 +398,21 @@ func (b Buffer) JQValueUpdate(key interface{}, u interface{}, delpath bool) inte
func (b Buffer) Display(w io.Writer, opts Options) error {
if opts.RawOutput {
bb, err := b.toBuffer()
br, err := b.toBuffer()
if err != nil {
return err
}
if _, err := io.Copy(w, bb.Clone()); err != nil {
if _, err := bitioextra.CopyBits(w, br); err != nil {
return err
}
return nil
}
return hexdump(w, b, opts)
}
func (b Buffer) toBuffer() (*bitio.Buffer, error) {
return b.bb.BitBufRange(b.r.Start, b.r.Len)
func (b Buffer) toBuffer() (bitio.ReaderAtSeeker, error) {
return bitioextra.Range(b.br, b.r.Start, b.r.Len)
}

26
pkg/interp/decode.go
View File

@ -11,6 +11,7 @@ import (
"time"
"github.com/mitchellh/mapstructure"
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/ioextra"
"github.com/wader/fq/pkg/bitio"
@ -201,7 +202,7 @@ func (i *Interp) _decode(c interface{}, a []interface{}) interface{} {
return err
}
dv, _, err := decode.Decode(i.evalContext.ctx, bv.bb, decodeFormat,
dv, _, err := decode.Decode(i.evalContext.ctx, bv.br, decodeFormat,
decode.Options{
IsRoot: true,
FillGaps: true,
@ -273,7 +274,7 @@ func makeDecodeValue(dv *decode.Value) interface{} {
return NewStructDecodeValue(dv, vv)
case *scalar.S:
switch vv := vv.Value().(type) {
case *bitio.Buffer:
case bitio.ReaderAtSeeker:
// is lazy so that in situations where the decode value is only used to
// create another buffer we don't have to read and create a string, ex:
// .unknown0 | tobytes[1:] | ...
@ -283,7 +284,11 @@ func makeDecodeValue(dv *decode.Value) interface{} {
IsScalar: true,
Fn: func() (gojq.JQValue, error) {
buf := &bytes.Buffer{}
if _, err := io.Copy(buf, vv.Clone()); err != nil {
vvC, err := bitioextra.Clone(vv)
if err != nil {
return nil, err
}
if _, err := bitioextra.CopyBits(buf, vvC); err != nil {
return nil, err
}
return gojqextra.String([]rune(buf.String())), nil
@ -360,7 +365,7 @@ func (dvb decodeValueBase) DecodeValue() *decode.Value {
func (dvb decodeValueBase) Display(w io.Writer, opts Options) error { return dump(dvb.dv, w, opts) }
func (dvb decodeValueBase) ToBuffer() (Buffer, error) {
return Buffer{bb: dvb.dv.RootBitBuf, r: dvb.dv.InnerRange(), unit: 8}, nil
return Buffer{br: dvb.dv.RootBitBuf, r: dvb.dv.InnerRange(), unit: 8}, nil
}
func (decodeValueBase) ExtType() string { return "decode_value" }
func (dvb decodeValueBase) ExtKeys() []string {
@ -475,13 +480,13 @@ func (dvb decodeValueBase) JQValueKey(name string) interface{} {
}
case "_bits":
return Buffer{
bb: dv.RootBitBuf,
br: dv.RootBitBuf,
r: dv.Range,
unit: 1,
}
case "_bytes":
return Buffer{
bb: dv.RootBitBuf,
br: dv.RootBitBuf,
r: dv.Range,
unit: 8,
}
@ -542,12 +547,17 @@ func (v decodeValue) JQValueToGoJQEx(optsFn func() Options) interface{} {
if err != nil {
return err
}
bb, err := bv.toBuffer()
br, err := bv.toBuffer()
if err != nil {
return err
}
s, err := optsFn().BitsFormatFn(bb.Clone())
brC, err := bitioextra.Clone(br)
if err != nil {
return err
}
s, err := optsFn().BitsFormatFn(brC)
if err != nil {
return err
}

2
pkg/interp/decorator.go
View File

@ -114,7 +114,7 @@ func decoratorFromOptions(opts Options) Decorator {
return d.True
}
return d.False
case string, *bitio.Buffer:
case string, bitio.Reader:
return d.String
case nil:
return d.Null

63
pkg/interp/dump.go
View File

@ -1,6 +1,7 @@
package interp
import (
"bytes"
"errors"
"fmt"
"io"
@ -8,6 +9,7 @@ import (
"strings"
"github.com/wader/fq/internal/asciiwriter"
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/internal/columnwriter"
"github.com/wader/fq/internal/hexpairwriter"
"github.com/wader/fq/internal/mathextra"
@ -217,7 +219,12 @@ func dumpEx(v *decode.Value, buf []byte, cw *columnwriter.Writer, depth int, roo
printErrs(depth, valueErr)
}
bufferLastBit := rootV.RootBitBuf.Len() - 1
rootBitLen, err := bitioextra.Len(rootV.RootBitBuf)
if err != nil {
return err
}
bufferLastBit := rootBitLen - 1
startBit := innerRange.Start
stopBit := innerRange.Stop() - 1
sizeBits := innerRange.Len
@ -260,7 +267,7 @@ func dumpEx(v *decode.Value, buf []byte, cw *columnwriter.Writer, depth int, roo
cfmt(colAddr, "%s%s\n",
rootIndent, deco.DumpAddr.F(mathextra.PadFormatInt(startLineByte, opts.AddrBase, true, addrWidth)))
vBitBuf, err := rootV.RootBitBuf.BitBufRange(startByte*8, displaySizeBits)
vBR, err := bitioextra.Range(rootV.RootBitBuf, startByte*8, displaySizeBits)
if err != nil {
return err
}
@ -269,19 +276,26 @@ func dumpEx(v *decode.Value, buf []byte, cw *columnwriter.Writer, depth int, roo
hexpairFn := func(b byte) string { return deco.ByteColor(b).Wrap(hexpairwriter.Pair(b)) }
asciiFn := func(b byte) string { return deco.ByteColor(b).Wrap(asciiwriter.SafeASCII(b)) }
if vBitBuf != nil {
if _, err := io.CopyBuffer(
hexpairwriter.New(cw.Columns[colHex], opts.LineBytes, int(startLineByteOffset), hexpairFn),
io.LimitReader(vBitBuf.Clone(), displaySizeBytes),
buf); err != nil {
return err
}
if _, err := io.CopyBuffer(
asciiwriter.New(cw.Columns[colASCII], opts.LineBytes, int(startLineByteOffset), asciiFn),
io.LimitReader(vBitBuf.Clone(), displaySizeBytes),
buf); err != nil {
return err
}
hexBR, err := bitioextra.Clone(vBR)
if err != nil {
return err
}
if _, err := bitioextra.CopyBitsBuffer(
hexpairwriter.New(cw.Columns[colHex], opts.LineBytes, int(startLineByteOffset), hexpairFn),
hexBR,
buf); err != nil {
return err
}
asciiBR, err := bitioextra.Clone(vBR)
if err != nil {
return err
}
if _, err := bitioextra.CopyBitsBuffer(
asciiwriter.New(cw.Columns[colASCII], opts.LineBytes, int(startLineByteOffset), asciiFn),
asciiBR,
buf); err != nil {
return err
}
for i := int64(1); i < addrLines; i++ {
@ -351,18 +365,31 @@ func dump(v *decode.Value, w io.Writer, opts Options) error {
}
func hexdump(w io.Writer, bv Buffer, opts Options) error {
bb, err := bv.toBuffer()
br, err := bv.toBuffer()
if err != nil {
return err
}
cBR, err := bitioextra.Clone(bv.br)
if err != nil {
return err
}
bytesB := &bytes.Buffer{}
if _, err := bitioextra.CopyBits(bytesB, cBR); err != nil {
return err
}
biib := bitio.NewBitReader(bytesB.Bytes(), -1)
// TODO: hack
opts.Verbose = true
return dump(
&decode.Value{
// TODO: hack
V: &scalar.S{Actual: bb},
V: &scalar.S{Actual: br},
Range: bv.r,
RootBitBuf: bv.bb.Clone(),
RootBitBuf: biib,
},
w,
opts,

49
pkg/interp/funcs.go
View File

@ -65,12 +65,12 @@ func makeStringBitBufTransformFn(
return func(c interface{}, a []interface{}) interface{} {
switch c := c.(type) {
case string:
bb, err := toBitBuf(c)
br, err := toBitBuf(c)
if err != nil {
return err
}
r, err := decodeFn(bb)
r, err := decodeFn(bitio.NewIOReader(br))
if err != nil {
return err
}
@ -79,11 +79,14 @@ func makeStringBitBufTransformFn(
if _, err := io.Copy(buf, r); err != nil {
return err
}
outBB := bitio.NewBufferFromBytes(buf.Bytes(), -1)
return newBufferFromBuffer(outBB, 8)
bb, err := newBufferFromBuffer(bitio.NewBitReader(buf.Bytes(), -1), 8)
if err != nil {
return err
}
return bb
default:
bb, err := toBitBuf(c)
br, err := toBitBuf(c)
if err != nil {
return err
}
@ -94,7 +97,7 @@ func makeStringBitBufTransformFn(
return err
}
if _, err := io.Copy(w, bb); err != nil {
if _, err := io.Copy(w, bitio.NewIOReader(br)); err != nil {
return err
}
@ -110,12 +113,12 @@ func makeStringBitBufTransformFn(
// transform to buffer using fn
func makeBitBufTransformFn(fn func(r io.Reader) (io.Reader, error)) func(c interface{}, a []interface{}) interface{} {
return func(c interface{}, a []interface{}) interface{} {
inBB, err := toBitBuf(c)
inBR, err := toBitBuf(c)
if err != nil {
return err
}
r, err := fn(inBB)
r, err := fn(bitio.NewIOReader(inBR))
if err != nil {
return err
}
@ -125,16 +128,20 @@ func makeBitBufTransformFn(fn func(r io.Reader) (io.Reader, error)) func(c inter
return err
}
outBB := bitio.NewBufferFromBytes(outBuf.Bytes(), -1)
outBR := bitio.NewBitReader(outBuf.Bytes(), -1)
return newBufferFromBuffer(outBB, 8)
bb, err := newBufferFromBuffer(outBR, 8)
if err != nil {
return err
}
return bb
}
}
// transform to buffer using fn
func makeHashFn(fn func() (hash.Hash, error)) func(c interface{}, a []interface{}) interface{} {
return func(c interface{}, a []interface{}) interface{} {
inBB, err := toBitBuf(c)
inBR, err := toBitBuf(c)
if err != nil {
return err
}
@ -143,13 +150,17 @@ func makeHashFn(fn func() (hash.Hash, error)) func(c interface{}, a []interface{
if err != nil {
return err
}
if _, err := io.Copy(h, inBB); err != nil {
if _, err := io.Copy(h, bitio.NewIOReader(inBR)); err != nil {
return err
}
outBB := bitio.NewBufferFromBytes(h.Sum(nil), -1)
outBR := bitio.NewBitReader(h.Sum(nil), -1)
return newBufferFromBuffer(outBB, 8)
bb, err := newBufferFromBuffer(outBR, 8)
if err != nil {
return err
}
return bb
}
}
@ -223,18 +234,22 @@ func (i *Interp) aesCtr(c interface{}, a []interface{}) interface{} {
ivBytes = make([]byte, block.BlockSize())
}
bb, err := toBitBuf(c)
br, err := toBitBuf(c)
if err != nil {
return err
}
buf := &bytes.Buffer{}
reader := &cipher.StreamReader{S: cipher.NewCTR(block, ivBytes), R: bb}
reader := &cipher.StreamReader{S: cipher.NewCTR(block, ivBytes), R: bitio.NewIOReader(br)}
if _, err := io.Copy(buf, reader); err != nil {
return err
}
return newBufferFromBuffer(bitio.NewBufferFromBytes(buf.Bytes(), -1), 8)
bb, err := newBufferFromBuffer(bitio.NewBitReader(buf.Bytes(), -1), 8)
if err != nil {
return err
}
return bb
}
func (i *Interp) _hexdump(c interface{}, a []interface{}) gojq.Iter {

37
pkg/interp/interp.go
View File

@ -19,6 +19,7 @@ import (
"github.com/mitchellh/mapstructure"
"github.com/wader/fq/internal/ansi"
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/internal/colorjson"
"github.com/wader/fq/internal/ctxstack"
"github.com/wader/fq/internal/ioextra"
@ -269,12 +270,12 @@ func toBigInt(v interface{}) (*big.Int, error) {
func toBytes(v interface{}) ([]byte, error) {
switch v := v.(type) {
default:
bb, err := toBitBuf(v)
br, err := toBitBuf(v)
if err != nil {
return nil, fmt.Errorf("value is not bytes")
}
buf := &bytes.Buffer{}
if _, err := io.Copy(buf, bb); err != nil {
if _, err := bitioextra.CopyBits(buf, br); err != nil {
return nil, err
}
@ -907,24 +908,24 @@ type Options struct {
SizeBase int `mapstructure:"sizebase"`
Decorator Decorator
BitsFormatFn func(bb *bitio.Buffer) (interface{}, error)
BitsFormatFn func(br bitio.ReaderAtSeeker) (interface{}, error)
}
func bitsFormatFnFromOptions(opts Options) func(bb *bitio.Buffer) (interface{}, error) {
func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (interface{}, error) {
switch opts.BitsFormat {
case "md5":
return func(bb *bitio.Buffer) (interface{}, error) {
return func(br bitio.ReaderAtSeeker) (interface{}, error) {
d := md5.New()
if _, err := io.Copy(d, bb); err != nil {
if _, err := bitioextra.CopyBits(d, br); err != nil {
return "", err
}
return hex.EncodeToString(d.Sum(nil)), nil
}
case "base64":
return func(bb *bitio.Buffer) (interface{}, error) {
return func(br bitio.ReaderAtSeeker) (interface{}, error) {
b := &bytes.Buffer{}
e := base64.NewEncoder(base64.StdEncoding, b)
if _, err := io.Copy(e, bb); err != nil {
if _, err := bitioextra.CopyBits(e, br); err != nil {
return "", err
}
e.Close()
@ -932,17 +933,17 @@ func bitsFormatFnFromOptions(opts Options) func(bb *bitio.Buffer) (interface{},
}
case "truncate":
// TODO: configure
return func(bb *bitio.Buffer) (interface{}, error) {
return func(br bitio.ReaderAtSeeker) (interface{}, error) {
b := &bytes.Buffer{}
if _, err := io.Copy(b, io.LimitReader(bb, 1024)); err != nil {
if _, err := bitioextra.CopyBits(b, bitio.NewLimitReader(br, 1024*8)); err != nil {
return "", err
}
return b.String(), nil
}
case "string":
return func(bb *bitio.Buffer) (interface{}, error) {
return func(br bitio.ReaderAtSeeker) (interface{}, error) {
b := &bytes.Buffer{}
if _, err := io.Copy(b, bb); err != nil {
if _, err := bitioextra.CopyBits(b, br); err != nil {
return "", err
}
return b.String(), nil
@ -950,14 +951,20 @@ func bitsFormatFnFromOptions(opts Options) func(bb *bitio.Buffer) (interface{},
case "snippet":
fallthrough
default:
return func(bb *bitio.Buffer) (interface{}, error) {
return func(br bitio.ReaderAtSeeker) (interface{}, error) {
b := &bytes.Buffer{}
e := base64.NewEncoder(base64.StdEncoding, b)
if _, err := io.Copy(e, io.LimitReader(bb, 256)); err != nil {
if _, err := bitioextra.CopyBits(e, bitio.NewLimitReader(br, 256*8)); err != nil {
return "", err
}
e.Close()
return fmt.Sprintf("<%s>%s", mathextra.Bits(bb.Len()).StringByteBits(opts.SizeBase), b.String()), nil
brLen, err := bitioextra.Len(br)
if err != nil {
return nil, err
}
return fmt.Sprintf("<%s>%s", mathextra.Bits(brLen).StringByteBits(opts.SizeBase), b.String()), nil
}
}
}

9
pkg/interp/match.go
View File

@ -7,6 +7,7 @@ import (
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/ioextra"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/ranges"
"github.com/wader/gojq"
)
@ -69,7 +70,7 @@ func (i *Interp) _bufferMatch(c interface{}, a []interface{}) gojq.Iter {
}
sreNames := sre.SubexpNames()
bb, err := bv.toBuffer()
br, err := bv.toBuffer()
if err != nil {
return gojq.NewIter(err)
}
@ -83,9 +84,9 @@ func (i *Interp) _bufferMatch(c interface{}, a []interface{}) gojq.Iter {
// will match the byte \0xff
if byteRunes {
// byte mode, read each byte as a rune
rr = ioextra.ByteRuneReader{RS: bb}
rr = ioextra.ByteRuneReader{RS: bitio.NewIOReadSeeker(br)}
} else {
rr = ioextra.RuneReadSeeker{RS: bb}
rr = ioextra.RuneReadSeeker{RS: bitio.NewIOReadSeeker(br)}
}
var off int64
@ -127,7 +128,7 @@ func (i *Interp) _bufferMatch(c interface{}, a []interface{}) gojq.Iter {
matchBitOff := (off + int64(start)) * 8
matchLength := int64(end-start) * 8
bbo := Buffer{
bb: bv.bb,
br: bv.br,
r: ranges.Range{
Start: bv.r.Start + matchBitOff,
Len: matchLength,

2
pkg/interp/preview.go
View File

@ -35,7 +35,7 @@ func previewValue(v interface{}, df scalar.DisplayFormat) string {
return fmt.Sprintf("%q", vv)
case nil:
return "null"
case *bitio.Buffer:
case bitio.Reader:
return "raw bits"
case *big.Int:
return mathextra.PadFormatBigInt(vv, df.FormatBase(), true, 0)

68
pkg/interp/testdata/buffer.fqtest vendored
View File

@ -28,7 +28,7 @@ $ fq -n '[("11" | hex), ("22" | hex)] | tobits | hd'
# TODO: bug, hexdump uses io.Copy which is byte oritneted
$ fq -n '[("12" | hex | .bits[4:]), ("34" | hex | .bits[0:4])] | tobits | hd'
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|20| | | |.: raw bits 0x0-0x0.7 (1)
0x0|23| |#| |.: raw bits 0x0-0x0.7 (1)
$ fq -d mp3 '.frames[]._bits[0:12] | tonumber' /test.mp3
4095
4095
@ -236,4 +236,70 @@ mp3> .frames[1] | tobytesrange | ., .start, .stop, .size, .[4:17], (tobits, toby
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0xe0| 00 00 0a 2c 43 2e 55 94 80| ...,C.U..|.: raw bits 0xe7-0xf3.7 (13)
0xf0|01 80 93 6b |...k |
mp3> "fq" | tobits | [.[range(.size)]] | map(tobits) | tobytes | tostring
"fq"
mp3> "fq" | tobits | chunk(range(17)+1) | tobytes | tostring
"fq"
"fq"
"fq"
"fq"
"fq"
"fq"
"fq"
"fq"
"fq"
"fq"
"fq"
"fq"
"fq"
"fq"
"fq"
"fq"
"fq"
mp3> range(17) | [range(.) | 1 | tobits] | tobytes
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
| | |.: raw bits 0x0-NA (0)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|80| |.| |.: raw bits 0x0-0x0 (0.1)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|c0| |.| |.: raw bits 0x0-0x0.1 (0.2)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|e0| |.| |.: raw bits 0x0-0x0.2 (0.3)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|f0| |.| |.: raw bits 0x0-0x0.3 (0.4)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|f8| |.| |.: raw bits 0x0-0x0.4 (0.5)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|fc| |.| |.: raw bits 0x0-0x0.5 (0.6)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|fe| |.| |.: raw bits 0x0-0x0.6 (0.7)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|ff| |.| |.: raw bits 0x0-0x0.7 (1)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|ff 80| |..| |.: raw bits 0x0-0x1 (1.1)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|ff c0| |..| |.: raw bits 0x0-0x1.1 (1.2)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|ff e0| |..| |.: raw bits 0x0-0x1.2 (1.3)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|ff f0| |..| |.: raw bits 0x0-0x1.3 (1.4)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|ff f8| |..| |.: raw bits 0x0-0x1.4 (1.5)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|ff fc| |..| |.: raw bits 0x0-0x1.5 (1.6)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|ff fe| |..| |.: raw bits 0x0-0x1.6 (1.7)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|ff ff| |..| |.: raw bits 0x0-0x1.7 (2)
mp3> "c9dfdac2f6ef68e5db666b6fbeee66d9c7deda66bebfbfe860bfbfbfe9d1636bbfbebf" | hex | tobits | reduce chunk(8)[] as $c ({h:[],g:[]}; .h += [(0|tobits), $c[0:7]] | .g |= . + [if length % 8 == 0 then (0|tobits) else empty end, $c[7:8]]) | .h, .g | tobytes
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x00|64 6f 6d 61 7b 77 34 72 6d 33 35 37 5f 77 33 6c|doma{w4rm357_w3l|.: raw bits 0x0-0x22.7 (35)
* |until 0x22.7 (end) (35) | |
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0|62 6c 30 67 7d| |bl0g}| |.: raw bits 0x0-0x4.7 (5)
mp3> .frames[1] | tobytes | mp3_frame | ., ((.header.bitrate | tobitsrange) as $v | tobitsrange | [.[:$v.start], (0xf | tobits), .[$v.start+$v.size:]] | mp3_frame) | .header.bitrate
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0| 50 | P |.header.bitrate: 64000 (5)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
0x0| f0 | . |.header.bitrate: 15 (bad)
mp3> ^D

97
pkg/scalar/scalar.go
View File

@ -11,6 +11,7 @@ import (
"strconv"
"strings"
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/pkg/bitio"
)
@ -41,7 +42,7 @@ func (df DisplayFormat) FormatBase() int {
}
type S struct {
Actual interface{} // int, int64, uint64, float64, string, bool, []byte, *bitio.Buffer
Actual interface{} // int, int64, uint64, float64, string, bool, []byte, bitio.BitReaderAtSeeker,
ActualDisplay DisplayFormat
Sym interface{}
SymDisplay DisplayFormat
@ -140,55 +141,6 @@ func StrFToSym(base int) Mapper {
return strMapToSym(func(s string) (interface{}, error) { return strconv.ParseFloat(s, base) })
}
//nolint:unparam
func rawSym(s S, nBytes int, fn func(b []byte) string) (S, error) {
bb, ok := s.Actual.(*bitio.Buffer)
if !ok {
return s, nil
}
bbLen := bb.Len()
if nBytes < 0 {
nBytes = int(bbLen) / 8
if bbLen%8 != 0 {
nBytes++
}
}
if bbLen < int64(nBytes)*8 {
return s, nil
}
// TODO: shared somehow?
b := make([]byte, nBytes)
if _, err := bb.ReadBitsAt(b, nBytes*8, 0); err != nil {
return s, err
}
s.Sym = fn(b[0:nBytes])
return s, nil
}
var RawSym = Fn(func(s S) (S, error) {
return rawSym(s, -1, func(b []byte) string {
return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
})
})
var RawUUID = Fn(func(s S) (S, error) {
return rawSym(s, -1, func(b []byte) string {
return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
})
})
var RawHex = Fn(func(s S) (S, error) {
return rawSym(s, -1, func(b []byte) string { return fmt.Sprintf("%x", b) })
})
var RawHexReverse = Fn(func(s S) (S, error) {
return rawSym(s, -1, func(b []byte) string {
return fmt.Sprintf("%x", bitio.ReverseBytes(append([]byte{}, b...)))
})
})
type URangeEntry struct {
Range [2]uint64
S S
@ -238,18 +190,57 @@ func (rs SRangeToScalar) MapScalar(s S) (S, error) {
return s, nil
}
func rawSym(s S, nBytes int, fn func(b []byte) string) (S, error) {
br, ok := s.Actual.(bitio.ReadAtSeeker)
if !ok {
return s, nil
}
brLen, err := bitioextra.Len(br)
if err != nil {
return S{}, err
}
if nBytes < 0 {
nBytes = int(brLen) / 8
if brLen%8 != 0 {
nBytes++
}
}
if brLen < int64(nBytes)*8 {
return s, nil
}
// TODO: shared somehow?
b := make([]byte, nBytes)
if _, err := br.ReadBitsAt(b, int64(nBytes)*8, 0); err != nil {
return s, err
}
s.Sym = fn(b[0:nBytes])
return s, nil
}
var RawUUID = Fn(func(s S) (S, error) {
return rawSym(s, -1, func(b []byte) string {
return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
})
})
var RawHex = Fn(func(s S) (S, error) {
return rawSym(s, -1, func(b []byte) string { return fmt.Sprintf("%x", b) })
})
type BytesToScalar []struct {
Bytes []byte
Scalar S
}
func (m BytesToScalar) MapScalar(s S) (S, error) {
ab, err := s.ActualBitBuf().Bytes()
if err != nil {
bb := &bytes.Buffer{}
if _, err := bitioextra.CopyBits(bb, s.ActualBitBuf()); err != nil {
return s, err
}
for _, bs := range m {
if bytes.Equal(ab, bs.Bytes) {
if bytes.Equal(bb.Bytes(), bs.Bytes) {
ns := bs.Scalar
ns.Actual = s.Actual
s = ns

12
pkg/scalar/scalar_gen.go
View File

@ -31,19 +31,19 @@ func (s S) SymBigInt() *big.Int {
// Type BitBuf
// ActualBitBuf asserts actual value is a BitBuf and returns it
func (s S) ActualBitBuf() *bitio.Buffer {
v, ok := s.Actual.(*bitio.Buffer)
func (s S) ActualBitBuf() bitio.ReaderAtSeeker {
v, ok := s.Actual.(bitio.ReaderAtSeeker)
if !ok {
panic(fmt.Sprintf("failed to type assert s.Actual %v as *bitio.Buffer", s.Actual))
panic(fmt.Sprintf("failed to type assert s.Actual %v as bitio.ReaderAtSeeker", s.Actual))
}
return v
}
// SymBitBuf asserts symbolic value is a BitBuf and returns it
func (s S) SymBitBuf() *bitio.Buffer {
v, ok := s.Sym.(*bitio.Buffer)
func (s S) SymBitBuf() bitio.ReaderAtSeeker {
v, ok := s.Sym.(bitio.ReaderAtSeeker)
if !ok {
panic(fmt.Sprintf("failed to type assert s.Sym %v as *bitio.Buffer", s.Sym))
panic(fmt.Sprintf("failed to type assert s.Sym %v as bitio.ReaderAtSeeker", s.Sym))
}
return v
}