mirror of
https://github.com/wader/fq.git
synced 2024-09-11 12:05:39 +03:00
bitio,decode: Refactor bitio usage and make buffer slicing more correct
Remove bitio.Buffer layer. bitio.Buffer was a kitchen sink layer with helpers now it's just a buffer and most functions have been moved to decode instead. bitio package now only have primitive types and functions simialar to standard library io and bytes packages. Make nearly eveything internally use bitio.Bit* interfaces so that slicing work correctly this will also make it possible to start experimenting with more complicated silcing helpers, ex things like: breplace(.header.bitrate; 123) to get a new buffer with bitrate changed.
This commit is contained in:
parent
9ed08ec108
commit
7c5215347d
@ -150,11 +150,7 @@ aheadreadseeker.Reader does readahead caching
|
||||
^
|
||||
| (io.ReadSeeker interface)
|
||||
|
|
||||
bitio.Reader (implements bitio.Bit* interfaces)
|
||||
^
|
||||
| (bitio.Bit* interfaces)
|
||||
|
|
||||
bitio.Buffer convenience wrapper to read bytes from bit reader, create section readers etc
|
||||
bitio.IOBitReader (implements bitio.Bit* interfaces)
|
||||
SectionBitReader
|
||||
MultiBitReader
|
||||
```
|
||||
|
@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/wader/fq/format"
|
||||
"github.com/wader/fq/format/registry"
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
"github.com/wader/fq/pkg/decode"
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
@ -108,14 +109,14 @@ func bzip2Decode(d *decode.D, in interface{}) interface{} {
|
||||
|
||||
compressedStart := d.Pos()
|
||||
|
||||
readCompressedSize, uncompressedBB, dv, _, _ := d.TryFieldReaderRangeFormat("uncompressed", 0, d.Len(), bzip2.NewReader, probeGroup, nil)
|
||||
if uncompressedBB != nil {
|
||||
readCompressedSize, uncompressedBR, dv, _, _ := d.TryFieldReaderRangeFormat("uncompressed", 0, d.Len(), bzip2.NewReader, probeGroup, nil)
|
||||
if uncompressedBR != nil {
|
||||
if dv == nil {
|
||||
d.FieldRootBitBuf("uncompressed", uncompressedBB)
|
||||
d.FieldRootBitBuf("uncompressed", uncompressedBR)
|
||||
}
|
||||
|
||||
blockCRC32W := crc32.NewIEEE()
|
||||
d.MustCopy(blockCRC32W, bitFlipReader{uncompressedBB.Clone()})
|
||||
d.MustCopy(blockCRC32W, bitFlipReader{bitio.NewIOReader(uncompressedBR)})
|
||||
blockCRC32N := bits.Reverse32(binary.BigEndian.Uint32(blockCRC32W.Sum(nil)))
|
||||
_ = blockCRCValue.TryScalarFn(d.ValidateU(uint64(blockCRC32N)))
|
||||
streamCRCN = blockCRC32N ^ ((streamCRCN << 1) | (streamCRCN >> 31))
|
||||
|
@ -135,17 +135,14 @@ func decodeCBORValue(d *decode.D) interface{} {
|
||||
})
|
||||
}
|
||||
})
|
||||
d.FieldRootBitBuf("value", bitio.NewBufferFromBytes(bb.Bytes(), -1))
|
||||
d.FieldRootBitBuf("value", bitio.NewBitReader(bb.Bytes(), -1))
|
||||
// nil, nested indefinite bytes is not allowed
|
||||
return nil
|
||||
}
|
||||
|
||||
bib := d.FieldRawLen("value", int64(count)*8)
|
||||
bs, err := bib.Bytes()
|
||||
if err != nil {
|
||||
d.IOPanic(err, "bytes bb.Bytes")
|
||||
}
|
||||
return bs
|
||||
buf := d.MustReadAllBits(d.FieldRawLen("value", int64(count)*8))
|
||||
|
||||
return buf
|
||||
}},
|
||||
majorTypeUTF8: {s: scalar.S{Sym: "utf8"}, d: func(d *decode.D, shortCount uint64, count uint64) interface{} {
|
||||
if shortCount == shortCountIndefinite {
|
||||
|
@ -80,7 +80,7 @@ func flacDecode(d *decode.D, in interface{}) interface{} {
|
||||
}
|
||||
})
|
||||
|
||||
md5CalcValue := d.FieldRootBitBuf("md5_calculated", bitio.NewBufferFromBytes(md5Samples.Sum(nil), -1))
|
||||
md5CalcValue := d.FieldRootBitBuf("md5_calculated", bitio.NewBitReader(md5Samples.Sum(nil), -1))
|
||||
_ = md5CalcValue.TryScalarFn(d.ValidateBitBuf(streamInfo.MD5), scalar.RawHex)
|
||||
d.FieldValueU("decoded_samples", framesNDecodedSamples)
|
||||
|
||||
|
@ -327,7 +327,7 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
|
||||
})
|
||||
|
||||
headerCRC := &checksum.CRC{Bits: 8, Table: checksum.ATM8Table}
|
||||
d.MustCopy(headerCRC, d.BitBufRange(frameStart, d.Pos()-frameStart))
|
||||
d.MustCopyBits(headerCRC, d.BitBufRange(frameStart, d.Pos()-frameStart))
|
||||
d.FieldU8("crc", d.ValidateUBytes(headerCRC.Sum(nil)), scalar.Hex)
|
||||
})
|
||||
|
||||
@ -573,7 +573,7 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
|
||||
d.FieldU("byte_align", d.ByteAlignBits(), d.AssertU(0))
|
||||
// <16> CRC-16 (polynomial = x^16 + x^15 + x^2 + x^0, initialized with 0) of everything before the crc, back to and including the frame header sync code
|
||||
footerCRC := &checksum.CRC{Bits: 16, Table: checksum.ANSI16Table}
|
||||
d.MustCopy(footerCRC, d.BitBufRange(frameStart, d.Pos()-frameStart))
|
||||
d.MustCopyBits(footerCRC, d.BitBufRange(frameStart, d.Pos()-frameStart))
|
||||
d.FieldRawLen("footer_crc", 16, d.ValidateBitBuf(footerCRC.Sum(nil)), scalar.RawHex)
|
||||
|
||||
streamSamples := len(channelSamples[0])
|
||||
|
@ -26,11 +26,8 @@ func streaminfoDecode(d *decode.D, in interface{}) interface{} {
|
||||
// <5> (bits per sample)-1. FLAC supports from 4 to 32 bits per sample. Currently the reference encoder and decoders only support up to 24 bits per sample.
|
||||
bitPerSample := d.FieldU5("bits_per_sample", scalar.UAdd(1))
|
||||
totalSamplesInStream := d.FieldU("total_samples_in_stream", 36)
|
||||
md5BB := d.FieldRawLen("md5", 16*8, scalar.RawHex)
|
||||
md5b, err := md5BB.Bytes()
|
||||
if err != nil {
|
||||
d.IOPanic(err, "md5BB.Bytes")
|
||||
}
|
||||
md5BR := d.FieldRawLen("md5", 16*8, scalar.RawHex)
|
||||
md5b := d.MustReadAllBits(md5BR)
|
||||
|
||||
return format.FlacStreaminfoOut{
|
||||
StreamInfo: format.FlacStreamInfo{
|
||||
|
@ -73,9 +73,9 @@ func gifDecode(d *decode.D, in interface{}) interface{} {
|
||||
blocks:
|
||||
for {
|
||||
switch d.PeekBits(8) {
|
||||
case 0x3b: /* ";" */
|
||||
case ';':
|
||||
break blocks
|
||||
case 0x21: /* "!" */
|
||||
case '!': /* "!" */
|
||||
d.FieldStruct("extension_block", func(d *decode.D) {
|
||||
d.FieldU8("introducer")
|
||||
functionCode := d.FieldU8("function_code", extensionNames, scalar.Hex)
|
||||
@ -93,7 +93,7 @@ func gifDecode(d *decode.D, in interface{}) interface{} {
|
||||
d.FieldU8("terminator")
|
||||
seenTerminator = true
|
||||
}
|
||||
d.MustCopy(dataBytes, b.Clone())
|
||||
d.MustCopyBits(dataBytes, d.MustClone(b))
|
||||
})
|
||||
}
|
||||
})
|
||||
@ -105,12 +105,12 @@ func gifDecode(d *decode.D, in interface{}) interface{} {
|
||||
// case extensionGraphicalControl:
|
||||
// d.FieldFormatBitBuf(
|
||||
// "graphics_control",
|
||||
// bitio.NewBufferFromBytes(dataBytes.Bytes(), -1),
|
||||
// bitio.NewReader(dataBytes.Bytes(), -1),
|
||||
// )
|
||||
|
||||
// }
|
||||
})
|
||||
case 0x2c: /* "," */
|
||||
case ',':
|
||||
d.FieldStruct("image", func(d *decode.D) {
|
||||
d.FieldU8("separator_character")
|
||||
d.FieldU16("left")
|
||||
|
@ -101,20 +101,21 @@ func gzDecode(d *decode.D, in interface{}) interface{} {
|
||||
var rFn func(r io.Reader) io.Reader
|
||||
switch compressionMethod {
|
||||
case delfateMethod:
|
||||
// *bitio.Buffer implements io.ByteReader so hat deflate don't do own
|
||||
// bitio.NewIOReadSeeker implements io.ByteReader so that deflate don't do own
|
||||
// buffering and might read more than needed messing up knowing compressed size
|
||||
rFn = func(r io.Reader) io.Reader { return flate.NewReader(r) }
|
||||
}
|
||||
|
||||
if rFn != nil {
|
||||
readCompressedSize, uncompressedBB, dv, _, _ := d.TryFieldReaderRangeFormat("uncompressed", d.Pos(), d.BitsLeft(), rFn, probeFormat, nil)
|
||||
if uncompressedBB != nil {
|
||||
readCompressedSize, uncompressedBR, dv, _, _ := d.TryFieldReaderRangeFormat("uncompressed", d.Pos(), d.BitsLeft(), rFn, probeFormat, nil)
|
||||
if uncompressedBR != nil {
|
||||
if dv == nil {
|
||||
d.FieldRootBitBuf("uncompressed", uncompressedBB)
|
||||
d.FieldRootBitBuf("uncompressed", uncompressedBR)
|
||||
}
|
||||
d.FieldRawLen("compressed", readCompressedSize)
|
||||
crc32W := crc32.NewIEEE()
|
||||
d.MustCopy(crc32W, uncompressedBB.Clone())
|
||||
// TODO: cleanup clone
|
||||
d.MustCopyBits(crc32W, d.MustClone(uncompressedBR))
|
||||
d.FieldU32("crc32", d.ValidateUBytes(crc32W.Sum(nil)), scalar.Hex)
|
||||
d.FieldU32("isize")
|
||||
}
|
||||
|
14
format/gzip/testdata/compressed_size.fqtest
vendored
Normal file
14
format/gzip/testdata/compressed_size.fqtest
vendored
Normal file
@ -0,0 +1,14 @@
|
||||
# this tests compressed size (TryFieldReaderRangeFormat)
|
||||
$ fq -d gzip 'tobits | chunk(3) | gzip' test.gz
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: (gzip)
|
||||
0x00|1f 8b |.. | identification: raw bits (valid)
|
||||
0x00| 08 | . | compression_method: "deflate" (8)
|
||||
0x00| 00 | . | flags{}:
|
||||
0x00| 41 02 ea 5f | A.._ | mtime: 1609171521
|
||||
0x00| 00 | . | extra_flags: 0
|
||||
0x00| 03 | . | os: "Unix" (3)
|
||||
0x0|74 65 73 74 0a| |test.| | uncompressed: raw bits
|
||||
0x00| 2b 49 2d 2e e1 02| +I-...| compressed: raw bits
|
||||
0x10|00 |. |
|
||||
0x10| c6 35 b9 3b | .5.; | crc32: 0x3bb935c6 (valid)
|
||||
0x10| 05 00 00 00| | ....| | isize: 5
|
@ -11,6 +11,7 @@ import (
|
||||
|
||||
"github.com/wader/fq/format"
|
||||
"github.com/wader/fq/format/registry"
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
"github.com/wader/fq/pkg/decode"
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
"golang.org/x/text/encoding"
|
||||
@ -533,8 +534,8 @@ func decodeFrame(d *decode.D, version int) uint64 {
|
||||
if unsyncFlag {
|
||||
// TODO: DecodeFn
|
||||
// TODO: unknown after frame decode
|
||||
unsyncedBb := d.MustNewBitBufFromReader(unsyncReader{Reader: d.BitBufRange(d.Pos(), int64(dataSize)*8)})
|
||||
d.FieldFormatBitBuf("unsync", unsyncedBb, decode.FormatFn(func(d *decode.D, in interface{}) interface{} {
|
||||
unsyncedBR := d.MustNewBitBufFromReader(unsyncReader{Reader: bitio.NewIOReader(d.BitBufRange(d.Pos(), int64(dataSize)*8))})
|
||||
d.FieldFormatBitBuf("unsync", unsyncedBR, decode.FormatFn(func(d *decode.D, in interface{}) interface{} {
|
||||
if fn, ok := frames[idNormalized]; ok {
|
||||
fn(d)
|
||||
} else {
|
||||
|
@ -6,6 +6,7 @@ import (
|
||||
|
||||
"github.com/wader/fq/format"
|
||||
"github.com/wader/fq/format/registry"
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
"github.com/wader/fq/pkg/checksum"
|
||||
"github.com/wader/fq/pkg/decode"
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
@ -98,8 +99,8 @@ func decodeIPv4(d *decode.D, in interface{}) interface{} {
|
||||
headerEnd := d.Pos()
|
||||
|
||||
ipv4Checksum := &checksum.IPv4{}
|
||||
d.MustCopy(ipv4Checksum, d.BitBufRange(0, checksumStart))
|
||||
d.MustCopy(ipv4Checksum, d.BitBufRange(checksumEnd, headerEnd-checksumEnd))
|
||||
d.MustCopy(ipv4Checksum, bitio.NewIOReader(d.BitBufRange(0, checksumStart)))
|
||||
d.MustCopy(ipv4Checksum, bitio.NewIOReader(d.BitBufRange(checksumEnd, headerEnd-checksumEnd)))
|
||||
_ = d.FieldMustGet("header_checksum").TryScalarFn(d.ValidateUBytes(ipv4Checksum.Sum(nil)), scalar.Hex)
|
||||
|
||||
dataLen := int64(totalLength-(ihl*4)) * 8
|
||||
|
@ -306,10 +306,7 @@ func jpegDecode(d *decode.D, in interface{}) interface{} {
|
||||
// TODO: FieldBitsLen? concat bitbuf?
|
||||
chunk := d.FieldRawLen("data", d.BitsLeft())
|
||||
// TODO: redo this? multi reader?
|
||||
chunkBytes, err := chunk.Bytes()
|
||||
if err != nil {
|
||||
d.Fatalf("failed to read xmp chunk: %s", err)
|
||||
}
|
||||
chunkBytes := d.MustReadAllBits(chunk)
|
||||
|
||||
if extendedXMP == nil {
|
||||
extendedXMP = make([]byte, fullLength)
|
||||
@ -348,7 +345,7 @@ func jpegDecode(d *decode.D, in interface{}) interface{} {
|
||||
}
|
||||
|
||||
if extendedXMP != nil {
|
||||
d.FieldRootBitBuf("extended_xmp", bitio.NewBufferFromBytes(extendedXMP, -1))
|
||||
d.FieldRootBitBuf("extended_xmp", bitio.NewBitReader(extendedXMP, -1))
|
||||
}
|
||||
|
||||
return nil
|
||||
|
@ -5,6 +5,7 @@ import (
|
||||
|
||||
"github.com/wader/fq/format"
|
||||
"github.com/wader/fq/format/registry"
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
"github.com/wader/fq/pkg/decode"
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
@ -25,8 +26,8 @@ func init() {
|
||||
}
|
||||
|
||||
func decodeJSON(d *decode.D, in interface{}) interface{} {
|
||||
bb := d.RawLen(d.Len())
|
||||
jd := stdjson.NewDecoder(bb)
|
||||
br := d.RawLen(d.Len())
|
||||
jd := stdjson.NewDecoder(bitio.NewIOReader(br))
|
||||
var s scalar.S
|
||||
if err := jd.Decode(&s.Actual); err != nil {
|
||||
d.Fatalf(err.Error())
|
||||
|
@ -964,12 +964,9 @@ func init() {
|
||||
|
||||
version := d.FieldU8("version")
|
||||
d.FieldU24("flags")
|
||||
systemIDBB := d.FieldRawLen("system_id", 6*8, systemIDNames)
|
||||
systemIDBR := d.FieldRawLen("system_id", 6*8, systemIDNames)
|
||||
// TODO: make nicer
|
||||
systemID, err := systemIDBB.Bytes()
|
||||
if err != nil {
|
||||
d.IOPanic(err, "systemIDBB.Bytes")
|
||||
}
|
||||
systemID := d.MustReadAllBits(systemIDBR)
|
||||
switch version {
|
||||
case 0:
|
||||
case 1:
|
||||
|
@ -6,6 +6,7 @@ import (
|
||||
"github.com/wader/fq/format"
|
||||
"github.com/wader/fq/format/registry"
|
||||
"github.com/wader/fq/internal/mathextra"
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
"github.com/wader/fq/pkg/decode"
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
@ -101,7 +102,7 @@ func avcNALUDecode(d *decode.D, in interface{}) interface{} {
|
||||
d.FieldBool("forbidden_zero_bit")
|
||||
d.FieldU2("nal_ref_idc")
|
||||
nalType := d.FieldU5("nal_unit_type", avcNALNames)
|
||||
unescapedBb := d.MustNewBitBufFromReader(decode.NALUnescapeReader{Reader: d.BitBufRange(d.Pos(), d.BitsLeft())})
|
||||
unescapedBR := d.MustNewBitBufFromReader(decode.NALUnescapeReader{Reader: bitio.NewIOReader(d.BitBufRange(d.Pos(), d.BitsLeft()))})
|
||||
|
||||
switch nalType {
|
||||
case avcNALCodedSliceNonIDR,
|
||||
@ -118,11 +119,11 @@ func avcNALUDecode(d *decode.D, in interface{}) interface{} {
|
||||
// TODO: if ( separate_colour_plane_flag from SPS ) colour_plane_id; frame_num
|
||||
})
|
||||
case avcNALSupplementalEnhancementInformation:
|
||||
d.FieldFormatBitBuf("sei", unescapedBb, avcSEIFormat, nil)
|
||||
d.FieldFormatBitBuf("sei", unescapedBR, avcSEIFormat, nil)
|
||||
case avcNALSequenceParameterSet:
|
||||
d.FieldFormatBitBuf("sps", unescapedBb, avcSPSFormat, nil)
|
||||
d.FieldFormatBitBuf("sps", unescapedBR, avcSPSFormat, nil)
|
||||
case avcNALPictureParameterSet:
|
||||
d.FieldFormatBitBuf("pps", unescapedBb, avcPPSFormat, nil)
|
||||
d.FieldFormatBitBuf("pps", unescapedBR, avcPPSFormat, nil)
|
||||
}
|
||||
d.FieldRawLen("data", d.BitsLeft())
|
||||
|
||||
|
@ -3,6 +3,7 @@ package mpeg
|
||||
import (
|
||||
"github.com/wader/fq/format"
|
||||
"github.com/wader/fq/format/registry"
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
"github.com/wader/fq/pkg/decode"
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
@ -72,9 +73,9 @@ func hevcNALUDecode(d *decode.D, in interface{}) interface{} {
|
||||
nalType := d.FieldU6("nal_unit_type", hevcNALNames)
|
||||
d.FieldU6("nuh_layer_id")
|
||||
d.FieldU3("nuh_temporal_id_plus1")
|
||||
unescapedBb := d.MustNewBitBufFromReader(decode.NALUnescapeReader{Reader: d.BitBufRange(d.Pos(), d.BitsLeft())})
|
||||
unescapedBR := d.MustNewBitBufFromReader(decode.NALUnescapeReader{Reader: bitio.NewIOReader(d.BitBufRange(d.Pos(), d.BitsLeft()))})
|
||||
|
||||
_ = unescapedBb
|
||||
_ = unescapedBR
|
||||
_ = nalType
|
||||
|
||||
d.FieldRawLen("data", d.BitsLeft())
|
||||
|
@ -386,8 +386,8 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
|
||||
|
||||
crcHash := &checksum.CRC{Bits: 16, Current: 0xffff, Table: checksum.ANSI16Table}
|
||||
// 2 bytes after sync and some other fields + all of side info
|
||||
d.MustCopy(crcHash, d.BitBufRange(2*8, 2*8))
|
||||
d.MustCopy(crcHash, d.BitBufRange(6*8, sideInfoBytes*8))
|
||||
d.MustCopyBits(crcHash, d.BitBufRange(2*8, 2*8))
|
||||
d.MustCopyBits(crcHash, d.BitBufRange(6*8, sideInfoBytes*8))
|
||||
|
||||
if crcValue != nil {
|
||||
_ = crcValue.TryScalarFn(d.ValidateUBytes(crcHash.Sum(nil)))
|
||||
|
@ -61,17 +61,16 @@ func pesDecode(d *decode.D, in interface{}) interface{} {
|
||||
s = &subStream{}
|
||||
substreams[dvv.number] = s
|
||||
}
|
||||
b, _ := dvv.bb.BytesRange(0, int(dvv.bb.Len()/8))
|
||||
s.b = append(s.b, b...)
|
||||
s.b = append(s.b, dvv.buf...)
|
||||
|
||||
if s.l == 0 && len(b) >= 2 {
|
||||
s.l = int(b[0])<<8 | int(b[1])
|
||||
if s.l == 0 && len(s.b) >= 2 {
|
||||
s.l = int(s.b[0])<<8 | int(s.b[1])
|
||||
// TODO: zero l?
|
||||
}
|
||||
|
||||
// TODO: is this how spu end is signalled?
|
||||
if s.l == len(s.b) {
|
||||
spuD.FieldFormatBitBuf("spu", bitio.NewBufferFromBytes(s.b, -1), spuFormat, nil)
|
||||
spuD.FieldFormatBitBuf("spu", bitio.NewBitReader(s.b, -1), spuFormat, nil)
|
||||
s.b = nil
|
||||
s.l = 0
|
||||
}
|
||||
|
@ -6,7 +6,6 @@ package mpeg
|
||||
import (
|
||||
"github.com/wader/fq/format"
|
||||
"github.com/wader/fq/format/registry"
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
"github.com/wader/fq/pkg/decode"
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
@ -28,7 +27,7 @@ const (
|
||||
|
||||
type subStreamPacket struct {
|
||||
number int
|
||||
bb *bitio.Buffer
|
||||
buf []byte
|
||||
}
|
||||
|
||||
var startAndStreamNames = scalar.URangeToScalar{
|
||||
@ -174,11 +173,11 @@ func pesPacketDecode(d *decode.D, in interface{}) interface{} {
|
||||
d.FieldStruct("data", func(d *decode.D) {
|
||||
d.LenFn(dataLen, func(d *decode.D) {
|
||||
substreamNumber := d.FieldU8("substream")
|
||||
substreamBB := d.FieldRawLen("data", dataLen-8)
|
||||
substreamBR := d.FieldRawLen("data", dataLen-8)
|
||||
|
||||
v = subStreamPacket{
|
||||
number: int(substreamNumber),
|
||||
bb: substreamBB,
|
||||
buf: d.MustReadAllBits(substreamBR),
|
||||
}
|
||||
})
|
||||
})
|
||||
|
@ -105,14 +105,14 @@ func decodeOgg(d *decode.D, in interface{}) interface{} {
|
||||
for _, bs := range oggPageOut.Segments {
|
||||
s.packetBuf = append(s.packetBuf, bs...)
|
||||
if len(bs) < 255 {
|
||||
bb := bitio.NewBufferFromBytes(s.packetBuf, -1)
|
||||
br := bitio.NewBitReader(s.packetBuf, -1)
|
||||
|
||||
if s.codec == codecUnknown {
|
||||
if b, err := bb.PeekBytes(len(vorbisIdentification)); err == nil && bytes.Equal(b, vorbisIdentification) {
|
||||
if bytes.HasPrefix(s.packetBuf, vorbisIdentification) {
|
||||
s.codec = codecVorbis
|
||||
} else if b, err := bb.PeekBytes(len(opusIdentification)); err == nil && bytes.Equal(b, opusIdentification) {
|
||||
} else if bytes.HasPrefix(s.packetBuf, opusIdentification) {
|
||||
s.codec = codecOpus
|
||||
} else if b, err := bb.PeekBytes(len(flacIdentification)); err == nil && bytes.Equal(b, flacIdentification) {
|
||||
} else if bytes.HasPrefix(s.packetBuf, flacIdentification) {
|
||||
s.codec = codecFlac
|
||||
}
|
||||
}
|
||||
@ -120,25 +120,22 @@ func decodeOgg(d *decode.D, in interface{}) interface{} {
|
||||
switch s.codec {
|
||||
case codecVorbis:
|
||||
// TODO: err
|
||||
if _, _, err := s.packetD.TryFieldFormatBitBuf("packet", bb, vorbisPacketFormat, nil); err != nil {
|
||||
s.packetD.FieldRootBitBuf("packet", bb)
|
||||
if _, _, err := s.packetD.TryFieldFormatBitBuf("packet", br, vorbisPacketFormat, nil); err != nil {
|
||||
s.packetD.FieldRootBitBuf("packet", br)
|
||||
}
|
||||
case codecOpus:
|
||||
// TODO: err
|
||||
if _, _, err := s.packetD.TryFieldFormatBitBuf("packet", bb, opusPacketFormat, nil); err != nil {
|
||||
s.packetD.FieldRootBitBuf("packet", bb)
|
||||
if _, _, err := s.packetD.TryFieldFormatBitBuf("packet", br, opusPacketFormat, nil); err != nil {
|
||||
s.packetD.FieldRootBitBuf("packet", br)
|
||||
}
|
||||
case codecFlac:
|
||||
var firstByte byte
|
||||
bs, err := bb.PeekBytes(1)
|
||||
if err != nil {
|
||||
if len(s.packetBuf) == 0 {
|
||||
return
|
||||
}
|
||||
firstByte = bs[0]
|
||||
|
||||
switch {
|
||||
case firstByte == 0x7f:
|
||||
s.packetD.FieldStructRootBitBufFn("packet", bb, func(d *decode.D) {
|
||||
case s.packetBuf[0] == 0x7f:
|
||||
s.packetD.FieldStructRootBitBufFn("packet", br, func(d *decode.D) {
|
||||
d.FieldU8("type")
|
||||
d.FieldUTF8("signature", 4)
|
||||
d.FieldU8("major")
|
||||
@ -152,13 +149,13 @@ func decodeOgg(d *decode.D, in interface{}) interface{} {
|
||||
}
|
||||
s.flacStreamInfo = flacMetadatablockOut.StreamInfo
|
||||
})
|
||||
case firstByte == 0xff:
|
||||
s.packetD.FieldFormatBitBuf("packet", bb, flacFrameFormat, nil)
|
||||
case s.packetBuf[0] == 0xff:
|
||||
s.packetD.FieldFormatBitBuf("packet", br, flacFrameFormat, nil)
|
||||
default:
|
||||
s.packetD.FieldFormatBitBuf("packet", bb, flacMetadatablockFormat, nil)
|
||||
s.packetD.FieldFormatBitBuf("packet", br, flacMetadatablockFormat, nil)
|
||||
}
|
||||
case codecUnknown:
|
||||
s.packetD.FieldRootBitBuf("packet", bb)
|
||||
s.packetD.FieldRootBitBuf("packet", br)
|
||||
}
|
||||
|
||||
s.packetBuf = nil
|
||||
|
@ -5,6 +5,7 @@ import (
|
||||
|
||||
"github.com/wader/fq/format"
|
||||
"github.com/wader/fq/format/registry"
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
"github.com/wader/fq/pkg/checksum"
|
||||
"github.com/wader/fq/pkg/decode"
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
@ -43,10 +44,7 @@ func pageDecode(d *decode.D, in interface{}) interface{} {
|
||||
})
|
||||
d.FieldArray("segments", func(d *decode.D) {
|
||||
for _, ss := range segmentTable {
|
||||
bs, err := d.FieldRawLen("segment", int64(ss)*8).Bytes()
|
||||
if err != nil {
|
||||
d.IOPanic(err, "d.BitBufRange segment")
|
||||
}
|
||||
bs := d.MustReadAllBits(d.FieldRawLen("segment", int64(ss)*8))
|
||||
p.Segments = append(p.Segments, bs)
|
||||
}
|
||||
})
|
||||
@ -54,9 +52,9 @@ func pageDecode(d *decode.D, in interface{}) interface{} {
|
||||
|
||||
pageChecksumValue := d.FieldGet("crc")
|
||||
pageCRC := &checksum.CRC{Bits: 32, Table: checksum.Poly04c11db7Table}
|
||||
d.MustCopy(pageCRC, d.BitBufRange(startPos, pageChecksumValue.Range.Start-startPos)) // header before checksum
|
||||
d.MustCopy(pageCRC, bytes.NewReader([]byte{0, 0, 0, 0})) // zero checksum bits
|
||||
d.MustCopy(pageCRC, d.BitBufRange(pageChecksumValue.Range.Stop(), endPos-pageChecksumValue.Range.Stop())) // rest of page
|
||||
d.MustCopy(pageCRC, bitio.NewIOReader(d.BitBufRange(startPos, pageChecksumValue.Range.Start-startPos))) // header before checksum
|
||||
d.MustCopy(pageCRC, bytes.NewReader([]byte{0, 0, 0, 0})) // zero checksum bits
|
||||
d.MustCopy(pageCRC, bitio.NewIOReader(d.BitBufRange(pageChecksumValue.Range.Stop(), endPos-pageChecksumValue.Range.Stop()))) // rest of page
|
||||
_ = pageChecksumValue.TryScalarFn(d.ValidateUBytes(pageCRC.Sum(nil)))
|
||||
|
||||
return p
|
||||
|
@ -78,12 +78,7 @@ func decodePcap(d *decode.D, in interface{}) interface{} {
|
||||
d.Errorf("incl_len %d > orig_len %d", inclLen, origLen)
|
||||
}
|
||||
|
||||
bb := d.BitBufRange(d.Pos(), int64(inclLen)*8)
|
||||
bs, err := bb.Bytes()
|
||||
if err != nil {
|
||||
// TODO:
|
||||
d.IOPanic(err, "BitBufRange: inclLen")
|
||||
}
|
||||
bs := d.MustReadAllBits(d.BitBufRange(d.Pos(), int64(inclLen)*8))
|
||||
|
||||
if fn, ok := linkToDecodeFn[linkType]; ok {
|
||||
// TODO: report decode errors
|
||||
|
@ -230,10 +230,7 @@ var blockFns = map[uint64]func(d *decode.D, dc *decodeContext){
|
||||
capturedLength := d.FieldU32("capture_packet_length")
|
||||
d.FieldU32("original_packet_length")
|
||||
|
||||
bs, err := d.BitBufRange(d.Pos(), int64(capturedLength)*8).Bytes()
|
||||
if err != nil {
|
||||
d.IOPanic(err, "d.BitBufRange")
|
||||
}
|
||||
bs := d.MustReadAllBits(d.BitBufRange(d.Pos(), int64(capturedLength)*8))
|
||||
|
||||
linkType := dc.interfaceTypes[int(interfaceID)]
|
||||
|
||||
|
@ -36,14 +36,14 @@ var linkToDecodeFn = map[int]func(fd *flowsdecoder.Decoder, bs []byte) error{
|
||||
func fieldFlows(d *decode.D, fd *flowsdecoder.Decoder, tcpStreamFormat decode.Group, ipv4PacketFormat decode.Group) {
|
||||
d.FieldArray("ipv4_reassembled", func(d *decode.D) {
|
||||
for _, p := range fd.IPV4Reassembled {
|
||||
bb := bitio.NewBufferFromBytes(p.Datagram, -1)
|
||||
br := bitio.NewBitReader(p.Datagram, -1)
|
||||
if dv, _, _ := d.TryFieldFormatBitBuf(
|
||||
"ipv4_packet",
|
||||
bb,
|
||||
br,
|
||||
ipv4PacketFormat,
|
||||
nil,
|
||||
); dv == nil {
|
||||
d.FieldRootBitBuf("ipv4_packet", bb)
|
||||
d.FieldRootBitBuf("ipv4_packet", br)
|
||||
}
|
||||
}
|
||||
})
|
||||
@ -55,30 +55,30 @@ func fieldFlows(d *decode.D, fd *flowsdecoder.Decoder, tcpStreamFormat decode.Gr
|
||||
d.FieldValueU("source_port", uint64(s.ClientEndpoint.Port), format.TCPPortMap)
|
||||
d.FieldValueStr("destination_ip", s.ServerEndpoint.IP.String())
|
||||
d.FieldValueU("destination_port", uint64(s.ServerEndpoint.Port), format.TCPPortMap)
|
||||
csBB := bitio.NewBufferFromBytes(s.ClientToServer.Bytes(), -1)
|
||||
csBR := bitio.NewBitReader(s.ClientToServer.Bytes(), -1)
|
||||
if dv, _, _ := d.TryFieldFormatBitBuf(
|
||||
"client_stream",
|
||||
csBB,
|
||||
csBR,
|
||||
tcpStreamFormat,
|
||||
format.TCPStreamIn{
|
||||
SourcePort: s.ClientEndpoint.Port,
|
||||
DestinationPort: s.ServerEndpoint.Port,
|
||||
},
|
||||
); dv == nil {
|
||||
d.FieldRootBitBuf("client_stream", csBB)
|
||||
d.FieldRootBitBuf("client_stream", csBR)
|
||||
}
|
||||
|
||||
scBB := bitio.NewBufferFromBytes(s.ServerToClient.Bytes(), -1)
|
||||
scBR := bitio.NewBitReader(s.ServerToClient.Bytes(), -1)
|
||||
if dv, _, _ := d.TryFieldFormatBitBuf(
|
||||
"server_stream",
|
||||
scBB,
|
||||
scBR,
|
||||
tcpStreamFormat,
|
||||
format.TCPStreamIn{
|
||||
SourcePort: s.ClientEndpoint.Port,
|
||||
DestinationPort: s.ServerEndpoint.Port,
|
||||
},
|
||||
); dv == nil {
|
||||
d.FieldRootBitBuf("server_stream", scBB)
|
||||
d.FieldRootBitBuf("server_stream", scBR)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/wader/fq/format"
|
||||
"github.com/wader/fq/format/registry"
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
"github.com/wader/fq/pkg/decode"
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
@ -227,7 +228,7 @@ func pngDecode(d *decode.D, in interface{}) interface{} {
|
||||
})
|
||||
|
||||
chunkCRC := crc32.NewIEEE()
|
||||
d.MustCopy(chunkCRC, d.BitBufRange(crcStartPos, d.Pos()-crcStartPos))
|
||||
d.MustCopy(chunkCRC, bitio.NewIOReader(d.BitBufRange(crcStartPos, d.Pos()-crcStartPos)))
|
||||
d.FieldU32("crc", d.ValidateUBytes(chunkCRC.Sum(nil)), scalar.Hex)
|
||||
})
|
||||
|
||||
|
@ -42,14 +42,14 @@ func commentDecode(d *decode.D, in interface{}) interface{} {
|
||||
|
||||
base64Offset := int64(len(metadataBlockPicturePreix)) * 8
|
||||
base64Len := int64(len(userComment))*8 - base64Offset
|
||||
_, base64BB, dv, _, _ := d.TryFieldReaderRangeFormat(
|
||||
_, base64Br, dv, _, _ := d.TryFieldReaderRangeFormat(
|
||||
"picture",
|
||||
userCommentStart+base64Offset, base64Len,
|
||||
func(r io.Reader) io.Reader { return base64.NewDecoder(base64.StdEncoding, r) },
|
||||
flacPicture, nil,
|
||||
)
|
||||
if dv == nil && base64BB != nil {
|
||||
d.FieldRootBitBuf("picture", base64BB)
|
||||
if dv == nil && base64Br != nil {
|
||||
d.FieldRootBitBuf("picture", base64Br)
|
||||
}
|
||||
}
|
||||
i++
|
||||
|
@ -285,15 +285,15 @@ func zipDecode(d *decode.D, in interface{}) interface{} {
|
||||
var rFn func(r io.Reader) io.Reader
|
||||
switch compressionMethod {
|
||||
case compressionMethodDeflated:
|
||||
// *bitio.Buffer implements io.ByteReader so hat deflate don't do own
|
||||
// bitio.NewIOReadSeeker implements io.ByteReader so that deflate don't do own
|
||||
// buffering and might read more than needed messing up knowing compressed size
|
||||
rFn = func(r io.Reader) io.Reader { return flate.NewReader(r) }
|
||||
}
|
||||
|
||||
if rFn != nil {
|
||||
readCompressedSize, uncompressedBB, dv, _, _ := d.TryFieldReaderRangeFormat("uncompressed", d.Pos(), compressedLimit, rFn, probeFormat, nil)
|
||||
if dv == nil && uncompressedBB != nil {
|
||||
d.FieldRootBitBuf("uncompressed", uncompressedBB)
|
||||
readCompressedSize, uncompressedBR, dv, _, _ := d.TryFieldReaderRangeFormat("uncompressed", d.Pos(), compressedLimit, rFn, probeFormat, nil)
|
||||
if dv == nil && uncompressedBR != nil {
|
||||
d.FieldRootBitBuf("uncompressed", uncompressedBR)
|
||||
}
|
||||
if compressedSize == 0 {
|
||||
compressedSize = readCompressedSize
|
||||
|
56
internal/bitioextra/bitioextra.go
Normal file
56
internal/bitioextra/bitioextra.go
Normal file
@ -0,0 +1,56 @@
|
||||
package bitioextra
|
||||
|
||||
// bitio helpers that don't really belong in bitio
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
)
|
||||
|
||||
func CopyBitsBuffer(dst io.Writer, src bitio.Reader, buf []byte) (int64, error) {
|
||||
return io.CopyBuffer(dst, bitio.NewIOReader(src), buf)
|
||||
}
|
||||
|
||||
func CopyBits(dst io.Writer, src bitio.Reader) (int64, error) {
|
||||
return CopyBitsBuffer(dst, src, nil)
|
||||
}
|
||||
|
||||
func Clone(br bitio.ReaderAtSeeker) (bitio.ReaderAtSeeker, error) {
|
||||
l, err := Len(br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bitio.NewSectionReader(br, 0, l), nil
|
||||
}
|
||||
|
||||
func Len(br bitio.ReadAtSeeker) (int64, error) {
|
||||
bPos, err := br.SeekBits(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
bEnd, err := br.SeekBits(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if _, err := br.SeekBits(bPos, io.SeekStart); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return bEnd, nil
|
||||
}
|
||||
|
||||
func Range(br bitio.ReadAtSeeker, firstBitOffset int64, nBits int64) (bitio.ReaderAtSeeker, error) {
|
||||
l, err := Len(br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// TODO: move error check?
|
||||
if nBits < 0 {
|
||||
return nil, errors.New("negative nBits")
|
||||
}
|
||||
if firstBitOffset+nBits > l {
|
||||
return nil, errors.New("outside buffer")
|
||||
}
|
||||
return bitio.NewSectionReader(br, firstBitOffset, nBits), nil
|
||||
}
|
@ -25,7 +25,7 @@ type Dumper struct {
|
||||
hasWrittenHeader bool
|
||||
|
||||
bitsBuf []byte
|
||||
bitsBufN int
|
||||
bitsBufN int64
|
||||
}
|
||||
|
||||
// TODO: something more generic? bin, octal, arbitrary base?
|
||||
@ -72,11 +72,11 @@ func (d *Dumper) flush() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Dumper) WriteBits(p []byte, nBits int) (n int, err error) {
|
||||
pos := 0
|
||||
func (d *Dumper) WriteBits(p []byte, nBits int64) (n int64, err error) {
|
||||
pos := int64(0)
|
||||
rBits := nBits
|
||||
if d.bitsBufN > 0 {
|
||||
r := mathextra.MinInt(8-d.bitsBufN, nBits)
|
||||
r := mathextra.MinInt64(8-d.bitsBufN, nBits)
|
||||
v := bitio.Read64(p, 0, r)
|
||||
bitio.Write64(v, r, d.bitsBuf, d.bitsBufN)
|
||||
|
||||
@ -86,7 +86,7 @@ func (d *Dumper) WriteBits(p []byte, nBits int) (n int, err error) {
|
||||
return nBits, nil
|
||||
}
|
||||
if n, err := d.Write(d.bitsBuf); err != nil {
|
||||
return n * 8, err
|
||||
return int64(n) * 8, err
|
||||
}
|
||||
pos = r
|
||||
rBits -= r
|
||||
@ -97,7 +97,7 @@ func (d *Dumper) WriteBits(p []byte, nBits int) (n int, err error) {
|
||||
|
||||
b[0] = byte(bitio.Read64(p, pos, 8))
|
||||
if n, err := d.Write(b[:]); err != nil {
|
||||
return n * 8, err
|
||||
return int64(n) * 8, err
|
||||
}
|
||||
|
||||
pos += 8
|
||||
|
@ -4,9 +4,6 @@ package hexpairwriter
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/wader/fq/internal/mathextra"
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
)
|
||||
|
||||
type Writer struct {
|
||||
@ -17,9 +14,6 @@ type Writer struct {
|
||||
offset int
|
||||
buf []byte
|
||||
bufOffset int
|
||||
|
||||
bitsBuf []byte
|
||||
bitsBufN int
|
||||
}
|
||||
|
||||
// for i in $(seq 0 255) ; do printf "%02x" $i ; done | while read -n 32 s ; do echo "\"$s\""+; done
|
||||
@ -53,9 +47,7 @@ func New(w io.Writer, width int, startLineOffset int, fn func(b byte) string) *W
|
||||
fn: fn,
|
||||
offset: 0,
|
||||
// TODO: ansi length? nicer reusable buffer?
|
||||
buf: make([]byte, width*200+1), // worst case " " or "\n" + width*(XX "+ansi) + "\n"
|
||||
bufOffset: 0,
|
||||
bitsBuf: make([]byte, 1),
|
||||
buf: make([]byte, width*200+1), // worst case " " or "\n" + width*(XX "+ansi) + "\n"
|
||||
}
|
||||
}
|
||||
|
||||
@ -110,46 +102,3 @@ func (h *Writer) Write(p []byte) (n int, err error) {
|
||||
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (h *Writer) WriteBits(p []byte, nBits int) (n int, err error) {
|
||||
pos := 0
|
||||
rBits := nBits
|
||||
if h.bitsBufN > 0 {
|
||||
r := mathextra.MinInt(8-h.bitsBufN, nBits)
|
||||
v := bitio.Read64(p, 0, r)
|
||||
bitio.Write64(v, r, h.bitsBuf, h.bitsBufN)
|
||||
|
||||
h.bitsBufN += r
|
||||
|
||||
if h.bitsBufN < 8 {
|
||||
return nBits, nil
|
||||
}
|
||||
if n, err := h.Write(h.bitsBuf); err != nil {
|
||||
return n * 8, err
|
||||
}
|
||||
pos = r
|
||||
rBits -= r
|
||||
}
|
||||
|
||||
for rBits >= 8 {
|
||||
b := [1]byte{0}
|
||||
|
||||
b[0] = byte(bitio.Read64(p, pos, 8))
|
||||
if n, err := h.Write(b[:]); err != nil {
|
||||
return n * 8, err
|
||||
}
|
||||
|
||||
pos += 8
|
||||
rBits -= 8
|
||||
|
||||
}
|
||||
|
||||
if rBits > 0 {
|
||||
h.bitsBuf[0] = byte(bitio.Read64(p, pos, rBits)) << (8 - rBits)
|
||||
h.bitsBufN = rBits
|
||||
} else {
|
||||
h.bitsBufN = 0
|
||||
}
|
||||
|
||||
return nBits, nil
|
||||
}
|
||||
|
18
pkg/bitio/README.md
Normal file
18
pkg/bitio/README.md
Normal file
@ -0,0 +1,18 @@
|
||||
The bitio package tries to mimic the standard library packages io and bytes as much as possible.
|
||||
|
||||
- bitio.Buffer same as bytes.Buffer
|
||||
- bitio.IOBitReadSeeker is a bitio.ReaderAtSeeker that from a io.ReadSeeker
|
||||
- bitio.IOBitWriter a bitio.BitWriter that write bytes to a io.Writer, use Flush() to write possible unaligned byte
|
||||
- bitio.IOReader is a io.Reader that reads bytes from a bit reader, will zero pad on unaligned byte eof
|
||||
- bitio.IOReadSeeker is a io.ReadSeeker that read/seek bytes in a bit stream, will zero pad on unaligned - bitio.NewBitReader same as bytes.NewReader
|
||||
- bitio.LimitReader same as io.LimitReader
|
||||
- bitio.MultiReader same as io.MultiReader
|
||||
- bitio.SectionReader same as io.SectionReader
|
||||
- bitio.Copy* same as io.Copy*
|
||||
- bitio.ReadFull same as io.ReadFull
|
||||
|
||||
TODO:
|
||||
- bitio.IOBitReader bitio.Reader that reads from a io.Reader
|
||||
- bitio.IOBitWriteSeeker bitio.BitWriteSeeker that writes to a io.WriteSeeker
|
||||
- bitio.CopyN
|
||||
- speed up read by using a cache somehow ([]byte or just a uint64?)
|
@ -1,40 +1,107 @@
|
||||
package bitio
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var ErrOffset = errors.New("invalid seek offset")
|
||||
var ErrNegativeNBits = errors.New("negative number of bits")
|
||||
|
||||
type BitReaderAt interface {
|
||||
ReadBitsAt(p []byte, nBits int, bitOff int64) (n int, err error)
|
||||
type ReaderAt interface {
|
||||
ReadBitsAt(p []byte, nBits int64, bitOff int64) (n int64, err error)
|
||||
}
|
||||
|
||||
type BitReader interface {
|
||||
ReadBits(p []byte, nBits int) (n int, err error)
|
||||
type Reader interface {
|
||||
ReadBits(p []byte, nBits int64) (n int64, err error)
|
||||
}
|
||||
|
||||
type BitSeeker interface {
|
||||
type Seeker interface {
|
||||
SeekBits(bitOffset int64, whence int) (int64, error)
|
||||
}
|
||||
|
||||
type BitReadSeeker interface {
|
||||
BitReader
|
||||
BitSeeker
|
||||
type ReadSeeker interface {
|
||||
Reader
|
||||
Seeker
|
||||
}
|
||||
|
||||
type BitReadAtSeeker interface {
|
||||
BitReaderAt
|
||||
BitSeeker
|
||||
type ReadAtSeeker interface {
|
||||
ReaderAt
|
||||
Seeker
|
||||
}
|
||||
|
||||
type BitWriter interface {
|
||||
WriteBits(p []byte, nBits int) (n int, err error)
|
||||
type ReaderAtSeeker interface {
|
||||
Reader
|
||||
ReaderAt
|
||||
Seeker
|
||||
}
|
||||
|
||||
func CopyBuffer(dst BitWriter, src BitReader, buf []byte) (n int64, err error) {
|
||||
type Writer interface {
|
||||
WriteBits(p []byte, nBits int64) (n int64, err error)
|
||||
}
|
||||
|
||||
// NewBitReader reading nBits bits from a []byte
|
||||
// If nBits is -1 all bits will be used.
|
||||
// Similar to bytes.NewReader
|
||||
func NewBitReader(buf []byte, nBits int64) *SectionReader {
|
||||
if nBits < 0 {
|
||||
nBits = int64(len(buf)) * 8
|
||||
}
|
||||
return NewSectionReader(
|
||||
NewIOBitReadSeeker(bytes.NewReader(buf)),
|
||||
0,
|
||||
nBits,
|
||||
)
|
||||
}
|
||||
|
||||
// BitsByteCount returns smallest amount of bytes to fit nBits bits
|
||||
func BitsByteCount(nBits int64) int64 {
|
||||
n := nBits / 8
|
||||
if nBits%8 != 0 {
|
||||
n++
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// BytesFromBitString []byte from bit string, ex: "0101" -> ([]byte{0x50}, 4)
|
||||
func BytesFromBitString(s string) ([]byte, int64) {
|
||||
r := len(s) % 8
|
||||
bufLen := len(s) / 8
|
||||
if r > 0 {
|
||||
bufLen++
|
||||
}
|
||||
buf := make([]byte, bufLen)
|
||||
|
||||
for i := 0; i < len(s); i++ {
|
||||
d := s[i] - '0'
|
||||
if d != 0 && d != 1 {
|
||||
panic(fmt.Sprintf("invalid bit string %q at index %d %q", s, i, s[i]))
|
||||
}
|
||||
buf[i/8] |= d << (7 - i%8)
|
||||
}
|
||||
|
||||
return buf, int64(len(s))
|
||||
}
|
||||
|
||||
// BitStringFromBytes string from []byte], ex: ([]byte{0x50}, 4) -> "0101"
|
||||
func BitStringFromBytes(buf []byte, nBits int64) string {
|
||||
sb := &strings.Builder{}
|
||||
for i := int64(0); i < nBits; i++ {
|
||||
if buf[i/8]&(1<<(7-i%8)) > 0 {
|
||||
sb.WriteString("1")
|
||||
} else {
|
||||
sb.WriteString("0")
|
||||
}
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// CopyBuffer bits from src to dst using provided buffer
|
||||
// Similar to io.CopyBuffer
|
||||
func CopyBuffer(dst Writer, src Reader, buf []byte) (n int64, err error) {
|
||||
// same default size as io.Copy
|
||||
if buf == nil {
|
||||
buf = make([]byte, 32*1024)
|
||||
@ -42,10 +109,10 @@ func CopyBuffer(dst BitWriter, src BitReader, buf []byte) (n int64, err error) {
|
||||
var written int64
|
||||
|
||||
for {
|
||||
rBits, rErr := src.ReadBits(buf, len(buf)*8)
|
||||
rBits, rErr := src.ReadBits(buf, int64(len(buf))*8)
|
||||
if rBits > 0 {
|
||||
wBits, wErr := dst.WriteBits(buf, rBits)
|
||||
written += int64(wBits)
|
||||
written += wBits
|
||||
if wErr != nil {
|
||||
err = wErr
|
||||
break
|
||||
@ -66,25 +133,41 @@ func CopyBuffer(dst BitWriter, src BitReader, buf []byte) (n int64, err error) {
|
||||
return written, err
|
||||
}
|
||||
|
||||
func Copy(dst BitWriter, src BitReader) (n int64, err error) {
|
||||
// Copy bits from src to dst
|
||||
// Similar to io.Copy
|
||||
func Copy(dst Writer, src Reader) (n int64, err error) {
|
||||
return CopyBuffer(dst, src, nil)
|
||||
}
|
||||
|
||||
// BitsByteCount returns smallest amount of bytes to fit nBits bits
|
||||
func BitsByteCount(nBits int64) int64 {
|
||||
n := nBits / 8
|
||||
if nBits%8 != 0 {
|
||||
n++
|
||||
// TODO: make faster, align and use copy()
|
||||
func copyBits(dst []byte, dstStart int64, src []byte, srcStart int64, n int64, zero bool) {
|
||||
l := n
|
||||
off := int64(0)
|
||||
for l > 0 {
|
||||
c := int64(64)
|
||||
if l < c {
|
||||
c = l
|
||||
}
|
||||
u := Read64(src, srcStart+off, c)
|
||||
Write64(u, c, dst, dstStart+off)
|
||||
off += c
|
||||
l -= c
|
||||
}
|
||||
|
||||
// zero fill last bits if not aligned
|
||||
e := dstStart + n
|
||||
if zero && e%8 != 0 {
|
||||
Write64(0, 8-(e%8), dst, e)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func readFull(p []byte, nBits int, bitOff int64, fn func(p []byte, nBits int, bitOff int64) (int, error)) (int, error) {
|
||||
// TODO: redo?
|
||||
func readFull(p []byte, nBits int64, bitOff int64, fn func(p []byte, nBits int64, bitOff int64) (int64, error)) (int64, error) {
|
||||
if nBits < 0 {
|
||||
return 0, ErrNegativeNBits
|
||||
}
|
||||
|
||||
readBitOffset := 0
|
||||
readBitOffset := int64(0)
|
||||
for readBitOffset < nBits {
|
||||
byteOffset := readBitOffset / 8
|
||||
byteBitsOffset := readBitOffset % 8
|
||||
@ -98,7 +181,7 @@ func readFull(p []byte, nBits int, bitOff int64, fn func(p []byte, nBits int, bi
|
||||
}
|
||||
|
||||
var pb [1]byte
|
||||
rBits, err := fn(pb[:], readBits, bitOff+int64(readBitOffset))
|
||||
rBits, err := fn(pb[:], readBits, bitOff+readBitOffset)
|
||||
Write64(uint64(pb[0]>>(8-rBits)), rBits, p, readBitOffset)
|
||||
readBitOffset += rBits
|
||||
|
||||
@ -109,7 +192,7 @@ func readFull(p []byte, nBits int, bitOff int64, fn func(p []byte, nBits int, bi
|
||||
continue
|
||||
}
|
||||
|
||||
rBits, err := fn(p[byteOffset:], nBits-readBitOffset, bitOff+int64(readBitOffset))
|
||||
rBits, err := fn(p[byteOffset:], nBits-readBitOffset, bitOff+readBitOffset)
|
||||
|
||||
readBitOffset += rBits
|
||||
if err != nil {
|
||||
@ -120,31 +203,14 @@ func readFull(p []byte, nBits int, bitOff int64, fn func(p []byte, nBits int, bi
|
||||
return nBits, nil
|
||||
}
|
||||
|
||||
func ReadAtFull(r BitReaderAt, p []byte, nBits int, bitOff int64) (int, error) {
|
||||
return readFull(p, nBits, bitOff, func(p []byte, nBits int, bitOff int64) (int, error) {
|
||||
func ReadAtFull(r ReaderAt, p []byte, nBits int64, bitOff int64) (int64, error) {
|
||||
return readFull(p, nBits, bitOff, func(p []byte, nBits int64, bitOff int64) (int64, error) {
|
||||
return r.ReadBitsAt(p, nBits, bitOff)
|
||||
})
|
||||
}
|
||||
|
||||
func ReadFull(r BitReader, p []byte, nBits int) (int, error) {
|
||||
return readFull(p, nBits, 0, func(p []byte, nBits int, bitOff int64) (int, error) {
|
||||
func ReadFull(r Reader, p []byte, nBits int64) (int64, error) {
|
||||
return readFull(p, nBits, 0, func(p []byte, nBits int64, bitOff int64) (int64, error) {
|
||||
return r.ReadBits(p, nBits)
|
||||
})
|
||||
}
|
||||
|
||||
// TODO: move?
|
||||
func EndPos(rs BitSeeker) (int64, error) {
|
||||
c, err := rs.SeekBits(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
e, err := rs.SeekBits(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
_, err = rs.SeekBits(c, io.SeekStart)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
@ -1,163 +0,0 @@
|
||||
package bitio_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
)
|
||||
|
||||
type shortBitReader struct {
|
||||
bitio.BitReaderAt
|
||||
}
|
||||
|
||||
func (b shortBitReader) ReadBitsAt(p []byte, nBits int, bitOff int64) (n int, err error) {
|
||||
if nBits > 3 {
|
||||
nBits = 3
|
||||
}
|
||||
return b.BitReaderAt.ReadBitsAt(p, nBits, bitOff)
|
||||
}
|
||||
|
||||
func TestReader(t *testing.T) {
|
||||
|
||||
bb, bbBits := bitio.BytesFromBitString("011011110")
|
||||
|
||||
br := bitio.NewReaderFromReadSeeker(bytes.NewReader(bb))
|
||||
sbr := bitio.NewSectionBitReader(br, 0, int64(bbBits))
|
||||
sbr2 := bitio.NewSectionBitReader(sbr, 0, 4)
|
||||
|
||||
ob := make([]byte, 2)
|
||||
|
||||
obBits, _ := sbr2.ReadBits(ob, 4)
|
||||
|
||||
obs := bitio.BitStringFromBytes(ob, obBits)
|
||||
|
||||
log.Printf("obs: %#+v\n", obs)
|
||||
}
|
||||
|
||||
func TestIOCopy(t *testing.T) {
|
||||
|
||||
br := bitio.NewReaderFromReadSeeker(bytes.NewReader([]byte{0xf0, 0xff, 0xff}))
|
||||
sbr := bitio.NewSectionBitReader(br, 0, 8*3-1)
|
||||
|
||||
b := &bytes.Buffer{}
|
||||
|
||||
n, err := io.Copy(b, sbr)
|
||||
log.Printf("n: %d err: %v b=%v\n", n, err, b.Bytes())
|
||||
|
||||
}
|
||||
|
||||
func TestReadFull(t *testing.T) {
|
||||
|
||||
bb, bbBits := bitio.BytesFromBitString("011011110")
|
||||
|
||||
br := bitio.NewReaderFromReadSeeker(bytes.NewReader(bb))
|
||||
|
||||
ob := make([]byte, 2)
|
||||
|
||||
obBits, _ := bitio.ReadAtFull(shortBitReader{br}, ob, bbBits, 0)
|
||||
|
||||
obs := bitio.BitStringFromBytes(ob, obBits)
|
||||
|
||||
log.Printf("obs: %#+v\n", obs)
|
||||
|
||||
}
|
||||
|
||||
func TestMultiBitReader(t *testing.T) {
|
||||
|
||||
bb1, bb1Bits := bitio.BytesFromBitString("101")
|
||||
br1 := bitio.NewSectionBitReader(bitio.NewReaderFromReadSeeker(bytes.NewReader(bb1)), 0, int64(bb1Bits))
|
||||
bb2, bb2Bits := bitio.BytesFromBitString("0001")
|
||||
br2 := bitio.NewSectionBitReader(bitio.NewReaderFromReadSeeker(bytes.NewReader(bb2)), 0, int64(bb2Bits))
|
||||
|
||||
mb, _ := bitio.NewMultiBitReader([]bitio.BitReadAtSeeker{br1, br2})
|
||||
|
||||
ob := make([]byte, 2)
|
||||
|
||||
obBits, _ := bitio.ReadAtFull(mb, ob, 7, 0)
|
||||
|
||||
obs := bitio.BitStringFromBytes(ob, obBits)
|
||||
|
||||
log.Printf("obs: %#+v\n", obs)
|
||||
|
||||
}
|
||||
|
||||
func TestMultiBitReader11(t *testing.T) {
|
||||
|
||||
bb1, bb1Bits := bitio.BytesFromBitString("11111111")
|
||||
br1 := bitio.NewSectionBitReader(bitio.NewReaderFromReadSeeker(bytes.NewReader(bb1)), 0, int64(bb1Bits))
|
||||
bb2, bb2Bits := bitio.BytesFromBitString("11111111")
|
||||
br2 := bitio.NewSectionBitReader(bitio.NewReaderFromReadSeeker(bytes.NewReader(bb2)), 0, int64(bb2Bits))
|
||||
|
||||
mb, _ := bitio.NewMultiBitReader([]bitio.BitReadAtSeeker{br1, br2})
|
||||
|
||||
ob := make([]byte, 2)
|
||||
|
||||
obBits, _ := bitio.ReadAtFull(mb, ob, 11, 0)
|
||||
|
||||
obs := bitio.BitStringFromBytes(ob, obBits)
|
||||
|
||||
log.Printf("obs: %#+v\n", obs)
|
||||
|
||||
}
|
||||
|
||||
type testBW struct{}
|
||||
|
||||
func (testBW) WriteBits(p []byte, nBits int) (n int, err error) {
|
||||
log.Printf("WriteBits p: %#+v nBits=%d\n", len(p), nBits)
|
||||
|
||||
for i := 0; i < nBits; i++ {
|
||||
if bitio.Read64(p, i, 1) != 0 {
|
||||
log.Print("1")
|
||||
} else {
|
||||
log.Print("0")
|
||||
}
|
||||
log.Println()
|
||||
}
|
||||
|
||||
return nBits, nil
|
||||
}
|
||||
|
||||
func TestCopy(t *testing.T) {
|
||||
bb1, bb1Bits := bitio.BytesFromBitString("101")
|
||||
br1 := bitio.NewSectionBitReader(bitio.NewReaderFromReadSeeker(bytes.NewReader(bb1)), 0, int64(bb1Bits))
|
||||
bb2, bb2Bits := bitio.BytesFromBitString("0001")
|
||||
br2 := bitio.NewSectionBitReader(bitio.NewReaderFromReadSeeker(bytes.NewReader(bb2)), 0, int64(bb2Bits))
|
||||
|
||||
mb, _ := bitio.NewMultiBitReader([]bitio.BitReadAtSeeker{br1, br2})
|
||||
|
||||
n, err := bitio.Copy(testBW{}, mb)
|
||||
log.Printf("n: %#+v\n", n)
|
||||
log.Printf("err: %#+v\n", err)
|
||||
}
|
||||
|
||||
func TestAlignBitWriter(t *testing.T) {
|
||||
bb1, bb1Bits := bitio.BytesFromBitString("101")
|
||||
br1 := bitio.NewSectionBitReader(bitio.NewReaderFromReadSeeker(bytes.NewReader(bb1)), 0, int64(bb1Bits))
|
||||
bb2, bb2Bits := bitio.BytesFromBitString("0001")
|
||||
br2 := bitio.NewSectionBitReader(bitio.NewReaderFromReadSeeker(bytes.NewReader(bb2)), 0, int64(bb2Bits))
|
||||
|
||||
mb, _ := bitio.NewMultiBitReader([]bitio.BitReadAtSeeker{br1, br2})
|
||||
|
||||
mbEnd, _ := bitio.EndPos(mb)
|
||||
|
||||
alignN := int64(8)
|
||||
|
||||
b := make([]byte, alignN/8+1)
|
||||
bLeft := (alignN - mbEnd%alignN) % alignN
|
||||
|
||||
log.Printf("bLeft: %#+v\n", bLeft)
|
||||
|
||||
mb, _ = bitio.NewMultiBitReader([]bitio.BitReadAtSeeker{mb, bitio.NewBufferFromBytes(b, bLeft)})
|
||||
|
||||
bw := testBW{}
|
||||
// ab := &bitio.AlignBitWriter{N: 8, W: bw}
|
||||
|
||||
n, err := bitio.Copy(bw, mb)
|
||||
// ab.Close()
|
||||
|
||||
log.Printf("n: %#+v\n", n)
|
||||
log.Printf("err: %#+v\n", err)
|
||||
}
|
@ -1,39 +0,0 @@
|
||||
package bitio
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// BytesFromBitString []byte from bit string, ex: "0101" -> ([]byte{0x50}, 4)
|
||||
func BytesFromBitString(s string) ([]byte, int) {
|
||||
r := len(s) % 8
|
||||
bufLen := len(s) / 8
|
||||
if r > 0 {
|
||||
bufLen++
|
||||
}
|
||||
buf := make([]byte, bufLen)
|
||||
|
||||
for i := 0; i < len(s); i++ {
|
||||
d := s[i] - '0'
|
||||
if d != 0 && d != 1 {
|
||||
panic(fmt.Sprintf("invalid bit string %q at index %d %q", s, i, s[i]))
|
||||
}
|
||||
buf[i/8] |= d << (7 - i%8)
|
||||
}
|
||||
|
||||
return buf, len(s)
|
||||
}
|
||||
|
||||
// BitStringFromBytes string from []byte], ex: ([]byte{0x50}, 4) -> "0101"
|
||||
func BitStringFromBytes(buf []byte, nBits int) string {
|
||||
sb := &strings.Builder{}
|
||||
for i := 0; i < nBits; i++ {
|
||||
if buf[i/8]&(1<<(7-i%8)) > 0 {
|
||||
sb.WriteString("1")
|
||||
} else {
|
||||
sb.WriteString("0")
|
||||
}
|
||||
}
|
||||
return sb.String()
|
||||
}
|
@ -1,36 +0,0 @@
|
||||
package bitio_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
)
|
||||
|
||||
func TestBitString(t *testing.T) {
|
||||
testCases := []string{
|
||||
"",
|
||||
"1",
|
||||
"0",
|
||||
"10",
|
||||
"01",
|
||||
"11",
|
||||
"00",
|
||||
"1000001",
|
||||
"0000000",
|
||||
"10000001",
|
||||
"00000000",
|
||||
"100000001",
|
||||
"000000000",
|
||||
"101010101",
|
||||
"111100000",
|
||||
}
|
||||
for _, tC := range testCases {
|
||||
t.Run(tC, func(t *testing.T) {
|
||||
bb, bbBits := bitio.BytesFromBitString(tC)
|
||||
actual := bitio.BitStringFromBytes(bb, bbBits)
|
||||
if tC != actual {
|
||||
t.Errorf("expected %s, got %s", tC, actual)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
@ -1,280 +1,71 @@
|
||||
package bitio
|
||||
|
||||
// TODO: NewBuffer with []byte arg to save on alloc
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type br interface {
|
||||
io.Reader // both Reader and SectionBitReader implement io.Reader
|
||||
io.Seeker
|
||||
BitReadSeeker
|
||||
BitReader
|
||||
BitReaderAt
|
||||
}
|
||||
|
||||
// Buffer is a bit buffer
|
||||
// Buffer is a bitio.Reader and bitio.Writer providing a bit buffer
|
||||
// Similar to bytes.Buffer
|
||||
type Buffer struct {
|
||||
// embed to forward to underlaying reader
|
||||
br
|
||||
|
||||
bitLen int64 // mostly to cache len
|
||||
buf []byte
|
||||
bufBits int64
|
||||
bitsOff int64
|
||||
}
|
||||
|
||||
// NewBufferFromReadSeeker new Buffer from io.ReadSeeker, start at firstBit with bit length lenBits
|
||||
// buf is not copied.
|
||||
func NewBufferFromReadSeeker(rs io.ReadSeeker) (*Buffer, error) {
|
||||
bPos, err := rs.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bEnd, err := rs.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := rs.Seek(bPos, io.SeekStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
func (b *Buffer) Len() int64 { return b.bufBits - b.bitsOff }
|
||||
|
||||
return &Buffer{
|
||||
br: NewReaderFromReadSeeker(rs),
|
||||
bitLen: bEnd * 8,
|
||||
}, nil
|
||||
func (b *Buffer) Reset() {
|
||||
b.bufBits = 0
|
||||
b.bitsOff = 0
|
||||
}
|
||||
|
||||
func NewBufferFromBitReadSeeker(br br) (*Buffer, error) {
|
||||
bPos, err := br.SeekBits(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bEnd, err := br.SeekBits(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := br.SeekBits(bPos, io.SeekStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Buffer{
|
||||
br: br,
|
||||
bitLen: bEnd,
|
||||
}, nil
|
||||
func (b *Buffer) Bytes() ([]byte, int64) {
|
||||
l := b.Len()
|
||||
buf := make([]byte, BitsByteCount(l))
|
||||
copyBits(buf, 0, b.buf, b.bitsOff, l, true)
|
||||
return buf, b.bufBits
|
||||
}
|
||||
|
||||
// NewBufferFromBytes new Buffer from bytes
|
||||
// if nBits is < 0 nBits is all bits in buf
|
||||
func NewBufferFromBytes(buf []byte, nBits int64) *Buffer {
|
||||
if nBits < 0 {
|
||||
nBits = int64(len(buf)) * 8
|
||||
}
|
||||
return &Buffer{
|
||||
br: NewSectionBitReader(NewReaderFromReadSeeker(bytes.NewReader(buf)), 0, nBits),
|
||||
bitLen: nBits,
|
||||
}
|
||||
}
|
||||
func (b *Buffer) WriteBits(p []byte, nBits int64) (n int64, err error) {
|
||||
tBytes := BitsByteCount(b.bufBits + nBits)
|
||||
|
||||
// NewBufferFromBitString new Buffer from bit string, ex: "0101"
|
||||
func NewBufferFromBitString(s string) *Buffer {
|
||||
b, bBits := BytesFromBitString(s)
|
||||
return NewBufferFromBytes(b, int64(bBits))
|
||||
}
|
||||
|
||||
// BitBufRange reads nBits bits starting from start
|
||||
// Does not update current position.
|
||||
// if nBits is < 0 nBits is all bits after firstBitOffset
|
||||
func (b *Buffer) BitBufRange(firstBitOffset int64, nBits int64) (*Buffer, error) {
|
||||
// TODO: move error check?
|
||||
if nBits < 0 {
|
||||
return nil, errors.New("negative nBits")
|
||||
}
|
||||
if firstBitOffset+nBits > b.bitLen {
|
||||
return nil, errors.New("outside buffer")
|
||||
}
|
||||
return &Buffer{
|
||||
br: NewSectionBitReader(b.br, firstBitOffset, nBits),
|
||||
bitLen: nBits,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Clone buffer and reset position to zero
|
||||
func (b *Buffer) Clone() *Buffer {
|
||||
return &Buffer{
|
||||
br: NewSectionBitReader(b.br, 0, b.bitLen),
|
||||
bitLen: b.bitLen,
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Buffer) Pos() (int64, error) {
|
||||
bPos, err := b.br.SeekBits(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return bPos, nil
|
||||
}
|
||||
|
||||
func (b *Buffer) Len() int64 {
|
||||
return b.bitLen
|
||||
}
|
||||
|
||||
// BitBufLen reads nBits
|
||||
func (b *Buffer) BitBufLen(nBits int64) (*Buffer, error) {
|
||||
bPos, err := b.Pos()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bb, err := b.BitBufRange(bPos, nBits)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := b.br.SeekBits(nBits, io.SeekCurrent); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return bb, nil
|
||||
}
|
||||
|
||||
// PeekBytes peek nBytes bytes from buffer
|
||||
func (b *Buffer) PeekBytes(nBytes int) ([]byte, error) {
|
||||
start, err := b.br.SeekBits(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bs, err := b.BytesLen(nBytes)
|
||||
if _, err := b.br.SeekBits(start, io.SeekStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bs, err
|
||||
}
|
||||
|
||||
// required by some decompressors (like deflate) to not do own buffering
|
||||
func (b *Buffer) ReadByte() (byte, error) {
|
||||
var rb [1]byte
|
||||
_, err := b.br.Read(rb[:])
|
||||
return rb[0], err
|
||||
}
|
||||
|
||||
// BytesRange reads nBytes bytes starting bit position start
|
||||
// Does not update current position.
|
||||
// TODO: swap args
|
||||
// TODO: nBytes -1?
|
||||
func (b *Buffer) BytesRange(bitOffset int64, nBytes int) ([]byte, error) {
|
||||
buf := make([]byte, nBytes)
|
||||
n, err := ReadAtFull(b.br, buf, nBytes*8, bitOffset)
|
||||
if n == nBytes*8 {
|
||||
err = nil
|
||||
}
|
||||
return buf, err
|
||||
}
|
||||
|
||||
// BytesLen reads nBytes bytes
|
||||
func (b *Buffer) BytesLen(nBytes int) ([]byte, error) {
|
||||
buf := make([]byte, nBytes)
|
||||
_, err := io.ReadAtLeast(b, buf, nBytes)
|
||||
return buf, err
|
||||
}
|
||||
|
||||
// Bytes all bytes
|
||||
func (b *Buffer) Bytes() ([]byte, error) {
|
||||
nBytes := int(b.bitLen) / 8
|
||||
if b.bitLen%8 != 0 {
|
||||
nBytes++
|
||||
}
|
||||
buf := make([]byte, nBytes)
|
||||
_, err := ReadAtFull(b, buf, int(b.bitLen), 0)
|
||||
return buf, err
|
||||
}
|
||||
|
||||
// End is true if current position is at the end
|
||||
func (b *Buffer) End() (bool, error) {
|
||||
bPos, err := b.Pos()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return bPos >= b.bitLen, nil
|
||||
}
|
||||
|
||||
// BitsLeft number of bits left until end
|
||||
func (b *Buffer) BitsLeft() (int64, error) {
|
||||
bPos, err := b.Pos()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return b.bitLen - bPos, nil
|
||||
}
|
||||
|
||||
// AlignBits number of bits to next nBits align
|
||||
func (b *Buffer) AlignBits(nBits int) (int, error) {
|
||||
bPos, err := b.Pos()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int((int64(nBits) - (bPos % int64(nBits))) % int64(nBits)), nil
|
||||
}
|
||||
|
||||
// ByteAlignBits number of bits to next byte align
|
||||
func (b *Buffer) ByteAlignBits() (int, error) {
|
||||
return b.AlignBits(8)
|
||||
}
|
||||
|
||||
// BytePos byte position of current bit position
|
||||
func (b *Buffer) BytePos() (int64, error) {
|
||||
bPos, err := b.Pos()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return bPos & 0x7, nil
|
||||
}
|
||||
|
||||
// SeekRel seeks relative to current bit position
|
||||
// TODO: better name?
|
||||
func (b *Buffer) SeekRel(delta int64) (int64, error) {
|
||||
return b.br.SeekBits(delta, io.SeekCurrent)
|
||||
}
|
||||
|
||||
// SeekAbs seeks to absolute position
|
||||
func (b *Buffer) SeekAbs(pos int64) (int64, error) {
|
||||
return b.br.SeekBits(pos, io.SeekStart)
|
||||
}
|
||||
|
||||
func (b *Buffer) String() string {
|
||||
truncLen := b.bitLen
|
||||
truncS := ""
|
||||
if truncLen > 64 {
|
||||
truncLen, truncS = 64, "..."
|
||||
}
|
||||
truncBB, err := b.BitBufRange(0, truncLen)
|
||||
if err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
bitString, err := truncBB.BitString()
|
||||
if err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
return fmt.Sprintf("0b%s%s /* %d bits */", bitString, truncS, b.bitLen)
|
||||
}
|
||||
|
||||
// BitString return bit string representation
|
||||
func (b *Buffer) BitString() (string, error) {
|
||||
var ss []string
|
||||
for {
|
||||
var buf [1]byte
|
||||
_, err := ReadFull(b, buf[:], 1)
|
||||
if err != nil {
|
||||
if !errors.Is(err, io.EOF) {
|
||||
return "", err
|
||||
}
|
||||
break
|
||||
}
|
||||
if buf[0] != 0b1000_0000 {
|
||||
ss = append(ss, "0")
|
||||
if tBytes > int64(len(b.buf)) {
|
||||
if tBytes <= int64(cap(b.buf)) {
|
||||
b.buf = b.buf[:tBytes]
|
||||
} else {
|
||||
ss = append(ss, "1")
|
||||
buf := make([]byte, tBytes, tBytes*2)
|
||||
copy(buf, b.buf)
|
||||
b.buf = buf
|
||||
}
|
||||
}
|
||||
|
||||
return strings.Join(ss, ""), nil
|
||||
copyBits(b.buf, b.bufBits, p, 0, nBits, true)
|
||||
b.bufBits += nBits
|
||||
|
||||
return nBits, nil
|
||||
}
|
||||
|
||||
func (b *Buffer) empty() bool { return b.bufBits <= b.bitsOff }
|
||||
|
||||
func (b *Buffer) ReadBits(p []byte, nBits int64) (n int64, err error) {
|
||||
if b.empty() {
|
||||
b.Reset()
|
||||
if nBits == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
c := nBits
|
||||
left := b.bufBits - b.bitsOff
|
||||
if c > left {
|
||||
c = left
|
||||
}
|
||||
|
||||
copyBits(p, 0, b.buf, b.bitsOff, c, true)
|
||||
b.bitsOff += c
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
@ -1,96 +0,0 @@
|
||||
package bitio_test
|
||||
|
||||
// TODO: unbreak, check err
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
)
|
||||
|
||||
func TestBufferBitString(t *testing.T) {
|
||||
testCases := []string{
|
||||
"",
|
||||
"1",
|
||||
"0",
|
||||
"10",
|
||||
"01",
|
||||
"11",
|
||||
"00",
|
||||
"1000001",
|
||||
"0000000",
|
||||
"10000001",
|
||||
"00000000",
|
||||
"100000001",
|
||||
"000000000",
|
||||
"101010101",
|
||||
"111100000",
|
||||
}
|
||||
for _, tC := range testCases {
|
||||
t.Run(tC, func(t *testing.T) {
|
||||
bb := bitio.NewBufferFromBitString(tC)
|
||||
actual, err := bb.BitString()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if tC != actual {
|
||||
t.Errorf("expected %s, got %s", tC, actual)
|
||||
}
|
||||
|
||||
for i := int64(0); i < bb.Len(); i++ {
|
||||
t.Run(fmt.Sprintf("%s_%d", tC, i), func(t *testing.T) {
|
||||
startBb, _ := bb.BitBufRange(i, bb.Len()-i)
|
||||
startExpected := tC[i : i+bb.Len()-i]
|
||||
startActual, _ := startBb.BitString()
|
||||
if startExpected != startActual {
|
||||
t.Errorf("startBb expected %s, got %s", startExpected, startActual)
|
||||
}
|
||||
|
||||
lenBb, _ := bb.BitBufRange(0, i)
|
||||
lenExpected := tC[0:i]
|
||||
lenActual, _ := lenBb.BitString()
|
||||
if lenExpected != lenActual {
|
||||
t.Errorf("lenBb expected %s, got %s", lenExpected, lenActual)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitStringRandom(t *testing.T) {
|
||||
r := rand.New(rand.NewSource(0)) //nolint:gosec
|
||||
|
||||
for i := 0; i < 10000; i++ {
|
||||
var ss []string
|
||||
for j := uint32(0); j < r.Uint32()%1000; j++ {
|
||||
switch r.Uint32() % 2 {
|
||||
case 0:
|
||||
ss = append(ss, "0")
|
||||
case 1:
|
||||
ss = append(ss, "1")
|
||||
}
|
||||
}
|
||||
expected := strings.Join(ss, "")
|
||||
bb := bitio.NewBufferFromBitString(expected)
|
||||
actual, _ := bb.BitString()
|
||||
if expected != actual {
|
||||
t.Errorf("expected %s, got %s", expected, actual)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidBitString(t *testing.T) {
|
||||
// TODO: check panic?
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
// nop
|
||||
} else {
|
||||
t.Error("did not panic")
|
||||
}
|
||||
}()
|
||||
bitio.NewBufferFromBitString("01invalid")
|
||||
}
|
85
pkg/bitio/iobitreadseeker.go
Normal file
85
pkg/bitio/iobitreadseeker.go
Normal file
@ -0,0 +1,85 @@
|
||||
package bitio
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
// IOBitReadSeeker is a bitio.BitReadAtSeeker reading from a io.ReadSeeker
|
||||
type IOBitReadSeeker struct {
|
||||
bitPos int64
|
||||
rs io.ReadSeeker
|
||||
buf []byte
|
||||
}
|
||||
|
||||
func NewIOBitReadSeeker(rs io.ReadSeeker) *IOBitReadSeeker {
|
||||
return &IOBitReadSeeker{
|
||||
bitPos: 0,
|
||||
rs: rs,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *IOBitReadSeeker) ReadBitsAt(p []byte, nBits int64, bitOffset int64) (int64, error) {
|
||||
if nBits < 0 {
|
||||
return 0, ErrNegativeNBits
|
||||
}
|
||||
|
||||
readBytePos := bitOffset / 8
|
||||
readSkipBits := bitOffset % 8
|
||||
wantReadBits := readSkipBits + nBits
|
||||
wantReadBytes := int(BitsByteCount(wantReadBits))
|
||||
|
||||
if wantReadBytes > len(r.buf) {
|
||||
// TODO: use append somehow?
|
||||
r.buf = make([]byte, wantReadBytes)
|
||||
}
|
||||
|
||||
_, err := r.rs.Seek(readBytePos, io.SeekStart)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// TODO: nBits should be available
|
||||
readBytes, err := io.ReadFull(r.rs, r.buf[0:wantReadBytes])
|
||||
if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return 0, err
|
||||
} else if errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
nBits = int64(readBytes) * 8
|
||||
err = io.EOF
|
||||
}
|
||||
|
||||
if readSkipBits == 0 && nBits%8 == 0 {
|
||||
copy(p[0:readBytes], r.buf[0:readBytes])
|
||||
return nBits, err
|
||||
}
|
||||
|
||||
nBytes := nBits / 8
|
||||
restBits := nBits % 8
|
||||
|
||||
// TODO: copy smartness if many bytes
|
||||
for i := int64(0); i < nBytes; i++ {
|
||||
p[i] = byte(Read64(r.buf, readSkipBits+i*8, 8))
|
||||
}
|
||||
if restBits != 0 {
|
||||
p[nBytes] = byte(Read64(r.buf, readSkipBits+nBytes*8, restBits)) << (8 - restBits)
|
||||
}
|
||||
|
||||
return nBits, err
|
||||
}
|
||||
|
||||
func (r *IOBitReadSeeker) ReadBits(p []byte, nBits int64) (n int64, err error) {
|
||||
rBits, err := r.ReadBitsAt(p, nBits, r.bitPos)
|
||||
r.bitPos += rBits
|
||||
return rBits, err
|
||||
}
|
||||
|
||||
func (r *IOBitReadSeeker) SeekBits(bitOff int64, whence int) (int64, error) {
|
||||
seekBytesPos, err := r.rs.Seek(bitOff/8, whence)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
seekBitPos := seekBytesPos*8 + bitOff%8
|
||||
r.bitPos = seekBitPos
|
||||
|
||||
return seekBitPos, nil
|
||||
}
|
63
pkg/bitio/iobitwriter.go
Normal file
63
pkg/bitio/iobitwriter.go
Normal file
@ -0,0 +1,63 @@
|
||||
package bitio
|
||||
|
||||
import (
|
||||
"io"
|
||||
)
|
||||
|
||||
// IOBitWriter is a bitio.BitWriter that writes to a io.Writer
|
||||
// Use Flush to write possible unaligned byte zero bit padded.
|
||||
type IOBitWriter struct {
|
||||
w io.Writer
|
||||
b Buffer
|
||||
}
|
||||
|
||||
func NewIOBitWriter(w io.Writer) *IOBitWriter {
|
||||
return &IOBitWriter{w: w}
|
||||
}
|
||||
|
||||
func (w *IOBitWriter) WriteBits(p []byte, nBits int64) (n int64, err error) {
|
||||
if n, err = w.b.WriteBits(p, nBits); err != nil {
|
||||
return n, err
|
||||
}
|
||||
|
||||
var sn int64
|
||||
|
||||
for {
|
||||
l := w.b.Len()
|
||||
if l < 8 {
|
||||
break
|
||||
}
|
||||
|
||||
var buf [32 * 1024]byte
|
||||
n, err := w.b.ReadBits(buf[:], l-(l%8))
|
||||
if err != nil {
|
||||
return sn, err
|
||||
}
|
||||
|
||||
rn, err := w.w.Write(buf[:n/8])
|
||||
sn += int64(rn)
|
||||
if err != nil {
|
||||
return sn, err
|
||||
}
|
||||
|
||||
sn += n
|
||||
}
|
||||
|
||||
return nBits, nil
|
||||
}
|
||||
|
||||
// Flush write possible unaligned byte zero bit padded.
|
||||
func (w *IOBitWriter) Flush() error {
|
||||
if w.b.Len() == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var buf [1]byte
|
||||
_, err := w.b.ReadBits(buf[:], w.b.Len())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = w.w.Write(buf[:])
|
||||
|
||||
return err
|
||||
}
|
76
pkg/bitio/ioreader.go
Normal file
76
pkg/bitio/ioreader.go
Normal file
@ -0,0 +1,76 @@
|
||||
package bitio
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
// IOReader is a io.Reader that reads bytes from a bitio.BitReader
|
||||
// Unaligned byte at EOF will be zero bit padded
|
||||
type IOReader struct {
|
||||
r Reader
|
||||
rErr error
|
||||
b Buffer
|
||||
}
|
||||
|
||||
func NewIOReader(r Reader) *IOReader {
|
||||
return &IOReader{r: r}
|
||||
}
|
||||
|
||||
func (r *IOReader) Read(p []byte) (n int, err error) {
|
||||
var ns int64
|
||||
|
||||
for {
|
||||
var err error
|
||||
|
||||
// uses p even if returning nothing, io.Reader docs says:
|
||||
// "it may use all of p as scratch space during the call"
|
||||
if r.rErr == nil {
|
||||
var rn int64
|
||||
rn, err = r.r.ReadBits(p, int64(len(p))*8)
|
||||
r.rErr = err
|
||||
ns += rn
|
||||
_, err = r.b.WriteBits(p, rn)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
if r.b.Len() >= 8 {
|
||||
// read whole bytes
|
||||
rBits := int64(len(p)) * 8
|
||||
bBits := r.b.Len()
|
||||
aBits := bBits - bBits%8
|
||||
if rBits > aBits {
|
||||
rBits = aBits
|
||||
}
|
||||
|
||||
rn, rErr := r.b.ReadBits(p, rBits)
|
||||
if rErr != nil {
|
||||
return int(rn / 8), rErr
|
||||
}
|
||||
return int(rn / 8), nil
|
||||
} else if r.rErr != nil {
|
||||
if errors.Is(r.rErr, io.EOF) && r.b.Len() > 0 {
|
||||
// TODO: hmm io.Buffer does this
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
_, err := r.b.ReadBits(p, r.b.Len())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
return 0, r.rErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// required to make some readers like deflate not do their own buffering
|
||||
func (r *IOReader) ReadByte() (byte, error) {
|
||||
var rb [1]byte
|
||||
_, err := r.Read(rb[:])
|
||||
return rb[0], err
|
||||
}
|
37
pkg/bitio/ioreadseeker.go
Normal file
37
pkg/bitio/ioreadseeker.go
Normal file
@ -0,0 +1,37 @@
|
||||
package bitio
|
||||
|
||||
// IOReadSeeker is a io.ReaSeeker that reads and seeks bytes from a bitio.BitReadSeeker
|
||||
// Unaligned byte at EOF will be zero bit padded
|
||||
type IOReadSeeker struct {
|
||||
IOReader
|
||||
s Seeker
|
||||
sPos int64
|
||||
}
|
||||
|
||||
func NewIOReadSeeker(rs ReadSeeker) *IOReadSeeker {
|
||||
return &IOReadSeeker{
|
||||
IOReader: IOReader{r: rs},
|
||||
s: rs,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *IOReadSeeker) Read(p []byte) (n int, err error) {
|
||||
n, err = r.IOReader.Read(p)
|
||||
r.sPos += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (r *IOReadSeeker) Seek(offset int64, whence int) (int64, error) {
|
||||
n, err := r.s.SeekBits(offset*8, whence)
|
||||
// TODO: reset last error on seek. some nicer way?
|
||||
r.IOReader.rErr = nil
|
||||
if n != r.sPos {
|
||||
r.b.Reset()
|
||||
r.sPos = n / 8
|
||||
}
|
||||
if err != nil {
|
||||
return n / 8, err
|
||||
}
|
||||
|
||||
return n / 8, err
|
||||
}
|
152
pkg/bitio/ioreadseeker_test.go
Normal file
152
pkg/bitio/ioreadseeker_test.go
Normal file
@ -0,0 +1,152 @@
|
||||
package bitio_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
)
|
||||
|
||||
func sb(s string) *bitio.SectionReader {
|
||||
buf, nBits := bitio.BytesFromBitString(s)
|
||||
return bitio.NewBitReader(buf, nBits)
|
||||
}
|
||||
|
||||
func bs(br bitio.Reader) string {
|
||||
bib := &bitio.Buffer{}
|
||||
_, err := bitio.Copy(bib, br)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
buf, nBits := bib.Bytes()
|
||||
return bitio.BitStringFromBytes(buf, nBits)
|
||||
}
|
||||
|
||||
func Test(t *testing.T) {
|
||||
testCases := []struct {
|
||||
bs string
|
||||
}{
|
||||
{""},
|
||||
{"|"},
|
||||
{"0"},
|
||||
{"1"},
|
||||
{"1|"},
|
||||
{"|1|"},
|
||||
{"|1|"},
|
||||
{"0|"},
|
||||
{"|0|"},
|
||||
{"0|"},
|
||||
{"1"},
|
||||
{"10"},
|
||||
{"101"},
|
||||
{"1011"},
|
||||
{"10110"},
|
||||
{"101101"},
|
||||
{"1011011"},
|
||||
{"10110110"},
|
||||
|
||||
{"101101101"},
|
||||
{"1011011010"},
|
||||
{"10110110101"},
|
||||
{"101101101011"},
|
||||
{"1011011010110"},
|
||||
{"10110110101101"},
|
||||
{"101101101011011"},
|
||||
{"1011011010110110"},
|
||||
|
||||
{"10110110|1"},
|
||||
{"10110110|10"},
|
||||
{"10110110|101"},
|
||||
{"10110110|1011"},
|
||||
{"10110110|10110"},
|
||||
{"10110110|101101"},
|
||||
{"10110110|1011011"},
|
||||
{"10110110|10110110"},
|
||||
|
||||
{"1|10110110|1"},
|
||||
{"10|10110110|10"},
|
||||
{"101|10110110|101"},
|
||||
{"1011|10110110|1011"},
|
||||
{"10110|10110110|10110"},
|
||||
{"101101|10110110|101101"},
|
||||
{"1011011|10110110|1011011"},
|
||||
{"10110110|10110110|10110110"},
|
||||
|
||||
{"1|1|0110110|1"},
|
||||
{"10|10|110110|10"},
|
||||
{"101|101|10110|101"},
|
||||
{"1011|1011|0110|1011"},
|
||||
{"10110|10110|110|10110"},
|
||||
{"101101|101101|10|101101"},
|
||||
{"1011011|1011011|0|1011011"},
|
||||
|
||||
{"1|10110110|10110110|1"},
|
||||
{"10|10110110|10110110|10"},
|
||||
{"101|10110110|10110110|101"},
|
||||
{"1011|10110110|10110110|1011"},
|
||||
{"10110|10110110|10110110|10110"},
|
||||
{"101101|10110110|10110110|101101"},
|
||||
{"1011011|10110110|10110110|1011011"},
|
||||
{"10110110|10110110|10110110|10110110"},
|
||||
|
||||
{"1|101101101011011010110110101101101011011010110110|1"},
|
||||
{"10|101101101011011010110110101101101011011010110110|10"},
|
||||
{"101|101101101011011010110110101101101011011010110110|101"},
|
||||
{"1011|101101101011011010110110101101101011011010110110|1011"},
|
||||
{"10110|101101101011011010110110101101101011011010110110|10110"},
|
||||
{"101101|101101101011011010110110101101101011011010110110|101101"},
|
||||
{"1011011|101101101011011010110110101101101011011010110110|1011011"},
|
||||
{"10110110|101101101011011010110110101101101011011010110110|10110110"},
|
||||
}
|
||||
for _, tC := range testCases {
|
||||
|
||||
bsParts := strings.Split(tC.bs, "|")
|
||||
var bsBRs []bitio.ReadAtSeeker
|
||||
for _, p := range bsParts {
|
||||
bsBRs = append(bsBRs, sb(p))
|
||||
}
|
||||
bsBR, err := bitio.NewMultiBitReader(bsBRs...)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
bsBitString := strings.ReplaceAll(tC.bs, "|", "")
|
||||
|
||||
for i := 0; i < len(bsBitString); i++ {
|
||||
t.Run(fmt.Sprintf("%s_%d", tC.bs, i), func(t *testing.T) {
|
||||
_, err = bsBR.SeekBits(int64(i), io.SeekStart)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
expectedBitString := bsBitString[i:]
|
||||
actualBitString := bs(bsBR)
|
||||
|
||||
if expectedBitString != actualBitString {
|
||||
t.Errorf("expected bits %q, got %q", expectedBitString, actualBitString)
|
||||
}
|
||||
|
||||
_, err = bsBR.SeekBits(int64(i), io.SeekStart)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
r := bitio.NewIOReader(bsBR)
|
||||
bb := &bytes.Buffer{}
|
||||
if _, err := io.Copy(bb, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
expecetdByteBitString := expectedBitString + strings.Repeat("0", (8-(len(expectedBitString)%8))%8)
|
||||
actualByteBitString := bitio.BitStringFromBytes(bb.Bytes(), int64(bb.Len()*8))
|
||||
|
||||
if expecetdByteBitString != actualByteBitString {
|
||||
t.Errorf("expected bytes %q, got %q", expecetdByteBitString, actualByteBitString)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
26
pkg/bitio/limitreader.go
Normal file
26
pkg/bitio/limitreader.go
Normal file
@ -0,0 +1,26 @@
|
||||
package bitio
|
||||
|
||||
import (
|
||||
"io"
|
||||
)
|
||||
|
||||
func NewLimitReader(r Reader, n int64) *LimitReader { return &LimitReader{r, n} }
|
||||
|
||||
// LimitReader is a bitio.Reader that reads a limited amount of bits from a bitio.Reader
|
||||
// Similar to bytes.LimitedReader but for bits
|
||||
type LimitReader struct {
|
||||
r Reader
|
||||
n int64
|
||||
}
|
||||
|
||||
func (l *LimitReader) ReadBits(p []byte, nBits int64) (n int64, err error) {
|
||||
if l.n <= 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
if nBits > l.n {
|
||||
nBits = l.n
|
||||
}
|
||||
n, err = l.r.ReadBits(p, nBits)
|
||||
l.n -= n
|
||||
return n, err
|
||||
}
|
108
pkg/bitio/multireader.go
Normal file
108
pkg/bitio/multireader.go
Normal file
@ -0,0 +1,108 @@
|
||||
package bitio
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
// TODO: smarter, track index?
|
||||
|
||||
func endPos(rs Seeker) (int64, error) {
|
||||
c, err := rs.SeekBits(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
e, err := rs.SeekBits(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
_, err = rs.SeekBits(c, io.SeekStart)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// MultiReader is a bitio.ReaderAtSeeker concatinating multiple bitio.ReadAtSeeker:s
|
||||
// Similar to io.MultiReader but for bits
|
||||
type MultiReader struct {
|
||||
pos int64
|
||||
readers []ReadAtSeeker
|
||||
readerEnds []int64
|
||||
}
|
||||
|
||||
func NewMultiBitReader(rs ...ReadAtSeeker) (*MultiReader, error) {
|
||||
readerEnds := make([]int64, len(rs))
|
||||
var esSum int64
|
||||
for i, r := range rs {
|
||||
e, err := endPos(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
esSum += e
|
||||
readerEnds[i] = esSum
|
||||
}
|
||||
return &MultiReader{readers: rs, readerEnds: readerEnds}, nil
|
||||
}
|
||||
|
||||
func (m *MultiReader) ReadBitsAt(p []byte, nBits int64, bitOff int64) (n int64, err error) {
|
||||
var end int64
|
||||
if len(m.readers) > 0 {
|
||||
end = m.readerEnds[len(m.readers)-1]
|
||||
}
|
||||
if end <= bitOff {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
prevAtEnd := int64(0)
|
||||
readerAt := m.readers[0]
|
||||
for i, end := range m.readerEnds {
|
||||
if bitOff < end {
|
||||
readerAt = m.readers[i]
|
||||
break
|
||||
}
|
||||
prevAtEnd = end
|
||||
}
|
||||
|
||||
rBits, err := readerAt.ReadBitsAt(p, nBits, bitOff-prevAtEnd)
|
||||
|
||||
if errors.Is(err, io.EOF) {
|
||||
if bitOff+rBits < end {
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
|
||||
return rBits, err
|
||||
}
|
||||
|
||||
func (m *MultiReader) ReadBits(p []byte, nBits int64) (n int64, err error) {
|
||||
n, err = m.ReadBitsAt(p, nBits, m.pos)
|
||||
m.pos += n
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (m *MultiReader) SeekBits(bitOff int64, whence int) (int64, error) {
|
||||
var p int64
|
||||
var end int64
|
||||
if len(m.readers) > 0 {
|
||||
end = m.readerEnds[len(m.readers)-1]
|
||||
}
|
||||
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
p = bitOff
|
||||
case io.SeekCurrent:
|
||||
p = m.pos + bitOff
|
||||
case io.SeekEnd:
|
||||
p = end + bitOff
|
||||
default:
|
||||
panic("unknown whence")
|
||||
}
|
||||
if p < 0 || p > end {
|
||||
return 0, ErrOffset
|
||||
}
|
||||
|
||||
m.pos = p
|
||||
|
||||
return p, nil
|
||||
}
|
@ -1,269 +0,0 @@
|
||||
package bitio
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
// Reader is a BitReadSeeker and BitReaderAt reading from a io.ReadSeeker
|
||||
type Reader struct {
|
||||
bitPos int64
|
||||
rs io.ReadSeeker
|
||||
buf []byte
|
||||
}
|
||||
|
||||
func NewReaderFromReadSeeker(rs io.ReadSeeker) *Reader {
|
||||
return &Reader{
|
||||
bitPos: 0,
|
||||
rs: rs,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reader) ReadBitsAt(p []byte, nBits int, bitOffset int64) (int, error) {
|
||||
if nBits < 0 {
|
||||
return 0, ErrNegativeNBits
|
||||
}
|
||||
|
||||
readBytePos := bitOffset / 8
|
||||
readSkipBits := int(bitOffset % 8)
|
||||
wantReadBits := readSkipBits + nBits
|
||||
wantReadBytes := int(BitsByteCount(int64(wantReadBits)))
|
||||
|
||||
if wantReadBytes > len(r.buf) {
|
||||
// TODO: use append somehow?
|
||||
r.buf = make([]byte, wantReadBytes)
|
||||
}
|
||||
|
||||
_, err := r.rs.Seek(readBytePos, io.SeekStart)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// TODO: nBits should be available
|
||||
readBytes, err := io.ReadFull(r.rs, r.buf[0:wantReadBytes])
|
||||
if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return 0, err
|
||||
} else if errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
nBits = readBytes * 8
|
||||
err = io.EOF
|
||||
}
|
||||
|
||||
if readSkipBits == 0 && nBits%8 == 0 {
|
||||
copy(p[0:readBytes], r.buf[0:readBytes])
|
||||
return nBits, err
|
||||
}
|
||||
|
||||
nBytes := nBits / 8
|
||||
restBits := nBits % 8
|
||||
|
||||
// TODO: copy smartness if many bytes
|
||||
for i := 0; i < nBytes; i++ {
|
||||
p[i] = byte(Read64(r.buf, readSkipBits+i*8, 8))
|
||||
}
|
||||
if restBits != 0 {
|
||||
p[nBytes] = byte(Read64(r.buf, readSkipBits+nBytes*8, restBits)) << (8 - restBits)
|
||||
}
|
||||
|
||||
return nBits, err
|
||||
}
|
||||
|
||||
func (r *Reader) ReadBits(p []byte, nBits int) (n int, err error) {
|
||||
rBits, err := r.ReadBitsAt(p, nBits, r.bitPos)
|
||||
r.bitPos += int64(rBits)
|
||||
return rBits, err
|
||||
}
|
||||
|
||||
func (r *Reader) SeekBits(bitOff int64, whence int) (int64, error) {
|
||||
seekBytesPos, err := r.rs.Seek(bitOff/8, whence)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
seekBitPos := seekBytesPos*8 + bitOff%8
|
||||
r.bitPos = seekBitPos
|
||||
|
||||
return seekBitPos, nil
|
||||
}
|
||||
|
||||
func (r *Reader) Read(p []byte) (n int, err error) {
|
||||
n, err = r.ReadBitsAt(p, len(p)*8, r.bitPos)
|
||||
r.bitPos += int64(n)
|
||||
if err != nil {
|
||||
return int(BitsByteCount(int64(n))), err
|
||||
}
|
||||
|
||||
return int(BitsByteCount(int64(n))), nil
|
||||
}
|
||||
|
||||
func (r *Reader) Seek(offset int64, whence int) (int64, error) {
|
||||
seekBytesPos, err := r.rs.Seek(offset, whence)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
r.bitPos = seekBytesPos * 8
|
||||
return seekBytesPos, nil
|
||||
}
|
||||
|
||||
// SectionBitReader is a BitReadSeeker reading from a BitReaderAt
|
||||
// modelled after io.SectionReader
|
||||
type SectionBitReader struct {
|
||||
r BitReaderAt
|
||||
bitBase int64
|
||||
bitOff int64
|
||||
bitLimit int64
|
||||
}
|
||||
|
||||
func NewSectionBitReader(r BitReaderAt, bitOff int64, nBits int64) *SectionBitReader {
|
||||
return &SectionBitReader{
|
||||
r: r,
|
||||
bitBase: bitOff,
|
||||
bitOff: bitOff,
|
||||
bitLimit: bitOff + nBits,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *SectionBitReader) ReadBitsAt(p []byte, nBits int, bitOff int64) (int, error) {
|
||||
if bitOff < 0 || bitOff >= r.bitLimit-r.bitBase {
|
||||
return 0, io.EOF
|
||||
}
|
||||
bitOff += r.bitBase
|
||||
if maxBits := int(r.bitLimit - bitOff); nBits > maxBits {
|
||||
nBits = maxBits
|
||||
rBits, err := r.r.ReadBitsAt(p, nBits, bitOff)
|
||||
return rBits, err
|
||||
}
|
||||
return r.r.ReadBitsAt(p, nBits, bitOff)
|
||||
}
|
||||
|
||||
func (r *SectionBitReader) ReadBits(p []byte, nBits int) (n int, err error) {
|
||||
rBits, err := r.ReadBitsAt(p, nBits, r.bitOff-r.bitBase)
|
||||
r.bitOff += int64(rBits)
|
||||
return rBits, err
|
||||
}
|
||||
|
||||
func (r *SectionBitReader) SeekBits(bitOff int64, whence int) (int64, error) {
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
bitOff += r.bitBase
|
||||
case io.SeekCurrent:
|
||||
bitOff += r.bitOff
|
||||
case io.SeekEnd:
|
||||
bitOff += r.bitLimit
|
||||
default:
|
||||
panic("unknown whence")
|
||||
}
|
||||
if bitOff < r.bitBase {
|
||||
return 0, ErrOffset
|
||||
}
|
||||
r.bitOff = bitOff
|
||||
return bitOff - r.bitBase, nil
|
||||
}
|
||||
|
||||
func (r *SectionBitReader) Read(p []byte) (n int, err error) {
|
||||
n, err = r.ReadBitsAt(p, len(p)*8, r.bitOff-r.bitBase)
|
||||
r.bitOff += int64(n)
|
||||
return int(BitsByteCount(int64(n))), err
|
||||
}
|
||||
|
||||
func (r *SectionBitReader) Seek(offset int64, whence int) (int64, error) {
|
||||
seekBitsPos, err := r.SeekBits(offset*8, whence)
|
||||
return seekBitsPos / 8, err
|
||||
}
|
||||
|
||||
// TODO: smart, track index?
|
||||
type MultiBitReader struct {
|
||||
pos int64
|
||||
readers []BitReadAtSeeker
|
||||
readerEnds []int64
|
||||
}
|
||||
|
||||
func NewMultiBitReader(rs []BitReadAtSeeker) (*MultiBitReader, error) {
|
||||
readerEnds := make([]int64, len(rs))
|
||||
var esSum int64
|
||||
for i, r := range rs {
|
||||
e, err := EndPos(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
esSum += e
|
||||
readerEnds[i] = esSum
|
||||
}
|
||||
return &MultiBitReader{readers: rs, readerEnds: readerEnds}, nil
|
||||
}
|
||||
|
||||
func (m *MultiBitReader) ReadBitsAt(p []byte, nBits int, bitOff int64) (n int, err error) {
|
||||
var end int64
|
||||
if len(m.readers) > 0 {
|
||||
end = m.readerEnds[len(m.readers)-1]
|
||||
}
|
||||
if end <= bitOff {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
prevAtEnd := int64(0)
|
||||
readerAt := m.readers[0]
|
||||
for i, end := range m.readerEnds {
|
||||
if bitOff < end {
|
||||
readerAt = m.readers[i]
|
||||
break
|
||||
}
|
||||
prevAtEnd = end
|
||||
}
|
||||
|
||||
rBits, err := readerAt.ReadBitsAt(p, nBits, bitOff-prevAtEnd)
|
||||
|
||||
if errors.Is(err, io.EOF) {
|
||||
if bitOff+int64(rBits) < end {
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
|
||||
return rBits, err
|
||||
}
|
||||
|
||||
func (m *MultiBitReader) ReadBits(p []byte, nBits int) (n int, err error) {
|
||||
n, err = m.ReadBitsAt(p, nBits, m.pos)
|
||||
m.pos += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (m *MultiBitReader) SeekBits(bitOff int64, whence int) (int64, error) {
|
||||
var p int64
|
||||
var end int64
|
||||
if len(m.readers) > 0 {
|
||||
end = m.readerEnds[len(m.readers)-1]
|
||||
}
|
||||
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
p = bitOff
|
||||
case io.SeekCurrent:
|
||||
p = m.pos + bitOff
|
||||
case io.SeekEnd:
|
||||
p = end + bitOff
|
||||
default:
|
||||
panic("unknown whence")
|
||||
}
|
||||
if p < 0 || p > end {
|
||||
return 0, ErrOffset
|
||||
}
|
||||
|
||||
m.pos = p
|
||||
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (m *MultiBitReader) Read(p []byte) (n int, err error) {
|
||||
n, err = m.ReadBitsAt(p, len(p)*8, m.pos)
|
||||
m.pos += int64(n)
|
||||
|
||||
if err != nil {
|
||||
return int(BitsByteCount(int64(n))), err
|
||||
}
|
||||
|
||||
return int(BitsByteCount(int64(n))), nil
|
||||
}
|
||||
|
||||
func (m *MultiBitReader) Seek(offset int64, whence int) (int64, error) {
|
||||
seekBitsPos, err := m.SeekBits(offset*8, whence)
|
||||
return seekBitsPos / 8, err
|
||||
}
|
@ -7,7 +7,7 @@ import (
|
||||
|
||||
// Read64 read nBits bits large unsigned integer from buf starting from firstBit.
|
||||
// Integer is read most significant bit first.
|
||||
func Read64(buf []byte, firstBit int, nBits int) uint64 {
|
||||
func Read64(buf []byte, firstBit int64, nBits int64) uint64 {
|
||||
if nBits < 0 || nBits > 64 {
|
||||
panic(fmt.Sprintf("nBits must be 0-64 (%d)", nBits))
|
||||
}
|
||||
@ -90,7 +90,7 @@ func Read64(buf []byte, firstBit int, nBits int) uint64 {
|
||||
return n
|
||||
}
|
||||
|
||||
func Write64(v uint64, nBits int, buf []byte, firstBit int) {
|
||||
func Write64(v uint64, nBits int64, buf []byte, firstBit int64) {
|
||||
if nBits < 0 || nBits > 64 {
|
||||
panic(fmt.Sprintf("nBits must be 0-64 (%d)", nBits))
|
||||
}
|
||||
@ -169,26 +169,3 @@ func Write64(v uint64, nBits int, buf []byte, firstBit int) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Uint64ReverseBytes(nBits int, n uint64) uint64 {
|
||||
switch {
|
||||
case nBits <= 8:
|
||||
return n
|
||||
case nBits <= 16:
|
||||
return n&0xff00>>8 | n&0xff<<8
|
||||
case nBits <= 24:
|
||||
return n&0xff<<16 | n&0xff00 | n&0xff0000>>16
|
||||
case nBits <= 32:
|
||||
return n&0xff<<24 | n&0xff00<<8 | n&0xff0000>>8 | n&0xff000000>>24
|
||||
case nBits <= 40:
|
||||
return n&0xff<<32 | n&0xff00<<16 | n&0xff0000 | n&0xff000000>>16 | n&0xff00000000>>32
|
||||
case nBits <= 48:
|
||||
return n&0xff<<40 | n&0xff00<<24 | n&0xff0000<<8 | n&0xff000000>>8 | n&0xff00000000>>24 | n&0xff0000000000>>40
|
||||
case nBits <= 56:
|
||||
return n&0xff<<48 | n&0xff00<<32 | n&0xff0000<<16 | n&0xff000000 | n&0xff00000000>>16 | n&0xff0000000000>>32 | n&0xff000000000000>>48
|
||||
case nBits <= 64:
|
||||
return n&0xff<<56 | n&0xff00<<40 | n&0xff0000<<24 | n&0xff000000<<8 | n&0xff00000000>>8 | n&0xff0000000000>>24 | n&0xff000000000000>>40 | n&0xff00000000000000>>56
|
||||
default:
|
||||
panic(fmt.Sprintf("unsupported bit length %d", nBits))
|
||||
}
|
||||
}
|
@ -12,8 +12,8 @@ import (
|
||||
func TestRead64(t *testing.T) {
|
||||
testCases := []struct {
|
||||
buf []byte
|
||||
firstBit int
|
||||
nBits int
|
||||
firstBit int64
|
||||
nBits int64
|
||||
expected uint64
|
||||
}{
|
||||
{buf: []byte{0xff}, firstBit: 0, nBits: 8, expected: 0b11111111},
|
||||
@ -93,9 +93,9 @@ func TestRead64(t *testing.T) {
|
||||
func TestWrite64(t *testing.T) {
|
||||
testCases := []struct {
|
||||
v uint64
|
||||
nBits int
|
||||
nBits int64
|
||||
buf []byte
|
||||
firstBit int
|
||||
firstBit int64
|
||||
expectedBuf []byte
|
||||
}{
|
||||
{0x0123456789abcdef, 8, []byte{0, 0, 0, 0, 0, 0, 0, 0}, 0, []byte{0xef, 0x00, 0x00, 0x0, 0x0, 0x00, 0x00, 0x00}},
|
||||
@ -145,44 +145,3 @@ func TestWrite64(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// func TestWrite64(t *testing.T) {
|
||||
|
||||
// b := make([]byte, 2)
|
||||
|
||||
// bitio.Write64(0xf, 4, b, 0)
|
||||
|
||||
// }
|
||||
|
||||
func TestUint64Panic(t *testing.T) {
|
||||
// TODO: check panic string
|
||||
defer func() { _ = recover() }()
|
||||
bitio.Read64([]byte{}, 0, 65)
|
||||
t.Error("should panic")
|
||||
}
|
||||
|
||||
func TestUint64ReverseBytes(t *testing.T) {
|
||||
testCases := []struct {
|
||||
nBits int
|
||||
n uint64
|
||||
expected uint64
|
||||
}{
|
||||
{nBits: 0, n: 0, expected: 0},
|
||||
{nBits: 8, n: 0x01, expected: 0x01},
|
||||
{nBits: 16, n: 0x0123, expected: 0x2301},
|
||||
{nBits: 24, n: 0x012345, expected: 0x452301},
|
||||
{nBits: 32, n: 0x01234567, expected: 0x67452301},
|
||||
{nBits: 40, n: 0x0123456789, expected: 0x8967452301},
|
||||
{nBits: 48, n: 0x0123456789ab, expected: 0xab8967452301},
|
||||
{nBits: 56, n: 0x0123456789abcd, expected: 0xcdab8967452301},
|
||||
{nBits: 64, n: 0x0123456789abcdef, expected: 0xefcdab8967452301},
|
||||
}
|
||||
for _, tC := range testCases {
|
||||
t.Run(fmt.Sprintf("%d %x %x", tC.nBits, tC.n, tC.expected), func(t *testing.T) {
|
||||
actual := bitio.Uint64ReverseBytes(tC.nBits, tC.n)
|
||||
if tC.expected != actual {
|
||||
t.Errorf("expected %x, got %x", tC.expected, actual)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
@ -1,9 +0,0 @@
|
||||
package bitio
|
||||
|
||||
func ReverseBytes(bs []byte) []byte {
|
||||
l := len(bs)
|
||||
for i := 0; i < l/2; i++ {
|
||||
bs[i], bs[l-i-1] = bs[l-i-1], bs[i]
|
||||
}
|
||||
return bs
|
||||
}
|
28
pkg/bitio/reversebytes64.go
Normal file
28
pkg/bitio/reversebytes64.go
Normal file
@ -0,0 +1,28 @@
|
||||
package bitio
|
||||
|
||||
import "fmt"
|
||||
|
||||
// ReverseBytes64 reverses the bytes part of the lowest nBits
|
||||
// Similar to bits.ReverseBytes64 but only rotates to lowest bytes and rest of bytes will be zero
|
||||
func ReverseBytes64(nBits int, n uint64) uint64 {
|
||||
switch {
|
||||
case nBits <= 8:
|
||||
return n
|
||||
case nBits <= 16:
|
||||
return n&0xff00>>8 | n&0xff<<8
|
||||
case nBits <= 24:
|
||||
return n&0xff<<16 | n&0xff00 | n&0xff0000>>16
|
||||
case nBits <= 32:
|
||||
return n&0xff<<24 | n&0xff00<<8 | n&0xff0000>>8 | n&0xff000000>>24
|
||||
case nBits <= 40:
|
||||
return n&0xff<<32 | n&0xff00<<16 | n&0xff0000 | n&0xff000000>>16 | n&0xff00000000>>32
|
||||
case nBits <= 48:
|
||||
return n&0xff<<40 | n&0xff00<<24 | n&0xff0000<<8 | n&0xff000000>>8 | n&0xff00000000>>24 | n&0xff0000000000>>40
|
||||
case nBits <= 56:
|
||||
return n&0xff<<48 | n&0xff00<<32 | n&0xff0000<<16 | n&0xff000000 | n&0xff00000000>>16 | n&0xff0000000000>>32 | n&0xff000000000000>>48
|
||||
case nBits <= 64:
|
||||
return n&0xff<<56 | n&0xff00<<40 | n&0xff0000<<24 | n&0xff000000<<8 | n&0xff00000000>>8 | n&0xff0000000000>>24 | n&0xff000000000000>>40 | n&0xff00000000000000>>56
|
||||
default:
|
||||
panic(fmt.Sprintf("unsupported bit length %d", nBits))
|
||||
}
|
||||
}
|
41
pkg/bitio/reversebytes64_test.go
Normal file
41
pkg/bitio/reversebytes64_test.go
Normal file
@ -0,0 +1,41 @@
|
||||
package bitio_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
)
|
||||
|
||||
func TestReverseBytes64Panic(t *testing.T) {
|
||||
// TODO: check panic string
|
||||
defer func() { _ = recover() }()
|
||||
bitio.Read64([]byte{}, 0, 65)
|
||||
t.Error("should panic")
|
||||
}
|
||||
|
||||
func TestReverseBytes64(t *testing.T) {
|
||||
testCases := []struct {
|
||||
nBits int
|
||||
n uint64
|
||||
expected uint64
|
||||
}{
|
||||
{nBits: 0, n: 0, expected: 0},
|
||||
{nBits: 8, n: 0x01, expected: 0x01},
|
||||
{nBits: 16, n: 0x0123, expected: 0x2301},
|
||||
{nBits: 24, n: 0x012345, expected: 0x452301},
|
||||
{nBits: 32, n: 0x01234567, expected: 0x67452301},
|
||||
{nBits: 40, n: 0x0123456789, expected: 0x8967452301},
|
||||
{nBits: 48, n: 0x0123456789ab, expected: 0xab8967452301},
|
||||
{nBits: 56, n: 0x0123456789abcd, expected: 0xcdab8967452301},
|
||||
{nBits: 64, n: 0x0123456789abcdef, expected: 0xefcdab8967452301},
|
||||
}
|
||||
for _, tC := range testCases {
|
||||
t.Run(fmt.Sprintf("%d %x %x", tC.nBits, tC.n, tC.expected), func(t *testing.T) {
|
||||
actual := bitio.ReverseBytes64(tC.nBits, tC.n)
|
||||
if tC.expected != actual {
|
||||
t.Errorf("expected %x, got %x", tC.expected, actual)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
@ -1,30 +0,0 @@
|
||||
package bitio_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
)
|
||||
|
||||
func TestReverseBytes(t *testing.T) {
|
||||
testCases := []struct {
|
||||
input []byte
|
||||
expected []byte
|
||||
}{
|
||||
{nil, nil},
|
||||
{[]byte{1}, []byte{1}},
|
||||
{[]byte{1, 2}, []byte{2, 1}},
|
||||
{[]byte{1, 2, 3}, []byte{3, 2, 1}},
|
||||
}
|
||||
for _, tC := range testCases {
|
||||
t.Run(fmt.Sprintf("%v", tC.input), func(t *testing.T) {
|
||||
actual := append([]byte(nil), tC.input...)
|
||||
bitio.ReverseBytes(actual)
|
||||
if !bytes.Equal(tC.expected, actual) {
|
||||
t.Errorf("expected %v, got %v", tC.expected, actual)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
60
pkg/bitio/sectiontreader.go
Normal file
60
pkg/bitio/sectiontreader.go
Normal file
@ -0,0 +1,60 @@
|
||||
package bitio
|
||||
|
||||
import (
|
||||
"io"
|
||||
)
|
||||
|
||||
// SectionReader is a bitio.BitReaderAtSeeker reading a section of a bitio.ReaderAt
|
||||
// Similar to io.SectionReader but for bits
|
||||
type SectionReader struct {
|
||||
r ReaderAt
|
||||
bitBase int64
|
||||
bitOff int64
|
||||
bitLimit int64
|
||||
}
|
||||
|
||||
func NewSectionReader(r ReaderAt, bitOff int64, nBits int64) *SectionReader {
|
||||
return &SectionReader{
|
||||
r: r,
|
||||
bitBase: bitOff,
|
||||
bitOff: bitOff,
|
||||
bitLimit: bitOff + nBits,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *SectionReader) ReadBitsAt(p []byte, nBits int64, bitOff int64) (int64, error) {
|
||||
if bitOff < 0 || bitOff >= r.bitLimit-r.bitBase {
|
||||
return 0, io.EOF
|
||||
}
|
||||
bitOff += r.bitBase
|
||||
if maxBits := r.bitLimit - bitOff; nBits > maxBits {
|
||||
nBits = maxBits
|
||||
rBits, err := r.r.ReadBitsAt(p, nBits, bitOff)
|
||||
return rBits, err
|
||||
}
|
||||
return r.r.ReadBitsAt(p, nBits, bitOff)
|
||||
}
|
||||
|
||||
func (r *SectionReader) ReadBits(p []byte, nBits int64) (n int64, err error) {
|
||||
rBits, err := r.ReadBitsAt(p, nBits, r.bitOff-r.bitBase)
|
||||
r.bitOff += rBits
|
||||
return rBits, err
|
||||
}
|
||||
|
||||
func (r *SectionReader) SeekBits(bitOff int64, whence int) (int64, error) {
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
bitOff += r.bitBase
|
||||
case io.SeekCurrent:
|
||||
bitOff += r.bitOff
|
||||
case io.SeekEnd:
|
||||
bitOff += r.bitLimit
|
||||
default:
|
||||
panic("unknown whence")
|
||||
}
|
||||
if bitOff < r.bitBase {
|
||||
return 0, ErrOffset
|
||||
}
|
||||
r.bitOff = bitOff
|
||||
return bitOff - r.bitBase, nil
|
||||
}
|
@ -8,6 +8,7 @@ import (
|
||||
"io/ioutil"
|
||||
"math/big"
|
||||
|
||||
"github.com/wader/fq/internal/bitioextra"
|
||||
"github.com/wader/fq/internal/recoverfn"
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
"github.com/wader/fq/pkg/ranges"
|
||||
@ -38,14 +39,19 @@ type Options struct {
|
||||
}
|
||||
|
||||
// Decode try decode group and return first success and all other decoder errors
|
||||
func Decode(ctx context.Context, bb *bitio.Buffer, group Group, opts Options) (*Value, interface{}, error) {
|
||||
return decode(ctx, bb, group, opts)
|
||||
func Decode(ctx context.Context, br bitio.ReaderAtSeeker, group Group, opts Options) (*Value, interface{}, error) {
|
||||
return decode(ctx, br, group, opts)
|
||||
}
|
||||
|
||||
func decode(ctx context.Context, bb *bitio.Buffer, group Group, opts Options) (*Value, interface{}, error) {
|
||||
func decode(ctx context.Context, br bitio.ReaderAtSeeker, group Group, opts Options) (*Value, interface{}, error) {
|
||||
brLen, err := bitioextra.Len(br)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
decodeRange := opts.Range
|
||||
if decodeRange.IsZero() {
|
||||
decodeRange = ranges.Range{Len: bb.Len()}
|
||||
decodeRange = ranges.Range{Len: brLen}
|
||||
}
|
||||
|
||||
if group == nil {
|
||||
@ -55,12 +61,12 @@ func decode(ctx context.Context, bb *bitio.Buffer, group Group, opts Options) (*
|
||||
formatsErr := FormatsError{}
|
||||
|
||||
for _, g := range group {
|
||||
cbb, err := bb.BitBufRange(decodeRange.Start, decodeRange.Len)
|
||||
cBR, err := bitioextra.Range(br, decodeRange.Start, decodeRange.Len)
|
||||
if err != nil {
|
||||
return nil, nil, IOError{Err: err, Op: "BitBufRange", ReadSize: decodeRange.Len, Pos: decodeRange.Start}
|
||||
}
|
||||
|
||||
d := newDecoder(ctx, g, cbb, opts)
|
||||
d := newDecoder(ctx, g, cBR, opts)
|
||||
|
||||
var decodeV interface{}
|
||||
r, rOk := recoverfn.Run(func() {
|
||||
@ -105,7 +111,7 @@ func decode(ctx context.Context, bb *bitio.Buffer, group Group, opts Options) (*
|
||||
if err := d.Value.WalkRootPreOrder(func(v *Value, rootV *Value, depth int, rootDepth int) error {
|
||||
minMaxRange = ranges.MinMax(minMaxRange, v.Range)
|
||||
v.Range.Start += decodeRange.Start
|
||||
v.RootBitBuf = bb
|
||||
v.RootBitBuf = br
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, nil, err
|
||||
@ -133,14 +139,14 @@ type D struct {
|
||||
Value *Value
|
||||
Options Options
|
||||
|
||||
bitBuf *bitio.Buffer
|
||||
bitBuf bitio.ReaderAtSeeker
|
||||
|
||||
readBuf *[]byte
|
||||
}
|
||||
|
||||
// TODO: new struct decoder?
|
||||
// note bb is assumed to be a non-shared buffer
|
||||
func newDecoder(ctx context.Context, format Format, bb *bitio.Buffer, opts Options) *D {
|
||||
// note br is assumed to be a non-shared buffer
|
||||
func newDecoder(ctx context.Context, format Format, br bitio.ReaderAtSeeker, opts Options) *D {
|
||||
name := format.RootName
|
||||
if opts.Name != "" {
|
||||
name = opts.Name
|
||||
@ -158,18 +164,18 @@ func newDecoder(ctx context.Context, format Format, bb *bitio.Buffer, opts Optio
|
||||
Value: &Value{
|
||||
Name: name,
|
||||
V: rootV,
|
||||
RootBitBuf: bb,
|
||||
RootBitBuf: br,
|
||||
Range: ranges.Range{Start: 0, Len: 0},
|
||||
IsRoot: opts.IsRoot,
|
||||
},
|
||||
Options: opts,
|
||||
|
||||
bitBuf: bb,
|
||||
bitBuf: br,
|
||||
readBuf: opts.ReadBuf,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *D) FieldDecoder(name string, bitBuf *bitio.Buffer, v interface{}) *D {
|
||||
func (d *D) FieldDecoder(name string, bitBuf bitio.ReaderAtSeeker, v interface{}) *D {
|
||||
return &D{
|
||||
Ctx: d.Ctx,
|
||||
Endian: d.Endian,
|
||||
@ -186,24 +192,63 @@ func (d *D) FieldDecoder(name string, bitBuf *bitio.Buffer, v interface{}) *D {
|
||||
}
|
||||
}
|
||||
|
||||
func (d *D) Copy(r io.Writer, w io.Reader) (int64, error) {
|
||||
func (d *D) CopyBits(w io.Writer, r bitio.Reader) (int64, error) {
|
||||
// TODO: what size? now same as io.Copy
|
||||
buf := d.SharedReadBuf(32 * 1024)
|
||||
return io.CopyBuffer(r, w, buf)
|
||||
return bitioextra.CopyBitsBuffer(w, r, buf)
|
||||
}
|
||||
|
||||
func (d *D) MustCopy(r io.Writer, w io.Reader) int64 {
|
||||
n, err := d.Copy(r, w)
|
||||
func (d *D) MustCopyBits(w io.Writer, r bitio.Reader) int64 {
|
||||
n, err := d.CopyBits(w, r)
|
||||
if err != nil {
|
||||
d.IOPanic(err, "MustCopy: Copy")
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (d *D) MustNewBitBufFromReader(r io.Reader) *bitio.Buffer {
|
||||
func (d *D) Copy(w io.Writer, r io.Reader) (int64, error) {
|
||||
// TODO: what size? now same as io.Copy
|
||||
buf := d.SharedReadBuf(32 * 1024)
|
||||
return io.CopyBuffer(w, r, buf)
|
||||
}
|
||||
|
||||
func (d *D) MustCopy(w io.Writer, r io.Reader) int64 {
|
||||
n, err := d.Copy(w, r)
|
||||
if err != nil {
|
||||
d.IOPanic(err, "MustCopy: Copy")
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (d *D) MustClone(br bitio.ReaderAtSeeker) bitio.ReaderAtSeeker {
|
||||
br, err := bitioextra.Clone(br)
|
||||
if err != nil {
|
||||
d.IOPanic(err, "MustClone")
|
||||
}
|
||||
return br
|
||||
}
|
||||
|
||||
func (d *D) MustNewBitBufFromReader(r io.Reader) bitio.ReaderAtSeeker {
|
||||
b := &bytes.Buffer{}
|
||||
d.MustCopy(b, r)
|
||||
return bitio.NewBufferFromBytes(b.Bytes(), -1)
|
||||
return bitio.NewBitReader(b.Bytes(), -1)
|
||||
}
|
||||
|
||||
func (d *D) ReadAllBits(r bitio.Reader) ([]byte, error) {
|
||||
bb := &bytes.Buffer{}
|
||||
buf := d.SharedReadBuf(32 * 1024)
|
||||
if _, err := bitioextra.CopyBitsBuffer(bb, r, buf); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bb.Bytes(), nil
|
||||
}
|
||||
|
||||
func (d *D) MustReadAllBits(r bitio.Reader) []byte {
|
||||
buf, err := d.ReadAllBits(r)
|
||||
if err != nil {
|
||||
d.IOPanic(err, "Bytes ReadAllBytes")
|
||||
}
|
||||
return buf
|
||||
}
|
||||
|
||||
func (d *D) SharedReadBuf(n int) []byte {
|
||||
@ -241,7 +286,7 @@ func (d *D) FillGaps(r ranges.Range, namePrefix string) {
|
||||
|
||||
gaps := ranges.Gaps(r, valueRanges)
|
||||
for i, gap := range gaps {
|
||||
bb, err := d.bitBuf.BitBufRange(gap.Start, gap.Len)
|
||||
br, err := bitioextra.Range(d.bitBuf, gap.Start, gap.Len)
|
||||
if err != nil {
|
||||
d.IOPanic(err, "FillGaps: BitBufRange")
|
||||
}
|
||||
@ -249,7 +294,7 @@ func (d *D) FillGaps(r ranges.Range, namePrefix string) {
|
||||
v := &Value{
|
||||
Name: fmt.Sprintf("%s%d", namePrefix, i),
|
||||
V: &scalar.S{
|
||||
Actual: bb,
|
||||
Actual: br,
|
||||
Unknown: true,
|
||||
},
|
||||
RootBitBuf: d.bitBuf,
|
||||
@ -283,12 +328,12 @@ func (d *D) bits(nBits int) (uint64, error) {
|
||||
}
|
||||
// 64 bits max, 9 byte worse case if not byte aligned
|
||||
buf := d.SharedReadBuf(9)
|
||||
_, err := bitio.ReadFull(d.bitBuf, buf, nBits)
|
||||
_, err := bitio.ReadFull(d.bitBuf, buf, int64(nBits)) // TODO: int64?
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return bitio.Read64(buf[:], 0, nBits), nil
|
||||
return bitio.Read64(buf[:], 0, int64(nBits)), nil // TODO: int64
|
||||
}
|
||||
|
||||
// Bits reads nBits bits from buffer
|
||||
@ -308,8 +353,20 @@ func (d *D) PeekBits(nBits int) uint64 {
|
||||
return n
|
||||
}
|
||||
|
||||
func (d *D) TryPeekBytes(nBytes int) ([]byte, error) {
|
||||
start, err := d.bitBuf.SeekBits(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bs, err := d.TryBytesLen(nBytes)
|
||||
if _, err := d.bitBuf.SeekBits(start, io.SeekStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bs, err
|
||||
}
|
||||
|
||||
func (d *D) PeekBytes(nBytes int) []byte {
|
||||
bs, err := d.bitBuf.PeekBytes(nBytes)
|
||||
bs, err := d.TryPeekBytes(nBytes)
|
||||
if err != nil {
|
||||
panic(IOError{Err: err, Op: "PeekBytes", ReadSize: int64(nBytes) * 8, Pos: d.Pos()})
|
||||
}
|
||||
@ -410,16 +467,34 @@ func (d *D) TryPeekFind(nBits int, seekBits int64, maxLen int64, fn func(v uint6
|
||||
return count, v, nil
|
||||
}
|
||||
|
||||
// BytesRange reads nBytes bytes starting bit position start
|
||||
// Does not update current position.
|
||||
// TODO: nBytes -1?
|
||||
func (d *D) TryBytesRange(bitOffset int64, nBytes int) ([]byte, error) {
|
||||
buf := make([]byte, nBytes)
|
||||
n, err := bitio.ReadAtFull(d.bitBuf, buf, int64(nBytes)*8, bitOffset)
|
||||
if n == int64(nBytes)*8 {
|
||||
err = nil
|
||||
}
|
||||
return buf, err
|
||||
}
|
||||
|
||||
func (d *D) BytesRange(firstBit int64, nBytes int) []byte {
|
||||
bs, err := d.bitBuf.BytesRange(firstBit, nBytes)
|
||||
bs, err := d.TryBytesRange(firstBit, nBytes)
|
||||
if err != nil {
|
||||
panic(IOError{Err: err, Op: "BytesRange", ReadSize: int64(nBytes) * 8, Pos: firstBit})
|
||||
}
|
||||
return bs
|
||||
}
|
||||
|
||||
func (d *D) TryBytesLen(nBytes int) ([]byte, error) {
|
||||
buf := make([]byte, nBytes)
|
||||
_, err := bitio.ReadFull(d.bitBuf, buf, int64(nBytes)*8)
|
||||
return buf, err
|
||||
}
|
||||
|
||||
func (d *D) BytesLen(nBytes int) []byte {
|
||||
bs, err := d.bitBuf.BytesLen(nBytes)
|
||||
bs, err := d.TryBytesLen(nBytes)
|
||||
if err != nil {
|
||||
panic(IOError{Err: err, Op: "BytesLen", ReadSize: int64(nBytes) * 8, Pos: d.Pos()})
|
||||
}
|
||||
@ -427,98 +502,202 @@ func (d *D) BytesLen(nBytes int) []byte {
|
||||
}
|
||||
|
||||
// TODO: rename/remove BitBuf name?
|
||||
func (d *D) BitBufRange(firstBit int64, nBits int64) *bitio.Buffer {
|
||||
bb, err := d.bitBuf.BitBufRange(firstBit, nBits)
|
||||
func (d *D) TryBitBufRange(firstBit int64, nBits int64) (bitio.ReaderAtSeeker, error) {
|
||||
return bitioextra.Range(d.bitBuf, firstBit, nBits)
|
||||
}
|
||||
|
||||
func (d *D) BitBufRange(firstBit int64, nBits int64) bitio.ReaderAtSeeker {
|
||||
br, err := bitioextra.Range(d.bitBuf, firstBit, nBits)
|
||||
if err != nil {
|
||||
panic(IOError{Err: err, Op: "BitBufRange", ReadSize: nBits, Pos: firstBit})
|
||||
}
|
||||
return bb
|
||||
return br
|
||||
}
|
||||
|
||||
func (d *D) TryPos() (int64, error) {
|
||||
return d.bitBuf.SeekBits(0, io.SeekCurrent)
|
||||
}
|
||||
|
||||
func (d *D) Pos() int64 {
|
||||
bPos, err := d.bitBuf.Pos()
|
||||
bPos, err := d.TryPos()
|
||||
if err != nil {
|
||||
panic(IOError{Err: err, Op: "Pos", ReadSize: 0, Pos: bPos})
|
||||
}
|
||||
return bPos
|
||||
}
|
||||
|
||||
func (d *D) TryLen() (int64, error) {
|
||||
return bitioextra.Len(d.bitBuf)
|
||||
}
|
||||
|
||||
func (d *D) Len() int64 {
|
||||
return d.bitBuf.Len()
|
||||
l, err := d.TryLen()
|
||||
if err != nil {
|
||||
panic(IOError{Err: err, Op: "Len"})
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
////
|
||||
|
||||
// BitBufLen reads nBits
|
||||
func (d *D) TryBitBufLen(nBits int64) (bitio.ReaderAtSeeker, error) {
|
||||
bPos, err := d.TryPos()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
br, err := d.TryBitBufRange(bPos, nBits)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := d.TrySeekRel(nBits); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return br, nil
|
||||
}
|
||||
|
||||
////
|
||||
|
||||
// End is true if current position is at the end
|
||||
func (d *D) TryEnd() (bool, error) {
|
||||
bPos, err := d.TryPos()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
bLen, err := d.TryLen()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return bPos >= bLen, nil
|
||||
}
|
||||
|
||||
func (d *D) End() bool {
|
||||
bEnd, err := d.bitBuf.End()
|
||||
bEnd, err := d.TryEnd()
|
||||
if err != nil {
|
||||
panic(IOError{Err: err, Op: "Len", ReadSize: 0, Pos: d.Pos()})
|
||||
panic(IOError{Err: err, Op: "End", ReadSize: 0, Pos: d.Pos()})
|
||||
}
|
||||
return bEnd
|
||||
}
|
||||
|
||||
func (d *D) NotEnd() bool { return !d.End() }
|
||||
|
||||
// BitsLeft number of bits left until end
|
||||
func (d *D) TryBitsLeft() (int64, error) {
|
||||
bPos, err := d.TryPos()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
bLen, err := d.TryLen()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return bLen - bPos, nil
|
||||
}
|
||||
|
||||
func (d *D) BitsLeft() int64 {
|
||||
bBitsLeft, err := d.bitBuf.BitsLeft()
|
||||
bBitsLeft, err := d.TryBitsLeft()
|
||||
if err != nil {
|
||||
panic(IOError{Err: err, Op: "BitsLeft", ReadSize: 0, Pos: d.Pos()})
|
||||
}
|
||||
return bBitsLeft
|
||||
}
|
||||
|
||||
// AlignBits number of bits to next nBits align
|
||||
func (d *D) TryAlignBits(nBits int) (int, error) {
|
||||
bPos, err := d.TryPos()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int((int64(nBits) - (bPos % int64(nBits))) % int64(nBits)), nil
|
||||
}
|
||||
|
||||
func (d *D) AlignBits(nBits int) int {
|
||||
bByteAlignBits, err := d.bitBuf.AlignBits(nBits)
|
||||
bByteAlignBits, err := d.TryAlignBits(nBits)
|
||||
if err != nil {
|
||||
panic(IOError{Err: err, Op: "AlignBits", ReadSize: 0, Pos: d.Pos()})
|
||||
}
|
||||
return bByteAlignBits
|
||||
}
|
||||
|
||||
// ByteAlignBits number of bits to next byte align
|
||||
func (d *D) TryByteAlignBits() (int, error) {
|
||||
return d.TryAlignBits(8)
|
||||
}
|
||||
|
||||
func (d *D) ByteAlignBits() int {
|
||||
bByteAlignBits, err := d.bitBuf.ByteAlignBits()
|
||||
bByteAlignBits, err := d.TryByteAlignBits()
|
||||
if err != nil {
|
||||
panic(IOError{Err: err, Op: "ByteAlignBits", ReadSize: 0, Pos: d.Pos()})
|
||||
}
|
||||
return bByteAlignBits
|
||||
}
|
||||
|
||||
// BytePos byte position of current bit position
|
||||
func (d *D) TryBytePos() (int64, error) {
|
||||
bPos, err := d.TryPos()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return bPos & 0x7, nil
|
||||
}
|
||||
|
||||
func (d *D) BytePos() int64 {
|
||||
bBytePos, err := d.bitBuf.BytePos()
|
||||
bBytePos, err := d.TryBytePos()
|
||||
if err != nil {
|
||||
panic(IOError{Err: err, Op: "BytePos", ReadSize: 0, Pos: d.Pos()})
|
||||
}
|
||||
return bBytePos
|
||||
}
|
||||
|
||||
func (d *D) seekAbs(pos int64, name string, fns ...func(d *D)) int64 {
|
||||
func (d *D) trySeekAbs(pos int64, fns ...func(d *D)) (int64, error) {
|
||||
var oldPos int64
|
||||
if len(fns) > 0 {
|
||||
oldPos = d.Pos()
|
||||
}
|
||||
|
||||
pos, err := d.bitBuf.SeekAbs(pos)
|
||||
pos, err := d.bitBuf.SeekBits(pos, io.SeekStart)
|
||||
if err != nil {
|
||||
panic(IOError{Err: err, Op: name, SeekPos: pos, Pos: d.Pos()})
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if len(fns) > 0 {
|
||||
for _, fn := range fns {
|
||||
fn(d)
|
||||
}
|
||||
_, err := d.bitBuf.SeekAbs(oldPos)
|
||||
_, err := d.bitBuf.SeekBits(oldPos, io.SeekStart)
|
||||
if err != nil {
|
||||
panic(IOError{Err: err, Op: name, SeekPos: pos, Pos: d.Pos()})
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
return pos
|
||||
return pos, nil
|
||||
}
|
||||
|
||||
func (d *D) SeekRel(deltaPos int64, fns ...func(d *D)) int64 {
|
||||
return d.seekAbs(d.Pos()+deltaPos, "SeekRel", fns...)
|
||||
// SeekRel seeks relative to current bit position
|
||||
func (d *D) TrySeekRel(delta int64, fns ...func(d *D)) (int64, error) {
|
||||
return d.trySeekAbs(d.Pos()+delta, fns...)
|
||||
}
|
||||
|
||||
func (d *D) SeekRel(delta int64, fns ...func(d *D)) int64 {
|
||||
n, err := d.trySeekAbs(d.Pos()+delta, fns...)
|
||||
if err != nil {
|
||||
d.IOPanic(err, "SeekRel")
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// SeekAbs seeks to absolute position
|
||||
func (d *D) TrySeekAbs(pos int64, fns ...func(d *D)) (int64, error) {
|
||||
return d.trySeekAbs(pos, fns...)
|
||||
}
|
||||
|
||||
func (d *D) SeekAbs(pos int64, fns ...func(d *D)) int64 {
|
||||
return d.seekAbs(pos, "SeekAbs", fns...)
|
||||
n, err := d.trySeekAbs(pos, fns...)
|
||||
if err != nil {
|
||||
d.IOPanic(err, "SeekAbs")
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (d *D) AddChild(v *Value) {
|
||||
@ -659,7 +838,7 @@ func (d *D) FieldValueNil(name string, sms ...scalar.Mapper) {
|
||||
|
||||
func (d *D) FieldValueRaw(name string, a []byte, sms ...scalar.Mapper) {
|
||||
d.FieldScalarFn(name, func(_ scalar.S) (scalar.S, error) {
|
||||
return scalar.S{Actual: bitio.NewBufferFromBytes(a, -1)}, nil
|
||||
return scalar.S{Actual: bitio.NewBitReader(a, -1)}, nil
|
||||
}, sms...)
|
||||
}
|
||||
|
||||
@ -685,11 +864,11 @@ func (d *D) RangeFn(firstBit int64, nBits int64, fn func(d *D)) {
|
||||
}
|
||||
|
||||
// TODO: do some kind of DecodeLimitedLen/RangeFn?
|
||||
bb := d.BitBufRange(0, firstBit+nBits)
|
||||
if _, err := bb.SeekAbs(firstBit); err != nil {
|
||||
br := d.BitBufRange(0, firstBit+nBits)
|
||||
if _, err := br.SeekBits(firstBit, io.SeekStart); err != nil {
|
||||
d.IOPanic(err, "RangeFn: SeekAbs")
|
||||
}
|
||||
sd := d.FieldDecoder("", bb, subV)
|
||||
sd := d.FieldDecoder("", br, subV)
|
||||
|
||||
fn(sd)
|
||||
|
||||
@ -735,7 +914,7 @@ func (d *D) Format(group Group, inArg interface{}) interface{} {
|
||||
panic("unreachable")
|
||||
}
|
||||
|
||||
if _, err := d.bitBuf.SeekRel(dv.Range.Len); err != nil {
|
||||
if _, err := d.bitBuf.SeekBits(dv.Range.Len, io.SeekCurrent); err != nil {
|
||||
d.IOPanic(err, "Format: SeekRel")
|
||||
}
|
||||
|
||||
@ -757,7 +936,7 @@ func (d *D) TryFieldFormat(name string, group Group, inArg interface{}) (*Value,
|
||||
}
|
||||
|
||||
d.AddChild(dv)
|
||||
if _, err := d.bitBuf.SeekRel(dv.Range.Len); err != nil {
|
||||
if _, err := d.bitBuf.SeekBits(dv.Range.Len, io.SeekCurrent); err != nil {
|
||||
d.IOPanic(err, "TryFieldFormat: SeekRel")
|
||||
}
|
||||
|
||||
@ -787,7 +966,7 @@ func (d *D) TryFieldFormatLen(name string, nBits int64, group Group, inArg inter
|
||||
}
|
||||
|
||||
d.AddChild(dv)
|
||||
if _, err := d.bitBuf.SeekRel(nBits); err != nil {
|
||||
if _, err := d.bitBuf.SeekBits(nBits, io.SeekCurrent); err != nil {
|
||||
d.IOPanic(err, "TryFieldFormatLen: SeekRel")
|
||||
}
|
||||
|
||||
@ -831,8 +1010,8 @@ func (d *D) FieldFormatRange(name string, firstBit int64, nBits int64, group Gro
|
||||
return dv, v
|
||||
}
|
||||
|
||||
func (d *D) TryFieldFormatBitBuf(name string, bb *bitio.Buffer, group Group, inArg interface{}) (*Value, interface{}, error) {
|
||||
dv, v, err := decode(d.Ctx, bb, group, Options{
|
||||
func (d *D) TryFieldFormatBitBuf(name string, br bitio.ReaderAtSeeker, group Group, inArg interface{}) (*Value, interface{}, error) {
|
||||
dv, v, err := decode(d.Ctx, br, group, Options{
|
||||
Name: name,
|
||||
Force: d.Options.Force,
|
||||
FillGaps: true,
|
||||
@ -851,8 +1030,8 @@ func (d *D) TryFieldFormatBitBuf(name string, bb *bitio.Buffer, group Group, inA
|
||||
return dv, v, err
|
||||
}
|
||||
|
||||
func (d *D) FieldFormatBitBuf(name string, bb *bitio.Buffer, group Group, inArg interface{}) (*Value, interface{}) {
|
||||
dv, v, err := d.TryFieldFormatBitBuf(name, bb, group, inArg)
|
||||
func (d *D) FieldFormatBitBuf(name string, br bitio.ReaderAtSeeker, group Group, inArg interface{}) (*Value, interface{}) {
|
||||
dv, v, err := d.TryFieldFormatBitBuf(name, br, group, inArg)
|
||||
if dv == nil || dv.Errors() != nil {
|
||||
d.IOPanic(err, "FieldFormatBitBuf: TryFieldFormatBitBuf")
|
||||
}
|
||||
@ -861,23 +1040,31 @@ func (d *D) FieldFormatBitBuf(name string, bb *bitio.Buffer, group Group, inArg
|
||||
}
|
||||
|
||||
// TODO: rethink these
|
||||
func (d *D) FieldRootBitBuf(name string, bb *bitio.Buffer, sms ...scalar.Mapper) *Value {
|
||||
|
||||
func (d *D) FieldRootBitBuf(name string, br bitio.ReaderAtSeeker, sms ...scalar.Mapper) *Value {
|
||||
brLen, err := bitioextra.Len(br)
|
||||
if err != nil {
|
||||
d.IOPanic(err, "br Len")
|
||||
}
|
||||
|
||||
v := &Value{}
|
||||
v.V = &scalar.S{Actual: bb}
|
||||
v.V = &scalar.S{Actual: br}
|
||||
v.Name = name
|
||||
v.RootBitBuf = bb
|
||||
v.RootBitBuf = br
|
||||
v.IsRoot = true
|
||||
v.Range = ranges.Range{Start: d.Pos(), Len: bb.Len()}
|
||||
v.Range = ranges.Range{Start: d.Pos(), Len: brLen}
|
||||
|
||||
if err := v.TryScalarFn(sms...); err != nil {
|
||||
d.Fatalf("%v", err)
|
||||
}
|
||||
|
||||
d.AddChild(v)
|
||||
|
||||
return v
|
||||
}
|
||||
|
||||
func (d *D) FieldArrayRootBitBufFn(name string, bb *bitio.Buffer, fn func(d *D)) *Value {
|
||||
cd := d.FieldDecoder(name, bb, &Compound{IsArray: true})
|
||||
func (d *D) FieldArrayRootBitBufFn(name string, br bitio.ReaderAtSeeker, fn func(d *D)) *Value {
|
||||
cd := d.FieldDecoder(name, br, &Compound{IsArray: true})
|
||||
cd.Value.IsRoot = true
|
||||
d.AddChild(cd.Value)
|
||||
fn(cd)
|
||||
@ -887,8 +1074,8 @@ func (d *D) FieldArrayRootBitBufFn(name string, bb *bitio.Buffer, fn func(d *D))
|
||||
return cd.Value
|
||||
}
|
||||
|
||||
func (d *D) FieldStructRootBitBufFn(name string, bb *bitio.Buffer, fn func(d *D)) *Value {
|
||||
cd := d.FieldDecoder(name, bb, &Compound{})
|
||||
func (d *D) FieldStructRootBitBufFn(name string, br bitio.ReaderAtSeeker, fn func(d *D)) *Value {
|
||||
cd := d.FieldDecoder(name, br, &Compound{})
|
||||
cd.Value.IsRoot = true
|
||||
d.AddChild(cd.Value)
|
||||
fn(cd)
|
||||
@ -900,55 +1087,56 @@ func (d *D) FieldStructRootBitBufFn(name string, bb *bitio.Buffer, fn func(d *D)
|
||||
|
||||
// TODO: range?
|
||||
func (d *D) FieldFormatReaderLen(name string, nBits int64, fn func(r io.Reader) (io.ReadCloser, error), group Group) (*Value, interface{}) {
|
||||
bb, err := d.bitBuf.BitBufLen(nBits)
|
||||
br, err := d.TryBitBufLen(nBits)
|
||||
if err != nil {
|
||||
d.IOPanic(err, "FieldFormatReaderLen: BitBufLen")
|
||||
}
|
||||
zr, err := fn(bb)
|
||||
|
||||
bbBR := bitio.NewIOReader(br)
|
||||
r, err := fn(bbBR)
|
||||
if err != nil {
|
||||
d.IOPanic(err, "FieldFormatReaderLen: fn")
|
||||
}
|
||||
zd, err := ioutil.ReadAll(zr)
|
||||
rBuf, err := ioutil.ReadAll(r)
|
||||
if err != nil {
|
||||
d.IOPanic(err, "FieldFormatReaderLen: ReadAll")
|
||||
}
|
||||
zbb := bitio.NewBufferFromBytes(zd, -1)
|
||||
rBR := bitio.NewBitReader(rBuf, -1)
|
||||
|
||||
return d.FieldFormatBitBuf(name, zbb, group, nil)
|
||||
return d.FieldFormatBitBuf(name, rBR, group, nil)
|
||||
}
|
||||
|
||||
// TODO: too mant return values
|
||||
func (d *D) TryFieldReaderRangeFormat(name string, startBit int64, nBits int64, fn func(r io.Reader) io.Reader, group Group, inArg interface{}) (int64, *bitio.Buffer, *Value, interface{}, error) {
|
||||
func (d *D) TryFieldReaderRangeFormat(name string, startBit int64, nBits int64, fn func(r io.Reader) io.Reader, group Group, inArg interface{}) (int64, bitio.ReaderAtSeeker, *Value, interface{}, error) {
|
||||
bitLen := nBits
|
||||
if bitLen == -1 {
|
||||
bitLen = d.BitsLeft()
|
||||
}
|
||||
bb, err := d.bitBuf.BitBufRange(startBit, bitLen)
|
||||
br, err := d.TryBitBufRange(startBit, bitLen)
|
||||
if err != nil {
|
||||
return 0, nil, nil, nil, err
|
||||
}
|
||||
r := fn(bb)
|
||||
// TODO: check if io.Closer?
|
||||
rb, err := ioutil.ReadAll(r)
|
||||
r := bitio.NewIOReadSeeker(br)
|
||||
rb, err := ioutil.ReadAll(fn(r))
|
||||
if err != nil {
|
||||
return 0, nil, nil, nil, err
|
||||
}
|
||||
cz, err := bb.Pos()
|
||||
rbb := bitio.NewBufferFromBytes(rb, -1)
|
||||
cz, err := r.Seek(0, io.SeekCurrent)
|
||||
rbr := bitio.NewBitReader(rb, -1)
|
||||
if err != nil {
|
||||
return 0, nil, nil, nil, err
|
||||
}
|
||||
dv, v, err := d.TryFieldFormatBitBuf(name, rbb, group, inArg)
|
||||
dv, v, err := d.TryFieldFormatBitBuf(name, rbr, group, inArg)
|
||||
|
||||
return cz, rbb, dv, v, err
|
||||
return cz * 8, rbr, dv, v, err
|
||||
}
|
||||
|
||||
func (d *D) FieldReaderRangeFormat(name string, startBit int64, nBits int64, fn func(r io.Reader) io.Reader, group Group, inArg interface{}) (int64, *bitio.Buffer, *Value, interface{}) {
|
||||
cz, rbb, dv, v, err := d.TryFieldReaderRangeFormat(name, startBit, nBits, fn, group, inArg)
|
||||
func (d *D) FieldReaderRangeFormat(name string, startBit int64, nBits int64, fn func(r io.Reader) io.Reader, group Group, inArg interface{}) (int64, bitio.ReaderAtSeeker, *Value, interface{}) {
|
||||
cz, rBR, dv, v, err := d.TryFieldReaderRangeFormat(name, startBit, nBits, fn, group, inArg)
|
||||
if err != nil {
|
||||
d.IOPanic(err, "TryFieldReaderRangeFormat")
|
||||
}
|
||||
return cz, rbb, dv, v
|
||||
return cz, rBR, dv, v
|
||||
}
|
||||
|
||||
func (d *D) TryFieldValue(name string, fn func() (*Value, error)) (*Value, error) {
|
||||
|
@ -62,7 +62,7 @@ func (d *D) FieldScalarBigIntFn(name string, fn func(d *D) *big.Int, sms ...scal
|
||||
// Type BitBuf
|
||||
|
||||
// TryFieldBitBufScalarFn tries to add a field, calls scalar functions and returns actual value as a BitBuf
|
||||
func (d *D) TryFieldBitBufScalarFn(name string, fn func(d *D) (scalar.S, error), sms ...scalar.Mapper) (*bitio.Buffer, error) {
|
||||
func (d *D) TryFieldBitBufScalarFn(name string, fn func(d *D) (scalar.S, error), sms ...scalar.Mapper) (bitio.ReaderAtSeeker, error) {
|
||||
v, err := d.TryFieldScalarFn(name, func(_ scalar.S) (scalar.S, error) { return fn(d) }, sms...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -71,7 +71,7 @@ func (d *D) TryFieldBitBufScalarFn(name string, fn func(d *D) (scalar.S, error),
|
||||
}
|
||||
|
||||
// FieldBitBufScalarFn adds a field, calls scalar functions and returns actual value as a BitBuf
|
||||
func (d *D) FieldBitBufScalarFn(name string, fn func(d *D) scalar.S, sms ...scalar.Mapper) *bitio.Buffer {
|
||||
func (d *D) FieldBitBufScalarFn(name string, fn func(d *D) scalar.S, sms ...scalar.Mapper) bitio.ReaderAtSeeker {
|
||||
v, err := d.TryFieldScalarFn(name, func(_ scalar.S) (scalar.S, error) { return fn(d), nil }, sms...)
|
||||
if err != nil {
|
||||
panic(IOError{Err: err, Name: name, Op: "BitBuf", Pos: d.Pos()})
|
||||
@ -79,30 +79,30 @@ func (d *D) FieldBitBufScalarFn(name string, fn func(d *D) scalar.S, sms ...scal
|
||||
return v.ActualBitBuf()
|
||||
}
|
||||
|
||||
// FieldBitBufFn adds a field, calls *bitio.Buffer decode function and returns actual value as a BitBuf
|
||||
func (d *D) FieldBitBufFn(name string, fn func(d *D) *bitio.Buffer, sms ...scalar.Mapper) *bitio.Buffer {
|
||||
// FieldBitBufFn adds a field, calls bitio.ReaderAtSeeker decode function and returns actual value as a BitBuf
|
||||
func (d *D) FieldBitBufFn(name string, fn func(d *D) bitio.ReaderAtSeeker, sms ...scalar.Mapper) bitio.ReaderAtSeeker {
|
||||
return d.FieldBitBufScalarFn(name, func(d *D) scalar.S { return scalar.S{Actual: fn(d)} }, sms...)
|
||||
}
|
||||
|
||||
// TryFieldBitBufFn tries to add a field, calls *bitio.Buffer decode function and returns actual value as a BitBuf
|
||||
func (d *D) TryFieldBitBufFn(name string, fn func(d *D) (*bitio.Buffer, error), sms ...scalar.Mapper) (*bitio.Buffer, error) {
|
||||
// TryFieldBitBufFn tries to add a field, calls bitio.ReaderAtSeeker decode function and returns actual value as a BitBuf
|
||||
func (d *D) TryFieldBitBufFn(name string, fn func(d *D) (bitio.ReaderAtSeeker, error), sms ...scalar.Mapper) (bitio.ReaderAtSeeker, error) {
|
||||
return d.TryFieldBitBufScalarFn(name, func(d *D) (scalar.S, error) {
|
||||
v, err := fn(d)
|
||||
return scalar.S{Actual: v}, err
|
||||
}, sms...)
|
||||
}
|
||||
|
||||
// TryFieldScalarBitBufFn tries to add a field, calls *bitio.Buffer decode function and returns scalar
|
||||
func (d *D) TryFieldScalarBitBufFn(name string, fn func(d *D) (*bitio.Buffer, error), sms ...scalar.Mapper) (*scalar.S, error) {
|
||||
// TryFieldScalarBitBufFn tries to add a field, calls bitio.ReaderAtSeeker decode function and returns scalar
|
||||
func (d *D) TryFieldScalarBitBufFn(name string, fn func(d *D) (bitio.ReaderAtSeeker, error), sms ...scalar.Mapper) (*scalar.S, error) {
|
||||
return d.TryFieldScalarFn(name, func(_ scalar.S) (scalar.S, error) {
|
||||
v, err := fn(d)
|
||||
return scalar.S{Actual: v}, err
|
||||
}, sms...)
|
||||
}
|
||||
|
||||
// FieldScalarBitBufFn tries to add a field, calls *bitio.Buffer decode function and returns scalar
|
||||
func (d *D) FieldScalarBitBufFn(name string, fn func(d *D) *bitio.Buffer, sms ...scalar.Mapper) *scalar.S {
|
||||
v, err := d.TryFieldScalarBitBufFn(name, func(d *D) (*bitio.Buffer, error) { return fn(d), nil }, sms...)
|
||||
// FieldScalarBitBufFn tries to add a field, calls bitio.ReaderAtSeeker decode function and returns scalar
|
||||
func (d *D) FieldScalarBitBufFn(name string, fn func(d *D) bitio.ReaderAtSeeker, sms ...scalar.Mapper) *scalar.S {
|
||||
v, err := d.TryFieldScalarBitBufFn(name, func(d *D) (bitio.ReaderAtSeeker, error) { return fn(d), nil }, sms...)
|
||||
if err != nil {
|
||||
panic(IOError{Err: err, Name: name, Op: "BitBuf", Pos: d.Pos()})
|
||||
}
|
||||
@ -758,10 +758,10 @@ func (d *D) ValidateURange(start, end uint64) scalar.Mapper {
|
||||
// Reader RawLen
|
||||
|
||||
// TryRawLen tries to read nBits raw bits
|
||||
func (d *D) TryRawLen(nBits int64) (*bitio.Buffer, error) { return d.tryBitBuf(nBits) }
|
||||
func (d *D) TryRawLen(nBits int64) (bitio.ReaderAtSeeker, error) { return d.tryBitBuf(nBits) }
|
||||
|
||||
// RawLen reads nBits raw bits
|
||||
func (d *D) RawLen(nBits int64) *bitio.Buffer {
|
||||
func (d *D) RawLen(nBits int64) bitio.ReaderAtSeeker {
|
||||
v, err := d.tryBitBuf(nBits)
|
||||
if err != nil {
|
||||
panic(IOError{Err: err, Op: "RawLen", Pos: d.Pos()})
|
||||
@ -792,13 +792,13 @@ func (d *D) FieldScalarRawLen(name string, nBits int64, sms ...scalar.Mapper) *s
|
||||
}
|
||||
|
||||
// TryFieldRawLen tries to add a field and read nBits raw bits
|
||||
func (d *D) TryFieldRawLen(name string, nBits int64, sms ...scalar.Mapper) (*bitio.Buffer, error) {
|
||||
func (d *D) TryFieldRawLen(name string, nBits int64, sms ...scalar.Mapper) (bitio.ReaderAtSeeker, error) {
|
||||
s, err := d.TryFieldScalarRawLen(name, nBits, sms...)
|
||||
return s.ActualBitBuf(), err
|
||||
}
|
||||
|
||||
// FieldRawLen adds a field and reads nBits raw bits
|
||||
func (d *D) FieldRawLen(name string, nBits int64, sms ...scalar.Mapper) *bitio.Buffer {
|
||||
func (d *D) FieldRawLen(name string, nBits int64, sms ...scalar.Mapper) bitio.ReaderAtSeeker {
|
||||
return d.FieldScalarRawLen(name, nBits, sms...).ActualBitBuf()
|
||||
}
|
||||
|
||||
|
@ -12,8 +12,8 @@ import (
|
||||
"golang.org/x/text/encoding/unicode"
|
||||
)
|
||||
|
||||
func (d *D) tryBitBuf(nBits int64) (*bitio.Buffer, error) {
|
||||
return d.bitBuf.BitBufLen(nBits)
|
||||
func (d *D) tryBitBuf(nBits int64) (bitio.ReaderAtSeeker, error) {
|
||||
return d.TryBitBufLen(nBits)
|
||||
}
|
||||
|
||||
func (d *D) tryUEndian(nBits int, endian Endian) (uint64, error) {
|
||||
@ -25,7 +25,7 @@ func (d *D) tryUEndian(nBits int, endian Endian) (uint64, error) {
|
||||
return 0, err
|
||||
}
|
||||
if endian == LittleEndian {
|
||||
n = bitio.Uint64ReverseBytes(nBits, n)
|
||||
n = bitio.ReverseBytes64(nBits, n)
|
||||
}
|
||||
|
||||
return n, nil
|
||||
@ -64,7 +64,7 @@ func (d *D) tryBigIntEndianSign(nBits int, endian Endian, sign bool) (*big.Int,
|
||||
}
|
||||
b := int(bitio.BitsByteCount(int64(nBits)))
|
||||
buf := d.SharedReadBuf(b)[0:b]
|
||||
_, err := bitio.ReadFull(d.bitBuf, buf, nBits)
|
||||
_, err := bitio.ReadFull(d.bitBuf, buf, int64(nBits))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -93,7 +93,7 @@ func (d *D) tryFEndian(nBits int, endian Endian) (float64, error) {
|
||||
return 0, err
|
||||
}
|
||||
if endian == LittleEndian {
|
||||
n = bitio.Uint64ReverseBytes(nBits, n)
|
||||
n = bitio.ReverseBytes64(nBits, n)
|
||||
}
|
||||
switch nBits {
|
||||
case 16:
|
||||
@ -116,7 +116,7 @@ func (d *D) tryFPEndian(nBits int, fBits int, endian Endian) (float64, error) {
|
||||
return 0, err
|
||||
}
|
||||
if endian == LittleEndian {
|
||||
n = bitio.Uint64ReverseBytes(nBits, n)
|
||||
n = bitio.ReverseBytes64(nBits, n)
|
||||
}
|
||||
return float64(n) / float64(uint64(1<<fBits)), nil
|
||||
}
|
||||
@ -135,7 +135,7 @@ func (d *D) tryText(nBytes int, e encoding.Encoding) (string, error) {
|
||||
return "", fmt.Errorf("tryText nBytes %d outside buffer, %d bytes left", nBytes, bytesLeft)
|
||||
}
|
||||
|
||||
bs, err := d.bitBuf.BytesLen(nBytes)
|
||||
bs, err := d.TryBytesLen(nBytes)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@ -173,7 +173,7 @@ func (d *D) tryTextLenPrefixed(lenBits int, fixedBytes int, e encoding.Encoding)
|
||||
}
|
||||
}
|
||||
|
||||
bs, err := d.bitBuf.BytesLen(n)
|
||||
bs, err := d.TryBytesLen(n)
|
||||
if err != nil {
|
||||
d.SeekAbs(p)
|
||||
return "", err
|
||||
@ -192,7 +192,7 @@ func (d *D) tryTextNull(nullBytes int, e encoding.Encoding) (string, error) {
|
||||
return "", err
|
||||
}
|
||||
n := (int(peekBits) / 8) + nullBytes
|
||||
bs, err := d.bitBuf.BytesLen(n)
|
||||
bs, err := d.TryBytesLen(n)
|
||||
if err != nil {
|
||||
d.SeekAbs(p)
|
||||
return "", err
|
||||
@ -210,7 +210,7 @@ func (d *D) tryTextNullLen(fixedBytes int, e encoding.Encoding) (string, error)
|
||||
return "", fmt.Errorf("tryTextNullLen fixedBytes %d outside, %d bytes left", fixedBytes, bytesLeft)
|
||||
}
|
||||
|
||||
bs, err := d.bitBuf.BytesLen(fixedBytes)
|
||||
bs, err := d.TryBytesLen(fixedBytes)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
@ -5,12 +5,13 @@ import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
|
||||
"github.com/wader/fq/internal/bitioextra"
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
|
||||
func bitBufIsZero(s scalar.S, isValidate bool) (scalar.S, error) {
|
||||
bb, ok := s.Actual.(*bitio.Buffer)
|
||||
br, ok := s.Actual.(bitio.ReaderAtSeeker)
|
||||
if !ok {
|
||||
return s, nil
|
||||
}
|
||||
@ -18,13 +19,17 @@ func bitBufIsZero(s scalar.S, isValidate bool) (scalar.S, error) {
|
||||
isZero := true
|
||||
// TODO: shared
|
||||
b := make([]byte, 32*1024)
|
||||
bLen := len(b) * 8
|
||||
bbLeft := int(bb.Len())
|
||||
bbPos := int64(0)
|
||||
bLen := int64(len(b)) * 8
|
||||
brLen, err := bitioextra.Len(br)
|
||||
if err != nil {
|
||||
return scalar.S{}, err
|
||||
}
|
||||
brLeft := brLen
|
||||
brPos := int64(0)
|
||||
|
||||
for bbLeft > 0 {
|
||||
rl := bbLeft
|
||||
if bbLeft > bLen {
|
||||
for brLeft > 0 {
|
||||
rl := brLeft
|
||||
if brLeft > bLen {
|
||||
rl = bLen
|
||||
}
|
||||
// zero last byte if uneven read
|
||||
@ -32,11 +37,11 @@ func bitBufIsZero(s scalar.S, isValidate bool) (scalar.S, error) {
|
||||
b[rl/8] = 0
|
||||
}
|
||||
|
||||
n, err := bitio.ReadAtFull(bb, b, rl, bbPos)
|
||||
n, err := bitio.ReadAtFull(br, b, rl, brPos)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
nb := int(bitio.BitsByteCount(int64(n)))
|
||||
nb := int(bitio.BitsByteCount(n))
|
||||
|
||||
for i := 0; i < nb; i++ {
|
||||
if b[i] != 0 {
|
||||
@ -45,7 +50,7 @@ func bitBufIsZero(s scalar.S, isValidate bool) (scalar.S, error) {
|
||||
}
|
||||
}
|
||||
|
||||
bbLeft -= n
|
||||
brLeft -= n
|
||||
}
|
||||
|
||||
if isZero {
|
||||
@ -74,12 +79,12 @@ func (d *D) BitBufValidateIsZero() scalar.Mapper {
|
||||
|
||||
// TODO: generate?
|
||||
func assertBitBuf(s scalar.S, isErr bool, bss ...[]byte) (scalar.S, error) {
|
||||
ab, err := s.ActualBitBuf().Bytes()
|
||||
if err != nil {
|
||||
bb := &bytes.Buffer{}
|
||||
if _, err := bitioextra.CopyBits(bb, s.ActualBitBuf()); err != nil {
|
||||
return s, err
|
||||
}
|
||||
for _, bs := range bss {
|
||||
if bytes.Equal(ab, bs) {
|
||||
if bytes.Equal(bb.Bytes(), bs) {
|
||||
s.Description = "valid"
|
||||
return s, nil
|
||||
}
|
||||
|
@ -5,7 +5,7 @@
|
||||
"S": {"go_type": "int64", "zero": "0", "compare": "a == b", "range": "a >= start && a <= end"},
|
||||
"F": {"go_type": "float64", "map_from": false, "zero": "0", "compare": "a == b", "range": "a >= start && a <= end"},
|
||||
"Bool": {"go_type": "bool", "zero": "false", "compare": "a == b"},
|
||||
"BitBuf": {"go_type": "*bitio.Buffer", "zero": "nil", "map_from": false, "map_to": false},
|
||||
"BitBuf": {"go_type": "bitio.ReaderAtSeeker", "zero": "nil", "map_from": false, "map_to": false},
|
||||
"BigInt": {"go_type": "*big.Int", "zero": "nil", "map_from": false, "map_to": false, "compare": "a.Cmp(b) == 0", "range": "a.Cmp(start) >= 0 && a.Cmp(end) <= 0"}
|
||||
},
|
||||
"readers": [
|
||||
@ -57,7 +57,13 @@
|
||||
"name": "UBigInt",
|
||||
"type": "BigInt",
|
||||
"variants": [
|
||||
{"name": "", "args": "nBits", "params": "nBits int", "call": "d.tryBigIntEndianSign(nBits, d.Endian, false)", "doc": "nBits bits signed integer in current endian"},
|
||||
{
|
||||
"name": "",
|
||||
"args": "nBits",
|
||||
"params": "nBits int",
|
||||
"call": "d.tryBigIntEndianSign(nBits, d.Endian, false)",
|
||||
"doc": "nBits bits signed integer in current endian"
|
||||
},
|
||||
{
|
||||
"name": "E",
|
||||
"args": "nBits, endian",
|
||||
@ -65,15 +71,33 @@
|
||||
"call": "d.tryBigIntEndianSign(nBits, endian, false)",
|
||||
"doc": "nBits signed integer in specified endian"
|
||||
},
|
||||
{"name": "LE", "args": "nBits", "params": "nBits int", "call": "d.tryBigIntEndianSign(nBits, LittleEndian, false)", "doc": "nBits bit signed integer in little-endian"},
|
||||
{"name": "BE", "args": "nBits", "params": "nBits int", "call": "d.tryBigIntEndianSign(nBits, BigEndian, false)", "doc": "nBits bit signed integer in big-endian"}
|
||||
{
|
||||
"name": "LE",
|
||||
"args": "nBits",
|
||||
"params": "nBits int",
|
||||
"call": "d.tryBigIntEndianSign(nBits, LittleEndian, false)",
|
||||
"doc": "nBits bit signed integer in little-endian"
|
||||
},
|
||||
{
|
||||
"name": "BE",
|
||||
"args": "nBits",
|
||||
"params": "nBits int",
|
||||
"call": "d.tryBigIntEndianSign(nBits, BigEndian, false)",
|
||||
"doc": "nBits bit signed integer in big-endian"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "SBigInt",
|
||||
"type": "BigInt",
|
||||
"variants": [
|
||||
{"name": "", "args": "nBits", "params": "nBits int", "call": "d.tryBigIntEndianSign(nBits, d.Endian, true)", "doc": "nBits bits signed integer in current endian"},
|
||||
{
|
||||
"name": "",
|
||||
"args": "nBits",
|
||||
"params": "nBits int",
|
||||
"call": "d.tryBigIntEndianSign(nBits, d.Endian, true)",
|
||||
"doc": "nBits bits signed integer in current endian"
|
||||
},
|
||||
{
|
||||
"name": "E",
|
||||
"args": "nBits, endian",
|
||||
@ -81,7 +105,13 @@
|
||||
"call": "d.tryBigIntEndianSign(nBits, endian, true)",
|
||||
"doc": "nBits signed integer in specified endian"
|
||||
},
|
||||
{"name": "LE", "args": "nBits", "params": "nBits int", "call": "d.tryBigIntEndianSign(nBits, LittleEndian, true)", "doc": "nBits bit signed integer in little-endian"},
|
||||
{
|
||||
"name": "LE",
|
||||
"args": "nBits",
|
||||
"params": "nBits int",
|
||||
"call": "d.tryBigIntEndianSign(nBits, LittleEndian, true)",
|
||||
"doc": "nBits bit signed integer in little-endian"
|
||||
},
|
||||
{"name": "BE", "args": "nBits", "params": "nBits int", "call": "d.tryBigIntEndianSign(nBits, BigEndian, true)", "doc": "nBits bit signed integer in big-endian"}
|
||||
]
|
||||
},
|
||||
@ -90,7 +120,13 @@
|
||||
"type": "F",
|
||||
"variants": [
|
||||
{"name": "", "args": "nBits", "params": "nBits int", "call": "d.tryFEndian(nBits, d.Endian)", "doc": "nBit IEEE 754 float in current endian"},
|
||||
{"name": "E", "args": "nBits, endian", "params": "nBits int, endian Endian", "call": "d.tryFEndian(nBits, endian)", "doc": "nBit IEEE 754 float in specified endian"},
|
||||
{
|
||||
"name": "E",
|
||||
"args": "nBits, endian",
|
||||
"params": "nBits int, endian Endian",
|
||||
"call": "d.tryFEndian(nBits, endian)",
|
||||
"doc": "nBit IEEE 754 float in specified endian"
|
||||
},
|
||||
{"name": "16", "args": "", "params": "", "call": "d.tryFEndian(16, d.Endian)", "doc": "16 bit IEEE 754 float in current endian"},
|
||||
{"name": "32", "args": "", "params": "", "call": "d.tryFEndian(32, d.Endian)", "doc": "32 bit IEEE 754 float in current endian"},
|
||||
{"name": "64", "args": "", "params": "", "call": "d.tryFEndian(64, d.Endian)", "doc": "64 bit IEEE 754 float in current endian"},
|
||||
|
@ -27,7 +27,7 @@ type Value struct {
|
||||
V interface{} // scalar.S or Compound (array/struct)
|
||||
Index int // index in parent array/struct
|
||||
Range ranges.Range
|
||||
RootBitBuf *bitio.Buffer
|
||||
RootBitBuf bitio.ReaderAtSeeker
|
||||
IsRoot bool // TODO: rework?
|
||||
}
|
||||
|
||||
|
@ -9,6 +9,7 @@ import (
|
||||
"math/big"
|
||||
|
||||
"github.com/wader/fq/internal/aheadreadseeker"
|
||||
"github.com/wader/fq/internal/bitioextra"
|
||||
"github.com/wader/fq/internal/ctxreadseeker"
|
||||
"github.com/wader/fq/internal/gojqextra"
|
||||
"github.com/wader/fq/internal/ioextra"
|
||||
@ -30,20 +31,20 @@ type ToBuffer interface {
|
||||
ToBuffer() (Buffer, error)
|
||||
}
|
||||
|
||||
func toBitBuf(v interface{}) (*bitio.Buffer, error) {
|
||||
func toBitBuf(v interface{}) (bitio.ReaderAtSeeker, error) {
|
||||
return toBitBufEx(v, false)
|
||||
}
|
||||
|
||||
func toBitBufEx(v interface{}, inArray bool) (*bitio.Buffer, error) {
|
||||
func toBitBufEx(v interface{}, inArray bool) (bitio.ReaderAtSeeker, error) {
|
||||
switch vv := v.(type) {
|
||||
case ToBuffer:
|
||||
bv, err := vv.ToBuffer()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bv.bb.BitBufRange(bv.r.Start, bv.r.Len)
|
||||
return bitioextra.Range(bv.br, bv.r.Start, bv.r.Len)
|
||||
case string:
|
||||
return bitio.NewBufferFromBytes([]byte(vv), -1), nil
|
||||
return bitio.NewBitReader([]byte(vv), -1), nil
|
||||
case int, float64, *big.Int:
|
||||
bi, err := toBigInt(v)
|
||||
if err != nil {
|
||||
@ -56,39 +57,40 @@ func toBitBufEx(v interface{}, inArray bool) (*bitio.Buffer, error) {
|
||||
}
|
||||
n := bi.Uint64()
|
||||
b := [1]byte{byte(n)}
|
||||
return bitio.NewBufferFromBytes(b[:], -1), nil
|
||||
return bitio.NewBitReader(b[:], -1), nil
|
||||
}
|
||||
|
||||
bitLen := int64(bi.BitLen())
|
||||
// bit.Int "The bit length of 0 is 0."
|
||||
if bitLen == 0 {
|
||||
var z [1]byte
|
||||
return bitio.NewBitReader(z[:], 1), nil
|
||||
}
|
||||
// TODO: how should this work? "0xf | tobytes" 4bits or 8bits? now 4
|
||||
//padBefore := (8 - (bi.BitLen() % 8)) % 8
|
||||
padBefore := 0
|
||||
bb, err := bitio.NewBufferFromBytes(bi.Bytes(), -1).BitBufRange(int64(padBefore), int64(bi.BitLen()))
|
||||
padBefore := (8 - (bitLen % 8)) % 8
|
||||
// padBefore := 0
|
||||
br, err := bitioextra.Range(bitio.NewBitReader(bi.Bytes(), -1), padBefore, bitLen)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bb, nil
|
||||
return br, nil
|
||||
case []interface{}:
|
||||
var rr []bitio.BitReadAtSeeker
|
||||
rr := make([]bitio.ReadAtSeeker, 0, len(vv))
|
||||
// TODO: optimize byte array case, flatten into one slice
|
||||
for _, e := range vv {
|
||||
eBB, eErr := toBitBufEx(e, true)
|
||||
eBR, eErr := toBitBufEx(e, true)
|
||||
if eErr != nil {
|
||||
return nil, eErr
|
||||
}
|
||||
rr = append(rr, eBB)
|
||||
rr = append(rr, eBR)
|
||||
}
|
||||
|
||||
mb, err := bitio.NewMultiBitReader(rr)
|
||||
mb, err := bitio.NewMultiBitReader(rr...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
bb, err := bitio.NewBufferFromBitReadSeeker(mb)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return bb, nil
|
||||
return mb, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("value can't be a buffer")
|
||||
}
|
||||
@ -99,11 +101,11 @@ func toBuffer(v interface{}) (Buffer, error) {
|
||||
case ToBuffer:
|
||||
return vv.ToBuffer()
|
||||
default:
|
||||
bb, err := toBitBuf(v)
|
||||
br, err := toBitBuf(v)
|
||||
if err != nil {
|
||||
return Buffer{}, err
|
||||
}
|
||||
return newBufferFromBuffer(bb, 8), nil
|
||||
return newBufferFromBuffer(br, 8)
|
||||
}
|
||||
}
|
||||
|
||||
@ -140,8 +142,15 @@ func (i *Interp) _toBitsRange(c interface{}, a []interface{}) interface{} {
|
||||
bv.unit = unit
|
||||
|
||||
if !r {
|
||||
bb, _ := bv.toBuffer()
|
||||
return newBufferFromBuffer(bb, unit)
|
||||
br, err := bv.toBuffer()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bb, err := newBufferFromBuffer(br, unit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return bb
|
||||
}
|
||||
|
||||
return bv
|
||||
@ -162,12 +171,12 @@ func (of *openFile) Display(w io.Writer, opts Options) error {
|
||||
}
|
||||
|
||||
func (of *openFile) ToBuffer() (Buffer, error) {
|
||||
return newBufferFromBuffer(of.bb, 8), nil
|
||||
return newBufferFromBuffer(of.br, 8)
|
||||
}
|
||||
|
||||
// def open: #:: string| => buffer
|
||||
// opens a file for reading from filesystem
|
||||
// TODO: when to close? when bb loses all refs? need to use finalizer somehow?
|
||||
// TODO: when to close? when br loses all refs? need to use finalizer somehow?
|
||||
func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
|
||||
var err error
|
||||
var f fs.File
|
||||
@ -237,7 +246,7 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
|
||||
|
||||
// bitio.Buffer -> (bitio.Reader) -> aheadreadseeker -> progressreadseeker -> ctxreadseeker -> readseeker
|
||||
|
||||
bbf.bb, err = bitio.NewBufferFromReadSeeker(aheadRs)
|
||||
bbf.br = bitio.NewIOBitReadSeeker(aheadRs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -249,28 +258,34 @@ var _ Value = Buffer{}
|
||||
var _ ToBuffer = Buffer{}
|
||||
|
||||
type Buffer struct {
|
||||
bb *bitio.Buffer
|
||||
br bitio.ReaderAtSeeker
|
||||
r ranges.Range
|
||||
unit int
|
||||
}
|
||||
|
||||
func newBufferFromBuffer(bb *bitio.Buffer, unit int) Buffer {
|
||||
return Buffer{
|
||||
bb: bb,
|
||||
r: ranges.Range{Start: 0, Len: bb.Len()},
|
||||
unit: unit,
|
||||
func newBufferFromBuffer(br bitio.ReaderAtSeeker, unit int) (Buffer, error) {
|
||||
l, err := bitioextra.Len(br)
|
||||
if err != nil {
|
||||
return Buffer{}, err
|
||||
}
|
||||
|
||||
return Buffer{
|
||||
br: br,
|
||||
r: ranges.Range{Start: 0, Len: l},
|
||||
unit: unit,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b Buffer) toBytesBuffer(r ranges.Range) (*bytes.Buffer, error) {
|
||||
bb, err := b.bb.BitBufRange(r.Start, r.Len)
|
||||
br, err := bitioextra.Range(b.br, r.Start, r.Len)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
buf := &bytes.Buffer{}
|
||||
if _, err := io.Copy(buf, bb.Clone()); err != nil {
|
||||
if _, err := bitioextra.CopyBits(buf, br); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
@ -307,7 +322,8 @@ func (b Buffer) JQValueIndex(index int) interface{} {
|
||||
return err
|
||||
}
|
||||
|
||||
extraBits := uint((8 - b.r.Len%8) % 8)
|
||||
extraBits := uint((8 - b.unit%8) % 8)
|
||||
|
||||
return new(big.Int).Rsh(new(big.Int).SetBytes(buf.Bytes()), extraBits)
|
||||
}
|
||||
func (b Buffer) JQValueSlice(start int, end int) interface{} {
|
||||
@ -315,7 +331,7 @@ func (b Buffer) JQValueSlice(start int, end int) interface{} {
|
||||
rLen := int64((end - start) * b.unit)
|
||||
|
||||
return Buffer{
|
||||
bb: b.bb,
|
||||
br: b.br,
|
||||
r: ranges.Range{Start: b.r.Start + rStart, Len: rLen},
|
||||
unit: b.unit,
|
||||
}
|
||||
@ -337,12 +353,12 @@ func (b Buffer) JQValueKey(name string) interface{} {
|
||||
if b.unit == 1 {
|
||||
return b
|
||||
}
|
||||
return Buffer{bb: b.bb, r: b.r, unit: 1}
|
||||
return Buffer{br: b.br, r: b.r, unit: 1}
|
||||
case "bytes":
|
||||
if b.unit == 8 {
|
||||
return b
|
||||
}
|
||||
return Buffer{bb: b.bb, r: b.r, unit: 8}
|
||||
return Buffer{br: b.br, r: b.r, unit: 8}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -382,19 +398,21 @@ func (b Buffer) JQValueUpdate(key interface{}, u interface{}, delpath bool) inte
|
||||
|
||||
func (b Buffer) Display(w io.Writer, opts Options) error {
|
||||
if opts.RawOutput {
|
||||
bb, err := b.toBuffer()
|
||||
br, err := b.toBuffer()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(w, bb.Clone()); err != nil {
|
||||
|
||||
if _, err := bitioextra.CopyBits(w, br); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return hexdump(w, b, opts)
|
||||
}
|
||||
|
||||
func (b Buffer) toBuffer() (*bitio.Buffer, error) {
|
||||
return b.bb.BitBufRange(b.r.Start, b.r.Len)
|
||||
func (b Buffer) toBuffer() (bitio.ReaderAtSeeker, error) {
|
||||
return bitioextra.Range(b.br, b.r.Start, b.r.Len)
|
||||
}
|
||||
|
@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/wader/fq/internal/bitioextra"
|
||||
"github.com/wader/fq/internal/gojqextra"
|
||||
"github.com/wader/fq/internal/ioextra"
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
@ -201,7 +202,7 @@ func (i *Interp) _decode(c interface{}, a []interface{}) interface{} {
|
||||
return err
|
||||
}
|
||||
|
||||
dv, _, err := decode.Decode(i.evalContext.ctx, bv.bb, decodeFormat,
|
||||
dv, _, err := decode.Decode(i.evalContext.ctx, bv.br, decodeFormat,
|
||||
decode.Options{
|
||||
IsRoot: true,
|
||||
FillGaps: true,
|
||||
@ -273,7 +274,7 @@ func makeDecodeValue(dv *decode.Value) interface{} {
|
||||
return NewStructDecodeValue(dv, vv)
|
||||
case *scalar.S:
|
||||
switch vv := vv.Value().(type) {
|
||||
case *bitio.Buffer:
|
||||
case bitio.ReaderAtSeeker:
|
||||
// is lazy so that in situations where the decode value is only used to
|
||||
// create another buffer we don't have to read and create a string, ex:
|
||||
// .unknown0 | tobytes[1:] | ...
|
||||
@ -283,7 +284,11 @@ func makeDecodeValue(dv *decode.Value) interface{} {
|
||||
IsScalar: true,
|
||||
Fn: func() (gojq.JQValue, error) {
|
||||
buf := &bytes.Buffer{}
|
||||
if _, err := io.Copy(buf, vv.Clone()); err != nil {
|
||||
vvC, err := bitioextra.Clone(vv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := bitioextra.CopyBits(buf, vvC); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return gojqextra.String([]rune(buf.String())), nil
|
||||
@ -360,7 +365,7 @@ func (dvb decodeValueBase) DecodeValue() *decode.Value {
|
||||
|
||||
func (dvb decodeValueBase) Display(w io.Writer, opts Options) error { return dump(dvb.dv, w, opts) }
|
||||
func (dvb decodeValueBase) ToBuffer() (Buffer, error) {
|
||||
return Buffer{bb: dvb.dv.RootBitBuf, r: dvb.dv.InnerRange(), unit: 8}, nil
|
||||
return Buffer{br: dvb.dv.RootBitBuf, r: dvb.dv.InnerRange(), unit: 8}, nil
|
||||
}
|
||||
func (decodeValueBase) ExtType() string { return "decode_value" }
|
||||
func (dvb decodeValueBase) ExtKeys() []string {
|
||||
@ -475,13 +480,13 @@ func (dvb decodeValueBase) JQValueKey(name string) interface{} {
|
||||
}
|
||||
case "_bits":
|
||||
return Buffer{
|
||||
bb: dv.RootBitBuf,
|
||||
br: dv.RootBitBuf,
|
||||
r: dv.Range,
|
||||
unit: 1,
|
||||
}
|
||||
case "_bytes":
|
||||
return Buffer{
|
||||
bb: dv.RootBitBuf,
|
||||
br: dv.RootBitBuf,
|
||||
r: dv.Range,
|
||||
unit: 8,
|
||||
}
|
||||
@ -542,12 +547,17 @@ func (v decodeValue) JQValueToGoJQEx(optsFn func() Options) interface{} {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bb, err := bv.toBuffer()
|
||||
br, err := bv.toBuffer()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s, err := optsFn().BitsFormatFn(bb.Clone())
|
||||
brC, err := bitioextra.Clone(br)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s, err := optsFn().BitsFormatFn(brC)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -114,7 +114,7 @@ func decoratorFromOptions(opts Options) Decorator {
|
||||
return d.True
|
||||
}
|
||||
return d.False
|
||||
case string, *bitio.Buffer:
|
||||
case string, bitio.Reader:
|
||||
return d.String
|
||||
case nil:
|
||||
return d.Null
|
||||
|
@ -1,6 +1,7 @@
|
||||
package interp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@ -8,6 +9,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/wader/fq/internal/asciiwriter"
|
||||
"github.com/wader/fq/internal/bitioextra"
|
||||
"github.com/wader/fq/internal/columnwriter"
|
||||
"github.com/wader/fq/internal/hexpairwriter"
|
||||
"github.com/wader/fq/internal/mathextra"
|
||||
@ -217,7 +219,12 @@ func dumpEx(v *decode.Value, buf []byte, cw *columnwriter.Writer, depth int, roo
|
||||
printErrs(depth, valueErr)
|
||||
}
|
||||
|
||||
bufferLastBit := rootV.RootBitBuf.Len() - 1
|
||||
rootBitLen, err := bitioextra.Len(rootV.RootBitBuf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
bufferLastBit := rootBitLen - 1
|
||||
startBit := innerRange.Start
|
||||
stopBit := innerRange.Stop() - 1
|
||||
sizeBits := innerRange.Len
|
||||
@ -260,7 +267,7 @@ func dumpEx(v *decode.Value, buf []byte, cw *columnwriter.Writer, depth int, roo
|
||||
cfmt(colAddr, "%s%s\n",
|
||||
rootIndent, deco.DumpAddr.F(mathextra.PadFormatInt(startLineByte, opts.AddrBase, true, addrWidth)))
|
||||
|
||||
vBitBuf, err := rootV.RootBitBuf.BitBufRange(startByte*8, displaySizeBits)
|
||||
vBR, err := bitioextra.Range(rootV.RootBitBuf, startByte*8, displaySizeBits)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -269,19 +276,26 @@ func dumpEx(v *decode.Value, buf []byte, cw *columnwriter.Writer, depth int, roo
|
||||
hexpairFn := func(b byte) string { return deco.ByteColor(b).Wrap(hexpairwriter.Pair(b)) }
|
||||
asciiFn := func(b byte) string { return deco.ByteColor(b).Wrap(asciiwriter.SafeASCII(b)) }
|
||||
|
||||
if vBitBuf != nil {
|
||||
if _, err := io.CopyBuffer(
|
||||
hexpairwriter.New(cw.Columns[colHex], opts.LineBytes, int(startLineByteOffset), hexpairFn),
|
||||
io.LimitReader(vBitBuf.Clone(), displaySizeBytes),
|
||||
buf); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.CopyBuffer(
|
||||
asciiwriter.New(cw.Columns[colASCII], opts.LineBytes, int(startLineByteOffset), asciiFn),
|
||||
io.LimitReader(vBitBuf.Clone(), displaySizeBytes),
|
||||
buf); err != nil {
|
||||
return err
|
||||
}
|
||||
hexBR, err := bitioextra.Clone(vBR)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := bitioextra.CopyBitsBuffer(
|
||||
hexpairwriter.New(cw.Columns[colHex], opts.LineBytes, int(startLineByteOffset), hexpairFn),
|
||||
hexBR,
|
||||
buf); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
asciiBR, err := bitioextra.Clone(vBR)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := bitioextra.CopyBitsBuffer(
|
||||
asciiwriter.New(cw.Columns[colASCII], opts.LineBytes, int(startLineByteOffset), asciiFn),
|
||||
asciiBR,
|
||||
buf); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i := int64(1); i < addrLines; i++ {
|
||||
@ -351,18 +365,31 @@ func dump(v *decode.Value, w io.Writer, opts Options) error {
|
||||
}
|
||||
|
||||
func hexdump(w io.Writer, bv Buffer, opts Options) error {
|
||||
bb, err := bv.toBuffer()
|
||||
br, err := bv.toBuffer()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cBR, err := bitioextra.Clone(bv.br)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
bytesB := &bytes.Buffer{}
|
||||
if _, err := bitioextra.CopyBits(bytesB, cBR); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
biib := bitio.NewBitReader(bytesB.Bytes(), -1)
|
||||
|
||||
// TODO: hack
|
||||
opts.Verbose = true
|
||||
return dump(
|
||||
&decode.Value{
|
||||
// TODO: hack
|
||||
V: &scalar.S{Actual: bb},
|
||||
V: &scalar.S{Actual: br},
|
||||
Range: bv.r,
|
||||
RootBitBuf: bv.bb.Clone(),
|
||||
RootBitBuf: biib,
|
||||
},
|
||||
w,
|
||||
opts,
|
||||
|
@ -65,12 +65,12 @@ func makeStringBitBufTransformFn(
|
||||
return func(c interface{}, a []interface{}) interface{} {
|
||||
switch c := c.(type) {
|
||||
case string:
|
||||
bb, err := toBitBuf(c)
|
||||
br, err := toBitBuf(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r, err := decodeFn(bb)
|
||||
r, err := decodeFn(bitio.NewIOReader(br))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -79,11 +79,14 @@ func makeStringBitBufTransformFn(
|
||||
if _, err := io.Copy(buf, r); err != nil {
|
||||
return err
|
||||
}
|
||||
outBB := bitio.NewBufferFromBytes(buf.Bytes(), -1)
|
||||
|
||||
return newBufferFromBuffer(outBB, 8)
|
||||
bb, err := newBufferFromBuffer(bitio.NewBitReader(buf.Bytes(), -1), 8)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return bb
|
||||
default:
|
||||
bb, err := toBitBuf(c)
|
||||
br, err := toBitBuf(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -94,7 +97,7 @@ func makeStringBitBufTransformFn(
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := io.Copy(w, bb); err != nil {
|
||||
if _, err := io.Copy(w, bitio.NewIOReader(br)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -110,12 +113,12 @@ func makeStringBitBufTransformFn(
|
||||
// transform to buffer using fn
|
||||
func makeBitBufTransformFn(fn func(r io.Reader) (io.Reader, error)) func(c interface{}, a []interface{}) interface{} {
|
||||
return func(c interface{}, a []interface{}) interface{} {
|
||||
inBB, err := toBitBuf(c)
|
||||
inBR, err := toBitBuf(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r, err := fn(inBB)
|
||||
r, err := fn(bitio.NewIOReader(inBR))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -125,16 +128,20 @@ func makeBitBufTransformFn(fn func(r io.Reader) (io.Reader, error)) func(c inter
|
||||
return err
|
||||
}
|
||||
|
||||
outBB := bitio.NewBufferFromBytes(outBuf.Bytes(), -1)
|
||||
outBR := bitio.NewBitReader(outBuf.Bytes(), -1)
|
||||
|
||||
return newBufferFromBuffer(outBB, 8)
|
||||
bb, err := newBufferFromBuffer(outBR, 8)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return bb
|
||||
}
|
||||
}
|
||||
|
||||
// transform to buffer using fn
|
||||
func makeHashFn(fn func() (hash.Hash, error)) func(c interface{}, a []interface{}) interface{} {
|
||||
return func(c interface{}, a []interface{}) interface{} {
|
||||
inBB, err := toBitBuf(c)
|
||||
inBR, err := toBitBuf(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -143,13 +150,17 @@ func makeHashFn(fn func() (hash.Hash, error)) func(c interface{}, a []interface{
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(h, inBB); err != nil {
|
||||
if _, err := io.Copy(h, bitio.NewIOReader(inBR)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
outBB := bitio.NewBufferFromBytes(h.Sum(nil), -1)
|
||||
outBR := bitio.NewBitReader(h.Sum(nil), -1)
|
||||
|
||||
return newBufferFromBuffer(outBB, 8)
|
||||
bb, err := newBufferFromBuffer(outBR, 8)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return bb
|
||||
}
|
||||
}
|
||||
|
||||
@ -223,18 +234,22 @@ func (i *Interp) aesCtr(c interface{}, a []interface{}) interface{} {
|
||||
ivBytes = make([]byte, block.BlockSize())
|
||||
}
|
||||
|
||||
bb, err := toBitBuf(c)
|
||||
br, err := toBitBuf(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
reader := &cipher.StreamReader{S: cipher.NewCTR(block, ivBytes), R: bb}
|
||||
reader := &cipher.StreamReader{S: cipher.NewCTR(block, ivBytes), R: bitio.NewIOReader(br)}
|
||||
if _, err := io.Copy(buf, reader); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return newBufferFromBuffer(bitio.NewBufferFromBytes(buf.Bytes(), -1), 8)
|
||||
bb, err := newBufferFromBuffer(bitio.NewBitReader(buf.Bytes(), -1), 8)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return bb
|
||||
}
|
||||
|
||||
func (i *Interp) _hexdump(c interface{}, a []interface{}) gojq.Iter {
|
||||
|
@ -19,6 +19,7 @@ import (
|
||||
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/wader/fq/internal/ansi"
|
||||
"github.com/wader/fq/internal/bitioextra"
|
||||
"github.com/wader/fq/internal/colorjson"
|
||||
"github.com/wader/fq/internal/ctxstack"
|
||||
"github.com/wader/fq/internal/ioextra"
|
||||
@ -269,12 +270,12 @@ func toBigInt(v interface{}) (*big.Int, error) {
|
||||
func toBytes(v interface{}) ([]byte, error) {
|
||||
switch v := v.(type) {
|
||||
default:
|
||||
bb, err := toBitBuf(v)
|
||||
br, err := toBitBuf(v)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("value is not bytes")
|
||||
}
|
||||
buf := &bytes.Buffer{}
|
||||
if _, err := io.Copy(buf, bb); err != nil {
|
||||
if _, err := bitioextra.CopyBits(buf, br); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@ -907,24 +908,24 @@ type Options struct {
|
||||
SizeBase int `mapstructure:"sizebase"`
|
||||
|
||||
Decorator Decorator
|
||||
BitsFormatFn func(bb *bitio.Buffer) (interface{}, error)
|
||||
BitsFormatFn func(br bitio.ReaderAtSeeker) (interface{}, error)
|
||||
}
|
||||
|
||||
func bitsFormatFnFromOptions(opts Options) func(bb *bitio.Buffer) (interface{}, error) {
|
||||
func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (interface{}, error) {
|
||||
switch opts.BitsFormat {
|
||||
case "md5":
|
||||
return func(bb *bitio.Buffer) (interface{}, error) {
|
||||
return func(br bitio.ReaderAtSeeker) (interface{}, error) {
|
||||
d := md5.New()
|
||||
if _, err := io.Copy(d, bb); err != nil {
|
||||
if _, err := bitioextra.CopyBits(d, br); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(d.Sum(nil)), nil
|
||||
}
|
||||
case "base64":
|
||||
return func(bb *bitio.Buffer) (interface{}, error) {
|
||||
return func(br bitio.ReaderAtSeeker) (interface{}, error) {
|
||||
b := &bytes.Buffer{}
|
||||
e := base64.NewEncoder(base64.StdEncoding, b)
|
||||
if _, err := io.Copy(e, bb); err != nil {
|
||||
if _, err := bitioextra.CopyBits(e, br); err != nil {
|
||||
return "", err
|
||||
}
|
||||
e.Close()
|
||||
@ -932,17 +933,17 @@ func bitsFormatFnFromOptions(opts Options) func(bb *bitio.Buffer) (interface{},
|
||||
}
|
||||
case "truncate":
|
||||
// TODO: configure
|
||||
return func(bb *bitio.Buffer) (interface{}, error) {
|
||||
return func(br bitio.ReaderAtSeeker) (interface{}, error) {
|
||||
b := &bytes.Buffer{}
|
||||
if _, err := io.Copy(b, io.LimitReader(bb, 1024)); err != nil {
|
||||
if _, err := bitioextra.CopyBits(b, bitio.NewLimitReader(br, 1024*8)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return b.String(), nil
|
||||
}
|
||||
case "string":
|
||||
return func(bb *bitio.Buffer) (interface{}, error) {
|
||||
return func(br bitio.ReaderAtSeeker) (interface{}, error) {
|
||||
b := &bytes.Buffer{}
|
||||
if _, err := io.Copy(b, bb); err != nil {
|
||||
if _, err := bitioextra.CopyBits(b, br); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return b.String(), nil
|
||||
@ -950,14 +951,20 @@ func bitsFormatFnFromOptions(opts Options) func(bb *bitio.Buffer) (interface{},
|
||||
case "snippet":
|
||||
fallthrough
|
||||
default:
|
||||
return func(bb *bitio.Buffer) (interface{}, error) {
|
||||
return func(br bitio.ReaderAtSeeker) (interface{}, error) {
|
||||
b := &bytes.Buffer{}
|
||||
e := base64.NewEncoder(base64.StdEncoding, b)
|
||||
if _, err := io.Copy(e, io.LimitReader(bb, 256)); err != nil {
|
||||
if _, err := bitioextra.CopyBits(e, bitio.NewLimitReader(br, 256*8)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
e.Close()
|
||||
return fmt.Sprintf("<%s>%s", mathextra.Bits(bb.Len()).StringByteBits(opts.SizeBase), b.String()), nil
|
||||
|
||||
brLen, err := bitioextra.Len(br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return fmt.Sprintf("<%s>%s", mathextra.Bits(brLen).StringByteBits(opts.SizeBase), b.String()), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -7,6 +7,7 @@ import (
|
||||
|
||||
"github.com/wader/fq/internal/gojqextra"
|
||||
"github.com/wader/fq/internal/ioextra"
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
"github.com/wader/fq/pkg/ranges"
|
||||
"github.com/wader/gojq"
|
||||
)
|
||||
@ -69,7 +70,7 @@ func (i *Interp) _bufferMatch(c interface{}, a []interface{}) gojq.Iter {
|
||||
}
|
||||
sreNames := sre.SubexpNames()
|
||||
|
||||
bb, err := bv.toBuffer()
|
||||
br, err := bv.toBuffer()
|
||||
if err != nil {
|
||||
return gojq.NewIter(err)
|
||||
}
|
||||
@ -83,9 +84,9 @@ func (i *Interp) _bufferMatch(c interface{}, a []interface{}) gojq.Iter {
|
||||
// will match the byte \0xff
|
||||
if byteRunes {
|
||||
// byte mode, read each byte as a rune
|
||||
rr = ioextra.ByteRuneReader{RS: bb}
|
||||
rr = ioextra.ByteRuneReader{RS: bitio.NewIOReadSeeker(br)}
|
||||
} else {
|
||||
rr = ioextra.RuneReadSeeker{RS: bb}
|
||||
rr = ioextra.RuneReadSeeker{RS: bitio.NewIOReadSeeker(br)}
|
||||
}
|
||||
|
||||
var off int64
|
||||
@ -127,7 +128,7 @@ func (i *Interp) _bufferMatch(c interface{}, a []interface{}) gojq.Iter {
|
||||
matchBitOff := (off + int64(start)) * 8
|
||||
matchLength := int64(end-start) * 8
|
||||
bbo := Buffer{
|
||||
bb: bv.bb,
|
||||
br: bv.br,
|
||||
r: ranges.Range{
|
||||
Start: bv.r.Start + matchBitOff,
|
||||
Len: matchLength,
|
||||
|
@ -35,7 +35,7 @@ func previewValue(v interface{}, df scalar.DisplayFormat) string {
|
||||
return fmt.Sprintf("%q", vv)
|
||||
case nil:
|
||||
return "null"
|
||||
case *bitio.Buffer:
|
||||
case bitio.Reader:
|
||||
return "raw bits"
|
||||
case *big.Int:
|
||||
return mathextra.PadFormatBigInt(vv, df.FormatBase(), true, 0)
|
||||
|
68
pkg/interp/testdata/buffer.fqtest
vendored
68
pkg/interp/testdata/buffer.fqtest
vendored
@ -28,7 +28,7 @@ $ fq -n '[("11" | hex), ("22" | hex)] | tobits | hd'
|
||||
# TODO: bug, hexdump uses io.Copy which is byte oritneted
|
||||
$ fq -n '[("12" | hex | .bits[4:]), ("34" | hex | .bits[0:4])] | tobits | hd'
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0|20| | | |.: raw bits 0x0-0x0.7 (1)
|
||||
0x0|23| |#| |.: raw bits 0x0-0x0.7 (1)
|
||||
$ fq -d mp3 '.frames[]._bits[0:12] | tonumber' /test.mp3
|
||||
4095
|
||||
4095
|
||||
@ -236,4 +236,70 @@ mp3> .frames[1] | tobytesrange | ., .start, .stop, .size, .[4:17], (tobits, toby
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0xe0| 00 00 0a 2c 43 2e 55 94 80| ...,C.U..|.: raw bits 0xe7-0xf3.7 (13)
|
||||
0xf0|01 80 93 6b |...k |
|
||||
mp3> "fq" | tobits | [.[range(.size)]] | map(tobits) | tobytes | tostring
|
||||
"fq"
|
||||
mp3> "fq" | tobits | chunk(range(17)+1) | tobytes | tostring
|
||||
"fq"
|
||||
"fq"
|
||||
"fq"
|
||||
"fq"
|
||||
"fq"
|
||||
"fq"
|
||||
"fq"
|
||||
"fq"
|
||||
"fq"
|
||||
"fq"
|
||||
"fq"
|
||||
"fq"
|
||||
"fq"
|
||||
"fq"
|
||||
"fq"
|
||||
"fq"
|
||||
"fq"
|
||||
mp3> range(17) | [range(.) | 1 | tobits] | tobytes
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
| | |.: raw bits 0x0-NA (0)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0|80| |.| |.: raw bits 0x0-0x0 (0.1)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0|c0| |.| |.: raw bits 0x0-0x0.1 (0.2)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0|e0| |.| |.: raw bits 0x0-0x0.2 (0.3)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0|f0| |.| |.: raw bits 0x0-0x0.3 (0.4)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0|f8| |.| |.: raw bits 0x0-0x0.4 (0.5)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0|fc| |.| |.: raw bits 0x0-0x0.5 (0.6)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0|fe| |.| |.: raw bits 0x0-0x0.6 (0.7)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0|ff| |.| |.: raw bits 0x0-0x0.7 (1)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0|ff 80| |..| |.: raw bits 0x0-0x1 (1.1)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0|ff c0| |..| |.: raw bits 0x0-0x1.1 (1.2)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0|ff e0| |..| |.: raw bits 0x0-0x1.2 (1.3)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0|ff f0| |..| |.: raw bits 0x0-0x1.3 (1.4)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0|ff f8| |..| |.: raw bits 0x0-0x1.4 (1.5)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0|ff fc| |..| |.: raw bits 0x0-0x1.5 (1.6)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0|ff fe| |..| |.: raw bits 0x0-0x1.6 (1.7)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0|ff ff| |..| |.: raw bits 0x0-0x1.7 (2)
|
||||
mp3> "c9dfdac2f6ef68e5db666b6fbeee66d9c7deda66bebfbfe860bfbfbfe9d1636bbfbebf" | hex | tobits | reduce chunk(8)[] as $c ({h:[],g:[]}; .h += [(0|tobits), $c[0:7]] | .g |= . + [if length % 8 == 0 then (0|tobits) else empty end, $c[7:8]]) | .h, .g | tobytes
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x00|64 6f 6d 61 7b 77 34 72 6d 33 35 37 5f 77 33 6c|doma{w4rm357_w3l|.: raw bits 0x0-0x22.7 (35)
|
||||
* |until 0x22.7 (end) (35) | |
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0|62 6c 30 67 7d| |bl0g}| |.: raw bits 0x0-0x4.7 (5)
|
||||
mp3> .frames[1] | tobytes | mp3_frame | ., ((.header.bitrate | tobitsrange) as $v | tobitsrange | [.[:$v.start], (0xf | tobits), .[$v.start+$v.size:]] | mp3_frame) | .header.bitrate
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0| 50 | P |.header.bitrate: 64000 (5)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|
|
||||
0x0| f0 | . |.header.bitrate: 15 (bad)
|
||||
mp3> ^D
|
||||
|
@ -11,6 +11,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/wader/fq/internal/bitioextra"
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
)
|
||||
|
||||
@ -41,7 +42,7 @@ func (df DisplayFormat) FormatBase() int {
|
||||
}
|
||||
|
||||
type S struct {
|
||||
Actual interface{} // int, int64, uint64, float64, string, bool, []byte, *bitio.Buffer
|
||||
Actual interface{} // int, int64, uint64, float64, string, bool, []byte, bitio.BitReaderAtSeeker,
|
||||
ActualDisplay DisplayFormat
|
||||
Sym interface{}
|
||||
SymDisplay DisplayFormat
|
||||
@ -140,55 +141,6 @@ func StrFToSym(base int) Mapper {
|
||||
return strMapToSym(func(s string) (interface{}, error) { return strconv.ParseFloat(s, base) })
|
||||
}
|
||||
|
||||
//nolint:unparam
|
||||
func rawSym(s S, nBytes int, fn func(b []byte) string) (S, error) {
|
||||
bb, ok := s.Actual.(*bitio.Buffer)
|
||||
if !ok {
|
||||
return s, nil
|
||||
}
|
||||
bbLen := bb.Len()
|
||||
if nBytes < 0 {
|
||||
nBytes = int(bbLen) / 8
|
||||
if bbLen%8 != 0 {
|
||||
nBytes++
|
||||
}
|
||||
}
|
||||
if bbLen < int64(nBytes)*8 {
|
||||
return s, nil
|
||||
}
|
||||
// TODO: shared somehow?
|
||||
b := make([]byte, nBytes)
|
||||
if _, err := bb.ReadBitsAt(b, nBytes*8, 0); err != nil {
|
||||
return s, err
|
||||
}
|
||||
|
||||
s.Sym = fn(b[0:nBytes])
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
var RawSym = Fn(func(s S) (S, error) {
|
||||
return rawSym(s, -1, func(b []byte) string {
|
||||
return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
|
||||
})
|
||||
})
|
||||
|
||||
var RawUUID = Fn(func(s S) (S, error) {
|
||||
return rawSym(s, -1, func(b []byte) string {
|
||||
return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
|
||||
})
|
||||
})
|
||||
|
||||
var RawHex = Fn(func(s S) (S, error) {
|
||||
return rawSym(s, -1, func(b []byte) string { return fmt.Sprintf("%x", b) })
|
||||
})
|
||||
|
||||
var RawHexReverse = Fn(func(s S) (S, error) {
|
||||
return rawSym(s, -1, func(b []byte) string {
|
||||
return fmt.Sprintf("%x", bitio.ReverseBytes(append([]byte{}, b...)))
|
||||
})
|
||||
})
|
||||
|
||||
type URangeEntry struct {
|
||||
Range [2]uint64
|
||||
S S
|
||||
@ -238,18 +190,57 @@ func (rs SRangeToScalar) MapScalar(s S) (S, error) {
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func rawSym(s S, nBytes int, fn func(b []byte) string) (S, error) {
|
||||
br, ok := s.Actual.(bitio.ReadAtSeeker)
|
||||
if !ok {
|
||||
return s, nil
|
||||
}
|
||||
brLen, err := bitioextra.Len(br)
|
||||
if err != nil {
|
||||
return S{}, err
|
||||
}
|
||||
if nBytes < 0 {
|
||||
nBytes = int(brLen) / 8
|
||||
if brLen%8 != 0 {
|
||||
nBytes++
|
||||
}
|
||||
}
|
||||
if brLen < int64(nBytes)*8 {
|
||||
return s, nil
|
||||
}
|
||||
// TODO: shared somehow?
|
||||
b := make([]byte, nBytes)
|
||||
if _, err := br.ReadBitsAt(b, int64(nBytes)*8, 0); err != nil {
|
||||
return s, err
|
||||
}
|
||||
|
||||
s.Sym = fn(b[0:nBytes])
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
var RawUUID = Fn(func(s S) (S, error) {
|
||||
return rawSym(s, -1, func(b []byte) string {
|
||||
return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
|
||||
})
|
||||
})
|
||||
|
||||
var RawHex = Fn(func(s S) (S, error) {
|
||||
return rawSym(s, -1, func(b []byte) string { return fmt.Sprintf("%x", b) })
|
||||
})
|
||||
|
||||
type BytesToScalar []struct {
|
||||
Bytes []byte
|
||||
Scalar S
|
||||
}
|
||||
|
||||
func (m BytesToScalar) MapScalar(s S) (S, error) {
|
||||
ab, err := s.ActualBitBuf().Bytes()
|
||||
if err != nil {
|
||||
bb := &bytes.Buffer{}
|
||||
if _, err := bitioextra.CopyBits(bb, s.ActualBitBuf()); err != nil {
|
||||
return s, err
|
||||
}
|
||||
for _, bs := range m {
|
||||
if bytes.Equal(ab, bs.Bytes) {
|
||||
if bytes.Equal(bb.Bytes(), bs.Bytes) {
|
||||
ns := bs.Scalar
|
||||
ns.Actual = s.Actual
|
||||
s = ns
|
||||
|
@ -31,19 +31,19 @@ func (s S) SymBigInt() *big.Int {
|
||||
// Type BitBuf
|
||||
|
||||
// ActualBitBuf asserts actual value is a BitBuf and returns it
|
||||
func (s S) ActualBitBuf() *bitio.Buffer {
|
||||
v, ok := s.Actual.(*bitio.Buffer)
|
||||
func (s S) ActualBitBuf() bitio.ReaderAtSeeker {
|
||||
v, ok := s.Actual.(bitio.ReaderAtSeeker)
|
||||
if !ok {
|
||||
panic(fmt.Sprintf("failed to type assert s.Actual %v as *bitio.Buffer", s.Actual))
|
||||
panic(fmt.Sprintf("failed to type assert s.Actual %v as bitio.ReaderAtSeeker", s.Actual))
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// SymBitBuf asserts symbolic value is a BitBuf and returns it
|
||||
func (s S) SymBitBuf() *bitio.Buffer {
|
||||
v, ok := s.Sym.(*bitio.Buffer)
|
||||
func (s S) SymBitBuf() bitio.ReaderAtSeeker {
|
||||
v, ok := s.Sym.(bitio.ReaderAtSeeker)
|
||||
if !ok {
|
||||
panic(fmt.Sprintf("failed to type assert s.Sym %v as *bitio.Buffer", s.Sym))
|
||||
panic(fmt.Sprintf("failed to type assert s.Sym %v as bitio.ReaderAtSeeker", s.Sym))
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user