wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-11-23 00:57:15 +03:00
Code Issues Projects Releases Wiki Activity

decode, interp: More buffer reuse

Browse Source
This commit is contained in:
Mattias Wadman 2021-09-16 12:24:53 +02:00
parent 6207fcc8f0
commit c7416e6dcb
7 changed files with 24 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
format/gzip/gzip.go
View File

@ -112,7 +112,7 @@ func gzDecode(d *decode.D, in interface{}) interface{} {
deflateR := flate.NewReader(compressedBB)
uncompressed := &bytes.Buffer{}
crc32W := crc32.NewIEEE()
if _, err := io.Copy(io.MultiWriter(uncompressed, crc32W), deflateR); err != nil { //nolint:gosec
if _, err := decode.Copy(d, io.MultiWriter(uncompressed, crc32W), deflateR); err != nil {
d.Invalid(err.Error())
}
calculatedCRC32 = crc32W.Sum(nil)

11
pkg/bitio/bitio.go
View File

@ -69,8 +69,11 @@ func (a *AlignBitReader) ReadBitsAt(p []byte, nBits int, bitOff int64) (n int, e
return n, err
}
func Copy(dst BitWriter, src BitReader) (n int64, err error) {
buf := make([]byte, 32*1024)
func CopyBuffer(dst BitWriter, src BitReader, buf []byte) (n int64, err error) {
// same default size as io.Copy
if buf == nil {
buf = make([]byte, 32*1024)
}
var written int64
for {
@ -98,6 +101,10 @@ func Copy(dst BitWriter, src BitReader) (n int64, err error) {
return written, err
}
func Copy(dst BitWriter, src BitReader) (n int64, err error) {
return CopyBuffer(dst, src, nil)
}
func BitsByteCount(nBits int64) int64 {
n := nBits / 8
if nBits%8 != 0 {

4
pkg/decode/decode.go
View File

@ -241,11 +241,11 @@ func NewDecoder(ctx context.Context, name string, description string, format *Fo
}
}
func (d *D) AllocReadBuf(n int) []byte {
func (d *D) SharedReadBuf(n int) []byte {
if d.readBuf == nil {
d.readBuf = new([]byte)
}
if cap(*d.readBuf) < n {
if len(*d.readBuf) < n {
*d.readBuf = make([]byte, n)
}
return *d.readBuf

2
pkg/decode/decode_readers.go
View File

@ -71,7 +71,7 @@ func (d *D) TryPeekBits(nBits int) (uint64, error) {
// Bits reads nBits bits from buffer
func (d *D) bits(nBits int) (uint64, error) {
// 64 bits max, 9 byte worse case if not byte aligned
buf := d.AllocReadBuf(9)
buf := d.SharedReadBuf(9)
_, err := bitio.ReadFull(d.bitBuf, buf, nBits)
if err != nil {
return 0, err

4
pkg/decode/io.go
View File

@ -8,8 +8,8 @@ import (
)
func Copy(d *D, r io.Writer, w io.Reader) (int64, error) {
// TODO: what size?
buf := d.AllocReadBuf(64 * 1024)
// TODO: what size? now same as io.Copy
buf := d.SharedReadBuf(32 * 1024)
return io.CopyBuffer(r, w, buf)
}

15
pkg/interp/dump.go
View File

@ -33,7 +33,7 @@ func isCompound(v *decode.Value) bool {
}
}
func dumpEx(v *decode.Value, cw *columnwriter.Writer, depth int, rootV *decode.Value, rootDepth int, addrWidth int, opts Options) error {
func dumpEx(v *decode.Value, buf []byte, cw *columnwriter.Writer, depth int, rootV *decode.Value, rootDepth int, addrWidth int, opts Options) error {
deco := opts.Decorator
// no error check as we write into buffering column
// we check for err later for Flush()
@ -265,14 +265,16 @@ func dumpEx(v *decode.Value, cw *columnwriter.Writer, depth int, rootV *decode.V
asciiFn := func(b byte) string { return deco.ByteColor(b).Wrap(asciiwriter.SafeASCII(b)) }
if vBitBuf != nil {
if _, err := io.Copy(
if _, err := io.CopyBuffer(
hexpairwriter.New(cw.Columns[colHex], opts.LineBytes, int(startLineByteOffset), hexpairFn),
io.LimitReader(vBitBuf.Copy(), displaySizeBytes)); err != nil {
io.LimitReader(vBitBuf.Copy(), displaySizeBytes),
buf); err != nil {
return err
}
if _, err := io.Copy(
if _, err := io.CopyBuffer(
asciiwriter.New(cw.Columns[colAscii], opts.LineBytes, int(startLineByteOffset), asciiFn),
io.LimitReader(vBitBuf.Copy(), displaySizeBytes)); err != nil {
io.LimitReader(vBitBuf.Copy(), displaySizeBytes),
buf); err != nil {
return err
}
}
@ -340,8 +342,9 @@ func dump(v *decode.Value, w io.Writer, opts Options) error {
}))
cw := columnwriter.New(w, []int{maxAddrIndentWidth, 1, opts.LineBytes*3 - 1, 1, opts.LineBytes, 1, -1})
buf := make([]byte, 32*1024)
return v.WalkPreOrder(makeWalkFn(func(v *decode.Value, rootV *decode.Value, depth int, rootDepth int) error {
return dumpEx(v, cw, depth, rootV, rootDepth, maxAddrIndentWidth-rootDepth, opts)
return dumpEx(v, buf, cw, depth, rootV, rootDepth, maxAddrIndentWidth-rootDepth, opts)
}))
}

13
pkg/interp/funcs.go
View File

@ -717,19 +717,6 @@ func (i *Interp) tovalue(c interface{}, a []interface{}) interface{} {
return v
}
// func (i *Interp) md5(c interface{}, a []interface{}) interface{} {
// bb, _, err := toBuffer(c)
// if err != nil {
// return err
// }
// if _, err := io.Copy(md5, bb); err != nil {
// return err
// }
// return md5.Sum(nil)
// }
func (i *Interp) queryEscape(c interface{}, a []interface{}) interface{} {
s, err := toString(c)
if err != nil {