1
1
mirror of https://github.com/wader/fq.git synced 2024-11-27 14:14:58 +03:00
fq/pkg/interp/binary.go
Mattias Wadman 0a043f9096 repl,interp: Refactor repl and slurp
Now repl, slurp and help implemented using same query rewrite.
Include filename context in error if possible.
Add spew function that does opposite of slurp.
Start of help infra, not done or documented yet.
Show error pointer on parse error.
Rename internal eval to _eval and make eval be wrapper that
does rewrite and has various eror handling etc.
Nicer repl, slupr and help errors.
2022-03-01 18:31:00 +01:00

436 lines
9.4 KiB
Go

package interp
import (
"bytes"
"errors"
"fmt"
"io"
"io/fs"
"io/ioutil"
"math/big"
"github.com/wader/fq/internal/aheadreadseeker"
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/internal/ctxreadseeker"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/ioextra"
"github.com/wader/fq/internal/progressreadseeker"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/ranges"
"github.com/wader/gojq"
)
func init() {
functionRegisterFns = append(functionRegisterFns, func(i *Interp) []Function {
return []Function{
{"_tobits", 3, 3, i._toBits, nil},
{"open", 0, 0, nil, i._open},
}
})
}
type ToBinary interface {
ToBinary() (Binary, error)
}
func toBinary(v interface{}) (Binary, error) {
switch vv := v.(type) {
case ToBinary:
return vv.ToBinary()
default:
br, err := toBitReader(v)
if err != nil {
return Binary{}, err
}
return newBinaryFromBitReader(br, 8, 0)
}
}
func toBitReader(v interface{}) (bitio.ReaderAtSeeker, error) {
return toBitReaderEx(v, false)
}
func toBitReaderEx(v interface{}, inArray bool) (bitio.ReaderAtSeeker, error) {
switch vv := v.(type) {
case ToBinary:
bv, err := vv.ToBinary()
if err != nil {
return nil, err
}
return bitioextra.Range(bv.br, bv.r.Start, bv.r.Len)
case string:
return bitio.NewBitReader([]byte(vv), -1), nil
case int, float64, *big.Int:
bi, err := toBigInt(v)
if err != nil {
return nil, err
}
if inArray {
if bi.Cmp(big.NewInt(255)) > 0 || bi.Cmp(big.NewInt(0)) < 0 {
return nil, fmt.Errorf("byte in binary list must be bytes (0-255) got %v", bi)
}
n := bi.Uint64()
b := [1]byte{byte(n)}
return bitio.NewBitReader(b[:], -1), nil
}
bitLen := int64(bi.BitLen())
// bit.Int "The bit length of 0 is 0."
if bitLen == 0 {
var z [1]byte
return bitio.NewBitReader(z[:], 1), nil
}
// TODO: how should this work? "0xf | tobytes" 4bits or 8bits? now 4
padBefore := (8 - (bitLen % 8)) % 8
// padBefore := 0
br, err := bitioextra.Range(bitio.NewBitReader(bi.Bytes(), -1), padBefore, bitLen)
if err != nil {
return nil, err
}
return br, nil
case []interface{}:
rr := make([]bitio.ReadAtSeeker, 0, len(vv))
// TODO: optimize byte array case, flatten into one slice
for _, e := range vv {
eBR, eErr := toBitReaderEx(e, true)
if eErr != nil {
return nil, eErr
}
rr = append(rr, eBR)
}
mb, err := bitio.NewMultiReader(rr...)
if err != nil {
return nil, err
}
return mb, nil
default:
return nil, fmt.Errorf("value can't be a binary")
}
}
// note is used to implement tobytes* also
func (i *Interp) _toBits(c interface{}, a []interface{}) interface{} {
unit, ok := gojqextra.ToInt(a[0])
if !ok {
return gojqextra.FuncTypeError{Name: "_tobits", V: a[0]}
}
keepRange, ok := gojqextra.ToBoolean(a[1])
if !ok {
return gojqextra.FuncTypeError{Name: "_tobits", V: a[1]}
}
padToUnits, ok := gojqextra.ToInt(a[2])
if !ok {
return gojqextra.FuncTypeError{Name: "_tobits", V: a[2]}
}
// TODO: unit > 8?
bv, err := toBinary(c)
if err != nil {
return err
}
pad := int64(unit * padToUnits)
if pad == 0 {
pad = int64(unit)
}
bv.unit = unit
bv.pad = (pad - bv.r.Len%pad) % pad
if keepRange {
return bv
}
br, err := bv.toReader()
if err != nil {
return err
}
bb, err := newBinaryFromBitReader(br, bv.unit, bv.pad)
if err != nil {
return err
}
return bb
}
type openFile struct {
Binary
filename string
progressFn progressreadseeker.ProgressFn
}
var _ Value = (*openFile)(nil)
var _ ToBinary = (*openFile)(nil)
func (of *openFile) Display(w io.Writer, opts Options) error {
_, err := fmt.Fprintf(w, "<openfile %q>\n", of.filename)
return err
}
func (of *openFile) ToBinary() (Binary, error) {
return newBinaryFromBitReader(of.br, 8, 0)
}
// opens a file for reading from filesystem
// TODO: when to close? when br loses all refs? need to use finalizer somehow?
func (i *Interp) _open(c interface{}, a []interface{}) gojq.Iter {
if i.evalInstance.isCompleting {
return gojq.NewIter()
}
var err error
var f fs.File
var path string
switch c.(type) {
case nil:
path = "<stdin>"
f = i.os.Stdin()
default:
path, err = toString(c)
if err != nil {
return gojq.NewIter(fmt.Errorf("%s: %w", path, err))
}
f, err = i.os.FS().Open(path)
if err != nil {
// path context added in jq error code
var pe *fs.PathError
if errors.As(err, &pe) {
return gojq.NewIter(pe.Err)
}
return gojq.NewIter(err)
}
}
var bEnd int64
var fRS io.ReadSeeker
fFI, err := f.Stat()
if err != nil {
f.Close()
return gojq.NewIter(err)
}
// ctxreadseeker is used to make sure any io calls can be canceled
// TODO: ctxreadseeker might leak if the underlaying call hangs forever
// a regular file should be seekable but fallback below to read whole file if not
if fFI.Mode().IsRegular() {
if rs, ok := f.(io.ReadSeeker); ok {
fRS = ctxreadseeker.New(i.evalInstance.ctx, rs)
bEnd = fFI.Size()
}
}
if fRS == nil {
buf, err := ioutil.ReadAll(ctxreadseeker.New(i.evalInstance.ctx, &ioextra.ReadErrSeeker{Reader: f}))
if err != nil {
f.Close()
return gojq.NewIter(err)
}
fRS = bytes.NewReader(buf)
bEnd = int64(len(buf))
}
bbf := &openFile{
filename: path,
}
const progressPrecision = 1024
fRS = progressreadseeker.New(fRS, progressPrecision, bEnd,
func(approxReadBytes int64, totalSize int64) {
// progressFn is assign by decode etc
if bbf.progressFn != nil {
bbf.progressFn(approxReadBytes, totalSize)
}
},
)
const cacheReadAheadSize = 512 * 1024
aheadRs := aheadreadseeker.New(fRS, cacheReadAheadSize)
// bitio.Buffer -> (bitio.Reader) -> aheadreadseeker -> progressreadseeker -> ctxreadseeker -> readseeker
bbf.br = bitio.NewIOBitReadSeeker(aheadRs)
if err != nil {
return gojq.NewIter(err)
}
return gojq.NewIter(bbf)
}
var _ Value = Binary{}
var _ ToBinary = Binary{}
type Binary struct {
br bitio.ReaderAtSeeker
r ranges.Range
unit int
pad int64
}
func newBinaryFromBitReader(br bitio.ReaderAtSeeker, unit int, pad int64) (Binary, error) {
l, err := bitioextra.Len(br)
if err != nil {
return Binary{}, err
}
return Binary{
br: br,
r: ranges.Range{Start: 0, Len: l},
unit: unit,
pad: pad,
}, nil
}
func (b Binary) toBytesBuffer(r ranges.Range) (*bytes.Buffer, error) {
br, err := bitioextra.Range(b.br, r.Start, r.Len)
if err != nil {
return nil, err
}
buf := &bytes.Buffer{}
if _, err := bitioextra.CopyBits(buf, br); err != nil {
return nil, err
}
return buf, nil
}
func (Binary) ExtType() string { return "binary" }
func (Binary) ExtKeys() []string {
return []string{
"size",
"start",
"stop",
"bits",
"bytes",
}
}
func (b Binary) ToBinary() (Binary, error) {
return b, nil
}
func (b Binary) JQValueLength() interface{} {
return int(b.r.Len / int64(b.unit))
}
func (b Binary) JQValueSliceLen() interface{} {
return b.JQValueLength()
}
func (b Binary) JQValueIndex(index int) interface{} {
if index < 0 {
return nil
}
buf, err := b.toBytesBuffer(ranges.Range{Start: b.r.Start + int64(index*b.unit), Len: int64(b.unit)})
if err != nil {
return err
}
extraBits := uint((8 - b.unit%8) % 8)
return new(big.Int).Rsh(new(big.Int).SetBytes(buf.Bytes()), extraBits)
}
func (b Binary) JQValueSlice(start int, end int) interface{} {
rStart := int64(start * b.unit)
rLen := int64((end - start) * b.unit)
return Binary{
br: b.br,
r: ranges.Range{Start: b.r.Start + rStart, Len: rLen},
unit: b.unit,
}
}
func (b Binary) JQValueKey(name string) interface{} {
switch name {
case "size":
return new(big.Int).SetInt64(b.r.Len / int64(b.unit))
case "start":
return new(big.Int).SetInt64(b.r.Start / int64(b.unit))
case "stop":
stop := b.r.Stop()
stopUnits := stop / int64(b.unit)
if stop%int64(b.unit) != 0 {
stopUnits++
}
return new(big.Int).SetInt64(stopUnits)
case "bits":
if b.unit == 1 {
return b
}
return Binary{br: b.br, r: b.r, unit: 1}
case "bytes":
if b.unit == 8 {
return b
}
return Binary{br: b.br, r: b.r, unit: 8}
}
return nil
}
func (b Binary) JQValueEach() interface{} {
return nil
}
func (b Binary) JQValueType() string {
return "binary"
}
func (b Binary) JQValueKeys() interface{} {
return gojqextra.FuncTypeNameError{Name: "keys", Typ: "binary"}
}
func (b Binary) JQValueHas(key interface{}) interface{} {
return gojqextra.HasKeyTypeError{L: "binary", R: fmt.Sprintf("%v", key)}
}
func (b Binary) JQValueToNumber() interface{} {
buf, err := b.toBytesBuffer(b.r)
if err != nil {
return err
}
extraBits := uint((8 - b.r.Len%8) % 8)
return new(big.Int).Rsh(new(big.Int).SetBytes(buf.Bytes()), extraBits)
}
func (b Binary) JQValueToString() interface{} {
return b.JQValueToGoJQ()
}
func (b Binary) JQValueToGoJQ() interface{} {
buf, err := b.toBytesBuffer(b.r)
if err != nil {
return err
}
return buf.String()
}
func (b Binary) JQValueUpdate(key interface{}, u interface{}, delpath bool) interface{} {
return gojqextra.NonUpdatableTypeError{Key: fmt.Sprintf("%v", key), Typ: "binary"}
}
func (b Binary) Display(w io.Writer, opts Options) error {
if opts.RawOutput {
br, err := b.toReader()
if err != nil {
return err
}
if _, err := bitioextra.CopyBits(w, br); err != nil {
return err
}
return nil
}
return hexdump(w, b, opts)
}
func (b Binary) toReader() (bitio.ReaderAtSeeker, error) {
br, err := bitioextra.Range(b.br, b.r.Start, b.r.Len)
if err != nil {
return nil, err
}
if b.pad == 0 {
return br, nil
}
return bitio.NewMultiReader(bitioextra.NewZeroAtSeeker(b.pad), br)
}