mirror of
https://github.com/wader/fq.git
synced 2024-12-29 16:42:06 +03:00
48a19cb82c
Also refactor readline and eval args into option struct and partinally start addressing some side effects during completion.
431 lines
9.3 KiB
Go
431 lines
9.3 KiB
Go
package interp
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"io/ioutil"
|
|
"math/big"
|
|
|
|
"github.com/wader/fq/internal/aheadreadseeker"
|
|
"github.com/wader/fq/internal/bitioextra"
|
|
"github.com/wader/fq/internal/ctxreadseeker"
|
|
"github.com/wader/fq/internal/gojqextra"
|
|
"github.com/wader/fq/internal/ioextra"
|
|
"github.com/wader/fq/internal/progressreadseeker"
|
|
"github.com/wader/fq/pkg/bitio"
|
|
"github.com/wader/fq/pkg/ranges"
|
|
"github.com/wader/gojq"
|
|
)
|
|
|
|
func init() {
|
|
functionRegisterFns = append(functionRegisterFns, func(i *Interp) []Function {
|
|
return []Function{
|
|
{"_tobits", 3, 3, i._toBits, nil},
|
|
{"open", 0, 0, nil, i._open},
|
|
}
|
|
})
|
|
}
|
|
|
|
type ToBinary interface {
|
|
ToBinary() (Binary, error)
|
|
}
|
|
|
|
func toBinary(v interface{}) (Binary, error) {
|
|
switch vv := v.(type) {
|
|
case ToBinary:
|
|
return vv.ToBinary()
|
|
default:
|
|
br, err := toBitReader(v)
|
|
if err != nil {
|
|
return Binary{}, err
|
|
}
|
|
return newBinaryFromBitReader(br, 8, 0)
|
|
}
|
|
}
|
|
|
|
func toBitReader(v interface{}) (bitio.ReaderAtSeeker, error) {
|
|
return toBitReaderEx(v, false)
|
|
}
|
|
|
|
func toBitReaderEx(v interface{}, inArray bool) (bitio.ReaderAtSeeker, error) {
|
|
switch vv := v.(type) {
|
|
case ToBinary:
|
|
bv, err := vv.ToBinary()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return bitioextra.Range(bv.br, bv.r.Start, bv.r.Len)
|
|
case string:
|
|
return bitio.NewBitReader([]byte(vv), -1), nil
|
|
case int, float64, *big.Int:
|
|
bi, err := toBigInt(v)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if inArray {
|
|
if bi.Cmp(big.NewInt(255)) > 0 || bi.Cmp(big.NewInt(0)) < 0 {
|
|
return nil, fmt.Errorf("byte in binary list must be bytes (0-255) got %v", bi)
|
|
}
|
|
n := bi.Uint64()
|
|
b := [1]byte{byte(n)}
|
|
return bitio.NewBitReader(b[:], -1), nil
|
|
}
|
|
|
|
bitLen := int64(bi.BitLen())
|
|
// bit.Int "The bit length of 0 is 0."
|
|
if bitLen == 0 {
|
|
var z [1]byte
|
|
return bitio.NewBitReader(z[:], 1), nil
|
|
}
|
|
// TODO: how should this work? "0xf | tobytes" 4bits or 8bits? now 4
|
|
padBefore := (8 - (bitLen % 8)) % 8
|
|
// padBefore := 0
|
|
br, err := bitioextra.Range(bitio.NewBitReader(bi.Bytes(), -1), padBefore, bitLen)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return br, nil
|
|
case []interface{}:
|
|
rr := make([]bitio.ReadAtSeeker, 0, len(vv))
|
|
// TODO: optimize byte array case, flatten into one slice
|
|
for _, e := range vv {
|
|
eBR, eErr := toBitReaderEx(e, true)
|
|
if eErr != nil {
|
|
return nil, eErr
|
|
}
|
|
rr = append(rr, eBR)
|
|
}
|
|
|
|
mb, err := bitio.NewMultiReader(rr...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return mb, nil
|
|
default:
|
|
return nil, fmt.Errorf("value can't be a binary")
|
|
}
|
|
}
|
|
|
|
// note is used to implement tobytes* also
|
|
func (i *Interp) _toBits(c interface{}, a []interface{}) interface{} {
|
|
unit, ok := gojqextra.ToInt(a[0])
|
|
if !ok {
|
|
return gojqextra.FuncTypeError{Name: "_tobits", V: a[0]}
|
|
}
|
|
keepRange, ok := gojqextra.ToBoolean(a[1])
|
|
if !ok {
|
|
return gojqextra.FuncTypeError{Name: "_tobits", V: a[1]}
|
|
}
|
|
padToUnits, ok := gojqextra.ToInt(a[2])
|
|
if !ok {
|
|
return gojqextra.FuncTypeError{Name: "_tobits", V: a[2]}
|
|
}
|
|
|
|
// TODO: unit > 8?
|
|
|
|
bv, err := toBinary(c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
pad := int64(unit * padToUnits)
|
|
if pad == 0 {
|
|
pad = int64(unit)
|
|
}
|
|
|
|
bv.unit = unit
|
|
bv.pad = (pad - bv.r.Len%pad) % pad
|
|
|
|
if keepRange {
|
|
return bv
|
|
}
|
|
|
|
br, err := bv.toReader()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
bb, err := newBinaryFromBitReader(br, bv.unit, bv.pad)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return bb
|
|
}
|
|
|
|
type openFile struct {
|
|
Binary
|
|
filename string
|
|
progressFn progressreadseeker.ProgressFn
|
|
}
|
|
|
|
var _ Value = (*openFile)(nil)
|
|
var _ ToBinary = (*openFile)(nil)
|
|
|
|
func (of *openFile) Display(w io.Writer, opts Options) error {
|
|
_, err := fmt.Fprintf(w, "<openfile %q>\n", of.filename)
|
|
return err
|
|
}
|
|
|
|
func (of *openFile) ToBinary() (Binary, error) {
|
|
return newBinaryFromBitReader(of.br, 8, 0)
|
|
}
|
|
|
|
// def open: #:: string| => binary
|
|
// opens a file for reading from filesystem
|
|
// TODO: when to close? when br loses all refs? need to use finalizer somehow?
|
|
func (i *Interp) _open(c interface{}, a []interface{}) gojq.Iter {
|
|
if i.evalContext.isCompleting {
|
|
return gojq.NewIter()
|
|
}
|
|
|
|
var err error
|
|
var f fs.File
|
|
var path string
|
|
|
|
switch c.(type) {
|
|
case nil:
|
|
path = "<stdin>"
|
|
f = i.os.Stdin()
|
|
default:
|
|
path, err = toString(c)
|
|
if err != nil {
|
|
return gojq.NewIter(fmt.Errorf("%s: %w", path, err))
|
|
}
|
|
f, err = i.os.FS().Open(path)
|
|
if err != nil {
|
|
return gojq.NewIter(err)
|
|
}
|
|
}
|
|
|
|
var bEnd int64
|
|
var fRS io.ReadSeeker
|
|
|
|
fFI, err := f.Stat()
|
|
if err != nil {
|
|
f.Close()
|
|
return gojq.NewIter(err)
|
|
}
|
|
|
|
// ctxreadseeker is used to make sure any io calls can be canceled
|
|
// TODO: ctxreadseeker might leak if the underlaying call hangs forever
|
|
|
|
// a regular file should be seekable but fallback below to read whole file if not
|
|
if fFI.Mode().IsRegular() {
|
|
if rs, ok := f.(io.ReadSeeker); ok {
|
|
fRS = ctxreadseeker.New(i.evalContext.ctx, rs)
|
|
bEnd = fFI.Size()
|
|
}
|
|
}
|
|
|
|
if fRS == nil {
|
|
buf, err := ioutil.ReadAll(ctxreadseeker.New(i.evalContext.ctx, &ioextra.ReadErrSeeker{Reader: f}))
|
|
if err != nil {
|
|
f.Close()
|
|
return gojq.NewIter(err)
|
|
}
|
|
fRS = bytes.NewReader(buf)
|
|
bEnd = int64(len(buf))
|
|
}
|
|
|
|
bbf := &openFile{
|
|
filename: path,
|
|
}
|
|
|
|
const progressPrecision = 1024
|
|
fRS = progressreadseeker.New(fRS, progressPrecision, bEnd,
|
|
func(approxReadBytes int64, totalSize int64) {
|
|
// progressFn is assign by decode etc
|
|
if bbf.progressFn != nil {
|
|
bbf.progressFn(approxReadBytes, totalSize)
|
|
}
|
|
},
|
|
)
|
|
|
|
const cacheReadAheadSize = 512 * 1024
|
|
aheadRs := aheadreadseeker.New(fRS, cacheReadAheadSize)
|
|
|
|
// bitio.Buffer -> (bitio.Reader) -> aheadreadseeker -> progressreadseeker -> ctxreadseeker -> readseeker
|
|
|
|
bbf.br = bitio.NewIOBitReadSeeker(aheadRs)
|
|
if err != nil {
|
|
return gojq.NewIter(err)
|
|
}
|
|
|
|
return gojq.NewIter(bbf)
|
|
}
|
|
|
|
var _ Value = Binary{}
|
|
var _ ToBinary = Binary{}
|
|
|
|
type Binary struct {
|
|
br bitio.ReaderAtSeeker
|
|
r ranges.Range
|
|
unit int
|
|
pad int64
|
|
}
|
|
|
|
func newBinaryFromBitReader(br bitio.ReaderAtSeeker, unit int, pad int64) (Binary, error) {
|
|
l, err := bitioextra.Len(br)
|
|
if err != nil {
|
|
return Binary{}, err
|
|
}
|
|
|
|
return Binary{
|
|
br: br,
|
|
r: ranges.Range{Start: 0, Len: l},
|
|
unit: unit,
|
|
pad: pad,
|
|
}, nil
|
|
}
|
|
|
|
func (b Binary) toBytesBuffer(r ranges.Range) (*bytes.Buffer, error) {
|
|
br, err := bitioextra.Range(b.br, r.Start, r.Len)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
buf := &bytes.Buffer{}
|
|
if _, err := bitioextra.CopyBits(buf, br); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return buf, nil
|
|
}
|
|
|
|
func (Binary) ExtType() string { return "binary" }
|
|
|
|
func (Binary) ExtKeys() []string {
|
|
return []string{
|
|
"size",
|
|
"start",
|
|
"stop",
|
|
"bits",
|
|
"bytes",
|
|
}
|
|
}
|
|
|
|
func (b Binary) ToBinary() (Binary, error) {
|
|
return b, nil
|
|
}
|
|
|
|
func (b Binary) JQValueLength() interface{} {
|
|
return int(b.r.Len / int64(b.unit))
|
|
}
|
|
func (b Binary) JQValueSliceLen() interface{} {
|
|
return b.JQValueLength()
|
|
}
|
|
|
|
func (b Binary) JQValueIndex(index int) interface{} {
|
|
if index < 0 {
|
|
return nil
|
|
}
|
|
|
|
buf, err := b.toBytesBuffer(ranges.Range{Start: b.r.Start + int64(index*b.unit), Len: int64(b.unit)})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
extraBits := uint((8 - b.unit%8) % 8)
|
|
|
|
return new(big.Int).Rsh(new(big.Int).SetBytes(buf.Bytes()), extraBits)
|
|
}
|
|
func (b Binary) JQValueSlice(start int, end int) interface{} {
|
|
rStart := int64(start * b.unit)
|
|
rLen := int64((end - start) * b.unit)
|
|
|
|
return Binary{
|
|
br: b.br,
|
|
r: ranges.Range{Start: b.r.Start + rStart, Len: rLen},
|
|
unit: b.unit,
|
|
}
|
|
}
|
|
func (b Binary) JQValueKey(name string) interface{} {
|
|
switch name {
|
|
case "size":
|
|
return new(big.Int).SetInt64(b.r.Len / int64(b.unit))
|
|
case "start":
|
|
return new(big.Int).SetInt64(b.r.Start / int64(b.unit))
|
|
case "stop":
|
|
stop := b.r.Stop()
|
|
stopUnits := stop / int64(b.unit)
|
|
if stop%int64(b.unit) != 0 {
|
|
stopUnits++
|
|
}
|
|
return new(big.Int).SetInt64(stopUnits)
|
|
case "bits":
|
|
if b.unit == 1 {
|
|
return b
|
|
}
|
|
return Binary{br: b.br, r: b.r, unit: 1}
|
|
case "bytes":
|
|
if b.unit == 8 {
|
|
return b
|
|
}
|
|
return Binary{br: b.br, r: b.r, unit: 8}
|
|
}
|
|
return nil
|
|
}
|
|
func (b Binary) JQValueEach() interface{} {
|
|
return nil
|
|
}
|
|
func (b Binary) JQValueType() string {
|
|
return "binary"
|
|
}
|
|
func (b Binary) JQValueKeys() interface{} {
|
|
return gojqextra.FuncTypeNameError{Name: "keys", Typ: "binary"}
|
|
}
|
|
func (b Binary) JQValueHas(key interface{}) interface{} {
|
|
return gojqextra.HasKeyTypeError{L: "binary", R: fmt.Sprintf("%v", key)}
|
|
}
|
|
func (b Binary) JQValueToNumber() interface{} {
|
|
buf, err := b.toBytesBuffer(b.r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
extraBits := uint((8 - b.r.Len%8) % 8)
|
|
return new(big.Int).Rsh(new(big.Int).SetBytes(buf.Bytes()), extraBits)
|
|
}
|
|
func (b Binary) JQValueToString() interface{} {
|
|
return b.JQValueToGoJQ()
|
|
}
|
|
func (b Binary) JQValueToGoJQ() interface{} {
|
|
buf, err := b.toBytesBuffer(b.r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return buf.String()
|
|
}
|
|
func (b Binary) JQValueUpdate(key interface{}, u interface{}, delpath bool) interface{} {
|
|
return gojqextra.NonUpdatableTypeError{Key: fmt.Sprintf("%v", key), Typ: "binary"}
|
|
}
|
|
|
|
func (b Binary) Display(w io.Writer, opts Options) error {
|
|
if opts.RawOutput {
|
|
br, err := b.toReader()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err := bitioextra.CopyBits(w, br); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
return hexdump(w, b, opts)
|
|
}
|
|
|
|
func (b Binary) toReader() (bitio.ReaderAtSeeker, error) {
|
|
br, err := bitioextra.Range(b.br, b.r.Start, b.r.Len)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if b.pad == 0 {
|
|
return br, nil
|
|
}
|
|
return bitio.NewMultiReader(bitioextra.NewZeroAtSeeker(b.pad), br)
|
|
}
|