mirror of
https://github.com/wader/fq.git
synced 2024-09-19 07:47:14 +03:00
498 lines
10 KiB
Go
498 lines
10 KiB
Go
package interp
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"math/big"
|
|
|
|
"github.com/wader/fq/internal/aheadreadseeker"
|
|
"github.com/wader/fq/internal/bitiox"
|
|
"github.com/wader/fq/internal/ctxreadseeker"
|
|
"github.com/wader/fq/internal/gojqx"
|
|
"github.com/wader/fq/internal/iox"
|
|
"github.com/wader/fq/internal/progressreadseeker"
|
|
"github.com/wader/fq/pkg/bitio"
|
|
"github.com/wader/fq/pkg/ranges"
|
|
"github.com/wader/gojq"
|
|
)
|
|
|
|
func init() {
|
|
RegisterFunc1("_tobits", (*Interp)._toBits)
|
|
RegisterFunc0("open", (*Interp)._open)
|
|
}
|
|
|
|
type ToBinary interface {
|
|
ToBinary() (Binary, error)
|
|
}
|
|
|
|
func toBinary(v any) (Binary, error) {
|
|
switch vv := v.(type) {
|
|
case ToBinary:
|
|
return vv.ToBinary()
|
|
default:
|
|
br, err := ToBitReader(v)
|
|
if err != nil {
|
|
return Binary{}, err
|
|
}
|
|
return NewBinaryFromBitReader(br, 8, 0)
|
|
}
|
|
}
|
|
|
|
func ToBitReader(v any) (bitio.ReaderAtSeeker, error) {
|
|
return toBitReaderEx(v, false)
|
|
}
|
|
|
|
type byteRangeError int
|
|
|
|
func (b byteRangeError) Error() string {
|
|
return fmt.Sprintf("byte in binary list must be bytes (0-255) got %d", int(b))
|
|
|
|
}
|
|
|
|
func toBitReaderEx(v any, inArray bool) (bitio.ReaderAtSeeker, error) {
|
|
switch vv := v.(type) {
|
|
case ToBinary:
|
|
bv, err := vv.ToBinary()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return bitiox.Range(bv.br, bv.r.Start, bv.r.Len)
|
|
case string:
|
|
return bitio.NewBitReader([]byte(vv), -1), nil
|
|
case int, float64, *big.Int:
|
|
bi, err := toBigInt(v)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if inArray {
|
|
if bi.Cmp(big.NewInt(255)) > 0 || bi.Cmp(big.NewInt(0)) < 0 {
|
|
return nil, byteRangeError(bi.Int64())
|
|
}
|
|
n := bi.Uint64()
|
|
b := [1]byte{byte(n)}
|
|
return bitio.NewBitReader(b[:], -1), nil
|
|
}
|
|
|
|
bitLen := int64(bi.BitLen())
|
|
// bit.Int "The bit length of 0 is 0."
|
|
if bitLen == 0 {
|
|
var z [1]byte
|
|
return bitio.NewBitReader(z[:], 1), nil
|
|
}
|
|
// TODO: how should this work? "0xf | tobytes" 4bits or 8bits? now 4
|
|
padBefore := (8 - (bitLen % 8)) % 8
|
|
// padBefore := 0
|
|
br, err := bitiox.Range(bitio.NewBitReader(bi.Bytes(), -1), padBefore, bitLen)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return br, nil
|
|
case []any:
|
|
rr := make([]bitio.ReadAtSeeker, 0, len(vv))
|
|
|
|
// fast path for slice containing only 0-255 numbers and strings
|
|
bs := &bytes.Buffer{}
|
|
for _, e := range vv {
|
|
if bs == nil {
|
|
break
|
|
}
|
|
switch ev := e.(type) {
|
|
case int:
|
|
if ev >= 0 && ev <= 255 {
|
|
bs.WriteByte(byte(ev))
|
|
continue
|
|
}
|
|
case float64:
|
|
b := int(ev)
|
|
if b >= 0 && b <= 255 {
|
|
bs.WriteByte(byte(ev))
|
|
continue
|
|
}
|
|
case *big.Int:
|
|
if ev.Cmp(big.NewInt(0)) >= 0 && ev.Cmp(big.NewInt(255)) <= 0 {
|
|
bs.WriteByte(byte(ev.Uint64()))
|
|
continue
|
|
}
|
|
case string:
|
|
// TODO: maybe only if less then some length?
|
|
bs.WriteString(ev)
|
|
continue
|
|
}
|
|
bs = nil
|
|
}
|
|
if bs != nil {
|
|
return bitio.NewBitReader(bs.Bytes(), -1), nil
|
|
}
|
|
|
|
// TODO: optimize byte array case, flatten into one slice
|
|
for _, e := range vv {
|
|
eBR, eErr := toBitReaderEx(e, true)
|
|
if eErr != nil {
|
|
return nil, eErr
|
|
}
|
|
rr = append(rr, eBR)
|
|
}
|
|
|
|
mb, err := bitio.NewMultiReader(rr...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return mb, nil
|
|
default:
|
|
return nil, fmt.Errorf("value can't be a binary")
|
|
}
|
|
}
|
|
|
|
type toBitsOpts struct {
|
|
Unit int
|
|
KeepRange bool
|
|
PadToUnits int
|
|
}
|
|
|
|
// note is used to implement tobytes* also
|
|
func (i *Interp) _toBits(c any, opts toBitsOpts) any {
|
|
// TODO: unit > 8?
|
|
|
|
bv, err := toBinary(c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
pad := int64(opts.Unit * opts.PadToUnits)
|
|
if pad == 0 {
|
|
pad = int64(opts.Unit)
|
|
}
|
|
|
|
bv.unit = opts.Unit
|
|
bv.pad = (pad - bv.r.Len%pad) % pad
|
|
|
|
if opts.KeepRange {
|
|
return bv
|
|
}
|
|
|
|
br, err := bv.toReader()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
bb, err := NewBinaryFromBitReader(br, bv.unit, 0)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return bb
|
|
}
|
|
|
|
type openFile struct {
|
|
Binary
|
|
filename string
|
|
progressFn progressreadseeker.ProgressFn
|
|
}
|
|
|
|
var _ Value = (*openFile)(nil)
|
|
var _ ToBinary = (*openFile)(nil)
|
|
|
|
func (of *openFile) Display(w io.Writer, opts *Options) error {
|
|
_, err := fmt.Fprintf(w, "<openfile %q>\n", of.filename)
|
|
return err
|
|
}
|
|
|
|
func (of *openFile) ToBinary() (Binary, error) {
|
|
return NewBinaryFromBitReader(of.br, 8, 0)
|
|
}
|
|
|
|
// opens a file for reading from filesystem
|
|
// TODO: when to close? when br loses all refs? need to use finalizer somehow?
|
|
func (i *Interp) _open(c any) any {
|
|
if i.EvalInstance.IsCompleting {
|
|
// TODO: have dummy values for each type for completion?
|
|
br, _ := NewBinaryFromBitReader(bitio.NewBitReader([]byte{}, -1), 8, 0)
|
|
return br
|
|
}
|
|
|
|
var err error
|
|
var f fs.File
|
|
var path string
|
|
|
|
switch c.(type) {
|
|
case nil:
|
|
path = "<stdin>"
|
|
f = i.OS.Stdin()
|
|
default:
|
|
path, err = toString(c)
|
|
if err != nil {
|
|
return fmt.Errorf("%s: %w", path, err)
|
|
}
|
|
f, err = i.OS.FS().Open(path)
|
|
if err != nil {
|
|
// path context added in jq error code
|
|
var pe *fs.PathError
|
|
if errors.As(err, &pe) {
|
|
return pe.Err
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
|
|
var bEnd int64
|
|
var fRS io.ReadSeeker
|
|
|
|
fFI, err := f.Stat()
|
|
if err != nil {
|
|
f.Close()
|
|
return err
|
|
}
|
|
|
|
// ctxreadseeker is used to make sure any io calls can be canceled
|
|
// TODO: ctxreadseeker might leak if the underlying call hangs forever
|
|
|
|
// a regular file should be seekable but fallback below to read whole file if not
|
|
if fFI.Mode().IsRegular() {
|
|
if rs, ok := f.(io.ReadSeeker); ok {
|
|
fRS = ctxreadseeker.New(i.EvalInstance.Ctx, rs)
|
|
bEnd = fFI.Size()
|
|
}
|
|
}
|
|
|
|
if fRS == nil {
|
|
buf, err := io.ReadAll(ctxreadseeker.New(i.EvalInstance.Ctx, &iox.ReadErrSeeker{Reader: f}))
|
|
if err != nil {
|
|
f.Close()
|
|
return err
|
|
}
|
|
fRS = bytes.NewReader(buf)
|
|
bEnd = int64(len(buf))
|
|
}
|
|
|
|
bbf := &openFile{
|
|
filename: path,
|
|
}
|
|
|
|
const progressPrecision = 1024
|
|
fRS = progressreadseeker.New(fRS, progressPrecision, bEnd,
|
|
func(approxReadBytes int64, totalSize int64) {
|
|
// progressFn is assign by decode etc
|
|
if bbf.progressFn != nil {
|
|
bbf.progressFn(approxReadBytes, totalSize)
|
|
}
|
|
},
|
|
)
|
|
|
|
const cacheReadAheadSize = 512 * 1024
|
|
aheadRs := aheadreadseeker.New(fRS, cacheReadAheadSize)
|
|
|
|
// bitio.Buffer -> (bitio.Reader) -> aheadreadseeker -> progressreadseeker -> ctxreadseeker -> readseeker
|
|
|
|
bbf.br = bitio.NewIOBitReadSeeker(aheadRs)
|
|
|
|
return bbf
|
|
}
|
|
|
|
var _ Value = Binary{}
|
|
var _ ToBinary = Binary{}
|
|
|
|
type Binary struct {
|
|
br bitio.ReaderAtSeeker
|
|
r ranges.Range
|
|
unit int
|
|
pad int64
|
|
}
|
|
|
|
func NewBinaryFromBitReader(br bitio.ReaderAtSeeker, unit int, pad int64) (Binary, error) {
|
|
l, err := bitiox.Len(br)
|
|
if err != nil {
|
|
return Binary{}, err
|
|
}
|
|
|
|
return Binary{
|
|
br: br,
|
|
r: ranges.Range{Start: 0, Len: l},
|
|
unit: unit,
|
|
pad: pad,
|
|
}, nil
|
|
}
|
|
|
|
func (b Binary) toBytesBuffer(r ranges.Range) (*bytes.Buffer, error) {
|
|
br, err := bitiox.Range(b.br, r.Start, r.Len)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
buf := &bytes.Buffer{}
|
|
if _, err := bitiox.CopyBits(buf, br); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return buf, nil
|
|
}
|
|
|
|
func (Binary) ExtType() string { return "binary" }
|
|
|
|
func (Binary) ExtKeys() []string {
|
|
return []string{
|
|
"bits",
|
|
"bytes",
|
|
"name",
|
|
"size",
|
|
"start",
|
|
"stop",
|
|
"unit",
|
|
}
|
|
}
|
|
|
|
func (b Binary) ToBinary() (Binary, error) {
|
|
return b, nil
|
|
}
|
|
|
|
func (b Binary) JQValueLength() any {
|
|
return int(b.r.Len / int64(b.unit))
|
|
}
|
|
func (b Binary) JQValueSliceLen() any {
|
|
return b.JQValueLength()
|
|
}
|
|
|
|
func (b Binary) JQValueIndex(index int) any {
|
|
if index < 0 {
|
|
return nil
|
|
}
|
|
|
|
buf, err := b.toBytesBuffer(ranges.Range{Start: b.r.Start + int64(index*b.unit), Len: int64(b.unit)})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
extraBits := uint((8 - b.unit%8) % 8)
|
|
|
|
return new(big.Int).Rsh(new(big.Int).SetBytes(buf.Bytes()), extraBits)
|
|
}
|
|
func (b Binary) JQValueSlice(start int, end int) any {
|
|
rStart := int64(start * b.unit)
|
|
rLen := int64((end - start) * b.unit)
|
|
|
|
return Binary{
|
|
br: b.br,
|
|
r: ranges.Range{Start: b.r.Start + rStart, Len: rLen},
|
|
unit: b.unit,
|
|
}
|
|
}
|
|
func (b Binary) JQValueKey(name string) any {
|
|
switch name {
|
|
case "bits":
|
|
if b.unit == 1 {
|
|
return b
|
|
}
|
|
return Binary{br: b.br, r: b.r, unit: 1}
|
|
case "bytes":
|
|
if b.unit == 8 {
|
|
return b
|
|
}
|
|
return Binary{br: b.br, r: b.r, unit: 8}
|
|
|
|
case "name":
|
|
f := iox.Unwrap(b.br)
|
|
// this exploits the fact that *os.File has Name()
|
|
if n, ok := f.(interface{ Name() string }); ok {
|
|
return n.Name()
|
|
}
|
|
return nil
|
|
case "size":
|
|
return new(big.Int).SetInt64(b.r.Len / int64(b.unit))
|
|
case "start":
|
|
return new(big.Int).SetInt64(b.r.Start / int64(b.unit))
|
|
case "stop":
|
|
stop := b.r.Stop()
|
|
stopUnits := stop / int64(b.unit)
|
|
if stop%int64(b.unit) != 0 {
|
|
stopUnits++
|
|
}
|
|
return new(big.Int).SetInt64(stopUnits)
|
|
case "unit":
|
|
return b.unit
|
|
}
|
|
return nil
|
|
}
|
|
func (b Binary) JQValueEach() any {
|
|
return nil
|
|
}
|
|
func (b Binary) JQValueType() string {
|
|
return gojq.JQTypeString
|
|
}
|
|
func (b Binary) JQValueKeys() any {
|
|
return gojqx.FuncTypeNameError{Name: "keys", Typ: gojq.JQTypeString}
|
|
}
|
|
func (b Binary) JQValueHas(key any) any {
|
|
return gojqx.HasKeyTypeError{L: gojq.JQTypeString, R: fmt.Sprintf("%v", key)}
|
|
}
|
|
func (b Binary) JQValueToNumber() any {
|
|
buf, err := b.toBytesBuffer(b.r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
extraBits := uint((8 - b.r.Len%8) % 8)
|
|
return new(big.Int).Rsh(new(big.Int).SetBytes(buf.Bytes()), extraBits)
|
|
}
|
|
func (b Binary) JQValueToString() any {
|
|
return b.JQValueToGoJQ()
|
|
}
|
|
func (b Binary) JQValueToGoJQEx(optsFn func() (*Options, error)) any {
|
|
br, err := b.toReader()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
brC, err := bitio.CloneReaderAtSeeker(br)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
opts, err := optsFn()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
s, err := opts.BitsFormatFn(brC)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return s
|
|
}
|
|
|
|
func (b Binary) JQValueToGoJQ() any {
|
|
buf, err := b.toBytesBuffer(b.r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return buf.String()
|
|
}
|
|
|
|
func (b Binary) Display(w io.Writer, opts *Options) error {
|
|
if opts.RawOutput {
|
|
br, err := b.toReader()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err := bitiox.CopyBits(w, br); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
return hexdump(w, b, opts)
|
|
}
|
|
|
|
func (b Binary) toReader() (bitio.ReaderAtSeeker, error) {
|
|
br, err := bitiox.Range(b.br, b.r.Start, b.r.Len)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if b.pad == 0 {
|
|
return br, nil
|
|
}
|
|
return bitio.NewMultiReader(bitiox.NewZeroAtSeeker(b.pad), br)
|
|
}
|