mirror of
https://github.com/wader/fq.git
synced 2024-11-23 00:57:15 +03:00
fc74f6857c
Is a bad idea, behaves confusingly with .. etc. Maybe can be done in the future via some option to make all raw bits behave as arrays are instroduced.
498 lines
10 KiB
Go
498 lines
10 KiB
Go
package interp
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"math/big"
|
|
|
|
"github.com/wader/fq/internal/aheadreadseeker"
|
|
"github.com/wader/fq/internal/bitiox"
|
|
"github.com/wader/fq/internal/ctxreadseeker"
|
|
"github.com/wader/fq/internal/gojqx"
|
|
"github.com/wader/fq/internal/iox"
|
|
"github.com/wader/fq/internal/progressreadseeker"
|
|
"github.com/wader/fq/pkg/bitio"
|
|
"github.com/wader/fq/pkg/ranges"
|
|
"github.com/wader/gojq"
|
|
)
|
|
|
|
func init() {
|
|
RegisterFunc1("_tobits", (*Interp)._toBits)
|
|
RegisterFunc0("open", (*Interp)._open)
|
|
}
|
|
|
|
type ToBinary interface {
|
|
ToBinary() (Binary, error)
|
|
}
|
|
|
|
func toBinary(v any) (Binary, error) {
|
|
switch vv := v.(type) {
|
|
case ToBinary:
|
|
return vv.ToBinary()
|
|
default:
|
|
br, err := ToBitReader(v)
|
|
if err != nil {
|
|
return Binary{}, err
|
|
}
|
|
return NewBinaryFromBitReader(br, 8, 0)
|
|
}
|
|
}
|
|
|
|
func ToBitReader(v any) (bitio.ReaderAtSeeker, error) {
|
|
return toBitReaderEx(v, false)
|
|
}
|
|
|
|
type byteRangeError int
|
|
|
|
func (b byteRangeError) Error() string {
|
|
return fmt.Sprintf("byte in binary list must be bytes (0-255) got %d", int(b))
|
|
|
|
}
|
|
|
|
func toBitReaderEx(v any, inArray bool) (bitio.ReaderAtSeeker, error) {
|
|
switch vv := v.(type) {
|
|
case ToBinary:
|
|
bv, err := vv.ToBinary()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return bitiox.Range(bv.br, bv.r.Start, bv.r.Len)
|
|
case string:
|
|
return bitio.NewBitReader([]byte(vv), -1), nil
|
|
case int, float64, *big.Int:
|
|
bi, err := toBigInt(v)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if inArray {
|
|
if bi.Cmp(big.NewInt(255)) > 0 || bi.Cmp(big.NewInt(0)) < 0 {
|
|
return nil, byteRangeError(bi.Int64())
|
|
}
|
|
n := bi.Uint64()
|
|
b := [1]byte{byte(n)}
|
|
return bitio.NewBitReader(b[:], -1), nil
|
|
}
|
|
|
|
bitLen := int64(bi.BitLen())
|
|
// bit.Int "The bit length of 0 is 0."
|
|
if bitLen == 0 {
|
|
var z [1]byte
|
|
return bitio.NewBitReader(z[:], 1), nil
|
|
}
|
|
// TODO: how should this work? "0xf | tobytes" 4bits or 8bits? now 4
|
|
padBefore := (8 - (bitLen % 8)) % 8
|
|
// padBefore := 0
|
|
br, err := bitiox.Range(bitio.NewBitReader(bi.Bytes(), -1), padBefore, bitLen)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return br, nil
|
|
case []any:
|
|
rr := make([]bitio.ReadAtSeeker, 0, len(vv))
|
|
|
|
// fast path for slice containing only 0-255 numbers and strings
|
|
bs := &bytes.Buffer{}
|
|
for _, e := range vv {
|
|
if bs == nil {
|
|
break
|
|
}
|
|
switch ev := e.(type) {
|
|
case int:
|
|
if ev >= 0 && ev <= 255 {
|
|
bs.WriteByte(byte(ev))
|
|
continue
|
|
}
|
|
case float64:
|
|
b := int(ev)
|
|
if b >= 0 && b <= 255 {
|
|
bs.WriteByte(byte(ev))
|
|
continue
|
|
}
|
|
case *big.Int:
|
|
if ev.Cmp(big.NewInt(0)) >= 0 && ev.Cmp(big.NewInt(255)) <= 0 {
|
|
bs.WriteByte(byte(ev.Uint64()))
|
|
continue
|
|
}
|
|
case string:
|
|
// TODO: maybe only if less then some length?
|
|
bs.WriteString(ev)
|
|
continue
|
|
}
|
|
bs = nil
|
|
}
|
|
if bs != nil {
|
|
return bitio.NewBitReader(bs.Bytes(), -1), nil
|
|
}
|
|
|
|
// TODO: optimize byte array case, flatten into one slice
|
|
for _, e := range vv {
|
|
eBR, eErr := toBitReaderEx(e, true)
|
|
if eErr != nil {
|
|
return nil, eErr
|
|
}
|
|
rr = append(rr, eBR)
|
|
}
|
|
|
|
mb, err := bitio.NewMultiReader(rr...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return mb, nil
|
|
default:
|
|
return nil, fmt.Errorf("value can't be a binary")
|
|
}
|
|
}
|
|
|
|
type toBitsOpts struct {
|
|
Unit int
|
|
KeepRange bool
|
|
PadToUnits int
|
|
}
|
|
|
|
// note is used to implement tobytes* also
|
|
func (i *Interp) _toBits(c any, opts toBitsOpts) any {
|
|
// TODO: unit > 8?
|
|
|
|
bv, err := toBinary(c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
pad := int64(opts.Unit * opts.PadToUnits)
|
|
if pad == 0 {
|
|
pad = int64(opts.Unit)
|
|
}
|
|
|
|
bv.unit = opts.Unit
|
|
bv.pad = (pad - bv.r.Len%pad) % pad
|
|
|
|
if opts.KeepRange {
|
|
return bv
|
|
}
|
|
|
|
br, err := bv.toReader()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
bb, err := NewBinaryFromBitReader(br, bv.unit, 0)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return bb
|
|
}
|
|
|
|
type openFile struct {
|
|
Binary
|
|
filename string
|
|
progressFn progressreadseeker.ProgressFn
|
|
}
|
|
|
|
var _ Value = (*openFile)(nil)
|
|
var _ ToBinary = (*openFile)(nil)
|
|
|
|
func (of *openFile) Display(w io.Writer, opts *Options) error {
|
|
_, err := fmt.Fprintf(w, "<openfile %q>\n", of.filename)
|
|
return err
|
|
}
|
|
|
|
func (of *openFile) ToBinary() (Binary, error) {
|
|
return NewBinaryFromBitReader(of.br, 8, 0)
|
|
}
|
|
|
|
// opens a file for reading from filesystem
|
|
// TODO: when to close? when br loses all refs? need to use finalizer somehow?
|
|
func (i *Interp) _open(c any) any {
|
|
if i.EvalInstance.IsCompleting {
|
|
// TODO: have dummy values for each type for completion?
|
|
br, _ := NewBinaryFromBitReader(bitio.NewBitReader([]byte{}, -1), 8, 0)
|
|
return br
|
|
}
|
|
|
|
var err error
|
|
var f fs.File
|
|
var path string
|
|
|
|
switch c.(type) {
|
|
case nil:
|
|
path = "<stdin>"
|
|
f = i.OS.Stdin()
|
|
default:
|
|
path, err = toString(c)
|
|
if err != nil {
|
|
return fmt.Errorf("%s: %w", path, err)
|
|
}
|
|
f, err = i.OS.FS().Open(path)
|
|
if err != nil {
|
|
// path context added in jq error code
|
|
var pe *fs.PathError
|
|
if errors.As(err, &pe) {
|
|
return pe.Err
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
|
|
var bEnd int64
|
|
var fRS io.ReadSeeker
|
|
|
|
fFI, err := f.Stat()
|
|
if err != nil {
|
|
f.Close()
|
|
return err
|
|
}
|
|
|
|
// ctxreadseeker is used to make sure any io calls can be canceled
|
|
// TODO: ctxreadseeker might leak if the underlying call hangs forever
|
|
|
|
// a regular file should be seekable but fallback below to read whole file if not
|
|
if fFI.Mode().IsRegular() {
|
|
if rs, ok := f.(io.ReadSeeker); ok {
|
|
fRS = ctxreadseeker.New(i.EvalInstance.Ctx, rs)
|
|
bEnd = fFI.Size()
|
|
}
|
|
}
|
|
|
|
if fRS == nil {
|
|
buf, err := io.ReadAll(ctxreadseeker.New(i.EvalInstance.Ctx, &iox.ReadErrSeeker{Reader: f}))
|
|
if err != nil {
|
|
f.Close()
|
|
return err
|
|
}
|
|
fRS = bytes.NewReader(buf)
|
|
bEnd = int64(len(buf))
|
|
}
|
|
|
|
bbf := &openFile{
|
|
filename: path,
|
|
}
|
|
|
|
const progressPrecision = 1024
|
|
fRS = progressreadseeker.New(fRS, progressPrecision, bEnd,
|
|
func(approxReadBytes int64, totalSize int64) {
|
|
// progressFn is assign by decode etc
|
|
if bbf.progressFn != nil {
|
|
bbf.progressFn(approxReadBytes, totalSize)
|
|
}
|
|
},
|
|
)
|
|
|
|
const cacheReadAheadSize = 512 * 1024
|
|
aheadRs := aheadreadseeker.New(fRS, cacheReadAheadSize)
|
|
|
|
// bitio.Buffer -> (bitio.Reader) -> aheadreadseeker -> progressreadseeker -> ctxreadseeker -> readseeker
|
|
|
|
bbf.br = bitio.NewIOBitReadSeeker(aheadRs)
|
|
|
|
return bbf
|
|
}
|
|
|
|
var _ Value = Binary{}
|
|
var _ ToBinary = Binary{}
|
|
|
|
type Binary struct {
|
|
br bitio.ReaderAtSeeker
|
|
r ranges.Range
|
|
unit int
|
|
pad int64
|
|
}
|
|
|
|
func NewBinaryFromBitReader(br bitio.ReaderAtSeeker, unit int, pad int64) (Binary, error) {
|
|
l, err := bitiox.Len(br)
|
|
if err != nil {
|
|
return Binary{}, err
|
|
}
|
|
|
|
return Binary{
|
|
br: br,
|
|
r: ranges.Range{Start: 0, Len: l},
|
|
unit: unit,
|
|
pad: pad,
|
|
}, nil
|
|
}
|
|
|
|
func (b Binary) toBytesBuffer(r ranges.Range) (*bytes.Buffer, error) {
|
|
br, err := bitiox.Range(b.br, r.Start, r.Len)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
buf := &bytes.Buffer{}
|
|
if _, err := bitiox.CopyBits(buf, br); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return buf, nil
|
|
}
|
|
|
|
func (Binary) ExtType() string { return "binary" }
|
|
|
|
func (Binary) ExtKeys() []string {
|
|
return []string{
|
|
"bits",
|
|
"bytes",
|
|
"name",
|
|
"size",
|
|
"start",
|
|
"stop",
|
|
"unit",
|
|
}
|
|
}
|
|
|
|
func (b Binary) ToBinary() (Binary, error) {
|
|
return b, nil
|
|
}
|
|
|
|
func (b Binary) JQValueLength() any {
|
|
return int(b.r.Len / int64(b.unit))
|
|
}
|
|
func (b Binary) JQValueSliceLen() any {
|
|
return b.JQValueLength()
|
|
}
|
|
|
|
func (b Binary) JQValueIndex(index int) any {
|
|
if index < 0 {
|
|
return nil
|
|
}
|
|
|
|
buf, err := b.toBytesBuffer(ranges.Range{Start: b.r.Start + int64(index*b.unit), Len: int64(b.unit)})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
extraBits := uint((8 - b.unit%8) % 8)
|
|
|
|
return new(big.Int).Rsh(new(big.Int).SetBytes(buf.Bytes()), extraBits)
|
|
}
|
|
func (b Binary) JQValueSlice(start int, end int) any {
|
|
rStart := int64(start * b.unit)
|
|
rLen := int64((end - start) * b.unit)
|
|
|
|
return Binary{
|
|
br: b.br,
|
|
r: ranges.Range{Start: b.r.Start + rStart, Len: rLen},
|
|
unit: b.unit,
|
|
}
|
|
}
|
|
func (b Binary) JQValueKey(name string) any {
|
|
switch name {
|
|
case "bits":
|
|
if b.unit == 1 {
|
|
return b
|
|
}
|
|
return Binary{br: b.br, r: b.r, unit: 1}
|
|
case "bytes":
|
|
if b.unit == 8 {
|
|
return b
|
|
}
|
|
return Binary{br: b.br, r: b.r, unit: 8}
|
|
|
|
case "name":
|
|
f := iox.Unwrap(b.br)
|
|
// this exploits the fact that *os.File has Name()
|
|
if n, ok := f.(interface{ Name() string }); ok {
|
|
return n.Name()
|
|
}
|
|
return nil
|
|
case "size":
|
|
return new(big.Int).SetInt64(b.r.Len / int64(b.unit))
|
|
case "start":
|
|
return new(big.Int).SetInt64(b.r.Start / int64(b.unit))
|
|
case "stop":
|
|
stop := b.r.Stop()
|
|
stopUnits := stop / int64(b.unit)
|
|
if stop%int64(b.unit) != 0 {
|
|
stopUnits++
|
|
}
|
|
return new(big.Int).SetInt64(stopUnits)
|
|
case "unit":
|
|
return b.unit
|
|
}
|
|
return nil
|
|
}
|
|
func (b Binary) JQValueEach() any {
|
|
return nil
|
|
}
|
|
func (b Binary) JQValueType() string {
|
|
return gojq.JQTypeString
|
|
}
|
|
func (b Binary) JQValueKeys() any {
|
|
return gojqx.FuncTypeNameError{Name: "keys", Typ: gojq.JQTypeString}
|
|
}
|
|
func (b Binary) JQValueHas(key any) any {
|
|
return gojqx.HasKeyTypeError{L: gojq.JQTypeString, R: fmt.Sprintf("%v", key)}
|
|
}
|
|
func (b Binary) JQValueToNumber() any {
|
|
buf, err := b.toBytesBuffer(b.r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
extraBits := uint((8 - b.r.Len%8) % 8)
|
|
return new(big.Int).Rsh(new(big.Int).SetBytes(buf.Bytes()), extraBits)
|
|
}
|
|
func (b Binary) JQValueToString() any {
|
|
return b.JQValueToGoJQ()
|
|
}
|
|
func (b Binary) JQValueToGoJQEx(optsFn func() (*Options, error)) any {
|
|
br, err := b.toReader()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
brC, err := bitio.CloneReaderAtSeeker(br)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
opts, err := optsFn()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
s, err := opts.BitsFormatFn(brC)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return s
|
|
}
|
|
|
|
func (b Binary) JQValueToGoJQ() any {
|
|
buf, err := b.toBytesBuffer(b.r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return buf.String()
|
|
}
|
|
|
|
func (b Binary) Display(w io.Writer, opts *Options) error {
|
|
if opts.RawOutput {
|
|
br, err := b.toReader()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err := bitiox.CopyBits(w, br); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
return hexdump(w, b, opts)
|
|
}
|
|
|
|
func (b Binary) toReader() (bitio.ReaderAtSeeker, error) {
|
|
br, err := bitiox.Range(b.br, b.r.Start, b.r.Len)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if b.pad == 0 {
|
|
return br, nil
|
|
}
|
|
return bitio.NewMultiReader(bitiox.NewZeroAtSeeker(b.pad), br)
|
|
}
|