package interp import ( "bytes" "fmt" "io" "io/fs" "io/ioutil" "math/big" "github.com/wader/fq/internal/aheadreadseeker" "github.com/wader/fq/internal/bitioextra" "github.com/wader/fq/internal/ctxreadseeker" "github.com/wader/fq/internal/gojqextra" "github.com/wader/fq/internal/ioextra" "github.com/wader/fq/internal/progressreadseeker" "github.com/wader/fq/pkg/bitio" "github.com/wader/fq/pkg/ranges" "github.com/wader/gojq" ) func init() { functionRegisterFns = append(functionRegisterFns, func(i *Interp) []Function { return []Function{ {"_tobits", 3, 3, i._toBits, nil}, {"open", 0, 0, nil, i._open}, } }) } type ToBinary interface { ToBinary() (Binary, error) } func toBinary(v interface{}) (Binary, error) { switch vv := v.(type) { case ToBinary: return vv.ToBinary() default: br, err := toBitReader(v) if err != nil { return Binary{}, err } return newBinaryFromBitReader(br, 8, 0) } } func toBitReader(v interface{}) (bitio.ReaderAtSeeker, error) { return toBitReaderEx(v, false) } func toBitReaderEx(v interface{}, inArray bool) (bitio.ReaderAtSeeker, error) { switch vv := v.(type) { case ToBinary: bv, err := vv.ToBinary() if err != nil { return nil, err } return bitioextra.Range(bv.br, bv.r.Start, bv.r.Len) case string: return bitio.NewBitReader([]byte(vv), -1), nil case int, float64, *big.Int: bi, err := toBigInt(v) if err != nil { return nil, err } if inArray { if bi.Cmp(big.NewInt(255)) > 0 || bi.Cmp(big.NewInt(0)) < 0 { return nil, fmt.Errorf("byte in binary list must be bytes (0-255) got %v", bi) } n := bi.Uint64() b := [1]byte{byte(n)} return bitio.NewBitReader(b[:], -1), nil } bitLen := int64(bi.BitLen()) // bit.Int "The bit length of 0 is 0." if bitLen == 0 { var z [1]byte return bitio.NewBitReader(z[:], 1), nil } // TODO: how should this work? "0xf | tobytes" 4bits or 8bits? now 4 padBefore := (8 - (bitLen % 8)) % 8 // padBefore := 0 br, err := bitioextra.Range(bitio.NewBitReader(bi.Bytes(), -1), padBefore, bitLen) if err != nil { return nil, err } return br, nil case []interface{}: rr := make([]bitio.ReadAtSeeker, 0, len(vv)) // TODO: optimize byte array case, flatten into one slice for _, e := range vv { eBR, eErr := toBitReaderEx(e, true) if eErr != nil { return nil, eErr } rr = append(rr, eBR) } mb, err := bitio.NewMultiReader(rr...) if err != nil { return nil, err } return mb, nil default: return nil, fmt.Errorf("value can't be a binary") } } // note is used to implement tobytes* also func (i *Interp) _toBits(c interface{}, a []interface{}) interface{} { unit, ok := gojqextra.ToInt(a[0]) if !ok { return gojqextra.FuncTypeError{Name: "_tobits", V: a[0]} } keepRange, ok := gojqextra.ToBoolean(a[1]) if !ok { return gojqextra.FuncTypeError{Name: "_tobits", V: a[1]} } padToUnits, ok := gojqextra.ToInt(a[2]) if !ok { return gojqextra.FuncTypeError{Name: "_tobits", V: a[2]} } // TODO: unit > 8? bv, err := toBinary(c) if err != nil { return err } pad := int64(unit * padToUnits) if pad == 0 { pad = int64(unit) } bv.unit = unit bv.pad = (pad - bv.r.Len%pad) % pad if keepRange { return bv } br, err := bv.toReader() if err != nil { return err } bb, err := newBinaryFromBitReader(br, bv.unit, bv.pad) if err != nil { return err } return bb } type openFile struct { Binary filename string progressFn progressreadseeker.ProgressFn } var _ Value = (*openFile)(nil) var _ ToBinary = (*openFile)(nil) func (of *openFile) Display(w io.Writer, opts Options) error { _, err := fmt.Fprintf(w, "\n", of.filename) return err } func (of *openFile) ToBinary() (Binary, error) { return newBinaryFromBitReader(of.br, 8, 0) } // def open: #:: string| => binary // opens a file for reading from filesystem // TODO: when to close? when br loses all refs? need to use finalizer somehow? func (i *Interp) _open(c interface{}, a []interface{}) gojq.Iter { if i.evalInstance.isCompleting { return gojq.NewIter() } var err error var f fs.File var path string switch c.(type) { case nil: path = "" f = i.os.Stdin() default: path, err = toString(c) if err != nil { return gojq.NewIter(fmt.Errorf("%s: %w", path, err)) } f, err = i.os.FS().Open(path) if err != nil { return gojq.NewIter(err) } } var bEnd int64 var fRS io.ReadSeeker fFI, err := f.Stat() if err != nil { f.Close() return gojq.NewIter(err) } // ctxreadseeker is used to make sure any io calls can be canceled // TODO: ctxreadseeker might leak if the underlaying call hangs forever // a regular file should be seekable but fallback below to read whole file if not if fFI.Mode().IsRegular() { if rs, ok := f.(io.ReadSeeker); ok { fRS = ctxreadseeker.New(i.evalInstance.ctx, rs) bEnd = fFI.Size() } } if fRS == nil { buf, err := ioutil.ReadAll(ctxreadseeker.New(i.evalInstance.ctx, &ioextra.ReadErrSeeker{Reader: f})) if err != nil { f.Close() return gojq.NewIter(err) } fRS = bytes.NewReader(buf) bEnd = int64(len(buf)) } bbf := &openFile{ filename: path, } const progressPrecision = 1024 fRS = progressreadseeker.New(fRS, progressPrecision, bEnd, func(approxReadBytes int64, totalSize int64) { // progressFn is assign by decode etc if bbf.progressFn != nil { bbf.progressFn(approxReadBytes, totalSize) } }, ) const cacheReadAheadSize = 512 * 1024 aheadRs := aheadreadseeker.New(fRS, cacheReadAheadSize) // bitio.Buffer -> (bitio.Reader) -> aheadreadseeker -> progressreadseeker -> ctxreadseeker -> readseeker bbf.br = bitio.NewIOBitReadSeeker(aheadRs) if err != nil { return gojq.NewIter(err) } return gojq.NewIter(bbf) } var _ Value = Binary{} var _ ToBinary = Binary{} type Binary struct { br bitio.ReaderAtSeeker r ranges.Range unit int pad int64 } func newBinaryFromBitReader(br bitio.ReaderAtSeeker, unit int, pad int64) (Binary, error) { l, err := bitioextra.Len(br) if err != nil { return Binary{}, err } return Binary{ br: br, r: ranges.Range{Start: 0, Len: l}, unit: unit, pad: pad, }, nil } func (b Binary) toBytesBuffer(r ranges.Range) (*bytes.Buffer, error) { br, err := bitioextra.Range(b.br, r.Start, r.Len) if err != nil { return nil, err } buf := &bytes.Buffer{} if _, err := bitioextra.CopyBits(buf, br); err != nil { return nil, err } return buf, nil } func (Binary) ExtType() string { return "binary" } func (Binary) ExtKeys() []string { return []string{ "size", "start", "stop", "bits", "bytes", } } func (b Binary) ToBinary() (Binary, error) { return b, nil } func (b Binary) JQValueLength() interface{} { return int(b.r.Len / int64(b.unit)) } func (b Binary) JQValueSliceLen() interface{} { return b.JQValueLength() } func (b Binary) JQValueIndex(index int) interface{} { if index < 0 { return nil } buf, err := b.toBytesBuffer(ranges.Range{Start: b.r.Start + int64(index*b.unit), Len: int64(b.unit)}) if err != nil { return err } extraBits := uint((8 - b.unit%8) % 8) return new(big.Int).Rsh(new(big.Int).SetBytes(buf.Bytes()), extraBits) } func (b Binary) JQValueSlice(start int, end int) interface{} { rStart := int64(start * b.unit) rLen := int64((end - start) * b.unit) return Binary{ br: b.br, r: ranges.Range{Start: b.r.Start + rStart, Len: rLen}, unit: b.unit, } } func (b Binary) JQValueKey(name string) interface{} { switch name { case "size": return new(big.Int).SetInt64(b.r.Len / int64(b.unit)) case "start": return new(big.Int).SetInt64(b.r.Start / int64(b.unit)) case "stop": stop := b.r.Stop() stopUnits := stop / int64(b.unit) if stop%int64(b.unit) != 0 { stopUnits++ } return new(big.Int).SetInt64(stopUnits) case "bits": if b.unit == 1 { return b } return Binary{br: b.br, r: b.r, unit: 1} case "bytes": if b.unit == 8 { return b } return Binary{br: b.br, r: b.r, unit: 8} } return nil } func (b Binary) JQValueEach() interface{} { return nil } func (b Binary) JQValueType() string { return "binary" } func (b Binary) JQValueKeys() interface{} { return gojqextra.FuncTypeNameError{Name: "keys", Typ: "binary"} } func (b Binary) JQValueHas(key interface{}) interface{} { return gojqextra.HasKeyTypeError{L: "binary", R: fmt.Sprintf("%v", key)} } func (b Binary) JQValueToNumber() interface{} { buf, err := b.toBytesBuffer(b.r) if err != nil { return err } extraBits := uint((8 - b.r.Len%8) % 8) return new(big.Int).Rsh(new(big.Int).SetBytes(buf.Bytes()), extraBits) } func (b Binary) JQValueToString() interface{} { return b.JQValueToGoJQ() } func (b Binary) JQValueToGoJQ() interface{} { buf, err := b.toBytesBuffer(b.r) if err != nil { return err } return buf.String() } func (b Binary) JQValueUpdate(key interface{}, u interface{}, delpath bool) interface{} { return gojqextra.NonUpdatableTypeError{Key: fmt.Sprintf("%v", key), Typ: "binary"} } func (b Binary) Display(w io.Writer, opts Options) error { if opts.RawOutput { br, err := b.toReader() if err != nil { return err } if _, err := bitioextra.CopyBits(w, br); err != nil { return err } return nil } return hexdump(w, b, opts) } func (b Binary) toReader() (bitio.ReaderAtSeeker, error) { br, err := bitioextra.Range(b.br, b.r.Start, b.r.Len) if err != nil { return nil, err } if b.pad == 0 { return br, nil } return bitio.NewMultiReader(bitioextra.NewZeroAtSeeker(b.pad), br) }