wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-12-25 14:23:18 +03:00
Code Issues Projects Releases Wiki Activity

Merge pull request #370 from wader/extra-rename-and-more-generics

Browse Source 
*extra: Rename <pkg>extra to just <pkg>ex and refactor to use generics
This commit is contained in:
Mattias Wadman 2022-08-12 16:45:02 +02:00 committed by GitHub
commit 13ae529ede
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
47 changed files with 249 additions and 264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.vscode/settings.json vendored
View File

@ -11,7 +11,7 @@
"BCDU",
"bgrep",
"bitio",
"bitioextra",
"bitioex",
"bson",
"bxor",
"bzip",
@ -53,7 +53,7 @@
"gofmt",
"goimports",
"gojq",
"gojqextra",
"gojqex",
"golangci",
"gopacket",
"GOPATH",
@ -73,7 +73,7 @@
"inband",
"infe",
"interp",
"ioextra",
"ioex",
"ipco",
"iprint",
"iprp",

4
format/cbor/cbor.go
View File

@ -13,7 +13,7 @@ import (
"strings"
"github.com/wader/fq/format"
"github.com/wader/fq/internal/mathextra"
"github.com/wader/fq/internal/mathex"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
@ -115,7 +115,7 @@ func decodeCBORValue(d *decode.D) any {
}},
majorTypeNegativeInt: {s: scalar.S{Sym: "negative_int"}, d: func(d *decode.D, shortCount uint64, count uint64) any {
n := new(big.Int)
n.SetUint64(count).Neg(n).Sub(n, mathextra.BigIntOne)
n.SetUint64(count).Neg(n).Sub(n, mathex.BigIntOne)
d.FieldValueBigInt("value", n)
return nil
}},

10
format/csv/csv.go
View File

@ -9,7 +9,7 @@ import (
"io"
"github.com/wader/fq/format"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/gojqex"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
@ -80,11 +80,11 @@ func toCSV(_ *interp.Interp, c []any, opts ToCSVOpts) any {
w.Comma = rune(opts.Comma[0])
}
for _, row := range c {
rs, ok := gojqextra.Cast[[]any](row)
rs, ok := gojqex.Cast[[]any](row)
if !ok {
return fmt.Errorf("expected row to be an array, got %s", gojqextra.TypeErrorPreview(row))
return fmt.Errorf("expected row to be an array, got %s", gojqex.TypeErrorPreview(row))
}
vs, ok := gojqextra.NormalizeToStrings(rs).([]any)
vs, ok := gojqex.NormalizeToStrings(rs).([]any)
if !ok {
panic("not array")
}
@ -92,7 +92,7 @@ func toCSV(_ *interp.Interp, c []any, opts ToCSVOpts) any {
for _, v := range vs {
s, ok := v.(string)
if !ok {
return fmt.Errorf("expected row record to be scalars, got %s", gojqextra.TypeErrorPreview(v))
return fmt.Errorf("expected row record to be scalars, got %s", gojqex.TypeErrorPreview(v))
}
ss = append(ss, s)
}

4
format/flac/flac.go
View File

@ -12,7 +12,7 @@ import (
"fmt"
"github.com/wader/fq/format"
"github.com/wader/fq/internal/mathextra"
"github.com/wader/fq/internal/mathex"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
@ -67,7 +67,7 @@ func flacDecode(d *decode.D, _ any) any {
samplesInFrame := ffo.Samples
if streamTotalSamples > 0 {
samplesInFrame = mathextra.MinUInt64(streamTotalSamples-streamDecodedSamples, ffo.Samples)
samplesInFrame = mathex.Min(streamTotalSamples-streamDecodedSamples, ffo.Samples)
}
frameStreamSamplesBuf := ffo.SamplesBuf[0 : samplesInFrame*uint64(ffo.Channels*ffo.BitsPerSample/8)]
framesNDecodedSamples += ffo.Samples

4
format/flac/flac_frame.go
View File

@ -5,7 +5,7 @@ import (
"math/bits"
"github.com/wader/fq/format"
"github.com/wader/fq/internal/mathextra"
"github.com/wader/fq/internal/mathex"
"github.com/wader/fq/pkg/checksum"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
@ -463,7 +463,7 @@ func frameDecode(d *decode.D, in any) any {
_ = high
low := d.U(riceParameter)
_ = low
samples[n] = mathextra.ZigZag(high<<riceParameter | low)
samples[n] = mathex.ZigZag(high<<riceParameter | low)
n++
}
samplesStop := d.Pos()

4
format/icc/icc_profile.go
View File

@ -5,7 +5,7 @@ package icc
import (
"github.com/wader/fq/format"
"github.com/wader/fq/internal/mathextra"
"github.com/wader/fq/internal/mathex"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
)
@ -165,7 +165,7 @@ func iccProfileDecode(d *decode.D, _ any) any {
// was. instead add alignment after if offset+size does not align and to be sure clamp it if outside buffer.
alignStart := int64(offset) + int64(size)
alignBytes := (4 - (int64(offset)+int64(size))%4) % 4
alignBytes = mathextra.MinInt64(d.Len()/8-alignStart, alignBytes)
alignBytes = mathex.Min(d.Len()/8-alignStart, alignBytes)
if alignBytes != 0 {
d.RangeFn(alignStart*8, alignBytes*8, func(d *decode.D) {
d.FieldRawLen("alignment", d.BitsLeft())

4
format/inet/ipv6_packet.go
View File

@ -5,7 +5,7 @@ import (
"net"
"github.com/wader/fq/format"
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/internal/bitioex"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
"github.com/wader/fq/pkg/scalar"
@ -101,7 +101,7 @@ var hopByHopTypeNames = scalar.UToSymStr{
var mapUToIPv6Sym = scalar.Fn(func(s scalar.S) (scalar.S, error) {
b := &bytes.Buffer{}
if _, err := bitioextra.CopyBits(b, s.ActualBitBuf()); err != nil {
if _, err := bitioex.CopyBits(b, s.ActualBitBuf()); err != nil {
return s, err
}
s.Sym = net.IP(b.Bytes()).String()

2
format/matroska/matroska.go
View File

@ -301,7 +301,7 @@ func decodeMaster(d *decode.D, bitsLimit int64, tag ebml.Tag, dc *decodeContext)
// crcValue := crcD.FieldMustRemove("value")
// elementCRC := &crc.CRC{Bits: 32, Current: 0xffff_ffff, Table: crc.IEEELETable}
// //log.Printf("crc: %x-%x %d\n", crcStart/8, d.Pos()/8, (d.Pos()-crcStart)/8)
// ioextra.MustCopy(elementCRC, d.BitBufRange(crcStart, d.Pos()-crcStart))
// ioex.MustCopy(elementCRC, d.BitBufRange(crcStart, d.Pos()-crcStart))
// crcD.FieldChecksumRange("value", crcValue.Range.Start, crcValue.Range.Len, elementCRC.Sum(nil), decode.LittleEndian)
// }
})

4
format/mpeg/avc_nalu.go
View File

@ -4,7 +4,7 @@ package mpeg
import (
"github.com/wader/fq/format"
"github.com/wader/fq/internal/mathextra"
"github.com/wader/fq/internal/mathex"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
@ -49,7 +49,7 @@ func uEV(d *decode.D) uint64 { return expGolomb(d) }
func sEV(d *decode.D) int64 {
v := expGolomb(d) + 1
return mathextra.ZigZag(v) - -int64(v&1)
return mathex.ZigZag(v) - -int64(v&1)
}
const (

6
format/protobuf/protobuf.go
View File

@ -6,7 +6,7 @@ import (
"embed"
"github.com/wader/fq/format"
"github.com/wader/fq/internal/mathextra"
"github.com/wader/fq/internal/mathex"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
"github.com/wader/fq/pkg/scalar"
@ -83,7 +83,7 @@ func protobufDecodeField(d *decode.D, pbm *format.ProtoBufMessage) {
switch pbf.Type {
case format.ProtoBufTypeInt32, format.ProtoBufTypeInt64:
v := mathextra.ZigZag(value)
v := mathex.ZigZag(value)
d.FieldValueS("value", v)
if len(pbf.Enums) > 0 {
d.FieldValueStr("enum", pbf.Enums[uint64(v)])
@ -95,7 +95,7 @@ func protobufDecodeField(d *decode.D, pbm *format.ProtoBufMessage) {
}
case format.ProtoBufTypeSInt32, format.ProtoBufTypeSInt64:
// TODO: correct? 32 different?
v := mathextra.TwosComplement(64, value)
v := mathex.TwosComplement(64, value)
d.FieldValueS("value", v)
if len(pbf.Enums) > 0 {
d.FieldValueStr("enum", pbf.Enums[uint64(v)])

20
format/text/url.go
View File

@ -3,7 +3,7 @@ package text
import (
"net/url"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/gojqex"
"github.com/wader/fq/pkg/interp"
)
@ -56,15 +56,15 @@ func init() {
toURLValues := func(c map[string]any) url.Values {
qv := url.Values{}
for k, v := range c {
if va, ok := gojqextra.Cast[[]any](v); ok {
if va, ok := gojqex.Cast[[]any](v); ok {
var ss []string
for _, s := range va {
if s, ok := gojqextra.Cast[string](s); ok {
if s, ok := gojqex.Cast[string](s); ok {
ss = append(ss, s)
}
}
qv[k] = ss
} else if vs, ok := gojqextra.Cast[string](v); ok {
} else if vs, ok := gojqex.Cast[string](v); ok {
qv[k] = []string{vs}
}
}
@ -72,7 +72,7 @@ func init() {
}
interp.RegisterFunc0("tourlquery", func(_ *interp.Interp, c map[string]any) any {
// TODO: nicer
c, ok := gojqextra.NormalizeToStrings(c).(map[string]any)
c, ok := gojqex.NormalizeToStrings(c).(map[string]any)
if !ok {
panic("not map")
}
@ -118,12 +118,12 @@ func init() {
})
interp.RegisterFunc0("tourl", func(_ *interp.Interp, c map[string]any) any {
// TODO: nicer
c, ok := gojqextra.NormalizeToStrings(c).(map[string]any)
c, ok := gojqex.NormalizeToStrings(c).(map[string]any)
if !ok {
panic("not map")
}
str := func(v any) string { s, _ := gojqextra.Cast[string](v); return s }
str := func(v any) string { s, _ := gojqex.Cast[string](v); return s }
u := url.URL{
Scheme: str(c["scheme"]),
Host: str(c["host"]),
@ -131,7 +131,7 @@ func init() {
Fragment: str(c["fragment"]),
}
if um, ok := gojqextra.Cast[map[string]any](c["user"]); ok {
if um, ok := gojqex.Cast[map[string]any](c["user"]); ok {
username, password := str(um["username"]), str(um["password"])
if username != "" {
if password == "" {
@ -141,10 +141,10 @@ func init() {
}
}
}
if s, ok := gojqextra.Cast[string](c["rawquery"]); ok {
if s, ok := gojqex.Cast[string](c["rawquery"]); ok {
u.RawQuery = s
}
if qm, ok := gojqextra.Cast[map[string]any](c["query"]); ok {
if qm, ok := gojqex.Cast[map[string]any](c["query"]); ok {
u.RawQuery = toURLValues(qm).Encode()
}

8
format/toml/toml.go
View File

@ -6,7 +6,7 @@ import (
"github.com/BurntSushi/toml"
"github.com/wader/fq/format"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/gojqex"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
@ -37,7 +37,7 @@ func decodeTOML(d *decode.D, _ any) any {
d.Fatalf("%s", err)
}
var s scalar.S
s.Actual = gojqextra.Normalize(r)
s.Actual = gojqex.Normalize(r)
// TODO: better way to handle that an empty file is valid toml and parsed as an object
switch v := s.Actual.(type) {
@ -58,11 +58,11 @@ func decodeTOML(d *decode.D, _ any) any {
func toTOML(_ *interp.Interp, c any) any {
if c == nil {
return gojqextra.FuncTypeError{Name: "totoml", V: c}
return gojqex.FuncTypeError{Name: "totoml", V: c}
}
b := &bytes.Buffer{}
if err := toml.NewEncoder(b).Encode(gojqextra.Normalize(c)); err != nil {
if err := toml.NewEncoder(b).Encode(gojqex.Normalize(c)); err != nil {
return err
}
return b.String()

20
format/xml/xml.go
View File

@ -18,9 +18,9 @@ import (
"strings"
"github.com/wader/fq/format"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/gojqex"
"github.com/wader/fq/internal/proxysort"
"github.com/wader/fq/internal/stringsextra"
"github.com/wader/fq/internal/stringsex"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
@ -258,7 +258,7 @@ func decodeXML(d *decode.D, in any) any {
switch t := t.(type) {
case xml.CharData:
if !whitespaceRE.Match([]byte(t)) {
d.Fatalf("root element has trailing non-whitespace %q", stringsextra.TrimN(string(t), 50, "..."))
d.Fatalf("root element has trailing non-whitespace %q", stringsex.TrimN(string(t), 50, "..."))
}
// ignore trailing whitespace
case xml.ProcInst:
@ -440,12 +440,12 @@ func toXMLFromArray(c any, opts ToXMLOpts) any {
ca, ok := c.([]any)
if !ok {
return gojqextra.FuncTypeError{Name: "toxml", V: c}
return gojqex.FuncTypeError{Name: "toxml", V: c}
}
n, ok := f(ca)
if !ok {
// TODO: better error
return gojqextra.FuncTypeError{Name: "toxml", V: c}
return gojqex.FuncTypeError{Name: "toxml", V: c}
}
bb := &bytes.Buffer{}
e := xml.NewEncoder(bb)
@ -461,10 +461,10 @@ func toXMLFromArray(c any, opts ToXMLOpts) any {
}
func toXML(_ *interp.Interp, c any, opts ToXMLOpts) any {
if v, ok := gojqextra.Cast[map[string]any](c); ok {
return toXMLFromObject(gojqextra.NormalizeToStrings(v), opts)
} else if v, ok := gojqextra.Cast[[]any](c); ok {
return toXMLFromArray(gojqextra.NormalizeToStrings(v), opts)
if v, ok := gojqex.Cast[map[string]any](c); ok {
return toXMLFromObject(gojqex.NormalizeToStrings(v), opts)
} else if v, ok := gojqex.Cast[[]any](c); ok {
return toXMLFromArray(gojqex.NormalizeToStrings(v), opts)
}
return gojqextra.FuncTypeError{Name: "toxml", V: c}
return gojqex.FuncTypeError{Name: "toxml", V: c}
}

4
format/yaml/yaml.go
View File

@ -6,7 +6,7 @@ import (
"embed"
"github.com/wader/fq/format"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/gojqex"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
@ -54,7 +54,7 @@ func decodeYAML(d *decode.D, _ any) any {
}
func toYAML(_ *interp.Interp, c any) any {
b, err := yaml.Marshal(gojqextra.Normalize(c))
b, err := yaml.Marshal(gojqex.Normalize(c))
if err != nil {
return err
}

3
go.mod
View File

@ -42,6 +42,9 @@ require (
// go get -d golang.org/x/crypto@master && go mod tidy
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d
// has no tags
// go get -d golang.org/x/exp@master && go mod tidy
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e
// has no tags
// go get -d golang.org/x/net@master && go mod tidy
golang.org/x/net v0.0.0-20220630215102-69896b714898
// bump: gomod-golang/text /golang\.org\/x\/text v(.*)/ https://github.com/golang/text.git|^0

2
go.sum
View File

@ -29,6 +29,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA=
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=

2
internal/bitioextra/bitioextra.go → internal/bitioex/bitioextra.go
View File

@ -1,4 +1,4 @@
package bitioextra
package bitioex
// bitio helpers that im not sure belong in bitio

2
internal/bitioextra/zeroreadatseeker.go → internal/bitioex/zeroreadatseeker.go
View File

@ -1,4 +1,4 @@
package bitioextra
package bitioex
import (
"io"

2
internal/gojqextra/error.go → internal/gojqex/error.go
View File

@ -1,4 +1,4 @@
package gojqextra
package gojqex
import (
"fmt"

2
internal/gojqextra/makefn.go → internal/gojqex/makefn.go
View File

@ -1,4 +1,4 @@
package gojqextra
package gojqex
//go:generate sh -c "cat makefn_gen.go.tmpl | go run ../../dev/tmpl.go | gofmt > makefn_gen.go"

2
internal/gojqextra/makefn_gen.go → internal/gojqex/makefn_gen.go
View File

@ -1,5 +1,5 @@
// Generated from makefn_gen.go.tmpl
package gojqextra
package gojqex
import (
"github.com/wader/fq/internal/mapstruct"

2
internal/gojqextra/makefn_gen.go.tmpl → internal/gojqex/makefn_gen.go.tmpl
View File

@ -1,5 +1,5 @@
// Generated from makefn_gen.go.tmpl
package gojqextra
package gojqex
import (
"github.com/wader/fq/internal/mapstruct"

2
internal/gojqextra/regexp.go → internal/gojqex/regexp.go
View File

@ -1,4 +1,4 @@
package gojqextra
package gojqex
import (
"fmt"

2
internal/gojqextra/totype.go → internal/gojqex/totype.go
View File

@ -3,7 +3,7 @@
// The MIT License (MIT)
// Copyright (c) 2019-2021 itchyny
package gojqextra
package gojqex
import (
"math"

2
internal/gojqextra/types.go → internal/gojqex/types.go
View File

@ -1,5 +1,5 @@
//nolint:gosimple
package gojqextra
package gojqex
import (
"bytes"

8
internal/hexdump/hexdump.go
View File

@ -5,7 +5,7 @@ import (
"strings"
"github.com/wader/fq/internal/columnwriter"
"github.com/wader/fq/internal/mathextra"
"github.com/wader/fq/internal/mathex"
"github.com/wader/fq/pkg/bitio"
)
@ -60,7 +60,7 @@ func New(w io.Writer, startOffset int64, addrLen int, addrBase int, lineBytes in
func (d *Dumper) flush() error {
if _, err := d.columnW.Columns[0].Write([]byte(
d.dumpAddrFn(mathextra.PadFormatInt(((d.offset-1)/d.lineBytes)*d.lineBytes, d.addrBase, true, d.addrLen)))); err != nil {
d.dumpAddrFn(mathex.PadFormatInt(((d.offset-1)/d.lineBytes)*d.lineBytes, d.addrBase, true, d.addrLen)))); err != nil {
return err
}
if _, err := d.separatorsW.Write([]byte(d.column)); err != nil {
@ -76,7 +76,7 @@ func (d *Dumper) WriteBits(p []byte, nBits int64) (n int64, err error) {
pos := int64(0)
rBits := nBits
if d.bitsBufN > 0 {
r := mathextra.MinInt64(8-d.bitsBufN, nBits)
r := mathex.Min(8-d.bitsBufN, nBits)
v := bitio.Read64(p, 0, r)
bitio.Write64(v, r, d.bitsBuf, d.bitsBufN)
@ -122,7 +122,7 @@ func (d *Dumper) Write(p []byte) (n int, err error) {
}
for i := int64(0); i < d.lineBytes; i++ {
headerSB := &strings.Builder{}
if _, err := headerSB.Write([]byte(mathextra.PadFormatInt(i, d.addrBase, false, 2))); err != nil {
if _, err := headerSB.Write([]byte(mathex.PadFormatInt(i, d.addrBase, false, 2))); err != nil {
return 0, err
}
if i < d.lineBytes-1 {

2
internal/ioextra/ioextra.go → internal/ioex/ioex.go
View File

@ -1,4 +1,4 @@
package ioextra
package ioex
import (
"context"

2
internal/ioextra/runereader.go → internal/ioex/runereader.go
View File

@ -1,4 +1,4 @@
package ioextra
package ioex
import (
"io"

2
internal/mathextra/big.go → internal/mathex/big.go
View File

@ -1,4 +1,4 @@
package mathextra
package mathex
import "math/big"

6
internal/mathextra/big_test.go → internal/mathex/big_test.go
View File

@ -1,11 +1,11 @@
package mathextra_test
package mathex_test
import (
"fmt"
"math/big"
"testing"
"github.com/wader/fq/internal/mathextra"
"github.com/wader/fq/internal/mathex"
)
func TestBigIntSetBytesSigned(t *testing.T) {
@ -28,7 +28,7 @@ func TestBigIntSetBytesSigned(t *testing.T) {
t.Run(fmt.Sprintf("%v %s", tC.buf, tC.s), func(t *testing.T) {
var n big.Int
expected := tC.s
actual := mathextra.BigIntSetBytesSigned(&n, tC.buf).String()
actual := mathex.BigIntSetBytesSigned(&n, tC.buf).String()
if expected != actual {
t.Errorf("expected %s, got %s", expected, actual)
}

2
internal/mathextra/float16.go → internal/mathex/float16.go
View File

@ -13,7 +13,7 @@
// limitations under the License.
//
//nolint:revive
package mathextra
package mathex
import "unsafe"

76
internal/mathextra/num.go → internal/mathex/num.go
View File

@ -1,4 +1,4 @@
package mathextra
package mathex
import (
"fmt"
@ -7,16 +7,22 @@ import (
"strconv"
"strings"
"golang.org/x/exp/constraints"
"github.com/wader/fq/pkg/ranges"
)
type Number interface {
constraints.Integer | constraints.Float
}
var BasePrefixMap = map[int]string{
2: "0b",
8: "0o",
16: "0x",
}
func DigitsInBase(n int64, basePrefix bool, base int) int {
func DigitsInBase[T constraints.Integer](n T, basePrefix bool, base int) int {
prefixLen := 0
if basePrefix {
prefixLen = len(BasePrefixMap[base])
@ -41,66 +47,40 @@ func padFormatNumber(s string, base int, basePrefix bool, width int) string {
return prefixStr + padStr + s
}
func PadFormatInt(i int64, base int, basePrefix bool, width int) string {
return padFormatNumber(strconv.FormatInt(i, base), base, basePrefix, width)
func PadFormatInt[T constraints.Signed](i T, base int, basePrefix bool, width int) string {
return padFormatNumber(strconv.FormatInt(int64(i), base), base, basePrefix, width)
}
func PadFormatUint(i uint64, base int, basePrefix bool, width int) string {
return padFormatNumber(strconv.FormatUint(i, base), base, basePrefix, width)
func PadFormatUint[T constraints.Unsigned](i T, base int, basePrefix bool, width int) string {
return padFormatNumber(strconv.FormatUint(uint64(i), base), base, basePrefix, width)
}
func PadFormatBigInt(i *big.Int, base int, basePrefix bool, width int) string {
return padFormatNumber(i.Text(base), base, basePrefix, width)
}
func MaxUInt64(a, b uint64) uint64 {
if a < b {
return b
func Max[T Number](v T, vs ...T) T {
m := v
for _, v := range vs {
if v > m {
m = v
}
}
return a
return m
}
func MinUInt64(a, b uint64) uint64 {
if a > b {
return b
func Min[T Number](v T, vs ...T) T {
m := v
for _, v := range vs {
if v < m {
m = v
}
}
return a
return m
}
func MaxInt64(a, b int64) int64 {
if a < b {
return b
}
return a
}
func MinInt64(a, b int64) int64 {
if a > b {
return b
}
return a
}
func MaxInt(a, b int) int {
if a < b {
return b
}
return a
}
func MinInt(a, b int) int {
if a > b {
return b
}
return a
}
func ClampInt(min, max, v int) int {
return MaxInt(min, MinInt(max, v))
}
func ClampInt64(min, max, v int64) int64 {
return MaxInt64(min, MinInt64(max, v))
func Clamp[T Number](min, max, v T) T {
return Max(min, Min(max, v))
}
type Bits uint64

6
internal/stringsextra/stringsextra.go → internal/stringsex/stringsextra.go
View File

@ -1,11 +1,11 @@
package stringsextra
package stringsex
import "github.com/wader/fq/internal/mathextra"
import "github.com/wader/fq/internal/mathex"
func TrimN(s string, n int, suffix string) string {
sl, tl := len(s), len(suffix)
if sl+tl <= n {
return s
}
return s[0:mathextra.MaxInt(n-tl, 0)] + suffix
return s[0:mathex.Max(n-tl, 0)] + suffix
}

24
pkg/decode/decode.go
View File

@ -8,8 +8,8 @@ import (
"math/big"
"regexp"
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/internal/ioextra"
"github.com/wader/fq/internal/bitioex"
"github.com/wader/fq/internal/ioex"
"github.com/wader/fq/internal/recoverfn"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/ranges"
@ -45,7 +45,7 @@ func Decode(ctx context.Context, br bitio.ReaderAtSeeker, group Group, opts Opti
}
func decode(ctx context.Context, br bitio.ReaderAtSeeker, group Group, opts Options) (*Value, any, error) {
brLen, err := bitioextra.Len(br)
brLen, err := bitioex.Len(br)
if err != nil {
return nil, nil, err
}
@ -76,7 +76,7 @@ func decode(ctx context.Context, br bitio.ReaderAtSeeker, group Group, opts Opti
}
}
cBR, err := bitioextra.Range(br, decodeRange.Start, decodeRange.Len)
cBR, err := bitioex.Range(br, decodeRange.Start, decodeRange.Len)
if err != nil {
return nil, nil, IOError{Err: err, Op: "BitBufRange", ReadSize: decodeRange.Len, Pos: decodeRange.Start}
}
@ -211,7 +211,7 @@ func (d *D) fieldDecoder(name string, bitBuf bitio.ReaderAtSeeker, v any) *D {
func (d *D) TryCopyBits(w io.Writer, r bitio.Reader) (int64, error) {
// TODO: what size? now same as io.Copy
buf := d.SharedReadBuf(32 * 1024)
return bitioextra.CopyBitsBuffer(w, r, buf)
return bitioex.CopyBitsBuffer(w, r, buf)
}
func (d *D) CopyBits(w io.Writer, r bitio.Reader) int64 {
@ -253,7 +253,7 @@ func (d *D) NewBitBufFromReader(r io.Reader) bitio.ReaderAtSeeker {
func (d *D) TryReadAllBits(r bitio.Reader) ([]byte, error) {
bb := &bytes.Buffer{}
buf := d.SharedReadBuf(32 * 1024)
if _, err := bitioextra.CopyBitsBuffer(bb, r, buf); err != nil {
if _, err := bitioex.CopyBitsBuffer(bb, r, buf); err != nil {
return nil, err
}
return bb.Bytes(), nil
@ -302,7 +302,7 @@ func (d *D) FillGaps(r ranges.Range, namePrefix string) {
gaps := ranges.Gaps(r, valueRanges)
for i, gap := range gaps {
br, err := bitioextra.Range(d.bitBuf, gap.Start, gap.Len)
br, err := bitioex.Range(d.bitBuf, gap.Start, gap.Len)
if err != nil {
d.IOPanic(err, "FillGaps: Range")
}
@ -520,11 +520,11 @@ func (d *D) BytesLen(nBytes int) []byte {
// TODO: rename/remove BitBuf name?
func (d *D) TryBitBufRange(firstBit int64, nBits int64) (bitio.ReaderAtSeeker, error) {
return bitioextra.Range(d.bitBuf, firstBit, nBits)
return bitioex.Range(d.bitBuf, firstBit, nBits)
}
func (d *D) BitBufRange(firstBit int64, nBits int64) bitio.ReaderAtSeeker {
br, err := bitioextra.Range(d.bitBuf, firstBit, nBits)
br, err := bitioex.Range(d.bitBuf, firstBit, nBits)
if err != nil {
panic(IOError{Err: err, Op: "BitBufRange", ReadSize: nBits, Pos: firstBit})
}
@ -544,7 +544,7 @@ func (d *D) Pos() int64 {
}
func (d *D) TryLen() (int64, error) {
return bitioextra.Len(d.bitBuf)
return bitioex.Len(d.bitBuf)
}
func (d *D) Len() int64 {
@ -1071,7 +1071,7 @@ func (d *D) FieldFormatBitBuf(name string, br bitio.ReaderAtSeeker, group Group,
// TODO: rethink these
func (d *D) FieldRootBitBuf(name string, br bitio.ReaderAtSeeker, sms ...scalar.Mapper) *Value {
brLen, err := bitioextra.Len(br)
brLen, err := bitioex.Len(br)
if err != nil {
d.IOPanic(err, "br Len")
}
@ -1233,7 +1233,7 @@ func (d *D) RE(reRef **regexp.Regexp, reStr string) []ranges.Range {
startPos := d.Pos()
rr := ioextra.ByteRuneReader{RS: bitio.NewIOReadSeeker(d.bitBuf)}
rr := ioex.ByteRuneReader{RS: bitio.NewIOReadSeeker(d.bitBuf)}
locs := (*reRef).FindReaderSubmatchIndex(rr)
if locs == nil {
return nil

6
pkg/decode/errors.go
View File

@ -4,7 +4,7 @@ import (
"fmt"
"strings"
"github.com/wader/fq/internal/mathextra"
"github.com/wader/fq/internal/mathex"
"github.com/wader/fq/internal/recoverfn"
)
@ -72,7 +72,7 @@ func (e IOError) Error() string {
}
return fmt.Sprintf("%s: failed at position %s (read size %s seek pos %s): %s",
prefix, mathextra.Bits(e.Pos).StringByteBits(10), mathextra.Bits(e.ReadSize).StringByteBits(10), mathextra.Bits(e.SeekPos).StringByteBits(10), e.Err)
prefix, mathex.Bits(e.Pos).StringByteBits(10), mathex.Bits(e.ReadSize).StringByteBits(10), mathex.Bits(e.SeekPos).StringByteBits(10), e.Err)
}
func (e IOError) Unwrap() error { return e.Err }
@ -84,7 +84,7 @@ type DecoderError struct {
}
func (e DecoderError) Error() string {
return fmt.Sprintf("error at position %s: %s", mathextra.Bits(e.Pos).StringByteBits(16), e.Reason)
return fmt.Sprintf("error at position %s: %s", mathex.Bits(e.Pos).StringByteBits(16), e.Reason)
}
func (DecoderError) IsRecoverableError() bool { return true }

6
pkg/decode/read.go
View File

@ -6,7 +6,7 @@ import (
"math"
"math/big"
"github.com/wader/fq/internal/mathextra"
"github.com/wader/fq/internal/mathex"
"github.com/wader/fq/pkg/bitio"
"golang.org/x/text/encoding"
"golang.org/x/text/encoding/unicode"
@ -75,7 +75,7 @@ func (d *D) tryBigIntEndianSign(nBits int, endian Endian, sign bool) (*big.Int,
n := new(big.Int)
if sign {
mathextra.BigIntSetBytesSigned(n, buf)
mathex.BigIntSetBytesSigned(n, buf)
} else {
n.SetBytes(buf)
}
@ -97,7 +97,7 @@ func (d *D) tryFEndian(nBits int, endian Endian) (float64, error) {
}
switch nBits {
case 16:
return float64(mathextra.Float16(uint16(n)).Float32()), nil
return float64(mathex.Float16(uint16(n)).Float32()), nil
case 32:
return float64(math.Float32frombits(uint32(n))), nil
case 64:

6
pkg/decode/scalar.go
View File

@ -5,7 +5,7 @@ import (
"encoding/binary"
"errors"
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/internal/bitioex"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/scalar"
)
@ -17,7 +17,7 @@ func bitBufIsZero(s scalar.S, isValidate bool) (scalar.S, error) {
// TODO: shared
b := make([]byte, 32*1024)
bLen := int64(len(b)) * 8
brLen, err := bitioextra.Len(br)
brLen, err := bitioex.Len(br)
if err != nil {
return scalar.S{}, err
}
@ -77,7 +77,7 @@ func (d *D) BitBufValidateIsZero() scalar.Mapper {
// TODO: generate?
func assertBitBuf(s scalar.S, isErr bool, bss ...[]byte) (scalar.S, error) {
bb := &bytes.Buffer{}
if _, err := bitioextra.CopyBits(bb, s.ActualBitBuf()); err != nil {
if _, err := bitioex.CopyBits(bb, s.ActualBitBuf()); err != nil {
return s, err
}
for _, bs := range bss {

28
pkg/interp/binary.go
View File

@ -9,10 +9,10 @@ import (
"math/big"
"github.com/wader/fq/internal/aheadreadseeker"
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/internal/bitioex"
"github.com/wader/fq/internal/ctxreadseeker"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/ioextra"
"github.com/wader/fq/internal/gojqex"
"github.com/wader/fq/internal/ioex"
"github.com/wader/fq/internal/progressreadseeker"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/ranges"
@ -52,7 +52,7 @@ func toBitReaderEx(v any, inArray bool) (bitio.ReaderAtSeeker, error) {
if err != nil {
return nil, err
}
return bitioextra.Range(bv.br, bv.r.Start, bv.r.Len)
return bitioex.Range(bv.br, bv.r.Start, bv.r.Len)
case string:
return bitio.NewBitReader([]byte(vv), -1), nil
case int, float64, *big.Int:
@ -79,7 +79,7 @@ func toBitReaderEx(v any, inArray bool) (bitio.ReaderAtSeeker, error) {
// TODO: how should this work? "0xf | tobytes" 4bits or 8bits? now 4
padBefore := (8 - (bitLen % 8)) % 8
// padBefore := 0
br, err := bitioextra.Range(bitio.NewBitReader(bi.Bytes(), -1), padBefore, bitLen)
br, err := bitioex.Range(bitio.NewBitReader(bi.Bytes(), -1), padBefore, bitLen)
if err != nil {
return nil, err
}
@ -216,7 +216,7 @@ func (i *Interp) _open(c any) any {
}
if fRS == nil {
buf, err := io.ReadAll(ctxreadseeker.New(i.EvalInstance.Ctx, &ioextra.ReadErrSeeker{Reader: f}))
buf, err := io.ReadAll(ctxreadseeker.New(i.EvalInstance.Ctx, &ioex.ReadErrSeeker{Reader: f}))
if err != nil {
f.Close()
return err
@ -263,7 +263,7 @@ type Binary struct {
}
func NewBinaryFromBitReader(br bitio.ReaderAtSeeker, unit int, pad int64) (Binary, error) {
l, err := bitioextra.Len(br)
l, err := bitioex.Len(br)
if err != nil {
return Binary{}, err
}
@ -277,12 +277,12 @@ func NewBinaryFromBitReader(br bitio.ReaderAtSeeker, unit int, pad int64) (Binar
}
func (b Binary) toBytesBuffer(r ranges.Range) (*bytes.Buffer, error) {
br, err := bitioextra.Range(b.br, r.Start, r.Len)
br, err := bitioex.Range(b.br, r.Start, r.Len)
if err != nil {
return nil, err
}
buf := &bytes.Buffer{}
if _, err := bitioextra.CopyBits(buf, br); err != nil {
if _, err := bitioex.CopyBits(buf, br); err != nil {
return nil, err
}
@ -369,10 +369,10 @@ func (b Binary) JQValueType() string {
return gojq.JQTypeString
}
func (b Binary) JQValueKeys() any {
return gojqextra.FuncTypeNameError{Name: "keys", Typ: gojq.JQTypeString}
return gojqex.FuncTypeNameError{Name: "keys", Typ: gojq.JQTypeString}
}
func (b Binary) JQValueHas(key any) any {
return gojqextra.HasKeyTypeError{L: gojq.JQTypeString, R: fmt.Sprintf("%v", key)}
return gojqex.HasKeyTypeError{L: gojq.JQTypeString, R: fmt.Sprintf("%v", key)}
}
func (b Binary) JQValueToNumber() any {
buf, err := b.toBytesBuffer(b.r)
@ -400,7 +400,7 @@ func (b Binary) Display(w io.Writer, opts Options) error {
return err
}
if _, err := bitioextra.CopyBits(w, br); err != nil {
if _, err := bitioex.CopyBits(w, br); err != nil {
return err
}
@ -411,12 +411,12 @@ func (b Binary) Display(w io.Writer, opts Options) error {
}
func (b Binary) toReader() (bitio.ReaderAtSeeker, error) {
br, err := bitioextra.Range(b.br, b.r.Start, b.r.Len)
br, err := bitioex.Range(b.br, b.r.Start, b.r.Len)
if err != nil {
return nil, err
}
if b.pad == 0 {
return br, nil
}
return bitio.NewMultiReader(bitioextra.NewZeroAtSeeker(b.pad), br)
return bitio.NewMultiReader(bitioex.NewZeroAtSeeker(b.pad), br)
}

44
pkg/interp/bitops.go
View File

@ -3,7 +3,7 @@ package interp
import (
"math/big"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/gojqex"
"github.com/wader/gojq"
)
@ -25,7 +25,7 @@ func (i *Interp) bnot(c any) any {
case gojq.JQValue:
return i.bnot(c.JQValueToGoJQ())
default:
return &gojqextra.UnaryTypeError{Name: "bnot", V: c}
return &gojqex.UnaryTypeError{Name: "bnot", V: c}
}
}
@ -39,12 +39,12 @@ func (i *Interp) bsl(c any, a any, b any) any {
},
func(l, r float64) any { return int(l) << int(r) },
func(l, r *big.Int) any { return new(big.Int).Lsh(l, uint(r.Uint64())) },
func(l, r string) any { return &gojqextra.BinopTypeError{Name: "bsl", L: l, R: r} },
func(l, r []any) any { return &gojqextra.BinopTypeError{Name: "bsl", L: l, R: r} },
func(l, r string) any { return &gojqex.BinopTypeError{Name: "bsl", L: l, R: r} },
func(l, r []any) any { return &gojqex.BinopTypeError{Name: "bsl", L: l, R: r} },
func(l, r map[string]any) any {
return &gojqextra.BinopTypeError{Name: "bsl", L: l, R: r}
return &gojqex.BinopTypeError{Name: "bsl", L: l, R: r}
},
func(l, r any) any { return &gojqextra.BinopTypeError{Name: "bsl", L: l, R: r} },
func(l, r any) any { return &gojqex.BinopTypeError{Name: "bsl", L: l, R: r} },
)
}
@ -53,12 +53,12 @@ func (i *Interp) bsr(c any, a any, b any) any {
func(l, r int) any { return l >> r },
func(l, r float64) any { return int(l) >> int(r) },
func(l, r *big.Int) any { return new(big.Int).Rsh(l, uint(r.Uint64())) },
func(l, r string) any { return &gojqextra.BinopTypeError{Name: "bsr", L: l, R: r} },
func(l, r []any) any { return &gojqextra.BinopTypeError{Name: "bsr", L: l, R: r} },
func(l, r string) any { return &gojqex.BinopTypeError{Name: "bsr", L: l, R: r} },
func(l, r []any) any { return &gojqex.BinopTypeError{Name: "bsr", L: l, R: r} },
func(l, r map[string]any) any {
return &gojqextra.BinopTypeError{Name: "bsr", L: l, R: r}
return &gojqex.BinopTypeError{Name: "bsr", L: l, R: r}
},
func(l, r any) any { return &gojqextra.BinopTypeError{Name: "bsr", L: l, R: r} },
func(l, r any) any { return &gojqex.BinopTypeError{Name: "bsr", L: l, R: r} },
)
}
@ -67,12 +67,12 @@ func (i *Interp) band(c any, a any, b any) any {
func(l, r int) any { return l & r },
func(l, r float64) any { return int(l) & int(r) },
func(l, r *big.Int) any { return new(big.Int).And(l, r) },
func(l, r string) any { return &gojqextra.BinopTypeError{Name: "band", L: l, R: r} },
func(l, r []any) any { return &gojqextra.BinopTypeError{Name: "band", L: l, R: r} },
func(l, r string) any { return &gojqex.BinopTypeError{Name: "band", L: l, R: r} },
func(l, r []any) any { return &gojqex.BinopTypeError{Name: "band", L: l, R: r} },
func(l, r map[string]any) any {
return &gojqextra.BinopTypeError{Name: "band", L: l, R: r}
return &gojqex.BinopTypeError{Name: "band", L: l, R: r}
},
func(l, r any) any { return &gojqextra.BinopTypeError{Name: "band", L: l, R: r} },
func(l, r any) any { return &gojqex.BinopTypeError{Name: "band", L: l, R: r} },
)
}
@ -81,12 +81,12 @@ func (i *Interp) bor(c any, a any, b any) any {
func(l, r int) any { return l | r },
func(l, r float64) any { return int(l) | int(r) },
func(l, r *big.Int) any { return new(big.Int).Or(l, r) },
func(l, r string) any { return &gojqextra.BinopTypeError{Name: "bor", L: l, R: r} },
func(l, r []any) any { return &gojqextra.BinopTypeError{Name: "bor", L: l, R: r} },
func(l, r string) any { return &gojqex.BinopTypeError{Name: "bor", L: l, R: r} },
func(l, r []any) any { return &gojqex.BinopTypeError{Name: "bor", L: l, R: r} },
func(l, r map[string]any) any {
return &gojqextra.BinopTypeError{Name: "bor", L: l, R: r}
return &gojqex.BinopTypeError{Name: "bor", L: l, R: r}
},
func(l, r any) any { return &gojqextra.BinopTypeError{Name: "bor", L: l, R: r} },
func(l, r any) any { return &gojqex.BinopTypeError{Name: "bor", L: l, R: r} },
)
}
@ -95,11 +95,11 @@ func (i *Interp) bxor(c any, a any, b any) any {
func(l, r int) any { return l ^ r },
func(l, r float64) any { return int(l) ^ int(r) },
func(l, r *big.Int) any { return new(big.Int).Xor(l, r) },
func(l, r string) any { return &gojqextra.BinopTypeError{Name: "bxor", L: l, R: r} },
func(l, r []any) any { return &gojqextra.BinopTypeError{Name: "bxor", L: l, R: r} },
func(l, r string) any { return &gojqex.BinopTypeError{Name: "bxor", L: l, R: r} },
func(l, r []any) any { return &gojqex.BinopTypeError{Name: "bxor", L: l, R: r} },
func(l, r map[string]any) any {
return &gojqextra.BinopTypeError{Name: "bxor", L: l, R: r}
return &gojqex.BinopTypeError{Name: "bxor", L: l, R: r}
},
func(l, r any) any { return &gojqextra.BinopTypeError{Name: "bxor", L: l, R: r} },
func(l, r any) any { return &gojqex.BinopTypeError{Name: "bxor", L: l, R: r} },
)
}

52
pkg/interp/decode.go
View File

@ -11,9 +11,9 @@ import (
"time"
"github.com/mitchellh/copystructure"
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/ioextra"
"github.com/wader/fq/internal/bitioex"
"github.com/wader/fq/internal/gojqex"
"github.com/wader/fq/internal/ioex"
"github.com/wader/fq/internal/mapstruct"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
@ -114,7 +114,7 @@ func (i *Interp) _registry(c any) any {
delete(args, k)
}
}
vf["decode_in_arg"] = gojqextra.Normalize(args)
vf["decode_in_arg"] = gojqex.Normalize(args)
}
if f.Functions != nil {
@ -195,7 +195,7 @@ func (i *Interp) _decode(c any, format string, opts decodeOpts) any {
c,
opts.Progress,
nil,
EvalOpts{output: ioextra.DiscardCtxWriter{Ctx: i.EvalInstance.Ctx}},
EvalOpts{output: ioex.DiscardCtxWriter{Ctx: i.EvalInstance.Ctx}},
)
}
lastProgress := time.Now()
@ -344,7 +344,7 @@ func makeDecodeValueOut(dv *decode.Value, out any) any {
// create another binary we don't have to read and create a string, ex:
// .unknown0 | tobytes[1:] | ...
return decodeValue{
JQValue: &gojqextra.Lazy{
JQValue: &gojqex.Lazy{
Type: "string",
IsScalar: true,
Fn: func() (gojq.JQValue, error) {
@ -353,10 +353,10 @@ func makeDecodeValueOut(dv *decode.Value, out any) any {
if err != nil {
return nil, err
}
if _, err := bitioextra.CopyBits(buf, vvC); err != nil {
if _, err := bitioex.CopyBits(buf, vvC); err != nil {
return nil, err
}
return gojqextra.String([]rune(buf.String())), nil
return gojqex.String([]rune(buf.String())), nil
},
},
decodeValueBase: decodeValueBase{dv: dv},
@ -364,52 +364,52 @@ func makeDecodeValueOut(dv *decode.Value, out any) any {
}
case bool:
return decodeValue{
JQValue: gojqextra.Boolean(vv),
JQValue: gojqex.Boolean(vv),
decodeValueBase: decodeValueBase{dv: dv},
}
case int:
return decodeValue{
JQValue: gojqextra.Number{V: vv},
JQValue: gojqex.Number{V: vv},
decodeValueBase: decodeValueBase{dv: dv},
}
case int64:
return decodeValue{
JQValue: gojqextra.Number{V: big.NewInt(vv)},
JQValue: gojqex.Number{V: big.NewInt(vv)},
decodeValueBase: decodeValueBase{dv: dv},
}
case uint64:
return decodeValue{
JQValue: gojqextra.Number{V: new(big.Int).SetUint64(vv)},
JQValue: gojqex.Number{V: new(big.Int).SetUint64(vv)},
decodeValueBase: decodeValueBase{dv: dv},
}
case float64:
return decodeValue{
JQValue: gojqextra.Number{V: vv},
JQValue: gojqex.Number{V: vv},
decodeValueBase: decodeValueBase{dv: dv},
}
case string:
return decodeValue{
JQValue: gojqextra.String(vv),
JQValue: gojqex.String(vv),
decodeValueBase: decodeValueBase{dv: dv},
}
case []any:
return decodeValue{
JQValue: gojqextra.Array(vv),
JQValue: gojqex.Array(vv),
decodeValueBase: decodeValueBase{dv: dv},
}
case map[string]any:
return decodeValue{
JQValue: gojqextra.Object(vv),
JQValue: gojqex.Object(vv),
decodeValueBase: decodeValueBase{dv: dv},
}
case nil:
return decodeValue{
JQValue: gojqextra.Null{},
JQValue: gojqex.Null{},
decodeValueBase: decodeValueBase{dv: dv},
}
case *big.Int:
return decodeValue{
JQValue: gojqextra.Number{V: vv},
JQValue: gojqex.Number{V: vv},
decodeValueBase: decodeValueBase{dv: dv},
}
default:
@ -497,7 +497,7 @@ func (dvb decodeValueBase) JQValueKey(name string) any {
case "_actual":
switch vv := dv.V.(type) {
case *scalar.S:
jv, ok := gojqextra.ToGoJQValue(vv.Actual)
jv, ok := gojqex.ToGoJQValue(vv.Actual)
if !ok {
return fmt.Errorf("can't convert actual value jq value %#+v", vv.Actual)
}
@ -508,7 +508,7 @@ func (dvb decodeValueBase) JQValueKey(name string) any {
case "_sym":
switch vv := dv.V.(type) {
case *scalar.S:
jv, ok := gojqextra.ToGoJQValue(vv.Sym)
jv, ok := gojqex.ToGoJQValue(vv.Sym)
if !ok {
return fmt.Errorf("can't convert sym value jq value %#+v", vv.Actual)
}
@ -619,7 +619,7 @@ func (v decodeValue) JQValueToGoJQEx(optsFn func() Options) any {
var _ DecodeValue = ArrayDecodeValue{}
type ArrayDecodeValue struct {
gojqextra.Base
gojqex.Base
decodeValueBase
*decode.Compound
}
@ -627,7 +627,7 @@ type ArrayDecodeValue struct {
func NewArrayDecodeValue(dv *decode.Value, out any, c *decode.Compound) ArrayDecodeValue {
return ArrayDecodeValue{
decodeValueBase: decodeValueBase{dv: dv, out: out},
Base: gojqextra.Base{Typ: gojq.JQTypeArray},
Base: gojqex.Base{Typ: gojq.JQTypeArray},
Compound: c,
}
}
@ -672,7 +672,7 @@ func (v ArrayDecodeValue) JQValueHas(key any) any {
func(key any) any {
intKey, ok := key.(int)
if !ok {
return gojqextra.HasKeyTypeError{L: gojq.JQTypeArray, R: fmt.Sprintf("%v", key)}
return gojqex.HasKeyTypeError{L: gojq.JQTypeArray, R: fmt.Sprintf("%v", key)}
}
return intKey >= 0 && intKey < len(v.Compound.Children)
})
@ -690,7 +690,7 @@ func (v ArrayDecodeValue) JQValueToGoJQ() any {
var _ DecodeValue = StructDecodeValue{}
type StructDecodeValue struct {
gojqextra.Base
gojqex.Base
decodeValueBase
*decode.Compound
}
@ -698,7 +698,7 @@ type StructDecodeValue struct {
func NewStructDecodeValue(dv *decode.Value, out any, c *decode.Compound) StructDecodeValue {
return StructDecodeValue{
decodeValueBase: decodeValueBase{dv: dv, out: out},
Base: gojqextra.Base{Typ: gojq.JQTypeObject},
Base: gojqex.Base{Typ: gojq.JQTypeObject},
Compound: c,
}
}
@ -738,7 +738,7 @@ func (v StructDecodeValue) JQValueHas(key any) any {
func(key any) any {
stringKey, ok := key.(string)
if !ok {
return gojqextra.HasKeyTypeError{L: gojq.JQTypeObject, R: fmt.Sprintf("%v", key)}
return gojqex.HasKeyTypeError{L: gojq.JQTypeObject, R: fmt.Sprintf("%v", key)}
}
for _, f := range v.Compound.Children {
if f.Name == stringKey {

14
pkg/interp/default_register.go
View File

@ -3,7 +3,7 @@ package interp
import (
"io/fs"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/gojqex"
"github.com/wader/fq/pkg/decode"
"github.com/wader/gojq"
)
@ -20,25 +20,25 @@ func RegisterFS(fs fs.ReadDirFS) {
}
func RegisterFunc0[Tc any](name string, fn func(e *Interp, c Tc) any) {
DefaultRegistry.Func(gojqextra.Func0(name, fn))
DefaultRegistry.Func(gojqex.Func0(name, fn))
}
func RegisterFunc1[Tc any, Ta0 any](name string, fn func(e *Interp, c Tc, a0 Ta0) any) {
DefaultRegistry.Func(gojqextra.Func1(name, fn))
DefaultRegistry.Func(gojqex.Func1(name, fn))
}
func RegisterFunc2[Tc any, Ta0 any, Ta1 any](name string, fn func(e *Interp, c Tc, a0 Ta0, a1 Ta1) any) {
DefaultRegistry.Func(gojqextra.Func2(name, fn))
DefaultRegistry.Func(gojqex.Func2(name, fn))
}
func RegisterIter0[Tc any](name string, fn func(e *Interp, c Tc) gojq.Iter) {
DefaultRegistry.Func(gojqextra.Iter0(name, fn))
DefaultRegistry.Func(gojqex.Iter0(name, fn))
}
func RegisterIter1[Tc any, Ta0 any](name string, fn func(e *Interp, c Tc, a0 Ta0) gojq.Iter) {
DefaultRegistry.Func(gojqextra.Iter1(name, fn))
DefaultRegistry.Func(gojqex.Iter1(name, fn))
}
func RegisterIter2[Tc any, Ta0 any, Ta1 any](name string, fn func(e *Interp, c Tc, a0 Ta0, a1 Ta1) gojq.Iter) {
DefaultRegistry.Func(gojqextra.Iter2(name, fn))
DefaultRegistry.Func(gojqex.Iter2(name, fn))
}

30
pkg/interp/dump.go
View File

@ -9,10 +9,10 @@ import (
"github.com/wader/fq/internal/ansi"
"github.com/wader/fq/internal/asciiwriter"
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/internal/bitioex"
"github.com/wader/fq/internal/columnwriter"
"github.com/wader/fq/internal/hexpairwriter"
"github.com/wader/fq/internal/mathextra"
"github.com/wader/fq/internal/mathex"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/scalar"
@ -190,7 +190,7 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD
if opts.Verbose {
cfmt(colField, " %s (%s)",
mathextra.BitRange(innerRange).StringByteBits(opts.Addrbase), mathextra.Bits(innerRange.Len).StringByteBits(opts.Sizebase))
mathex.BitRange(innerRange).StringByteBits(opts.Addrbase), mathex.Bits(innerRange.Len).StringByteBits(opts.Sizebase))
}
cprint(colField, "\n")
@ -234,7 +234,7 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD
printErrs(depth, valueErr)
}
rootBitLen, err := bitioextra.Len(rootV.RootReader)
rootBitLen, err := bitioex.Len(rootV.RootReader)
if err != nil {
return err
}
@ -280,9 +280,9 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD
// has length and is not compound or a collapsed struct/array (max depth)
if innerRange.Len > 0 && (!isCompound(v) || (opts.Depth != 0 && opts.Depth == depth)) {
cfmt(colAddr, "%s%s\n",
rootIndent, deco.DumpAddr.F(mathextra.PadFormatInt(startLineByte, opts.Addrbase, true, addrWidth)))
rootIndent, deco.DumpAddr.F(mathex.PadFormatInt(startLineByte, opts.Addrbase, true, addrWidth)))
vBR, err := bitioextra.Range(rootV.RootReader, startByte*8, displaySizeBits)
vBR, err := bitioex.Range(rootV.RootReader, startByte*8, displaySizeBits)
if err != nil {
return err
}
@ -295,7 +295,7 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD
if err != nil {
return err
}
if _, err := bitioextra.CopyBitsBuffer(
if _, err := bitioex.CopyBitsBuffer(
hexpairwriter.New(cw.Columns[colHex], opts.LineBytes, int(startLineByteOffset), hexpairFn),
hexBR,
buf); err != nil {
@ -306,7 +306,7 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD
if err != nil {
return err
}
if _, err := bitioextra.CopyBitsBuffer(
if _, err := bitioex.CopyBitsBuffer(
asciiwriter.New(cw.Columns[colASCII], opts.LineBytes, int(startLineByteOffset), asciiFn),
asciiBR,
buf); err != nil {
@ -316,7 +316,7 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD
for i := int64(1); i < addrLines; i++ {
lineStartByte := startLineByte + i*int64(opts.LineBytes)
columns()
cfmt(colAddr, "%s%s\n", rootIndent, deco.DumpAddr.F(mathextra.PadFormatInt(lineStartByte, opts.Addrbase, true, addrWidth)))
cfmt(colAddr, "%s%s\n", rootIndent, deco.DumpAddr.F(mathex.PadFormatInt(lineStartByte, opts.Addrbase, true, addrWidth)))
}
// TODO: correct? should rethink columnwriter api maybe?
lastLineStopByte := startLineByte + addrLines*int64(opts.LineBytes) - 1
@ -337,9 +337,9 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD
cprint(colHex, "\n")
// TODO: truncate if display_bytes is small?
cfmt(colHex, "until %s%s (%s)",
mathextra.Bits(stopBit).StringByteBits(opts.Addrbase),
mathex.Bits(stopBit).StringByteBits(opts.Addrbase),
isEnd,
mathextra.PadFormatInt(bitio.BitsByteCount(sizeBits), opts.Sizebase, true, 0))
mathex.PadFormatInt(bitio.BitsByteCount(sizeBits), opts.Sizebase, true, 0))
// TODO: dump last line?
}
}
@ -364,9 +364,9 @@ func dump(v *decode.Value, w io.Writer, opts Options) error {
}
_ = v.WalkPreOrder(makeWalkFn(func(v *decode.Value, _ *decode.Value, _ int, rootDepth int) error {
maxAddrIndentWidth = mathextra.MaxInt(
maxAddrIndentWidth = mathex.Max(
maxAddrIndentWidth,
rootIndentWidth*rootDepth+mathextra.DigitsInBase(bitio.BitsByteCount(v.InnerRange().Stop()), true, opts.Addrbase),
rootIndentWidth*rootDepth+mathex.DigitsInBase(bitio.BitsByteCount(v.InnerRange().Stop()), true, opts.Addrbase),
)
return nil
}))
@ -392,7 +392,7 @@ func dump(v *decode.Value, w io.Writer, opts Options) error {
var hexHeader string
var asciiHeader string
for i := 0; i < opts.LineBytes; i++ {
s := mathextra.PadFormatInt(int64(i), opts.Addrbase, false, 2)
s := mathex.PadFormatInt(int64(i), opts.Addrbase, false, 2)
hexHeader += s
if i < opts.LineBytes-1 {
hexHeader += " "
@ -414,7 +414,7 @@ func dump(v *decode.Value, w io.Writer, opts Options) error {
}
func hexdump(w io.Writer, bv Binary, opts Options) error {
br, err := bitioextra.Range(bv.br, bv.r.Start, bv.r.Len)
br, err := bitioex.Range(bv.br, bv.r.Start, bv.r.Len)
if err != nil {
return err
}

42
pkg/interp/interp.go
View File

@ -18,13 +18,13 @@ import (
"time"
"github.com/wader/fq/internal/ansi"
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/internal/bitioex"
"github.com/wader/fq/internal/colorjson"
"github.com/wader/fq/internal/ctxstack"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/ioextra"
"github.com/wader/fq/internal/gojqex"
"github.com/wader/fq/internal/ioex"
"github.com/wader/fq/internal/mapstruct"
"github.com/wader/fq/internal/mathextra"
"github.com/wader/fq/internal/mathex"
"github.com/wader/fq/internal/pos"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
@ -287,7 +287,7 @@ func toBytes(v any) ([]byte, error) {
return nil, fmt.Errorf("value is not bytes")
}
buf := &bytes.Buffer{}
if _, err := bitioextra.CopyBits(buf, br); err != nil {
if _, err := bitioex.CopyBits(buf, br); err != nil {
return nil, err
}
@ -456,7 +456,7 @@ func (i *Interp) _readline(c any, opts readlineOpts) gojq.Iter {
opts.Complete,
[]any{line, pos},
EvalOpts{
output: ioextra.DiscardCtxWriter{Ctx: completeCtx},
output: ioex.DiscardCtxWriter{Ctx: completeCtx},
isCompleting: true,
},
)
@ -472,7 +472,7 @@ func (i *Interp) _readline(c any, opts readlineOpts) gojq.Iter {
}
// {abc: 123, abd: 123} | complete(".ab"; 3) will return {prefix: "ab", names: ["abc", "abd"]}
r, ok := gojqextra.CastFn[completionResult](v, mapstruct.ToStruct)
r, ok := gojqex.CastFn[completionResult](v, mapstruct.ToStruct)
if !ok {
return nil, pos, fmt.Errorf("completion result not a map")
}
@ -921,7 +921,7 @@ func (i *Interp) Eval(ctx context.Context, c any, expr string, opts EvalOpts) (g
runCtx, runCtxCancelFn := i.interruptStack.Push(ctx)
ni.EvalInstance.Ctx = runCtx
ni.EvalInstance.Output = ioextra.CtxWriter{Writer: output, Ctx: runCtx}
ni.EvalInstance.Output = ioex.CtxWriter{Writer: output, Ctx: runCtx}
// inherit or maybe set
ni.EvalInstance.IsCompleting = i.EvalInstance.IsCompleting || opts.isCompleting
iter := gc.RunWithContext(runCtx, c, variableValues...)
@ -1013,12 +1013,12 @@ type Options struct {
func OptionsFromValue(v any) Options {
var opts Options
_ = mapstruct.ToStruct(v, &opts)
opts.ArrayTruncate = mathextra.MaxInt(0, opts.ArrayTruncate)
opts.Depth = mathextra.MaxInt(0, opts.Depth)
opts.Addrbase = mathextra.ClampInt(2, 36, opts.Addrbase)
opts.Sizebase = mathextra.ClampInt(2, 36, opts.Sizebase)
opts.LineBytes = mathextra.MaxInt(0, opts.LineBytes)
opts.DisplayBytes = mathextra.MaxInt(0, opts.DisplayBytes)
opts.ArrayTruncate = mathex.Max(0, opts.ArrayTruncate)
opts.Depth = mathex.Max(0, opts.Depth)
opts.Addrbase = mathex.Clamp(2, 36, opts.Addrbase)
opts.Sizebase = mathex.Clamp(2, 36, opts.Sizebase)
opts.LineBytes = mathex.Max(0, opts.LineBytes)
opts.DisplayBytes = mathex.Max(0, opts.DisplayBytes)
opts.Decorator = decoratorFromOptions(opts)
opts.BitsFormatFn = bitsFormatFnFromOptions(opts)
@ -1030,7 +1030,7 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
case "md5":
return func(br bitio.ReaderAtSeeker) (any, error) {
d := md5.New()
if _, err := bitioextra.CopyBits(d, br); err != nil {
if _, err := bitioex.CopyBits(d, br); err != nil {
return "", err
}
return hex.EncodeToString(d.Sum(nil)), nil
@ -1039,7 +1039,7 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
return func(br bitio.ReaderAtSeeker) (any, error) {
b := &bytes.Buffer{}
e := base64.NewEncoder(base64.StdEncoding, b)
if _, err := bitioextra.CopyBits(e, br); err != nil {
if _, err := bitioex.CopyBits(e, br); err != nil {
return "", err
}
e.Close()
@ -1049,7 +1049,7 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
// TODO: configure
return func(br bitio.ReaderAtSeeker) (any, error) {
b := &bytes.Buffer{}
if _, err := bitioextra.CopyBits(b, bitio.NewLimitReader(br, 1024*8)); err != nil {
if _, err := bitioex.CopyBits(b, bitio.NewLimitReader(br, 1024*8)); err != nil {
return "", err
}
return b.String(), nil
@ -1057,7 +1057,7 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
case "string":
return func(br bitio.ReaderAtSeeker) (any, error) {
b := &bytes.Buffer{}
if _, err := bitioextra.CopyBits(b, br); err != nil {
if _, err := bitioex.CopyBits(b, br); err != nil {
return "", err
}
return b.String(), nil
@ -1068,17 +1068,17 @@ func bitsFormatFnFromOptions(opts Options) func(br bitio.ReaderAtSeeker) (any, e
return func(br bitio.ReaderAtSeeker) (any, error) {
b := &bytes.Buffer{}
e := base64.NewEncoder(base64.StdEncoding, b)
if _, err := bitioextra.CopyBits(e, bitio.NewLimitReader(br, 256*8)); err != nil {
if _, err := bitioex.CopyBits(e, bitio.NewLimitReader(br, 256*8)); err != nil {
return "", err
}
e.Close()
brLen, err := bitioextra.Len(br)
brLen, err := bitioex.Len(br)
if err != nil {
return nil, err
}
return fmt.Sprintf("<%s>%s", mathextra.Bits(brLen).StringByteBits(opts.Sizebase), b.String()), nil
return fmt.Sprintf("<%s>%s", mathex.Bits(brLen).StringByteBits(opts.Sizebase), b.String()), nil
}
}
}

10
pkg/interp/match.go
View File

@ -5,8 +5,8 @@ import (
"regexp"
"strings"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/ioextra"
"github.com/wader/fq/internal/gojqex"
"github.com/wader/fq/internal/ioex"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/ranges"
"github.com/wader/gojq"
@ -50,7 +50,7 @@ func (i *Interp) _binaryMatch(c any, pattern any, flags string) gojq.Iter {
// TODO: err to string
// TODO: extract to regexpextra? "all" FindReaderSubmatchIndex that can iter?
sre, err := gojqextra.CompileRegexp(re, "gimb", flags)
sre, err := gojqex.CompileRegexp(re, "gimb", flags)
if err != nil {
return gojq.NewIter(err)
}
@ -70,9 +70,9 @@ func (i *Interp) _binaryMatch(c any, pattern any, flags string) gojq.Iter {
// will match the byte \0xff
if byteRunes {
// byte mode, read each byte as a rune
rr = ioextra.ByteRuneReader{RS: bitio.NewIOReadSeeker(br)}
rr = ioex.ByteRuneReader{RS: bitio.NewIOReadSeeker(br)}
} else {
rr = ioextra.RuneReadSeeker{RS: bitio.NewIOReadSeeker(br)}
rr = ioex.RuneReadSeeker{RS: bitio.NewIOReadSeeker(br)}
}
var off int64

14
pkg/interp/preview.go
View File

@ -5,8 +5,8 @@ import (
"math/big"
"strconv"
"github.com/wader/fq/internal/mathextra"
"github.com/wader/fq/internal/stringsextra"
"github.com/wader/fq/internal/mathex"
"github.com/wader/fq/internal/stringsex"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/scalar"
)
@ -20,23 +20,23 @@ func previewValue(v any, df scalar.DisplayFormat) string {
return "false"
case int:
// TODO: DisplayFormat is weird
return mathextra.PadFormatInt(int64(vv), df.FormatBase(), true, 0)
return mathex.PadFormatInt(int64(vv), df.FormatBase(), true, 0)
case int64:
// TODO: DisplayFormat is weird
return mathextra.PadFormatInt(vv, df.FormatBase(), true, 0)
return mathex.PadFormatInt(vv, df.FormatBase(), true, 0)
case uint64:
return mathextra.PadFormatUint(vv, df.FormatBase(), true, 0)
return mathex.PadFormatUint(vv, df.FormatBase(), true, 0)
case float64:
// TODO: float32? better truncated to significant digits?
return strconv.FormatFloat(vv, 'g', -1, 64)
case string:
return fmt.Sprintf("%q", stringsextra.TrimN(vv, 50, "..."))
return fmt.Sprintf("%q", stringsex.TrimN(vv, 50, "..."))
case nil:
return "null"
case bitio.Reader:
return "raw bits"
case *big.Int:
return mathextra.PadFormatBigInt(vv, df.FormatBase(), true, 0)
return mathex.PadFormatBigInt(vv, df.FormatBase(), true, 0)
default:
panic("unreachable")
}

4
pkg/interp/registry.go
View File

@ -7,11 +7,11 @@ import (
"sort"
"sync"
"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/internal/gojqex"
"github.com/wader/fq/pkg/decode"
)
type EnvFuncFn func(env *Interp) gojqextra.Function
type EnvFuncFn func(env *Interp) gojqex.Function
type Registry struct {
FormatGroups map[string]decode.Group

6
pkg/scalar/scalar.go
View File

@ -12,7 +12,7 @@ import (
"strings"
"time"
"github.com/wader/fq/internal/bitioextra"
"github.com/wader/fq/internal/bitioex"
"github.com/wader/fq/pkg/bitio"
)
@ -172,7 +172,7 @@ func (rs SRangeToScalar) MapScalar(s S) (S, error) {
func RawSym(s S, nBytes int, fn func(b []byte) string) (S, error) {
br := s.ActualBitBuf()
brLen, err := bitioextra.Len(br)
brLen, err := bitioex.Len(br)
if err != nil {
return S{}, err
}
@ -217,7 +217,7 @@ func (m BytesToScalar) MapScalar(s S) (S, error) {
return s, err
}
bb := &bytes.Buffer{}
if _, err := bitioextra.CopyBits(bb, rc); err != nil {
if _, err := bitioex.CopyBits(bb, rc); err != nil {
return s, err
}
for _, bs := range m {