wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-12-22 21:01:37 +03:00
Code Issues Projects Releases Wiki Activity

doc: Document io packages a bit

Browse Source
This commit is contained in:
Mattias Wadman 2021-11-21 21:55:53 +01:00
parent 7c4b0b3844
commit 1047d909ac
3 changed files with 28 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
doc/dev.md
View File

@ -60,3 +60,25 @@ main:main()
print tree print tree
empty output empty output
``` ```
## bitio and other io packages
```
*os.File, *bytes.Buffer
^
ctxreadseeker.Reader defers blocking io operations to a goroutine to make them cancellable
^
progressreadseeker.Reader approximates how much of a file has been read
^
aheadreadseeker.Reader does readahead caching
^
| (io.ReadSeeker interface)
|
bitio.Reader (implements bitio.Bit* interfaces)
^
| (bitio.Bit* interfaces)
|
bitio.Buffer convenience wrapper to read bytes from bit reader, create section readers etc
SectionBitReader
MultiBitReader
```

10
internal/progressreadseeker/progressreaderseeker.go
View File

@ -6,7 +6,7 @@ import (
type ProgressFn func(approxReadBytes int64, totalSize int64) type ProgressFn func(approxReadBytes int64, totalSize int64)
type ProgressReaderSeeker struct { type Reader struct {
rs io.ReadSeeker rs io.ReadSeeker
pos int64 pos int64
totalSize int64 totalSize int64
@ -16,12 +16,12 @@ type ProgressReaderSeeker struct {
progressFn ProgressFn progressFn ProgressFn
} }
func New(rs io.ReadSeeker, precision int64, totalSize int64, fn ProgressFn) *ProgressReaderSeeker { func New(rs io.ReadSeeker, precision int64, totalSize int64, fn ProgressFn) *Reader {
partitionSize := totalSize / precision partitionSize := totalSize / precision
if totalSize%precision != 0 { if totalSize%precision != 0 {
partitionSize++ partitionSize++
} }
return &ProgressReaderSeeker{ return &Reader{
rs: rs, rs: rs,
totalSize: totalSize, totalSize: totalSize,
partitionSize: partitionSize, partitionSize: partitionSize,
@ -30,7 +30,7 @@ func New(rs io.ReadSeeker, precision int64, totalSize int64, fn ProgressFn) *Pro
} }
} }
func (prs *ProgressReaderSeeker) Read(p []byte) (n int, err error) { func (prs *Reader) Read(p []byte) (n int, err error) {
n, err = prs.rs.Read(p) n, err = prs.rs.Read(p)
newPos := prs.pos + int64(n) newPos := prs.pos + int64(n)
lastPartitionsReadCount := prs.partitionsReadCount lastPartitionsReadCount := prs.partitionsReadCount
@ -59,7 +59,7 @@ func (prs *ProgressReaderSeeker) Read(p []byte) (n int, err error) {
return n, err return n, err
} }
func (prs *ProgressReaderSeeker) Seek(offset int64, whence int) (int64, error) { func (prs *Reader) Seek(offset int64, whence int) (int64, error) {
pos, err := prs.rs.Seek(offset, whence) pos, err := prs.rs.Seek(offset, whence)
prs.pos = pos prs.pos = pos
return pos, err return pos, err

2
pkg/interp/bufferrange.go
View File

@ -151,7 +151,7 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
const cacheReadAheadSize = 512 * 1024 const cacheReadAheadSize = 512 * 1024
aheadRs := aheadreadseeker.New(fRS, cacheReadAheadSize) aheadRs := aheadreadseeker.New(fRS, cacheReadAheadSize)
// bitio.Buffer -> aheadreadseeker -> progressreadseeker -> ctxreadseeker -> readseeker // bitio.Buffer -> (bitio.Reader) -> aheadreadseeker -> progressreadseeker -> ctxreadseeker -> readseeker
bbf.bb, err = bitio.NewBufferFromReadSeeker(aheadRs) bbf.bb, err = bitio.NewBufferFromReadSeeker(aheadRs)
if err != nil { if err != nil {