walles/moar
1
1
Fork 0
mirror of https://github.com/walles/moar.git synced 2025-01-08 14:30:57 +03:00
Code Issues Projects Releases Wiki Activity

Speed up line counting 4x

Browse Source 
And improve large file loading performance in the process.
This commit is contained in:
Johan Walles 2021-05-08 14:16:38 +02:00
parent 09ab8a4072
commit 5d5ecae7c0
1 changed files with 3 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
m/reader.go
View File

@ -339,8 +339,8 @@ func tryOpen(filename string) error {
// From: https://stackoverflow.com/a/52153000/473672
func countLines(filename string) (uint64, error) {
var count uint64
const lineBreak = '\n'
sliceWithSingleLineBreak := []byte{lineBreak}
reader, err := os.Open(filename)
if err != nil {
@ -353,6 +353,7 @@ func countLines(filename string) (uint64, error) {
}
}()
var count uint64
t0 := time.Now().UnixNano()
buf := make([]byte, bufio.MaxScanTokenSize)
lastReadEndsInNewline := true
@ -366,15 +367,7 @@ func countLines(filename string) (uint64, error) {
lastReadEndsInNewline = (buf[bufferSize-1] == lineBreak)
}
var bufPosition int
for {
i := bytes.IndexByte(buf[bufPosition:], lineBreak)
if i == -1 || bufferSize == bufPosition {
break
}
bufPosition += i + 1
count++
}
count += uint64(bytes.Count(buf[:bufferSize], sliceWithSingleLineBreak))
if err == io.EOF {
break
}