mirror of
https://github.com/walles/moar.git
synced 2024-09-11 12:15:43 +03:00
Add tokenizer tests
This commit is contained in:
parent
e473c1bc60
commit
1103ebf5bc
@ -202,6 +202,8 @@ func _UpdateStyle(logger *log.Logger, style tcell.Style, escapeSequence string)
|
||||
style = style.Foreground(6)
|
||||
case "37":
|
||||
style = style.Foreground(7)
|
||||
case "39":
|
||||
style = style.Foreground(tcell.ColorDefault)
|
||||
|
||||
// Background colors
|
||||
case "40":
|
||||
@ -220,6 +222,8 @@ func _UpdateStyle(logger *log.Logger, style tcell.Style, escapeSequence string)
|
||||
style = style.Background(6)
|
||||
case "47":
|
||||
style = style.Background(7)
|
||||
case "49":
|
||||
style = style.Background(tcell.ColorDefault)
|
||||
|
||||
default:
|
||||
logger.Printf("Unrecognized ANSI SGI code <%s>", number)
|
||||
|
@ -1,4 +1,46 @@
|
||||
package m
|
||||
|
||||
// FIXME: Verify that we can tokenize all lines
|
||||
// in ../sample-files/* without logging any errors
|
||||
import (
|
||||
"bufio"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Verify that we can tokenize all lines in ../sample-files/*
|
||||
// without logging any errors
|
||||
func TestTokenize(t *testing.T) {
|
||||
for _, fileName := range _GetTestFiles() {
|
||||
file, err := os.Open(fileName)
|
||||
if err != nil {
|
||||
t.Errorf("Error opening file <%s>: %s", fileName, err.Error())
|
||||
continue
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
lineNumber := 0
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
lineNumber++
|
||||
|
||||
var loglines strings.Builder
|
||||
logger := log.New(&loglines, "", 0)
|
||||
|
||||
tokens, plainString := TokensFromString(logger, line)
|
||||
if len(tokens) != utf8.RuneCountInString(*plainString) {
|
||||
t.Errorf("%s:%d: len(tokens)=%d, len(plainString)=%d for: <%s>",
|
||||
fileName, lineNumber,
|
||||
len(tokens), utf8.RuneCountInString(*plainString), line)
|
||||
continue
|
||||
}
|
||||
|
||||
if len(loglines.String()) != 0 {
|
||||
t.Errorf("%s: %s", fileName, loglines.String())
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user