Add tokenizer tests

m/ansiTokenizer.go

@@ -202,6 +202,8 @@ func _UpdateStyle(logger *log.Logger, style tcell.Style, escapeSequence string)
 			style = style.Foreground(6)
 		case "37":
 			style = style.Foreground(7)
+		case "39":
+			style = style.Foreground(tcell.ColorDefault)
 
 		// Background colors
 		case "40":
@@ -220,6 +222,8 @@ func _UpdateStyle(logger *log.Logger, style tcell.Style, escapeSequence string)
 			style = style.Background(6)
 		case "47":
 			style = style.Background(7)
+		case "49":
+			style = style.Background(tcell.ColorDefault)
 
 		default:
 			logger.Printf("Unrecognized ANSI SGI code <%s>", number)

m/ansiTokenizer_test.go

@@ -1,4 +1,46 @@
 package m
 
-// FIXME: Verify that we can tokenize all lines
-// in ../sample-files/* without logging any errors
+import (
+	"bufio"
+	"log"
+	"os"
+	"strings"
+	"testing"
+	"unicode/utf8"
+)
+
+// Verify that we can tokenize all lines in ../sample-files/*
+// without logging any errors
+func TestTokenize(t *testing.T) {
+	for _, fileName := range _GetTestFiles() {
+		file, err := os.Open(fileName)
+		if err != nil {
+			t.Errorf("Error opening file <%s>: %s", fileName, err.Error())
+			continue
+		}
+		defer file.Close()
+
+		scanner := bufio.NewScanner(file)
+		lineNumber := 0
+		for scanner.Scan() {
+			line := scanner.Text()
+			lineNumber++
+
+			var loglines strings.Builder
+			logger := log.New(&loglines, "", 0)
+
+			tokens, plainString := TokensFromString(logger, line)
+			if len(tokens) != utf8.RuneCountInString(*plainString) {
+				t.Errorf("%s:%d: len(tokens)=%d, len(plainString)=%d for: <%s>",
+					fileName, lineNumber,
+					len(tokens), utf8.RuneCountInString(*plainString), line)
+				continue
+			}
+
+			if len(loglines.String()) != 0 {
+				t.Errorf("%s: %s", fileName, loglines.String())
+				continue
+			}
+		}
+	}
+}