Add line numbers to escape sequence log messages

m/ansiTokenizer.go

@@ -40,11 +40,11 @@ func NewLine(raw string) Line {
 
 // Returns a representation of the string split into styled tokens. Any regexp
 // matches are highlighted. A nil regexp means no highlighting.
-func (line *Line) HighlightedTokens(linePrefix string, search *regexp.Regexp) cellsWithTrailer {
+func (line *Line) HighlightedTokens(linePrefix string, search *regexp.Regexp, lineNumberOneBased *int) cellsWithTrailer {
-	plain := line.Plain()
+	plain := line.Plain(lineNumberOneBased)
 	matchRanges := getMatchRanges(&plain, search)
 
-	fromString := cellsFromString(linePrefix + line.raw)
+	fromString := cellsFromString(linePrefix+line.raw, lineNumberOneBased)
 	returnCells := make([]twin.Cell, 0, len(fromString.Cells))
 	for _, token := range fromString.Cells {
 		style := token.Style
@@ -69,9 +69,9 @@ func (line *Line) HighlightedTokens(linePrefix string, search *regexp.Regexp) ce
 }
 
 // Plain returns a plain text representation of the initial string
-func (line *Line) Plain() string {
+func (line *Line) Plain(lineNumberOneBased *int) string {
 	if line.plain == nil {
-		plain := withoutFormatting(line.raw)
+		plain := withoutFormatting(line.raw, lineNumberOneBased)
 		line.plain = &plain
 	}
 	return *line.plain
@@ -103,7 +103,7 @@ func ConsumeLessTermcapEnvs() {
 
 func termcapToStyle(termcap string) twin.Style {
 	// Add a character to be sure we have one to take the format from
-	cells := cellsFromString(termcap + "x").Cells
+	cells := cellsFromString(termcap+"x", nil).Cells
 	return cells[len(cells)-1].Style
 }
 
@@ -121,7 +121,7 @@ func isPlain(s string) bool {
 	return true
 }
 
-func withoutFormatting(s string) string {
+func withoutFormatting(s string, lineNumberOneBased *int) string {
 	if isPlain(s) {
 		return s
 	}
@@ -134,7 +134,7 @@ func withoutFormatting(s string) string {
 	// runes.
 	stripped.Grow(len(s) * 2)
 
-	for _, styledString := range styledStringsFromString(s).styledStrings {
+	for _, styledString := range styledStringsFromString(s, lineNumberOneBased).styledStrings {
 		for _, runeValue := range runesFromStyledString(styledString) {
 			switch runeValue {
 
@@ -179,13 +179,13 @@ func withoutFormatting(s string) string {
 }
 
 // Turn a (formatted) string into a series of screen cells
-func cellsFromString(s string) cellsWithTrailer {
+func cellsFromString(s string, lineNumberOneBased *int) cellsWithTrailer {
 	var cells []twin.Cell
 
 	// Specs: https://en.wikipedia.org/wiki/ANSI_escape_code#3-bit_and_4-bit
 	styleUnprintable := twin.StyleDefault.Background(twin.NewColor16(1)).Foreground(twin.NewColor16(7))
 
-	stringsWithTrailer := styledStringsFromString(s)
+	stringsWithTrailer := styledStringsFromString(s, lineNumberOneBased)
 	for _, styledString := range stringsWithTrailer.styledStrings {
 		for _, token := range tokensFromStyledString(styledString) {
 			switch token.Rune {

m/ansiTokenizer_test.go

@@ -51,8 +51,8 @@ func TestTokenize(t *testing.T) {
 				var loglines strings.Builder
 				log.SetOutput(&loglines)
 
-				tokens := cellsFromString(line.raw).Cells
+				tokens := cellsFromString(line.raw, &lineNumber).Cells
-				plainString := withoutFormatting(line.raw)
+				plainString := withoutFormatting(line.raw, &lineNumber)
 				if len(tokens) != utf8.RuneCountInString(plainString) {
 					t.Errorf("%s:%d: len(tokens)=%d, len(plainString)=%d for: <%s>",
 						fileName, lineNumber,
@@ -104,7 +104,7 @@ func TestTokenize(t *testing.T) {
 }
 
 func TestUnderline(t *testing.T) {
-	tokens := cellsFromString("a\x1b[4mb\x1b[24mc").Cells
+	tokens := cellsFromString("a\x1b[4mb\x1b[24mc", nil).Cells
 	assert.Equal(t, len(tokens), 3)
 	assert.Equal(t, tokens[0], twin.Cell{Rune: 'a', Style: twin.StyleDefault})
 	assert.Equal(t, tokens[1], twin.Cell{Rune: 'b', Style: twin.StyleDefault.WithAttr(twin.AttrUnderline)})
@@ -113,14 +113,14 @@ func TestUnderline(t *testing.T) {
 
 func TestManPages(t *testing.T) {
 	// Bold
-	tokens := cellsFromString("ab\bbc").Cells
+	tokens := cellsFromString("ab\bbc", nil).Cells
 	assert.Equal(t, len(tokens), 3)
 	assert.Equal(t, tokens[0], twin.Cell{Rune: 'a', Style: twin.StyleDefault})
 	assert.Equal(t, tokens[1], twin.Cell{Rune: 'b', Style: twin.StyleDefault.WithAttr(twin.AttrBold)})
 	assert.Equal(t, tokens[2], twin.Cell{Rune: 'c', Style: twin.StyleDefault})
 
 	// Underline
-	tokens = cellsFromString("a_\bbc").Cells
+	tokens = cellsFromString("a_\bbc", nil).Cells
 	assert.Equal(t, len(tokens), 3)
 	assert.Equal(t, tokens[0], twin.Cell{Rune: 'a', Style: twin.StyleDefault})
 	assert.Equal(t, tokens[1], twin.Cell{Rune: 'b', Style: twin.StyleDefault.WithAttr(twin.AttrUnderline)})
@@ -128,7 +128,7 @@ func TestManPages(t *testing.T) {
 
 	// Bullet point 1, taken from doing this on my macOS system:
 	// env PAGER="hexdump -C" man printf | moar
-	tokens = cellsFromString("a+\b+\bo\bob").Cells
+	tokens = cellsFromString("a+\b+\bo\bob", nil).Cells
 	assert.Equal(t, len(tokens), 3)
 	assert.Equal(t, tokens[0], twin.Cell{Rune: 'a', Style: twin.StyleDefault})
 	assert.Equal(t, tokens[1], twin.Cell{Rune: '•', Style: twin.StyleDefault})
@@ -136,7 +136,7 @@ func TestManPages(t *testing.T) {
 
 	// Bullet point 2, taken from doing this using the "fish" shell on my macOS system:
 	// man printf | hexdump -C | moar
-	tokens = cellsFromString("a+\bob").Cells
+	tokens = cellsFromString("a+\bob", nil).Cells
 	assert.Equal(t, len(tokens), 3)
 	assert.Equal(t, tokens[0], twin.Cell{Rune: 'a', Style: twin.StyleDefault})
 	assert.Equal(t, tokens[1], twin.Cell{Rune: '•', Style: twin.StyleDefault})
@@ -203,7 +203,7 @@ func TestRawUpdateStyle(t *testing.T) {
 func TestHyperlink_escBackslash(t *testing.T) {
 	url := "http://example.com"
 
-	tokens := cellsFromString("a\x1b]8;;" + url + "\x1b\\bc\x1b]8;;\x1b\\d").Cells
+	tokens := cellsFromString("a\x1b]8;;"+url+"\x1b\\bc\x1b]8;;\x1b\\d", nil).Cells
 
 	assert.DeepEqual(t, tokens, []twin.Cell{
 		{Rune: 'a', Style: twin.StyleDefault},
@@ -219,7 +219,7 @@ func TestHyperlink_escBackslash(t *testing.T) {
 func TestHyperlink_bell(t *testing.T) {
 	url := "http://example.com"
 
-	tokens := cellsFromString("a\x1b]8;;" + url + "\x07bc\x1b]8;;\x07d").Cells
+	tokens := cellsFromString("a\x1b]8;;"+url+"\x07bc\x1b]8;;\x07d", nil).Cells
 
 	assert.DeepEqual(t, tokens, []twin.Cell{
 		{Rune: 'a', Style: twin.StyleDefault},
@@ -232,7 +232,7 @@ func TestHyperlink_bell(t *testing.T) {
 // Test with some other ESC sequence than ESC-backslash
 func TestHyperlink_nonTerminatingEsc(t *testing.T) {
 	complete := "a\x1b]8;;https://example.com\x1bbc"
-	tokens := cellsFromString(complete).Cells
+	tokens := cellsFromString(complete, nil).Cells
 
 	// This should not be treated as any link
 	for i := 0; i < len(complete); i++ {
@@ -252,7 +252,7 @@ func TestHyperlink_incomplete(t *testing.T) {
 	for l := len(complete) - 1; l >= 0; l-- {
 		incomplete := complete[:l]
 		t.Run(fmt.Sprintf("l=%d incomplete=<%s>", l, strings.ReplaceAll(incomplete, "\x1b", "ESC")), func(t *testing.T) {
-			tokens := cellsFromString(incomplete).Cells
+			tokens := cellsFromString(incomplete, nil).Cells
 
 			for i := 0; i < l; i++ {
 				if complete[i] == '\x1b' {

m/linewrapper_test.go

@@ -9,7 +9,7 @@ import (
 
 func tokenize(input string) []twin.Cell {
 	line := NewLine(input)
-	return line.HighlightedTokens("", nil).Cells
+	return line.HighlightedTokens("", nil, nil).Cells
 }
 
 func rowsToString(cellLines [][]twin.Cell) string {

m/pager_test.go

@@ -515,7 +515,7 @@ func TestPageSamples(t *testing.T) {
 			continue
 		}
 		firstPagerLine := rowToString(screen.GetRow(0))
-		assert.Assert(t, strings.HasPrefix(firstReaderLine.Plain(), firstPagerLine))
+		assert.Assert(t, strings.HasPrefix(firstReaderLine.Plain(nil), firstPagerLine))
 	}
 }
 

m/reader_test.go

@@ -250,10 +250,10 @@ func TestGetLongLine(t *testing.T) {
 	assert.Equal(t, overflow, didFit)
 
 	line := lines.lines[0]
-	assert.Assert(t, strings.HasPrefix(line.Plain(), "1 2 3 4"), "<%s>", line)
+	assert.Assert(t, strings.HasPrefix(line.Plain(nil), "1 2 3 4"), "<%s>", line)
-	assert.Assert(t, strings.HasSuffix(line.Plain(), "0123456789"), line)
+	assert.Assert(t, strings.HasSuffix(line.Plain(nil), "0123456789"), line)
 
-	assert.Equal(t, len(line.Plain()), 100021)
+	assert.Equal(t, len(line.Plain(nil)), 100021)
 }
 
 func getReaderWithLineCount(totalLines int) *Reader {
@@ -307,7 +307,7 @@ func testCompressedFile(t *testing.T, filename string) {
 	}
 
 	lines, _ := reader.GetLines(1, 5)
-	assert.Equal(t, lines.lines[0].Plain(), "This is a compressed file", "%s", filename)
+	assert.Equal(t, lines.lines[0].Plain(nil), "This is a compressed file", "%s", filename)
 }
 
 func TestCompressedFiles(t *testing.T) {

m/screenLines.go

@@ -51,7 +51,7 @@ func (p *Pager) redraw(spinner string) overflowState {
 		// This happens when we're done
 		eofSpinner = "---"
 	}
-	spinnerLine := cellsFromString(_EofMarkerFormat + eofSpinner).Cells
+	spinnerLine := cellsFromString(_EofMarkerFormat+eofSpinner, nil).Cells
 	for column, cell := range spinnerLine {
 		p.screen.SetCell(column, lastUpdatedScreenLineNumber+1, cell)
 	}
@@ -204,7 +204,7 @@ func (p *Pager) renderLines() ([]renderedLine, string, overflowState) {
 // lineNumber and numberPrefixLength are required for knowing how much to
 // indent, and to (optionally) render the line number.
 func (p *Pager) renderLine(line *Line, lineNumber int) ([]renderedLine, overflowState) {
-	highlighted := line.HighlightedTokens(p.linePrefix, p.searchPattern)
+	highlighted := line.HighlightedTokens(p.linePrefix, p.searchPattern, &lineNumber)
 	var wrapped [][]twin.Cell
 	overflow := didFit
 	if p.WrapLongLines {

m/scrollPosition.go

@@ -165,7 +165,7 @@ func (si *scrollPositionInternal) emptyBottomLinesCount(pager *Pager) int {
 			break
 		}
 
-		subLines, _ := pager.renderLine(line, 0)
+		subLines, _ := pager.renderLine(line, lineNumberOneBased)
 		unclaimedViewportLines -= len(subLines)
 		if unclaimedViewportLines <= 0 {
 			return 0

m/search.go

@@ -64,7 +64,7 @@ func (p *Pager) findFirstHit(startPosition scrollPosition, backwards bool) *scro
 			return nil
 		}
 
-		lineText := line.Plain()
+		lineText := line.Plain(&searchPosition)
 		if p.searchPattern.MatchString(lineText) {
 			return scrollPositionFromLineNumber("findFirstHit", searchPosition)
 		}

m/styledStringSplitter.go

@@ -12,7 +12,9 @@ import (
 const esc = '\x1b'
 
 type styledStringSplitter struct {
-	input             string
+	input              string
+	lineNumberOneBased *int
+
 	nextByteIndex     int
 	previousByteIndex int
 
@@ -23,7 +25,7 @@ type styledStringSplitter struct {
 	trailer twin.Style
 }
 
-func styledStringsFromString(s string) styledStringsWithTrailer {
+func styledStringsFromString(s string, lineNumberOneBased *int) styledStringsWithTrailer {
 	if !strings.ContainsAny(s, "\x1b") {
 		// This shortcut makes BenchmarkPlainTextSearch() perform a lot better
 		return styledStringsWithTrailer{
@@ -36,7 +38,8 @@ func styledStringsFromString(s string) styledStringsWithTrailer {
 	}
 
 	splitter := styledStringSplitter{
-		input: s,
+		input:              s,
+		lineNumberOneBased: lineNumberOneBased,
 	}
 	splitter.run()
 
@@ -80,8 +83,13 @@ func (s *styledStringSplitter) run() {
 			escIndex := s.previousByteIndex
 			err := s.handleEscape()
 			if err != nil {
+				header := ""
+				if s.lineNumberOneBased != nil {
+					header = fmt.Sprintf("Line %d: ", *s.lineNumberOneBased)
+				}
+
 				failed := s.input[escIndex:s.nextByteIndex]
-				log.Debug("Failed to parse <", strings.ReplaceAll(failed, "\x1b", "ESC"), ">: ", err)
+				log.Debug(header, "<", strings.ReplaceAll(failed, "\x1b", "ESC"), ">: ", err)
 
 				// Somewhere in handleEscape(), we got a character that was
 				// unexpected. We need to treat everything up to before that
@@ -113,7 +121,7 @@ func (s *styledStringSplitter) handleEscape() error {
 		return s.consumeControlSequence(char)
 	}
 
-	return fmt.Errorf("Unhandled char after ESC: %q", char)
+	return fmt.Errorf("Unhandled Fe sequence ESC%c", char)
 }
 
 func (s *styledStringSplitter) consumeControlSequence(charAfterEsc rune) error {
@@ -172,7 +180,7 @@ func (s *styledStringSplitter) handleCompleteControlSequence(charAfterEsc rune,
 		return nil
 	}
 
-	return fmt.Errorf("Expected 'm' at the end of the control sequence, got %q", lastChar)
+	return fmt.Errorf("Unhandled CSI type %q", lastChar)
 }
 
 func (s *styledStringSplitter) handleOsc(sequence string) error {

m/styledStringSplitter_test.go

@@ -33,14 +33,14 @@ func TestNextCharLastChar_empty(t *testing.T) {
 // https://gitlab.freedesktop.org/Per_Bothner/specifications/blob/master/proposals/semantic-prompts.md
 func TestIgnorePromptHints(t *testing.T) {
 	// From an e-mail I got titled "moar question: "--RAW-CONTROL-CHARS" equivalent"
-	result := styledStringsFromString("\x1b]133;A\x1b\\hello")
+	result := styledStringsFromString("\x1b]133;A\x1b\\hello", nil)
 	assert.Equal(t, twin.StyleDefault, result.trailer)
 	assert.Equal(t, 1, len(result.styledStrings))
 	assert.Equal(t, "hello", result.styledStrings[0].String)
 	assert.Equal(t, twin.StyleDefault, result.styledStrings[0].Style)
 
 	// C rather than A, different end-of-sequence, should also be ignored
-	result = styledStringsFromString("\x1b]133;C\x07hello")
+	result = styledStringsFromString("\x1b]133;C\x07hello", nil)
 	assert.Equal(t, twin.StyleDefault, result.trailer)
 	assert.Equal(t, 1, len(result.styledStrings))
 	assert.Equal(t, "hello", result.styledStrings[0].String)
@@ -54,7 +54,7 @@ func TestIgnorePromptHints(t *testing.T) {
 // Johan got an e-mail titled "moar question: "--RAW-CONTROL-CHARS" equivalent"
 // about the sequence we're testing here.
 func TestColonColors(t *testing.T) {
-	result := styledStringsFromString("\x1b[38:5:238mhello")
+	result := styledStringsFromString("\x1b[38:5:238mhello", nil)
 	assert.Equal(t, twin.StyleDefault, result.trailer)
 	assert.Equal(t, 1, len(result.styledStrings))
 	assert.Equal(t, "hello", result.styledStrings[0].String)

moar.go

@@ -200,7 +200,7 @@ func parseUnprintableStyle(styleOption string) (m.UnprintableStyle, error) {
 func parseScrollHint(scrollHint string) (twin.Cell, error) {
 	scrollHint = strings.ReplaceAll(scrollHint, "ESC", "\x1b")
 	hintAsLine := m.NewLine(scrollHint)
-	parsedTokens := hintAsLine.HighlightedTokens("", nil).Cells
+	parsedTokens := hintAsLine.HighlightedTokens("", nil, nil).Cells
 	if len(parsedTokens) == 1 {
 		return parsedTokens[0], nil
 	}