Display Private Use Unicode characters

Before this change, we rendered those as highlighted question marks.

With this change in place, we just send them to the screen for
displaying.

The reason is that Font Awesome uses these for symbols, this one for
example: https://fontawesome.com/v4/icon/battery-empty

More reading: https://en.wikipedia.org/wiki/Private_Use_Areas

m/ansiTokenizer.go

@@ -6,7 +6,6 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
-	"unicode"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/walles/moar/twin"
@@ -166,7 +165,7 @@ func withoutFormatting(s string) string {
 				runeCount++
 
 			default:
-				if !unicode.IsPrint(runeValue) {
+				if !twin.Printable(runeValue) {
 					stripped.WriteRune('?')
 					runeCount++
 					continue
@@ -227,7 +226,7 @@ func cellsFromString(s string) cellsWithTrailer {
 				})
 
 			default:
-				if !unicode.IsPrint(token.Rune) {
+				if !twin.Printable(token.Rune) {
 					if unprintableStyle == UNPRINTABLE_STYLE_HIGHLIGHT {
 						cells = append(cells, twin.Cell{
 							Rune:  '?',

m/ansiTokenizer_test.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"strings"
 	"testing"
-	"unicode"
 	"unicode/utf8"
 
 	log "github.com/sirupsen/logrus"
@@ -74,11 +73,11 @@ func TestTokenize(t *testing.T) {
 				plainStringFromCells := cellsToPlainString(tokens)
 				positionMarker := strings.Repeat(" ", index) + "^"
 				cellCharString := string(cellChar.Rune)
-				if !unicode.IsPrint(cellChar.Rune) {
+				if !twin.Printable(cellChar.Rune) {
 					cellCharString = fmt.Sprint(int(cellChar.Rune))
 				}
 				plainCharString := string(plainChar)
-				if !unicode.IsPrint(plainChar) {
+				if !twin.Printable(plainChar) {
 					plainCharString = fmt.Sprint(int(plainChar))
 				}
 				t.Errorf("%s:%d, 0-based column %d: cell char <%s> != plain char <%s>:\nPlain: %s\nCells: %s\n       %s",

m/pager_test.go

@@ -29,11 +29,11 @@ func TestUnicodeRendering(t *testing.T) {
 
 	contents := startPaging(t, reader).GetRow(0)
 	for pos, expected := range answers {
-		logDifference(t, expected, contents[pos])
+		assertCellsEqual(t, expected, contents[pos])
 	}
 }
 
-func logDifference(t *testing.T, expected twin.Cell, actual twin.Cell) {
+func assertCellsEqual(t *testing.T, expected twin.Cell, actual twin.Cell) {
 	if actual.Rune == expected.Rune && actual.Style == expected.Style {
 		return
 	}
@@ -59,7 +59,7 @@ func TestFgColorRendering(t *testing.T) {
 
 	contents := startPaging(t, reader).GetRow(0)
 	for pos, expected := range answers {
-		logDifference(t, expected, contents[pos])
+		assertCellsEqual(t, expected, contents[pos])
 	}
 }
 
@@ -88,7 +88,7 @@ func TestBrokenUtf8(t *testing.T) {
 
 	contents := startPaging(t, reader).GetRow(0)
 	for pos, expected := range answers {
-		logDifference(t, expected, contents[pos])
+		assertCellsEqual(t, expected, contents[pos])
 	}
 }
 
@@ -184,10 +184,25 @@ func TestCodeHighlighting(t *testing.T) {
 
 	contents := startPaging(t, reader).GetRow(0)
 	for pos, expected := range answers {
-		logDifference(t, expected, contents[pos])
+		assertCellsEqual(t, expected, contents[pos])
 	}
 }
 
+func TestUnicodePrivateUse(t *testing.T) {
+	// This character lives in a Private Use Area:
+	// https://codepoints.net/U+f244
+	//
+	// It's used by Font Awesome as "fa-battery-empty":
+	// https://fontawesome.com/v4/icon/battery-empty
+	char := '\uf244'
+
+	reader := NewReaderFromText("hello", string(char))
+	renderedCell := startPaging(t, reader).GetRow(0)[0]
+
+	// Make sure we display this character unmodified
+	assertCellsEqual(t, twin.NewCell(char, twin.StyleDefault), renderedCell)
+}
+
 func resetManPageFormat() {
 	manPageBold = twin.StyleDefault.WithAttr(twin.AttrBold)
 	manPageUnderline = twin.StyleDefault.WithAttr(twin.AttrUnderline)
@@ -210,7 +225,7 @@ func testManPageFormatting(t *testing.T, input string, expected twin.Cell) {
 	resetManPageFormat()
 
 	contents := startPaging(t, reader).GetRow(0)
-	logDifference(t, expected, contents[0])
+	assertCellsEqual(t, expected, contents[0])
 	assert.Equal(t, contents[1].Rune, ' ')
 }
 
@@ -365,7 +380,7 @@ func TestScrollToEndLongInput(t *testing.T) {
 	// line holds the last contents line.
 	lastContentsLine := screen.GetRow(screenHeight - 2)
 	firstContentsColumn := len("10_100 ")
-	logDifference(t, twin.NewCell('X', twin.StyleDefault), lastContentsLine[firstContentsColumn])
+	assertCellsEqual(t, twin.NewCell('X', twin.StyleDefault), lastContentsLine[firstContentsColumn])
 }
 
 func TestIsScrolledToEnd_LongFile(t *testing.T) {

sample-files/invalid-utf8.txt

@@ -0,0 +1,3 @@
+start
+
+end

twin/cell.go

@@ -51,3 +51,22 @@ func TrimSpaceLeft(cells []Cell) []Cell {
 	// All whitespace, return empty
 	return []Cell{}
 }
+
+func Printable(char rune) bool {
+	if unicode.IsPrint(char) {
+		return true
+	}
+
+	if unicode.Is(unicode.Co, char) {
+		// Co == "Private Use": https://www.compart.com/en/unicode/category
+		//
+		// This space is used by Font Awesome, for "fa-battery-empty" for
+		// example: https://fontawesome.com/v4/icon/battery-empty
+		//
+		// So we want to print these and let the rendering engine deal with
+		// outputting them in a way that's helpful to the user.
+		return true
+	}
+
+	return false
+}

twin/screen.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"regexp"
 	"strings"
-	"unicode"
 	"unicode/utf8"
 
 	log "github.com/sirupsen/logrus"
@@ -421,7 +420,7 @@ func renderLine(row []Cell) (string, int) {
 
 		style := cell.Style
 		runeToWrite := cell.Rune
-		if !unicode.IsPrint(runeToWrite) {
+		if !Printable(runeToWrite) {
 			// Highlight unprintable runes
 			style = Style{
 				fg:    NewColor16(7), // White