Merge pull request #205 from walles/johan/decompression-improvements

Fix highlighting of compressed files

m/pager_test.go

@@ -193,6 +193,32 @@ func TestCodeHighlighting(t *testing.T) {
 	}
 }
 
+func TestCodeHighlight_compressed(t *testing.T) {
+	// Same as TestCodeHighlighting but with "compressed-markdown.md.gz"
+	reader, err := NewReaderFromFilename("../sample-files/compressed-markdown.md.gz", *styles.Get("native"), formatters.TTY16m, nil)
+	assert.NilError(t, err)
+	assert.NilError(t, reader._wait())
+
+	markdownHeading1Style := twin.StyleDefault.WithAttr(twin.AttrBold).WithForeground(twin.NewColorHex(0xffffff))
+	var answers = []twin.Cell{
+		twin.NewCell('#', markdownHeading1Style),
+		twin.NewCell(' ', markdownHeading1Style),
+		twin.NewCell('M', markdownHeading1Style),
+		twin.NewCell('a', markdownHeading1Style),
+		twin.NewCell('r', markdownHeading1Style),
+		twin.NewCell('k', markdownHeading1Style),
+		twin.NewCell('d', markdownHeading1Style),
+		twin.NewCell('o', markdownHeading1Style),
+		twin.NewCell('w', markdownHeading1Style),
+		twin.NewCell('n', markdownHeading1Style),
+	}
+
+	contents := startPaging(t, reader).GetRow(0)
+	for pos, expected := range answers {
+		assertCellsEqual(t, expected, contents[pos])
+	}
+}
+
 func TestUnicodePrivateUse(t *testing.T) {
 	// This character lives in a Private Use Area:
 	// https://codepoints.net/U+f244

m/reader.go

@@ -331,7 +331,7 @@ func countLines(filename string) (uint64, error) {
 	const lineBreak = '\n'
 	sliceWithSingleLineBreak := []byte{lineBreak}
 
-	reader, err := ZOpen(filename)
+	reader, _, err := ZOpen(filename)
 	if err != nil {
 		return 0, err
 	}
@@ -384,21 +384,24 @@ func NewReaderFromFilenameWithoutStyle(filename string, formatter chroma.Formatt
 		return nil, fileError
 	}
 
-	stream, err := ZOpen(filename)
+	stream, highlightingFilename, err := ZOpen(filename)
 	if err != nil {
 		return nil, err
 	}
 
-	// Set lexer to nil in this call since we want to do our own highlighting in
-	// parallel with the stream being read. See the call to
-	// StartHighlightingFromFile() below.
-	returnMe := newReaderFromStream(stream, &filename, nil, nil)
+	if lexer == nil {
+		lexer = lexers.Match(highlightingFilename)
+	}
+
+	returnMe := newReaderFromStream(stream, &filename, formatter, lexer)
 
 	returnMe.Lock()
 	returnMe.name = &filename
 	returnMe.Unlock()
 
-	startHighlightingFromFile(returnMe, filename, formatter, lexer)
+	if lexer == nil {
+		returnMe.highlightingDone.Store(true)
+	}
 
 	return returnMe, nil
 }
@@ -419,58 +422,6 @@ func NewReaderFromFilename(filename string, style chroma.Style, formatter chroma
 	return mReader, nil
 }
 
-func startHighlightingFromFile(reader *Reader, filename string, formatter chroma.Formatter, lexer chroma.Lexer) {
-	reportDone := func() {
-		reader.highlightingDone.Store(true)
-		select {
-		case reader.maybeDone <- true:
-		default:
-		}
-
-		log.Trace("Highlighting done")
-	}
-
-	fileInfo, err := os.Stat(filename)
-	if err != nil {
-		log.Warn("Failed to stat file for highlighting: ", err)
-		reportDone()
-		return
-	}
-	if fileInfo.Size() > MAX_HIGHLIGHT_SIZE {
-		log.Debug("File too large for highlighting: ", fileInfo.Size())
-		reportDone()
-		return
-	}
-
-	go func() {
-		defer reportDone()
-
-		fileBytes, err := os.ReadFile(filename)
-		if err != nil {
-			log.Warn("Failed to read file for highlighting: ", err)
-			return
-		}
-
-		if lexer == nil {
-			// Try auto detecting by filename
-			lexer = lexers.Match(filename)
-		}
-
-		highlighted, err := highlight(string(fileBytes), <-reader.highlightingStyle, formatter, lexer)
-		if err != nil {
-			log.Warn("Highlighting failed: ", err)
-			return
-		}
-
-		if highlighted == nil {
-			// No highlighting would be done, never mind
-			return
-		}
-
-		reader.setText(*highlighted)
-	}()
-}
-
 // We expect this to be executed in a goroutine
 func highlightFromMemory(reader *Reader, style chroma.Style, formatter chroma.Formatter, lexer chroma.Lexer) {
 	if lexer == nil {

m/reader_test.go

@@ -144,22 +144,6 @@ func (r *Reader) _wait() error {
 
 func TestGetLines(t *testing.T) {
 	for _, file := range getTestFiles(t) {
-		if strings.HasSuffix(file, ".xz") {
-			_, err := exec.LookPath("xz")
-			if err != nil {
-				t.Log("Not testing xz compressed file, xz not found in $PATH: ", file)
-				continue
-			}
-		}
-
-		if strings.HasSuffix(file, ".bz2") {
-			_, err := exec.LookPath("bzip2")
-			if err != nil {
-				t.Log("Not testing bzip2 compressed file, bzip2 not found in $PATH: ", file)
-				continue
-			}
-		}
-
 		reader, err := NewReaderFromFilename(file, *styles.Get("native"), formatters.TTY16m, nil)
 		if err != nil {
 			t.Errorf("Error opening file <%s>: %s", file, err.Error())

m/zopen.go

@@ -11,44 +11,47 @@ import (
 	"github.com/ulikunitz/xz"
 )
 
-func ZOpen(filename string) (io.ReadCloser, error) {
+// The second return value is the file name with any compression extension removed.
+func ZOpen(filename string) (io.ReadCloser, string, error) {
 	file, err := os.Open(filename)
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 
 	switch {
 	case strings.HasSuffix(filename, ".gz"):
-		return gzip.NewReader(file)
+		reader, err := gzip.NewReader(file)
+		return reader, strings.TrimSuffix(filename, ".gz"), err
 
 	// Ref: https://github.com/walles/moar/issues/194
 	case strings.HasSuffix(filename, ".tgz"):
-		return gzip.NewReader(file)
+		reader, err := gzip.NewReader(file)
+		return reader, strings.TrimSuffix(filename, ".tgz"), err
 
 	case strings.HasSuffix(filename, ".bz2"):
 		return struct {
 			io.Reader
 			io.Closer
-		}{bzip2.NewReader(file), file}, nil
+		}{bzip2.NewReader(file), file}, strings.TrimSuffix(filename, ".bz2"), nil
 
 	case strings.HasSuffix(filename, ".zst") || strings.HasSuffix(filename, ".zstd"):
 		decoder, err := zstd.NewReader(file)
 		if err != nil {
-			return nil, err
+			return nil, "", err
 		}
-		return decoder.IOReadCloser(), nil
+		return decoder.IOReadCloser(), strings.TrimSuffix(filename, ".zst"), nil
 
 	case strings.HasSuffix(filename, ".xz"):
 		xzReader, err := xz.NewReader(file)
 		if err != nil {
-			return nil, err
+			return nil, "", err
 		}
 
 		return struct {
 			io.Reader
 			io.Closer
-		}{xzReader, file}, nil
+		}{xzReader, file}, strings.TrimSuffix(filename, ".xz"), nil
 	}
 
-	return file, nil
+	return file, filename, nil
 }

moar.go

@@ -406,7 +406,7 @@ func pumpToStdout(inputFilenames ...string) error {
 		// If we get both redirected stdin and an input filenames, should only
 		// copy the files and ignore stdin, because that's how less works.
 		for _, inputFilename := range inputFilenames {
-			inputFile, err := m.ZOpen(inputFilename)
+			inputFile, _, err := m.ZOpen(inputFilename)
 			if err != nil {
 				return fmt.Errorf("Failed to open %s: %w", inputFilename, err)
 			}