Merge pull request #19403 from atom/fix-binary-files

Fix handling of binary files when using ripgrep scanner

spec/fixtures/dir/file-detected-as-binary

@@ -0,0 +1,3 @@
+asciiProperty=Foo
+utf8Property=Fòò
+latin1Property=Fòò

spec/workspace-spec.js

@@ -2424,7 +2424,7 @@ describe('Workspace', () => {
 
           results.sort((a, b) => a.filePath.localeCompare(b.filePath))
 
-          expect(results).toHaveLength(3)
+          expect(results.length).toBeGreaterThan(0)
           expect(results[0].filePath).toBe(
             atom.project.getDirectories()[0].resolve('a')
           )
@@ -2669,6 +2669,33 @@ describe('Workspace', () => {
           })
         })
 
+        it('returns results on files detected as binary', async () => {
+          const results = []
+
+          await scan(
+            /asciiProperty=Foo/,
+            {
+              trailingContextLineCount: 2
+            },
+            result => results.push(result)
+          )
+          expect(results.length).toBe(1)
+          const { filePath, matches } = results[0]
+          expect(filePath).toBe(atom.project.getDirectories()[0].resolve('file-detected-as-binary'))
+          expect(matches).toHaveLength(1)
+          expect(matches[0]).toEqual({
+            matchText: 'asciiProperty=Foo',
+            lineText: 'asciiProperty=Foo',
+            lineTextOffset: 0,
+            range: [[0, 0], [0, 17]],
+            leadingContextLines: [],
+            trailingContextLines: [
+              'utf8Property=Fòò',
+              'latin1Property=F<><46>'
+            ]
+          })
+        })
+
         describe('when the core.excludeVcsIgnoredPaths config is truthy', () => {
           let projectPath
           let ignoredPath
@@ -2777,7 +2804,7 @@ describe('Workspace', () => {
 
           await scan(/a|Elephant/, {}, result => results.push(result))
 
-          expect(results).toHaveLength(3)
+          expect(results.length).toBeGreaterThan(0)
           const resultForA = _.find(
             results,
             ({ filePath }) => path.basename(filePath) === 'a'

src/ripgrep-directory-searcher.js

@@ -50,7 +50,7 @@ function updateLeadingContext (message, pendingLeadingContext, options) {
   }
 
   if (options.leadingContextLineCount) {
-    pendingLeadingContext.push(cleanResultLine(message.data.lines.text))
+    pendingLeadingContext.push(cleanResultLine(message.data.lines))
 
     if (pendingLeadingContext.length > options.leadingContextLineCount) {
       pendingLeadingContext.shift()
@@ -65,7 +65,7 @@ function updateTrailingContexts (message, pendingTrailingContexts, options) {
 
   if (options.trailingContextLineCount) {
     for (const trailingContextLines of pendingTrailingContexts) {
-      trailingContextLines.push(cleanResultLine(message.data.lines.text))
+      trailingContextLines.push(cleanResultLine(message.data.lines))
 
       if (trailingContextLines.length === options.trailingContextLineCount) {
         pendingTrailingContexts.delete(trailingContextLines)
@@ -75,6 +75,8 @@ function updateTrailingContexts (message, pendingTrailingContexts, options) {
 }
 
 function cleanResultLine (resultLine) {
+  resultLine = getText(resultLine)
+
   return resultLine[resultLine.length - 1] === '\n' ? resultLine.slice(0, -1) : resultLine
 }
 
@@ -93,12 +95,14 @@ function getPositionFromColumn (lines, column) {
 }
 
 function processUnicodeMatch (match) {
-  if (match.lines.text.length === Buffer.byteLength(match.lines.text)) {
+  const text = getText(match.lines)
+
+  if (text.length === Buffer.byteLength(text)) {
     // fast codepath for lines that only contain characters of 1 byte length.
     return
   }
 
-  let remainingBuffer = Buffer.from(match.lines.text)
+  let remainingBuffer = Buffer.from(text)
   let currentLength = 0
   let previousPosition = 0
 
@@ -146,10 +150,14 @@ function processSubmatch (submatch, lineText, offsetRow) {
 
   return {
     range: [start, end],
-    lineText: cleanResultLine(lineParts.join('\n'))
+    lineText: cleanResultLine({ text: lineParts.join('\n') })
   }
 }
 
+function getText (input) {
+  return input.text ? input.text : Buffer.from(input.bytes, 'base64').toString()
+}
+
 module.exports = class RipgrepDirectorySearcher {
   canSearchDirectory () {
     return true
@@ -280,7 +288,7 @@ module.exports = class RipgrepDirectorySearcher {
 
           if (message.type === 'begin') {
             pendingEvent = {
-              filePath: message.data.path.text,
+              filePath: getText(message.data.path),
               matches: []
             }
             pendingLeadingContext = []
@@ -294,12 +302,12 @@ module.exports = class RipgrepDirectorySearcher {
             for (const submatch of message.data.submatches) {
               const { lineText, range } = processSubmatch(
                 submatch,
-                message.data.lines.text,
+                getText(message.data.lines),
                 message.data.line_number - 1
               )
 
               pendingEvent.matches.push({
-                matchText: submatch.match.text,
+                matchText: getText(submatch.match),
                 lineText,
                 lineTextOffset: 0,
                 range,