git.iko.soy/crush
on main help sign in
fix(grep): check mime type (#1228)
1a40fbaCarlos Alexandro Becker
internal/llm/tools/grep.go+15−29
compare with main
unifiedsplit
@@ -7,6 +7,7 @@ import (
7 "encoding/json"
7 "encoding/json"
8 "fmt"
8 "fmt"
9 "io"
9 "io"
10+ "net/http"
10 "os"
11 "os"
11 "os/exec"
12 "os/exec"
12 "path/filepath"
13 "path/filepath"
@@ -390,8 +391,8 @@ func searchFilesWithRegex(pattern, rootPath, include string) ([]grepMatch, error
390}
391}
391
392
392func fileContainsPattern(filePath string, pattern *regexp.Regexp) (bool, int, string, error) {
393func fileContainsPattern(filePath string, pattern *regexp.Regexp) (bool, int, string, error) {
393− // Quick binary file detection
394+ // Only search text files.
394− if isBinaryFile(filePath) {
395+ if !isTextFile(filePath) {
395 return false, 0, "", nil
396 return false, 0, "", nil
396 }
397 }
397
398
@@ -414,45 +415,30 @@ func fileContainsPattern(filePath string, pattern *regexp.Regexp) (bool, int, st
414 return false, 0, "", scanner.Err()
415 return false, 0, "", scanner.Err()
415}
416}
416
417
417−var binaryExts = map[string]struct{}{
418+// isTextFile checks if a file is a text file by examining its MIME type.
418− ".exe": {}, ".dll": {}, ".so": {}, ".dylib": {},
419− ".bin": {}, ".obj": {}, ".o": {}, ".a": {},
420− ".zip": {}, ".tar": {}, ".gz": {}, ".bz2": {},
421− ".jpg": {}, ".jpeg": {}, ".png": {}, ".gif": {},
422− ".pdf": {}, ".doc": {}, ".docx": {}, ".xls": {},
423− ".mp3": {}, ".mp4": {}, ".avi": {}, ".mov": {},
424−}
425−
426−// isBinaryFile performs a quick check to determine if a file is binary
427−func isBinaryFile(filePath string) bool {
419+func isTextFile(filePath string) bool {
428− // Check file extension first (fastest)
429− ext := strings.ToLower(filepath.Ext(filePath))
430− if _, isBinary := binaryExts[ext]; isBinary {
431− return true
432− }
433−
434− // Quick content check for files without clear extensions
435 file, err := os.Open(filePath)
420 file, err := os.Open(filePath)
436 if err != nil {
421 if err != nil {
437− return false // If we can't open it, let the caller handle the error
422+ return false
438 }
423 }
439 defer file.Close()
424 defer file.Close()
440
425
441− // Read first 512 bytes to check for null bytes
426+ // Read first 512 bytes for MIME type detection.
442 buffer := make([]byte, 512)
427 buffer := make([]byte, 512)
443 n, err := file.Read(buffer)
428 n, err := file.Read(buffer)
444 if err != nil && err != io.EOF {
429 if err != nil && err != io.EOF {
445 return false
430 return false
446 }
431 }
447
432
448− // Check for null bytes (common in binary files)
433+ // Detect content type.
449− for i := range n {
434+ contentType := http.DetectContentType(buffer[:n])
450− if buffer[i] == 0 {
451− return true
452− }
453− }
454
435
455− return false
436+ // Check if it's a text MIME type.
437+ return strings.HasPrefix(contentType, "text/") ||
438+ contentType == "application/json" ||
439+ contentType == "application/xml" ||
440+ contentType == "application/javascript" ||
441+ contentType == "application/x-sh"
456}
442}
457
443
458func globToRegex(glob string) string {
444func globToRegex(glob string) string {
internal/llm/tools/grep_test.go+192
@@ -198,3 +198,195 @@ func BenchmarkRegexCacheVsCompile(b *testing.B) {
198 }
198 }
199 })
199 })
200}
200}
201+
202+func TestIsTextFile(t *testing.T) {
203+ t.Parallel()
204+ tempDir := t.TempDir()
205+
206+ tests := []struct {
207+ name string
208+ filename string
209+ content []byte
210+ wantText bool
211+ }{
212+ {
213+ name: "go file",
214+ filename: "test.go",
215+ content: []byte("package main\n\nfunc main() {}\n"),
216+ wantText: true,
217+ },
218+ {
219+ name: "yaml file",
220+ filename: "config.yaml",
221+ content: []byte("key: value\nlist:\n - item1\n - item2\n"),
222+ wantText: true,
223+ },
224+ {
225+ name: "yml file",
226+ filename: "config.yml",
227+ content: []byte("key: value\n"),
228+ wantText: true,
229+ },
230+ {
231+ name: "json file",
232+ filename: "data.json",
233+ content: []byte(`{"key": "value"}`),
234+ wantText: true,
235+ },
236+ {
237+ name: "javascript file",
238+ filename: "script.js",
239+ content: []byte("console.log('hello');\n"),
240+ wantText: true,
241+ },
242+ {
243+ name: "typescript file",
244+ filename: "script.ts",
245+ content: []byte("const x: string = 'hello';\n"),
246+ wantText: true,
247+ },
248+ {
249+ name: "markdown file",
250+ filename: "README.md",
251+ content: []byte("# Title\n\nSome content\n"),
252+ wantText: true,
253+ },
254+ {
255+ name: "shell script",
256+ filename: "script.sh",
257+ content: []byte("#!/bin/bash\necho 'hello'\n"),
258+ wantText: true,
259+ },
260+ {
261+ name: "python file",
262+ filename: "script.py",
263+ content: []byte("print('hello')\n"),
264+ wantText: true,
265+ },
266+ {
267+ name: "xml file",
268+ filename: "data.xml",
269+ content: []byte("<?xml version=\"1.0\"?>\n<root></root>\n"),
270+ wantText: true,
271+ },
272+ {
273+ name: "plain text",
274+ filename: "file.txt",
275+ content: []byte("plain text content\n"),
276+ wantText: true,
277+ },
278+ {
279+ name: "css file",
280+ filename: "style.css",
281+ content: []byte("body { color: red; }\n"),
282+ wantText: true,
283+ },
284+ {
285+ name: "scss file",
286+ filename: "style.scss",
287+ content: []byte("$primary: blue;\nbody { color: $primary; }\n"),
288+ wantText: true,
289+ },
290+ {
291+ name: "sass file",
292+ filename: "style.sass",
293+ content: []byte("$primary: blue\nbody\n color: $primary\n"),
294+ wantText: true,
295+ },
296+ {
297+ name: "rust file",
298+ filename: "main.rs",
299+ content: []byte("fn main() {\n println!(\"Hello, world!\");\n}\n"),
300+ wantText: true,
301+ },
302+ {
303+ name: "zig file",
304+ filename: "main.zig",
305+ content: []byte("const std = @import(\"std\");\npub fn main() void {}\n"),
306+ wantText: true,
307+ },
308+ {
309+ name: "java file",
310+ filename: "Main.java",
311+ content: []byte("public class Main {\n public static void main(String[] args) {}\n}\n"),
312+ wantText: true,
313+ },
314+ {
315+ name: "c file",
316+ filename: "main.c",
317+ content: []byte("#include <stdio.h>\nint main() { return 0; }\n"),
318+ wantText: true,
319+ },
320+ {
321+ name: "cpp file",
322+ filename: "main.cpp",
323+ content: []byte("#include <iostream>\nint main() { return 0; }\n"),
324+ wantText: true,
325+ },
326+ {
327+ name: "fish shell",
328+ filename: "script.fish",
329+ content: []byte("#!/usr/bin/env fish\necho 'hello'\n"),
330+ wantText: true,
331+ },
332+ {
333+ name: "powershell file",
334+ filename: "script.ps1",
335+ content: []byte("Write-Host 'Hello, World!'\n"),
336+ wantText: true,
337+ },
338+ {
339+ name: "cmd batch file",
340+ filename: "script.bat",
341+ content: []byte("@echo off\necho Hello, World!\n"),
342+ wantText: true,
343+ },
344+ {
345+ name: "cmd file",
346+ filename: "script.cmd",
347+ content: []byte("@echo off\necho Hello, World!\n"),
348+ wantText: true,
349+ },
350+ {
351+ name: "binary exe",
352+ filename: "binary.exe",
353+ content: []byte{0x4D, 0x5A, 0x90, 0x00, 0x03, 0x00, 0x00, 0x00},
354+ wantText: false,
355+ },
356+ {
357+ name: "png image",
358+ filename: "image.png",
359+ content: []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A},
360+ wantText: false,
361+ },
362+ {
363+ name: "jpeg image",
364+ filename: "image.jpg",
365+ content: []byte{0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46},
366+ wantText: false,
367+ },
368+ {
369+ name: "zip archive",
370+ filename: "archive.zip",
371+ content: []byte{0x50, 0x4B, 0x03, 0x04, 0x14, 0x00, 0x00, 0x00},
372+ wantText: false,
373+ },
374+ {
375+ name: "pdf file",
376+ filename: "document.pdf",
377+ content: []byte("%PDF-1.4\n%âãÏÓ\n"),
378+ wantText: false,
379+ },
380+ }
381+
382+ for _, tt := range tests {
383+ t.Run(tt.name, func(t *testing.T) {
384+ t.Parallel()
385+ filePath := filepath.Join(tempDir, tt.filename)
386+ require.NoError(t, os.WriteFile(filePath, tt.content, 0o644))
387+
388+ got := isTextFile(filePath)
389+ require.Equal(t, tt.wantText, got, "isTextFile(%s) = %v, want %v", tt.filename, got, tt.wantText)
390+ })
391+ }
392+}