on main help sign in
refactor(fsext): improve hierarchical ignore handling and consolidate file exclusion logic (#999)

* refactor(fsext): improve hierarchical ignore handling and consolidate file exclusion logic - Refactor FastGlobWalker to use directoryLister for consistent ignore handling - Add ShouldExcludeFile function for unified file exclusion checking - Add WalkDirectories function for directory traversal with ignore support - Improve directory pattern matching by checking both with and without trailing slash - Add comprehensive tests for hierarchical ignore behavior and common patterns - Remove direct dependency on go-gitignore in favor of existing directoryLister implementation ๐Ÿ’– Generated with Crush Co-Authored-By: Crush <crush@charm.land> * fix: improvements Signed-off-by: Carlos Alexandro Becker <caarlos0@users.noreply.github.com> * chore: t.Context() Signed-off-by: Carlos Alexandro Becker <caarlos0@users.noreply.github.com> * fix: tests Signed-off-by: Carlos Alexandro Becker <caarlos0@users.noreply.github.com> --------- Signed-off-by: Carlos Alexandro Becker <caarlos0@users.noreply.github.com> Co-authored-by: Crush <crush@charm.land>

commitdf2e64ead1cc499c222ac57ed9bab0424dfac62f authorCarlos Alexandro Becker <caarlos0@users.noreply.github.com> dateWed 10 Sep 2025, 08:03 (1 year ago) parent442867d
8 files changed+213โˆ’89
internal/fsext/fileutil.go+45โˆ’48 internal/fsext/ignore_test.go+96 internal/fsext/ls.go+14โˆ’3 internal/llm/provider/openai_test.go+1โˆ’1 internal/llm/tools/grep.go+19โˆ’7 internal/llm/tools/grep_test.go+35โˆ’26 internal/shell/command_block_test.go+1โˆ’2 internal/shell/shell_test.go+2โˆ’2
internal/fsext/fileutil.go+45โˆ’48
@@ -11,8 +11,6 @@ import (
1111 "github.com/bmatcuk/doublestar/v4"
1212 "github.com/charlievieth/fastwalk"
1313 "github.com/charmbracelet/crush/internal/home"
14โˆ’
15โˆ’ ignore "github.com/sabhiram/go-gitignore"
1614)
1715
1816type FileInfo struct {
@@ -58,60 +56,22 @@ func SkipHidden(path string) bool {
5856}
5957
6058// FastGlobWalker provides gitignore-aware file walking with fastwalk
59+// It uses hierarchical ignore checking like git does, checking .gitignore/.crushignore
60+// files in each directory from the root to the target path.
6161type FastGlobWalker struct {
62โˆ’ gitignore *ignore.GitIgnore
63โˆ’ crushignore *ignore.GitIgnore
64โˆ’ rootPath string
62+ directoryLister *directoryLister
6563}
6664
6765func NewFastGlobWalker(searchPath string) *FastGlobWalker {
68โˆ’ walker := &FastGlobWalker{
69โˆ’ rootPath: searchPath,
70โˆ’ }
71โˆ’
72โˆ’ // Load gitignore if it exists
73โˆ’ gitignorePath := filepath.Join(searchPath, ".gitignore")
74โˆ’ if _, err := os.Stat(gitignorePath); err == nil {
75โˆ’ if gi, err := ignore.CompileIgnoreFile(gitignorePath); err == nil {
76โˆ’ walker.gitignore = gi
77โˆ’ }
66+ return &FastGlobWalker{
67+ directoryLister: NewDirectoryLister(searchPath),
7868 }
79โˆ’
80โˆ’ // Load crushignore if it exists
81โˆ’ crushignorePath := filepath.Join(searchPath, ".crushignore")
82โˆ’ if _, err := os.Stat(crushignorePath); err == nil {
83โˆ’ if ci, err := ignore.CompileIgnoreFile(crushignorePath); err == nil {
84โˆ’ walker.crushignore = ci
85โˆ’ }
86โˆ’ }
87โˆ’
88โˆ’ return walker
8969}
9070
91โˆ’// ShouldSkip checks if a path should be skipped based on gitignore, crushignore, and hidden file rules
71+// ShouldSkip checks if a path should be skipped based on hierarchical gitignore,
72+// crushignore, and hidden file rules
9273func (w *FastGlobWalker) ShouldSkip(path string) bool {
93โˆ’ if SkipHidden(path) {
94โˆ’ return true
95โˆ’ }
96โˆ’
97โˆ’ relPath, err := filepath.Rel(w.rootPath, path)
98โˆ’ if err != nil {
99โˆ’ return false
100โˆ’ }
101โˆ’
102โˆ’ if w.gitignore != nil {
103โˆ’ if w.gitignore.MatchesPath(relPath) {
104โˆ’ return true
105โˆ’ }
106โˆ’ }
107โˆ’
108โˆ’ if w.crushignore != nil {
109โˆ’ if w.crushignore.MatchesPath(relPath) {
110โˆ’ return true
111โˆ’ }
112โˆ’ }
113โˆ’
114โˆ’ return false
74+ return w.directoryLister.shouldIgnore(path, nil)
11575}
11676
11777func GlobWithDoubleStar(pattern, searchPath string, limit int) ([]string, bool, error) {
@@ -182,6 +142,43 @@ func GlobWithDoubleStar(pattern, searchPath string, limit int) ([]string, bool,
182142 return results, truncated, nil
183143}
184144
145+// ShouldExcludeFile checks if a file should be excluded from processing
146+// based on common patterns and ignore rules
147+func ShouldExcludeFile(rootPath, filePath string) bool {
148+ return NewDirectoryLister(rootPath).
149+ shouldIgnore(filePath, nil)
150+}
151+
152+// WalkDirectories walks a directory tree and calls the provided function for each directory,
153+// respecting hierarchical .gitignore/.crushignore files like git does.
154+func WalkDirectories(rootPath string, fn func(path string, d os.DirEntry, err error) error) error {
155+ dl := NewDirectoryLister(rootPath)
156+
157+ conf := fastwalk.Config{
158+ Follow: true,
159+ ToSlash: fastwalk.DefaultToSlash(),
160+ Sort: fastwalk.SortDirsFirst,
161+ }
162+
163+ return fastwalk.Walk(&conf, rootPath, func(path string, d os.DirEntry, err error) error {
164+ if err != nil {
165+ return fn(path, d, err)
166+ }
167+
168+ // Only process directories
169+ if !d.IsDir() {
170+ return nil
171+ }
172+
173+ // Check if directory should be ignored
174+ if dl.shouldIgnore(path, nil) {
175+ return filepath.SkipDir
176+ }
177+
178+ return fn(path, d, err)
179+ })
180+}
181+
185182func PrettyPath(path string) string {
186183 return home.Short(path)
187184}
internal/fsext/ignore_test.go+96
@@ -2,6 +2,7 @@ package fsext
22
33import (
44 "os"
5+ "path/filepath"
56 "testing"
67
78 "github.com/stretchr/testify/require"
@@ -30,3 +31,98 @@ func TestCrushIgnore(t *testing.T) {
3031 require.False(t, dl.shouldIgnore("test1.txt", nil), ".txt files should not be ignored")
3132 require.True(t, dl.shouldIgnore("test3.tmp", nil), ".tmp files should be ignored by common patterns")
3233}
34+
35+func TestShouldExcludeFile(t *testing.T) {
36+ t.Parallel()
37+
38+ // Create a temporary directory structure for testing
39+ tempDir := t.TempDir()
40+
41+ // Create directories that should be ignored
42+ nodeModules := filepath.Join(tempDir, "node_modules")
43+ target := filepath.Join(tempDir, "target")
44+ customIgnored := filepath.Join(tempDir, "custom_ignored")
45+ normalDir := filepath.Join(tempDir, "src")
46+
47+ for _, dir := range []string{nodeModules, target, customIgnored, normalDir} {
48+ if err := os.MkdirAll(dir, 0o755); err != nil {
49+ t.Fatalf("Failed to create directory %s: %v", dir, err)
50+ }
51+ }
52+
53+ // Create .gitignore file
54+ gitignoreContent := "node_modules/\ntarget/\n"
55+ if err := os.WriteFile(filepath.Join(tempDir, ".gitignore"), []byte(gitignoreContent), 0o644); err != nil {
56+ t.Fatalf("Failed to create .gitignore: %v", err)
57+ }
58+
59+ // Create .crushignore file
60+ crushignoreContent := "custom_ignored/\n"
61+ if err := os.WriteFile(filepath.Join(tempDir, ".crushignore"), []byte(crushignoreContent), 0o644); err != nil {
62+ t.Fatalf("Failed to create .crushignore: %v", err)
63+ }
64+
65+ // Test that ignored directories are properly ignored
66+ require.True(t, ShouldExcludeFile(tempDir, nodeModules), "Expected node_modules to be ignored by .gitignore")
67+ require.True(t, ShouldExcludeFile(tempDir, target), "Expected target to be ignored by .gitignore")
68+ require.True(t, ShouldExcludeFile(tempDir, customIgnored), "Expected custom_ignored to be ignored by .crushignore")
69+
70+ // Test that normal directories are not ignored
71+ require.False(t, ShouldExcludeFile(tempDir, normalDir), "Expected src directory to not be ignored")
72+
73+ // Test that the workspace root itself is not ignored
74+ require.False(t, ShouldExcludeFile(tempDir, tempDir), "Expected workspace root to not be ignored")
75+}
76+
77+func TestShouldExcludeFileHierarchical(t *testing.T) {
78+ t.Parallel()
79+
80+ // Create a nested directory structure for testing hierarchical ignore
81+ tempDir := t.TempDir()
82+
83+ // Create nested directories
84+ subDir := filepath.Join(tempDir, "subdir")
85+ nestedNormal := filepath.Join(subDir, "normal_nested")
86+
87+ for _, dir := range []string{subDir, nestedNormal} {
88+ if err := os.MkdirAll(dir, 0o755); err != nil {
89+ t.Fatalf("Failed to create directory %s: %v", dir, err)
90+ }
91+ }
92+
93+ // Create .crushignore in subdir that ignores normal_nested
94+ subCrushignore := "normal_nested/\n"
95+ if err := os.WriteFile(filepath.Join(subDir, ".crushignore"), []byte(subCrushignore), 0o644); err != nil {
96+ t.Fatalf("Failed to create subdir .crushignore: %v", err)
97+ }
98+
99+ // Test hierarchical ignore behavior - this should work because the .crushignore is in the parent directory
100+ require.True(t, ShouldExcludeFile(tempDir, nestedNormal), "Expected normal_nested to be ignored by subdir .crushignore")
101+ require.False(t, ShouldExcludeFile(tempDir, subDir), "Expected subdir itself to not be ignored")
102+}
103+
104+func TestShouldExcludeFileCommonPatterns(t *testing.T) {
105+ t.Parallel()
106+
107+ tempDir := t.TempDir()
108+
109+ // Create directories that should be ignored by common patterns
110+ commonIgnored := []string{
111+ filepath.Join(tempDir, ".git"),
112+ filepath.Join(tempDir, "node_modules"),
113+ filepath.Join(tempDir, "__pycache__"),
114+ filepath.Join(tempDir, "target"),
115+ filepath.Join(tempDir, ".vscode"),
116+ }
117+
118+ for _, dir := range commonIgnored {
119+ if err := os.MkdirAll(dir, 0o755); err != nil {
120+ t.Fatalf("Failed to create directory %s: %v", dir, err)
121+ }
122+ }
123+
124+ // Test that common patterns are ignored even without explicit ignore files
125+ for _, dir := range commonIgnored {
126+ require.True(t, ShouldExcludeFile(tempDir, dir), "Expected %s to be ignored by common patterns", filepath.Base(dir))
127+ }
128+}
internal/fsext/ls.go+14โˆ’3
@@ -141,8 +141,16 @@ func (dl *directoryLister) shouldIgnore(path string, ignorePatterns []string) bo
141141 return true
142142 }
143143
144โˆ’ if dl.getIgnore(filepath.Dir(path)).MatchesPath(relPath) {
145โˆ’ slog.Debug("ignoring dir pattern", "path", relPath, "dir", filepath.Dir(path))
144+ parentDir := filepath.Dir(path)
145+ ignoreParser := dl.getIgnore(parentDir)
146+ if ignoreParser.MatchesPath(relPath) {
147+ slog.Debug("ignoring dir pattern", "path", relPath, "dir", parentDir)
148+ return true
149+ }
150+
151+ // For directories, also check with trailing slash (gitignore convention)
152+ if ignoreParser.MatchesPath(relPath + "/") {
153+ slog.Debug("ignoring dir pattern with slash", "path", relPath+"/", "dir", parentDir)
146154 return true
147155 }
148156
@@ -160,11 +168,14 @@ func (dl *directoryLister) shouldIgnore(path string, ignorePatterns []string) bo
160168
161169func (dl *directoryLister) checkParentIgnores(path string) bool {
162170 parent := filepath.Dir(filepath.Dir(path))
163โˆ’ for parent != dl.rootPath && parent != "." && path != "." {
171+ for parent != "." && path != "." {
164172 if dl.getIgnore(parent).MatchesPath(path) {
165173 slog.Debug("ingoring parent dir pattern", "path", path, "dir", parent)
166174 return true
167175 }
176+ if parent == dl.rootPath {
177+ break
178+ }
168179 parent = filepath.Dir(parent)
169180 }
170181 return false
internal/llm/provider/openai_test.go+1โˆ’1
@@ -75,7 +75,7 @@ func TestOpenAIClientStreamChoices(t *testing.T) {
7575 },
7676 }
7777
78โˆ’ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
78+ ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
7979 defer cancel()
8080
8181 eventsChan := client.stream(ctx, messages, nil)
internal/llm/tools/grep.go+19โˆ’7
@@ -279,11 +279,13 @@ func searchWithRipgrep(ctx context.Context, pattern, path, include string) ([]gr
279279 return nil, fmt.Errorf("ripgrep not found in $PATH")
280280 }
281281
282โˆ’ cmd.Args = append(
283โˆ’ cmd.Args,
284โˆ’ "--ignore-file", filepath.Join(path, ".gitignore"),
285โˆ’ "--ignore-file", filepath.Join(path, ".crushignore"),
286โˆ’ )
282+ // Only add ignore files if they exist
283+ for _, ignoreFile := range []string{".gitignore", ".crushignore"} {
284+ ignorePath := filepath.Join(path, ignoreFile)
285+ if _, err := os.Stat(ignorePath); err == nil {
286+ cmd.Args = append(cmd.Args, "--ignore-file", ignorePath)
287+ }
288+ }
287289
288290 output, err := cmd.Output()
289291 if err != nil {
@@ -357,14 +359,24 @@ func searchFilesWithRegex(pattern, rootPath, include string) ([]grepMatch, error
357359 }
358360
359361 if info.IsDir() {
360โˆ’ return nil // Skip directories
362+ // Check if directory should be skipped
363+ if walker.ShouldSkip(path) {
364+ return filepath.SkipDir
365+ }
366+ return nil // Continue into directory
361367 }
362368
363โˆ’ // Use walker's shouldSkip method instead of just SkipHidden
369+ // Use walker's shouldSkip method for files
364370 if walker.ShouldSkip(path) {
365371 return nil
366372 }
367373
374+ // Skip hidden files (starting with a dot) to match ripgrep's default behavior
375+ base := filepath.Base(path)
376+ if base != "." && strings.HasPrefix(base, ".") {
377+ return nil
378+ }
379+
368380 if includePattern != nil && !includePattern.MatchString(path) {
369381 return nil
370382 }
internal/llm/tools/grep_test.go+35โˆ’26
@@ -1,8 +1,6 @@
11package tools
22
33import (
4โˆ’ "context"
5โˆ’ "encoding/json"
64 "os"
75 "path/filepath"
86 "regexp"
@@ -59,6 +57,7 @@ func TestGlobToRegexCaching(t *testing.T) {
5957}
6058
6159func TestGrepWithIgnoreFiles(t *testing.T) {
60+ t.Parallel()
6261 tempDir := t.TempDir()
6362
6463 // Create test files
@@ -84,32 +83,42 @@ func TestGrepWithIgnoreFiles(t *testing.T) {
8483 crushignoreContent := "node_modules/\n"
8584 require.NoError(t, os.WriteFile(filepath.Join(tempDir, ".crushignore"), []byte(crushignoreContent), 0o644))
8685
87โˆ’ // Create grep tool
88โˆ’ grepTool := NewGrepTool(tempDir)
86+ // Test both implementations
87+ for name, fn := range map[string]func(pattern, path, include string) ([]grepMatch, error){
88+ "regex": searchFilesWithRegex,
89+ "rg": func(pattern, path, include string) ([]grepMatch, error) {
90+ return searchWithRipgrep(t.Context(), pattern, path, include)
91+ },
92+ } {
93+ t.Run(name, func(t *testing.T) {
94+ t.Parallel()
95+
96+ if name == "rg" && getRg() == "" {
97+ t.Skip("rg is not in $PATH")
98+ }
99+
100+ matches, err := fn("hello world", tempDir, "")
101+ require.NoError(t, err)
89102
90โˆ’ // Create grep parameters
91โˆ’ params := GrepParams{
92โˆ’ Pattern: "hello world",
93โˆ’ Path: tempDir,
103+ // Convert matches to a set of file paths for easier testing
104+ foundFiles := make(map[string]bool)
105+ for _, match := range matches {
106+ foundFiles[filepath.Base(match.path)] = true
107+ }
108+
109+ // Should find file1.txt and file2.txt
110+ require.True(t, foundFiles["file1.txt"], "Should find file1.txt")
111+ require.True(t, foundFiles["file2.txt"], "Should find file2.txt")
112+
113+ // Should NOT find ignored files
114+ require.False(t, foundFiles["file3.txt"], "Should not find file3.txt (ignored by .gitignore)")
115+ require.False(t, foundFiles["lib.js"], "Should not find lib.js (ignored by .crushignore)")
116+ require.False(t, foundFiles["secret.key"], "Should not find secret.key (ignored by .gitignore)")
117+
118+ // Should find exactly 2 matches
119+ require.Equal(t, 2, len(matches), "Should find exactly 2 matches")
120+ })
94121 }
95โˆ’ paramsJSON, err := json.Marshal(params)
96โˆ’ require.NoError(t, err)
97โˆ’
98โˆ’ // Run grep
99โˆ’ call := ToolCall{Input: string(paramsJSON)}
100โˆ’ response, err := grepTool.Run(context.Background(), call)
101โˆ’ require.NoError(t, err)
102โˆ’
103โˆ’ // Check results - should only find file1.txt and file2.txt
104โˆ’ // ignored/file3.txt should be ignored by .gitignore
105โˆ’ // node_modules/lib.js should be ignored by .crushignore
106โˆ’ // secret.key should be ignored by .gitignore
107โˆ’ result := response.Content
108โˆ’ require.Contains(t, result, "file1.txt")
109โˆ’ require.Contains(t, result, "file2.txt")
110โˆ’ require.NotContains(t, result, "file3.txt")
111โˆ’ require.NotContains(t, result, "lib.js")
112โˆ’ require.NotContains(t, result, "secret.key")
113122}
114123
115124func TestSearchImplementations(t *testing.T) {
internal/shell/command_block_test.go+1โˆ’2
@@ -1,7 +1,6 @@
11package shell
22
33import (
4โˆ’ "context"
54 "strings"
65 "testing"
76
@@ -92,7 +91,7 @@ func TestCommandBlocking(t *testing.T) {
9291 BlockFuncs: tt.blockFuncs,
9392 })
9493
95โˆ’ _, _, err := shell.Exec(context.Background(), tt.command)
94+ _, _, err := shell.Exec(t.Context(), tt.command)
9695
9796 if tt.shouldBlock {
9897 if err == nil {
internal/shell/shell_test.go+2โˆ’2
@@ -16,7 +16,7 @@ func BenchmarkShellQuickCommands(b *testing.B) {
1616 b.ReportAllocs()
1717
1818 for b.Loop() {
19โˆ’ _, _, err := shell.Exec(context.Background(), "echo test")
19+ _, _, err := shell.Exec(b.Context(), "echo test")
2020 exitCode := ExitCode(err)
2121 if err != nil || exitCode != 0 {
2222 b.Fatalf("Command failed: %v, exit code: %d", err, exitCode)
@@ -100,7 +100,7 @@ func TestRunContinuity(t *testing.T) {
100100
101101func TestCrossPlatformExecution(t *testing.T) {
102102 shell := NewShell(&Options{WorkingDir: "."})
103โˆ’ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
103+ ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
104104 defer cancel()
105105
106106 // Test a simple command that should work on all platforms