on main help sign in
fix(glob): keep file search fast and bounded on large directories

Pointing the glob tool at a huge or symlink-heavy directory (a home folder, a module cache) could peg every CPU core, balloon memory to several GB, and freeze the session until manually interrupted. It could also wander outside the project by following symlinks. Glob now applies a timeout, streams results instead of buffering the whole listing, scopes the walk to the pattern's literal prefix, and no longer follows symlinks. A search over a giant tree now returns quickly or times out cleanly instead of hanging.

commit67f50014e81be7fbdf1187fa78e65279b87b7908 authorKieran Klukas <kieran@dunkirk.sh> dateMon 22 Jun 2026, 15:45 (4 months ago) parent6f8b104
10 files changed+248−26
internal/agent/agentic_fetch_tool.go+1−1 internal/agent/common_test.go+1−1 internal/agent/coordinator.go+1−1 internal/agent/tools/glob.go+75−18 internal/agent/tools/glob_test.go+74 internal/agent/tools/rg.go+6−1 internal/config/config.go+10 internal/filepathext/filepath.go+25 internal/filepathext/filepath_test.go+37 internal/fsext/fileutil.go+18−4
internal/agent/agentic_fetch_tool.go+1−1
@@ -167,7 +167,7 @@ func (c *coordinator) agenticFetchTool(_ context.Context, client *http.Client) (
167167 fetchTools := []fantasy.AgentTool{
168168 webFetchTool,
169169 webSearchTool,
170− tools.NewGlobTool(tmpDir),
170+ tools.NewGlobTool(tmpDir, c.cfg.Config().Tools.Glob),
171171 tools.NewGrepTool(tmpDir, c.cfg.Config().Tools.Grep),
172172 tools.NewSourcegraphTool(client),
173173 tools.NewViewTool(c.lspManager, c.permissions, c.filetracker, nil, tmpDir),
internal/agent/common_test.go+1−1
@@ -172,7 +172,7 @@ func coderAgent(r *vcr.Recorder, env fakeEnv, large, small fantasy.LanguageModel
172172 tools.NewEditTool(nil, env.permissions, env.history, *env.filetracker, env.workingDir),
173173 tools.NewMultiEditTool(nil, env.permissions, env.history, *env.filetracker, env.workingDir),
174174 tools.NewFetchTool(env.permissions, env.workingDir, r.GetDefaultClient()),
175− tools.NewGlobTool(env.workingDir),
175+ tools.NewGlobTool(env.workingDir, cfg.Config().Tools.Glob),
176176 tools.NewGrepTool(env.workingDir, cfg.Config().Tools.Grep),
177177 tools.NewLsTool(env.permissions, env.workingDir, cfg.Config().Tools.Ls),
178178 tools.NewSourcegraphTool(r.GetDefaultClient()),
internal/agent/coordinator.go+1−1
@@ -634,7 +634,7 @@ func (c *coordinator) buildTools(ctx context.Context, agent config.Agent, isSubA
634634 tools.NewEditTool(c.lspManager, c.permissions, c.history, c.filetracker, c.cfg.WorkingDir()),
635635 tools.NewMultiEditTool(c.lspManager, c.permissions, c.history, c.filetracker, c.cfg.WorkingDir()),
636636 tools.NewFetchTool(c.permissions, c.cfg.WorkingDir(), nil),
637− tools.NewGlobTool(c.cfg.WorkingDir()),
637+ tools.NewGlobTool(c.cfg.WorkingDir(), c.cfg.Config().Tools.Glob),
638638 tools.NewGrepTool(c.cfg.WorkingDir(), c.cfg.Config().Tools.Grep),
639639 tools.NewLsTool(c.permissions, c.cfg.WorkingDir(), c.cfg.Config().Tools.Ls),
640640 tools.NewSourcegraphTool(nil),
internal/agent/tools/glob.go+75−18
@@ -1,6 +1,7 @@
11package tools
22
33import (
4+ "bufio"
45 "bytes"
56 "cmp"
67 "context"
@@ -14,6 +15,7 @@ import (
1415 "strings"
1516
1617 "charm.land/fantasy"
18+ "github.com/charmbracelet/crush/internal/config"
1719 "github.com/charmbracelet/crush/internal/filepathext"
1820 "github.com/charmbracelet/crush/internal/fsext"
1921)
@@ -48,7 +50,7 @@ type GlobResponseMetadata struct {
4850 Truncated bool `json:"truncated"`
4951}
5052
51−func NewGlobTool(workingDir string) fantasy.AgentTool {
53+func NewGlobTool(workingDir string, cfg config.ToolGlob) fantasy.AgentTool {
5254 return fantasy.NewAgentTool(
5355 GlobToolName,
5456 globDescription(),
@@ -59,7 +61,13 @@ func NewGlobTool(workingDir string) fantasy.AgentTool {
5961
6062 searchPath := cmp.Or(params.Path, workingDir)
6163
62− files, truncated, err := globFiles(ctx, params.Pattern, searchPath, 100)
64+ // Bound the search so a huge or symlink-heavy root (e.g. $HOME
65+ // or a module cache) fails cleanly instead of pinning the CPU
66+ // and hanging the agent.
67+ searchCtx, cancel := context.WithTimeout(ctx, cfg.GetTimeout())
68+ defer cancel()
69+
70+ files, truncated, err := globFiles(searchCtx, params.Pattern, searchPath, 100)
6371 if err != nil {
6472 return fantasy.NewTextErrorResponse(fmt.Sprintf("error finding files: %v", err)), nil
6573 }
@@ -87,38 +95,87 @@ func NewGlobTool(workingDir string) fantasy.AgentTool {
8795}
8896
8997func globFiles(ctx context.Context, pattern, searchPath string, limit int) ([]string, bool, error) {
90− cmdRg := getRgCmd(ctx, pattern)
98+ // Scope the walk to the pattern's literal directory prefix. A pattern
99+ // like "internal/agent/*.go" only needs to walk "internal/agent", so we
100+ // start there instead of enumerating the entire tree and filtering.
101+ // Patterns that begin with a wildcard (e.g. "**/foo.go") have no prefix
102+ // and still walk from searchPath.
103+ prefix, rest := filepathext.SplitGlobPrefix(pattern)
104+ walkRoot := searchPath
105+ walkPattern := pattern
106+ if prefix != "" {
107+ walkRoot = filepath.Join(searchPath, prefix)
108+ walkPattern = rest
109+ }
110+
111+ cmdRg := getRgCmd(ctx, walkPattern)
91112 if cmdRg != nil {
92− cmdRg.Dir = searchPath
93− matches, err := runRipgrep(cmdRg, searchPath, limit)
113+ cmdRg.Dir = walkRoot
114+ matches, err := runRipgrep(cmdRg, walkRoot, limit)
94115 if err == nil {
95116 return matches, len(matches) >= limit && limit > 0, nil
96117 }
97118 slog.Warn("Ripgrep execution failed, falling back to doublestar", "error", err)
98119 }
99120
100− return fsext.GlobGitignoreAware(pattern, searchPath, limit)
121+ return fsext.GlobGitignoreAwareCtx(ctx, walkPattern, walkRoot, limit)
101122}
102123
103124func runRipgrep(cmd *exec.Cmd, searchRoot string, limit int) ([]string, error) {
104− out, err := cmd.CombinedOutput()
125+ // Stream ripgrep's stdout instead of buffering the whole file list.
126+ // Over a huge root (e.g. $HOME) the full --files listing can be
127+ // hundreds of MB; reading it all at once and then sorting allocated
128+ // gigabytes. We read incrementally and stop once we have a bounded
129+ // pool of candidates.
130+ //
131+ // We collect more than `limit` so the shortest-path preference below
132+ // still has something to choose from, but the pool is capped so memory
133+ // stays small (a few thousand paths) no matter how large the tree is.
134+ candidatePool := max(limit*20, 1000)
135+
136+ stdout, err := cmd.StdoutPipe()
105137 if err != nil {
106− if ee, ok := err.(*exec.ExitError); ok && ee.ExitCode() == 1 {
107− return nil, nil
108− }
109− return nil, fmt.Errorf("ripgrep: %w\n%s", err, out)
138+ return nil, fmt.Errorf("ripgrep: %w", err)
139+ }
140+ var stderr bytes.Buffer
141+ cmd.Stderr = &stderr
142+
143+ if err := cmd.Start(); err != nil {
144+ return nil, fmt.Errorf("ripgrep: %w", err)
110145 }
111146
112147 var matches []string
113− for p := range bytes.SplitSeq(out, []byte{0}) {
114− if len(p) == 0 {
115− continue
148+ reader := bufio.NewReader(stdout)
149+ for {
150+ path, err := reader.ReadString(0)
151+ if len(path) > 0 {
152+ path = strings.TrimRight(path, "\x00")
153+ if path != "" {
154+ absPath := filepathext.SmartJoin(searchRoot, path)
155+ if !fsext.SkipHidden(absPath) {
156+ matches = append(matches, absPath)
157+ }
158+ }
159+ }
160+ if err != nil {
161+ break // EOF or read error; drain handled by Wait below.
116162 }
117− absPath := filepathext.SmartJoin(searchRoot, string(p))
118− if fsext.SkipHidden(absPath) {
119− continue
163+ if len(matches) >= candidatePool {
164+ // Enough candidates; stop reading and let the process be
165+ // killed by the command context / Wait. Draining the rest
166+ // would just buffer paths we are going to discard.
167+ break
168+ }
169+ }
170+
171+ // Close our end so ripgrep gets SIGPIPE and stops, then reap it.
172+ _ = stdout.Close()
173+ waitErr := cmd.Wait()
174+ if waitErr != nil && len(matches) == 0 {
175+ if ee, ok := waitErr.(*exec.ExitError); ok && ee.ExitCode() == 1 {
176+ return nil, nil // No matches.
120177 }
121− matches = append(matches, absPath)
178+ return nil, fmt.Errorf("ripgrep: %w\n%s", waitErr, stderr.String())
122179 }
123180
124181 sort.SliceStable(matches, func(i, j int) bool {
internal/agent/tools/glob_test.gonew file+74
@@ -0,0 +1,74 @@
1+package tools
2+
3+import (
4+ "context"
5+ "os"
6+ "path/filepath"
7+ "testing"
8+
9+ "github.com/stretchr/testify/require"
10+)
11+
12+func TestGlobFilesScopedPrefixMatchesUnscoped(t *testing.T) {
13+ t.Parallel()
14+
15+ root := t.TempDir()
16+ mkfile := func(rel string) {
17+ full := filepath.Join(root, rel)
18+ require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755))
19+ require.NoError(t, os.WriteFile(full, []byte("x"), 0o644))
20+ }
21+ mkfile("a/b/one.go")
22+ mkfile("a/b/c/two.go")
23+ mkfile("a/other.txt")
24+ mkfile("z/three.go")
25+
26+ got, _, err := globFiles(context.Background(), "a/**/*.go", root, 100)
27+ require.NoError(t, err)
28+ require.Len(t, got, 2)
29+ for _, p := range got {
30+ require.Contains(t, p, filepath.Join("a", "b"))
31+ }
32+}
33+
34+func TestGlobFilesDoesNotFollowSymlinkEscape(t *testing.T) {
35+ t.Parallel()
36+
37+ // Build a project dir with a symlink pointing outside it. With symlink
38+ // following disabled, the glob must not pick up files behind the link.
39+ outside := t.TempDir()
40+ require.NoError(t, os.WriteFile(filepath.Join(outside, "secret.go"), []byte("x"), 0o644))
41+
42+ project := t.TempDir()
43+ require.NoError(t, os.WriteFile(filepath.Join(project, "in.go"), []byte("x"), 0o644))
44+ link := filepath.Join(project, "escape")
45+ if err := os.Symlink(outside, link); err != nil {
46+ t.Skipf("symlinks unsupported: %v", err)
47+ }
48+
49+ got, _, err := globFiles(context.Background(), "**/*.go", project, 100)
50+ require.NoError(t, err)
51+ for _, p := range got {
52+ require.NotContains(t, p, "secret.go", "glob followed a symlink out of the search root")
53+ }
54+}
55+
56+func TestGlobFilesCapsResultsOnLargeTree(t *testing.T) {
57+ t.Parallel()
58+
59+ // A tree with far more matches than the limit must still return at
60+ // most `limit` results and report truncation, regardless of which
61+ // backend (ripgrep or the doublestar fallback) runs.
62+ root := t.TempDir()
63+ for i := range 500 {
64+ dir := filepath.Join(root, "pkg")
65+ require.NoError(t, os.MkdirAll(dir, 0o755))
66+ name := filepath.Join(dir, "file"+string(rune('a'+i%26))+string(rune('a'+(i/26)%26))+"_"+string(rune('0'+i%10))+".go")
67+ require.NoError(t, os.WriteFile(name, []byte("x"), 0o644))
68+ }
69+
70+ got, truncated, err := globFiles(context.Background(), "**/*.go", root, 10)
71+ require.NoError(t, err)
72+ require.LessOrEqual(t, len(got), 10, "must not exceed limit")
73+ require.True(t, truncated, "should report truncation on an over-limit tree")
74+}
internal/agent/tools/rg.go+6−1
@@ -31,7 +31,12 @@ func getRgCmd(ctx context.Context, globPattern string) *exec.Cmd {
3131 if name == "" {
3232 return nil
3333 }
34− args := []string{"--files", "-L", "--null"}
34+ // Note: we intentionally do not pass -L (follow symlinks). Following
35+ // symlinks lets rg escape the search root (into module caches, the nix
36+ // store, $HOME, etc.) and chase cycles, which pins all cores and can
37+ // hang. This keeps glob scoped to the tree it was pointed at, matching
38+ // the grep search command.
39+ args := []string{"--files", "--null"}
3540 if globPattern != "" {
3641 if !filepath.IsAbs(globPattern) && !strings.HasPrefix(globPattern, "/") {
3742 globPattern = "/" + globPattern
internal/config/config.go+10
@@ -527,6 +527,7 @@ type Agent struct {
527527type Tools struct {
528528 Ls ToolLs `json:"ls,omitzero"`
529529 Grep ToolGrep `json:"grep,omitzero"`
530+ Glob ToolGlob `json:"glob,omitzero"`
530531}
531532
532533type ToolLs struct {
@@ -548,6 +549,15 @@ func (t ToolGrep) GetTimeout() time.Duration {
548549 return ptrValOr(t.Timeout, 5*time.Second)
549550}
550551
552+type ToolGlob struct {
553+ Timeout *time.Duration `json:"timeout,omitempty" jsonschema:"description=Timeout for the glob tool call,default=30s,example=10s"`
554+}
555+
556+// GetTimeout returns the user-defined timeout or the default.
557+func (t ToolGlob) GetTimeout() time.Duration {
558+ return ptrValOr(t.Timeout, 30*time.Second)
559+}
560+
551561// HookConfig defines a user-configured shell command that fires on a hook
552562// event (e.g. PreToolUse). This is a pure-data struct: matcher compilation
553563// is owned by hooks.Runner so a JSON round-trip, merge, or reload can't
internal/filepathext/filepath.go+25
@@ -25,3 +25,28 @@ func SmartIsAbs(path string) bool {
2525 return filepath.IsAbs(path)
2626 }
2727}
28+
29+// SplitGlobPrefix splits a glob pattern into the longest leading run of
30+// literal path segments and the remaining pattern. The prefix contains no
31+// glob metacharacters, so callers can safely use it as a directory to start
32+// a walk from. For "internal/agent/*.go" it returns ("internal/agent",
33+// "*.go"); for "**/foo.go" it returns ("", "**/foo.go").
34+func SplitGlobPrefix(pattern string) (prefix, rest string) {
35+ pattern = filepath.ToSlash(pattern)
36+ segments := strings.Split(pattern, "/")
37+ var literal []string
38+ for i, seg := range segments {
39+ if strings.ContainsAny(seg, "*?[{\\") {
40+ rest = strings.Join(segments[i:], "/")
41+ return strings.Join(literal, "/"), rest
42+ }
43+ literal = append(literal, seg)
44+ }
45+ // Whole pattern is literal (a plain path); walk its parent and match
46+ // the basename so the existing match logic still applies.
47+ if len(literal) == 0 {
48+ return "", pattern
49+ }
50+ parent := strings.Join(literal[:len(literal)-1], "/")
51+ return parent, literal[len(literal)-1]
52+}
internal/filepathext/filepath_test.gonew file+37
@@ -0,0 +1,37 @@
1+package filepathext
2+
3+import (
4+ "testing"
5+
6+ "github.com/stretchr/testify/require"
7+)
8+
9+func TestSplitGlobPrefix(t *testing.T) {
10+ t.Parallel()
11+
12+ cases := []struct {
13+ pattern string
14+ wantPrefix string
15+ wantRest string
16+ }{
17+ {"*.go", "", "*.go"},
18+ {"**/foo.go", "", "**/foo.go"},
19+ {"internal/agent/*.go", "internal/agent", "*.go"},
20+ {"internal/**/*.go", "internal", "**/*.go"},
21+ {"a/b/c/*.txt", "a/b/c", "*.txt"},
22+ // A fully literal path walks its parent and matches the basename.
23+ {"internal/agent/glob.go", "internal/agent", "glob.go"},
24+ {"glob.go", "", "glob.go"},
25+ // A brace or bracket in the first segment means no literal prefix.
26+ {"{a,b}/x.go", "", "{a,b}/x.go"},
27+ }
28+
29+ for _, tc := range cases {
30+ t.Run(tc.pattern, func(t *testing.T) {
31+ t.Parallel()
32+ gotPrefix, gotRest := SplitGlobPrefix(tc.pattern)
33+ require.Equal(t, tc.wantPrefix, gotPrefix, "prefix")
34+ require.Equal(t, tc.wantRest, gotRest, "rest")
35+ })
36+ }
37+}
internal/fsext/fileutil.go+18−4
@@ -1,6 +1,7 @@
11package fsext
22
33import (
4+ "context"
45 "errors"
56 "fmt"
67 "os"
@@ -86,15 +87,21 @@ func (w *FastGlobWalker) ShouldSkipDir(path string) bool {
8687//
8788// Does not respect gitignore.
8889func Glob(pattern string, cwd string, limit int) ([]string, bool, error) {
89− return globWithDoubleStar(pattern, cwd, limit, false)
90+ return globWithDoubleStar(context.Background(), pattern, cwd, limit, false)
9091}
9192
9293// GlobGitignoreAware globs files respecting gitignore.
9394func GlobGitignoreAware(pattern string, cwd string, limit int) ([]string, bool, error) {
94− return globWithDoubleStar(pattern, cwd, limit, true)
95+ return globWithDoubleStar(context.Background(), pattern, cwd, limit, true)
9596}
9697
97−func globWithDoubleStar(pattern, searchPath string, limit int, gitignore bool) ([]string, bool, error) {
98+// GlobGitignoreAwareCtx is like [GlobGitignoreAware] but stops early when ctx
99+// is cancelled (e.g. on timeout), returning whatever was found so far.
100+func GlobGitignoreAwareCtx(ctx context.Context, pattern, cwd string, limit int) ([]string, bool, error) {
101+ return globWithDoubleStar(ctx, pattern, cwd, limit, true)
102+}
103+
104+func globWithDoubleStar(ctx context.Context, pattern, searchPath string, limit int, gitignore bool) ([]string, bool, error) {
98105 // Normalize pattern to forward slashes on Windows so their config can use
99106 // backslashes
100107 pattern = filepath.ToSlash(pattern)
@@ -102,11 +109,18 @@ func globWithDoubleStar(pattern, searchPath string, limit int, gitignore bool) (
102109 walker := NewFastGlobWalker(searchPath)
103110 found := csync.NewSlice[FileInfo]()
104111 conf := fastwalk.Config{
105− Follow: true,
112+ // Do not follow symlinks: following them lets the walk escape the
113+ // search root (into module caches, the nix store, $HOME, etc.) and
114+ // chase cycles, which is slow and can hang. Mirrors the rg path,
115+ // which no longer passes -L.
116+ Follow: false,
106117 ToSlash: fastwalk.DefaultToSlash(),
107118 Sort: fastwalk.SortFilesFirst,
108119 }
109120 err := fastwalk.Walk(&conf, searchPath, func(path string, d os.DirEntry, err error) error {
121+ if ctx.Err() != nil {
122+ return filepath.SkipAll // Timed out or cancelled; stop walking.
123+ }
110124 if err != nil {
111125 return nil // Skip files we can't access
112126 }