git.iko.soy/crush
on main help sign in
[feature/ripgrep-glob] Add ripgrep-based file globbing to improve performance
3c2b0f4isaac-scarrott
internal/llm/tools/glob.go+69
compare with main
unifiedsplit
@@ -1,11 +1,13 @@
1package tools
1package tools
2
2
3import (
3import (
4+ "bytes"
4 "context"
5 "context"
5 "encoding/json"
6 "encoding/json"
6 "fmt"
7 "fmt"
7 "io/fs"
8 "io/fs"
8 "os"
9 "os"
10+ "os/exec"
9 "path/filepath"
11 "path/filepath"
10 "sort"
12 "sort"
11 "strings"
13 "strings"
@@ -132,6 +134,73 @@ func (g *globTool) Run(ctx context.Context, call ToolCall) (ToolResponse, error)
132}
134}
133
135
134func globFiles(pattern, searchPath string, limit int) ([]string, bool, error) {
136func globFiles(pattern, searchPath string, limit int) ([]string, bool, error) {
137+ matches, err := globWithRipgrep(pattern, searchPath, limit)
138+ if err == nil {
139+ return matches, len(matches) >= limit, nil
140+ }
141+
142+ return globWithDoublestar(pattern, searchPath, limit)
143+}
144+
145+func globWithRipgrep(
146+ pattern, searchRoot string,
147+ limit int,
148+) ([]string, error) {
149+
150+ if searchRoot == "" {
151+ searchRoot = "."
152+ }
153+
154+ rgBin, err := exec.LookPath("rg")
155+ if err != nil {
156+ return nil, fmt.Errorf("ripgrep not found in $PATH: %w", err)
157+ }
158+
159+ if !filepath.IsAbs(pattern) && !strings.HasPrefix(pattern, "/") {
160+ pattern = "/" + pattern
161+ }
162+
163+ args := []string{
164+ "--files",
165+ "--null",
166+ "--glob", pattern,
167+ "-L",
168+ }
169+
170+ cmd := exec.Command(rgBin, args...)
171+ cmd.Dir = searchRoot
172+
173+ out, err := cmd.CombinedOutput()
174+ if err != nil {
175+ if ee, ok := err.(*exec.ExitError); ok && ee.ExitCode() == 1 {
176+ return nil, nil
177+ }
178+ return nil, fmt.Errorf("ripgrep: %w\n%s", err, out)
179+ }
180+
181+ var matches []string
182+ for _, p := range bytes.Split(out, []byte{0}) {
183+ if len(p) == 0 {
184+ continue
185+ }
186+ abs := filepath.Join(searchRoot, string(p))
187+ if skipHidden(abs) {
188+ continue
189+ }
190+ matches = append(matches, abs)
191+ }
192+
193+ sort.SliceStable(matches, func(i, j int) bool {
194+ return len(matches[i]) < len(matches[j])
195+ })
196+
197+ if len(matches) > limit {
198+ matches = matches[:limit]
199+ }
200+ return matches, nil
201+}
202+
203+func globWithDoublestar(pattern, searchPath string, limit int) ([]string, bool, error) {
135 if !strings.HasPrefix(pattern, "/") && !strings.HasPrefix(pattern, searchPath) {
204 if !strings.HasPrefix(pattern, "/") && !strings.HasPrefix(pattern, searchPath) {
136 if !strings.HasSuffix(searchPath, "/") {
205 if !strings.HasSuffix(searchPath, "/") {
137 searchPath += "/"
206 searchPath += "/"