git.iko.soy/crush
on main help sign in
fix: try to make the search tool more reliable (#1779)
ab6d971Kujtim Hoxha
internal/agent/tools/search.go+101−71
compare with main
unifiedsplit
@@ -4,10 +4,13 @@ import (
4 "context"
4 "context"
5 "fmt"
5 "fmt"
6 "io"
6 "io"
7+ "math/rand/v2"
7 "net/http"
8 "net/http"
8 "net/url"
9 "net/url"
9 "slices"
10 "slices"
10 "strings"
11 "strings"
12+ "sync"
13+ "time"
11
14
12 "golang.org/x/net/html"
15 "golang.org/x/net/html"
13)
16)
@@ -20,28 +23,41 @@ type SearchResult struct {
20 Position int
23 Position int
21}
24}
22
25
23−// searchDuckDuckGo performs a web search using DuckDuckGo's HTML endpoint.
26+var userAgents = []string{
27+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
28+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
29+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
30+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
31+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
32+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
33+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.0) Gecko/20100101 Firefox/132.0",
34+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0",
35+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15",
36+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15",
37+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
38+}
39+
40+var acceptLanguages = []string{
41+ "en-US,en;q=0.9",
42+ "en-US,en;q=0.9,es;q=0.8",
43+ "en-GB,en;q=0.9,en-US;q=0.8",
44+ "en-US,en;q=0.5",
45+ "en-CA,en;q=0.9,en-US;q=0.8",
46+}
47+
24func searchDuckDuckGo(ctx context.Context, client *http.Client, query string, maxResults int) ([]SearchResult, error) {
48func searchDuckDuckGo(ctx context.Context, client *http.Client, query string, maxResults int) ([]SearchResult, error) {
25 if maxResults <= 0 {
49 if maxResults <= 0 {
26 maxResults = 10
50 maxResults = 10
27 }
51 }
28
52
29− formData := url.Values{}
53+ searchURL := "https://lite.duckduckgo.com/lite/?q=" + url.QueryEscape(query)
30− formData.Set("q", query)
31− formData.Set("b", "")
32− formData.Set("kl", "")
33
54
34− req, err := http.NewRequestWithContext(ctx, "POST", "https://html.duckduckgo.com/html", strings.NewReader(formData.Encode()))
55+ req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
35 if err != nil {
56 if err != nil {
36 return nil, fmt.Errorf("failed to create request: %w", err)
57 return nil, fmt.Errorf("failed to create request: %w", err)
37 }
58 }
38
59
39− req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
60+ setRandomizedHeaders(req)
40− req.Header.Set("User-Agent", BrowserUserAgent)
41− req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
42− req.Header.Set("Accept-Language", "en-US,en;q=0.5")
43− req.Header.Set("Accept-Encoding", "gzip, deflate")
44− req.Header.Set("Referer", "https://duckduckgo.com/")
45
61
46 resp, err := client.Do(req)
62 resp, err := client.Do(req)
47 if err != nil {
63 if err != nil {
@@ -49,10 +65,8 @@ func searchDuckDuckGo(ctx context.Context, client *http.Client, query string, ma
49 }
65 }
50 defer resp.Body.Close()
66 defer resp.Body.Close()
51
67
52− // Accept both 200 (OK) and 202 (Accepted).
53− // DuckDuckGo may still return 202 for rate limiting or bot detection.
54 if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
68 if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
55− return nil, fmt.Errorf("search failed with status code: %d (DuckDuckGo may be rate limiting requests)", resp.StatusCode)
69+ return nil, fmt.Errorf("search failed with status code: %d", resp.StatusCode)
56 }
70 }
57
71
58 body, err := io.ReadAll(resp.Body)
72 body, err := io.ReadAll(resp.Body)
@@ -60,85 +74,90 @@ func searchDuckDuckGo(ctx context.Context, client *http.Client, query string, ma
60 return nil, fmt.Errorf("failed to read response: %w", err)
74 return nil, fmt.Errorf("failed to read response: %w", err)
61 }
75 }
62
76
63− return parseSearchResults(string(body), maxResults)
77+ return parseLiteSearchResults(string(body), maxResults)
64}
78}
65
79
66−// parseSearchResults extracts search results from DuckDuckGo HTML response.
80+func setRandomizedHeaders(req *http.Request) {
81+ req.Header.Set("User-Agent", userAgents[rand.IntN(len(userAgents))])
82+ req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
83+ req.Header.Set("Accept-Language", acceptLanguages[rand.IntN(len(acceptLanguages))])
84+ req.Header.Set("Accept-Encoding", "identity")
85+ req.Header.Set("Connection", "keep-alive")
86+ req.Header.Set("Upgrade-Insecure-Requests", "1")
87+ req.Header.Set("Sec-Fetch-Dest", "document")
88+ req.Header.Set("Sec-Fetch-Mode", "navigate")
89+ req.Header.Set("Sec-Fetch-Site", "none")
90+ req.Header.Set("Sec-Fetch-User", "?1")
91+ req.Header.Set("Cache-Control", "max-age=0")
92+ if rand.IntN(2) == 0 {
93+ req.Header.Set("DNT", "1")
94+ }
95+}
96+
67−func parseSearchResults(htmlContent string, maxResults int) ([]SearchResult, error) {
97+func parseLiteSearchResults(htmlContent string, maxResults int) ([]SearchResult, error) {
68 doc, err := html.Parse(strings.NewReader(htmlContent))
98 doc, err := html.Parse(strings.NewReader(htmlContent))
69 if err != nil {
99 if err != nil {
70 return nil, fmt.Errorf("failed to parse HTML: %w", err)
100 return nil, fmt.Errorf("failed to parse HTML: %w", err)
71 }
101 }
72
102
73 var results []SearchResult
103 var results []SearchResult
74− var traverse func(*html.Node)
104+ var currentResult *SearchResult
75
105
106+ var traverse func(*html.Node)
76 traverse = func(n *html.Node) {
107 traverse = func(n *html.Node) {
77− if n.Type == html.ElementNode && n.Data == "div" && hasClass(n, "result") {
108+ if n.Type == html.ElementNode {
78− result := extractResult(n)
109+ if n.Data == "a" && hasClass(n, "result-link") {
79− if result != nil && result.Link != "" && !strings.Contains(result.Link, "y.js") {
110+ if currentResult != nil && currentResult.Link != "" {
80− result.Position = len(results) + 1
111+ currentResult.Position = len(results) + 1
81− results = append(results, *result)
112+ results = append(results, *currentResult)
82− if len(results) >= maxResults {
113+ if len(results) >= maxResults {
83− return
114+ return
115+ }
116+ }
117+ currentResult = &SearchResult{Title: getTextContent(n)}
118+ for _, attr := range n.Attr {
119+ if attr.Key == "href" {
120+ currentResult.Link = cleanDuckDuckGoURL(attr.Val)
121+ break
122+ }
84 }
123 }
85 }
124 }
125+ if n.Data == "td" && hasClass(n, "result-snippet") && currentResult != nil {
126+ currentResult.Snippet = getTextContent(n)
127+ }
86 }
128 }
129+ for c := n.FirstChild; c != nil; c = c.NextSibling {
87− for c := n.FirstChild; c != nil && len(results) < maxResults; c = c.NextSibling {
130+ if len(results) >= maxResults {
131+ return
132+ }
88 traverse(c)
133 traverse(c)
89 }
134 }
90 }
135 }
91
136
92 traverse(doc)
137 traverse(doc)
138+
139+ if currentResult != nil && currentResult.Link != "" && len(results) < maxResults {
140+ currentResult.Position = len(results) + 1
141+ results = append(results, *currentResult)
142+ }
143+
93 return results, nil
144 return results, nil
94}
145}
95
146
96−// hasClass checks if an HTML node has a specific class.
97func hasClass(n *html.Node, class string) bool {
147func hasClass(n *html.Node, class string) bool {
98 for _, attr := range n.Attr {
148 for _, attr := range n.Attr {
99 if attr.Key == "class" {
149 if attr.Key == "class" {
100− return slices.Contains(strings.Fields(attr.Val), class)
150+ if slices.Contains(strings.Fields(attr.Val), class) {
101− }
151+ return true
102− }
103− return false
104−}
105−
106−// extractResult extracts a search result from a result div node.
107−func extractResult(n *html.Node) *SearchResult {
108− result := &SearchResult{}
109−
110− var traverse func(*html.Node)
111− traverse = func(node *html.Node) {
112− if node.Type == html.ElementNode {
113− // Look for title link.
114− if node.Data == "a" && hasClass(node, "result__a") {
115− result.Title = getTextContent(node)
116− for _, attr := range node.Attr {
117− if attr.Key == "href" {
118− result.Link = cleanDuckDuckGoURL(attr.Val)
119− break
120− }
121− }
122− }
123− // Look for snippet.
124− if node.Data == "a" && hasClass(node, "result__snippet") {
125− result.Snippet = getTextContent(node)
126 }
152 }
127 }
153 }
128− for c := node.FirstChild; c != nil; c = c.NextSibling {
129− traverse(c)
130− }
131 }
154 }
132−
133− traverse(n)
134− return result
155+ return false
135}
156}
136
157
137−// getTextContent extracts all text content from a node and its children.
138func getTextContent(n *html.Node) string {
158func getTextContent(n *html.Node) string {
139 var text strings.Builder
159 var text strings.Builder
140 var traverse func(*html.Node)
160 var traverse func(*html.Node)
141−
142 traverse = func(node *html.Node) {
161 traverse = func(node *html.Node) {
143 if node.Type == html.TextNode {
162 if node.Type == html.TextNode {
144 text.WriteString(node.Data)
163 text.WriteString(node.Data)
@@ -147,22 +166,18 @@ func getTextContent(n *html.Node) string {
147 traverse(c)
166 traverse(c)
148 }
167 }
149 }
168 }
150−
151 traverse(n)
169 traverse(n)
152 return strings.TrimSpace(text.String())
170 return strings.TrimSpace(text.String())
153}
171}
154
172
155−// cleanDuckDuckGoURL extracts the actual URL from DuckDuckGo's redirect URL.
156func cleanDuckDuckGoURL(rawURL string) string {
173func cleanDuckDuckGoURL(rawURL string) string {
157 if strings.HasPrefix(rawURL, "//duckduckgo.com/l/?uddg=") {
174 if strings.HasPrefix(rawURL, "//duckduckgo.com/l/?uddg=") {
158− // Extract the actual URL from the redirect.
159 if idx := strings.Index(rawURL, "uddg="); idx != -1 {
175 if idx := strings.Index(rawURL, "uddg="); idx != -1 {
160 encoded := rawURL[idx+5:]
176 encoded := rawURL[idx+5:]
161 if ampIdx := strings.Index(encoded, "&"); ampIdx != -1 {
177 if ampIdx := strings.Index(encoded, "&"); ampIdx != -1 {
162 encoded = encoded[:ampIdx]
178 encoded = encoded[:ampIdx]
163 }
179 }
164− decoded, err := url.QueryUnescape(encoded)
180+ if decoded, err := url.QueryUnescape(encoded); err == nil {
165− if err == nil {
166 return decoded
181 return decoded
167 }
182 }
168 }
183 }
@@ -170,20 +185,35 @@ func cleanDuckDuckGoURL(rawURL string) string {
170 return rawURL
185 return rawURL
171}
186}
172
187
173−// formatSearchResults formats search results for LLM consumption.
174func formatSearchResults(results []SearchResult) string {
188func formatSearchResults(results []SearchResult) string {
175 if len(results) == 0 {
189 if len(results) == 0 {
176− return "No results were found for your search query. This could be due to DuckDuckGo's bot detection or the query returned no matches. Please try rephrasing your search or try again in a few minutes."
190+ return "No results found. Try rephrasing your search."
177 }
191 }
178
192
179 var sb strings.Builder
193 var sb strings.Builder
180 sb.WriteString(fmt.Sprintf("Found %d search results:\n\n", len(results)))
194 sb.WriteString(fmt.Sprintf("Found %d search results:\n\n", len(results)))
181−
182 for _, result := range results {
195 for _, result := range results {
183 sb.WriteString(fmt.Sprintf("%d. %s\n", result.Position, result.Title))
196 sb.WriteString(fmt.Sprintf("%d. %s\n", result.Position, result.Title))
184 sb.WriteString(fmt.Sprintf(" URL: %s\n", result.Link))
197 sb.WriteString(fmt.Sprintf(" URL: %s\n", result.Link))
185 sb.WriteString(fmt.Sprintf(" Summary: %s\n\n", result.Snippet))
198 sb.WriteString(fmt.Sprintf(" Summary: %s\n\n", result.Snippet))
186 }
199 }
187−
188 return sb.String()
200 return sb.String()
189}
201}
202+
203+var (
204+ lastSearchMu sync.Mutex
205+ lastSearchTime time.Time
206+)
207+
208+// maybeDelaySearch adds a random delay if the last search was recent.
209+func maybeDelaySearch() {
210+ lastSearchMu.Lock()
211+ defer lastSearchMu.Unlock()
212+
213+ minGap := time.Duration(500+rand.IntN(1500)) * time.Millisecond
214+ elapsed := time.Since(lastSearchTime)
215+ if elapsed < minGap {
216+ time.Sleep(minGap - elapsed)
217+ }
218+ lastSearchTime = time.Now()
219+}
internal/agent/tools/web_search.go+3
@@ -3,6 +3,7 @@ package tools
3import (
3import (
4 "context"
4 "context"
5 _ "embed"
5 _ "embed"
6+ "log/slog"
6 "net/http"
7 "net/http"
7 "time"
8 "time"
8
9
@@ -41,7 +42,9 @@ func NewWebSearchTool(client *http.Client) fantasy.AgentTool {
41 maxResults = 20
42 maxResults = 20
42 }
43 }
43
44
45+ maybeDelaySearch()
44 results, err := searchDuckDuckGo(ctx, client, params.Query, maxResults)
46 results, err := searchDuckDuckGo(ctx, client, params.Query, maxResults)
47+ slog.Debug("Web search completed", "query", params.Query, "results", len(results), "err", err)
45 if err != nil {
48 if err != nil {
46 return fantasy.NewTextErrorResponse("Failed to search: " + err.Error()), nil
49 return fantasy.NewTextErrorResponse("Failed to search: " + err.Error()), nil
47 }
50 }