@@ -4,10 +4,13 @@ import (
44 "context"
55 "fmt"
66 "io"
7+ "math/rand/v2"
78 "net/http"
89 "net/url"
910 "slices"
1011 "strings"
12+ "sync"
13+ "time"
1114
1215 "golang.org/x/net/html"
1316)
@@ -20,28 +23,41 @@ type SearchResult struct {
2023 Position int
2124}
2225
23−// searchDuckDuckGo performs a web search using DuckDuckGo's HTML endpoint.
26+var userAgents = []string{
27+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
28+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
29+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
30+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
31+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
32+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
33+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.0) Gecko/20100101 Firefox/132.0",
34+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0",
35+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15",
36+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15",
37+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
38+}
39+
40+var acceptLanguages = []string{
41+ "en-US,en;q=0.9",
42+ "en-US,en;q=0.9,es;q=0.8",
43+ "en-GB,en;q=0.9,en-US;q=0.8",
44+ "en-US,en;q=0.5",
45+ "en-CA,en;q=0.9,en-US;q=0.8",
46+}
47+
2448func searchDuckDuckGo(ctx context.Context, client *http.Client, query string, maxResults int) ([]SearchResult, error) {
2549 if maxResults <= 0 {
2650 maxResults = 10
2751 }
2852
29− formData := url.Values{}
30− formData.Set("q", query)
31− formData.Set("b", "")
32− formData.Set("kl", "")
53+ searchURL := "https://lite.duckduckgo.com/lite/?q=" + url.QueryEscape(query)
3354
34− req, err := http.NewRequestWithContext(ctx, "POST", "https://html.duckduckgo.com/html", strings.NewReader(formData.Encode()))
55+ req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
3556 if err != nil {
3657 return nil, fmt.Errorf("failed to create request: %w", err)
3758 }
3859
39− req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
40− req.Header.Set("User-Agent", BrowserUserAgent)
41− req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
42− req.Header.Set("Accept-Language", "en-US,en;q=0.5")
43− req.Header.Set("Accept-Encoding", "gzip, deflate")
44− req.Header.Set("Referer", "https://duckduckgo.com/")
60+ setRandomizedHeaders(req)
4561
4662 resp, err := client.Do(req)
4763 if err != nil {
@@ -49,10 +65,8 @@ func searchDuckDuckGo(ctx context.Context, client *http.Client, query string, ma
4965 }
5066 defer resp.Body.Close()
5167
52− // Accept both 200 (OK) and 202 (Accepted).
53− // DuckDuckGo may still return 202 for rate limiting or bot detection.
5468 if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
55− return nil, fmt.Errorf("search failed with status code: %d (DuckDuckGo may be rate limiting requests)", resp.StatusCode)
69+ return nil, fmt.Errorf("search failed with status code: %d", resp.StatusCode)
5670 }
5771
5872 body, err := io.ReadAll(resp.Body)
@@ -60,85 +74,90 @@ func searchDuckDuckGo(ctx context.Context, client *http.Client, query string, ma
6074 return nil, fmt.Errorf("failed to read response: %w", err)
6175 }
6276
63− return parseSearchResults(string(body), maxResults)
77+ return parseLiteSearchResults(string(body), maxResults)
6478}
6579
66−// parseSearchResults extracts search results from DuckDuckGo HTML response.
67−func parseSearchResults(htmlContent string, maxResults int) ([]SearchResult, error) {
80+func setRandomizedHeaders(req *http.Request) {
81+ req.Header.Set("User-Agent", userAgents[rand.IntN(len(userAgents))])
82+ req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
83+ req.Header.Set("Accept-Language", acceptLanguages[rand.IntN(len(acceptLanguages))])
84+ req.Header.Set("Accept-Encoding", "identity")
85+ req.Header.Set("Connection", "keep-alive")
86+ req.Header.Set("Upgrade-Insecure-Requests", "1")
87+ req.Header.Set("Sec-Fetch-Dest", "document")
88+ req.Header.Set("Sec-Fetch-Mode", "navigate")
89+ req.Header.Set("Sec-Fetch-Site", "none")
90+ req.Header.Set("Sec-Fetch-User", "?1")
91+ req.Header.Set("Cache-Control", "max-age=0")
92+ if rand.IntN(2) == 0 {
93+ req.Header.Set("DNT", "1")
94+ }
95+}
96+
97+func parseLiteSearchResults(htmlContent string, maxResults int) ([]SearchResult, error) {
6898 doc, err := html.Parse(strings.NewReader(htmlContent))
6999 if err != nil {
70100 return nil, fmt.Errorf("failed to parse HTML: %w", err)
71101 }
72102
73103 var results []SearchResult
74− var traverse func(*html.Node)
104+ var currentResult *SearchResult
75105
106+ var traverse func(*html.Node)
76107 traverse = func(n *html.Node) {
77− if n.Type == html.ElementNode && n.Data == "div" && hasClass(n, "result") {
78− result := extractResult(n)
79− if result != nil && result.Link != "" && !strings.Contains(result.Link, "y.js") {
80− result.Position = len(results) + 1
81− results = append(results, *result)
82− if len(results) >= maxResults {
83− return
108+ if n.Type == html.ElementNode {
109+ if n.Data == "a" && hasClass(n, "result-link") {
110+ if currentResult != nil && currentResult.Link != "" {
111+ currentResult.Position = len(results) + 1
112+ results = append(results, *currentResult)
113+ if len(results) >= maxResults {
114+ return
115+ }
116+ }
117+ currentResult = &SearchResult{Title: getTextContent(n)}
118+ for _, attr := range n.Attr {
119+ if attr.Key == "href" {
120+ currentResult.Link = cleanDuckDuckGoURL(attr.Val)
121+ break
122+ }
84123 }
85124 }
125+ if n.Data == "td" && hasClass(n, "result-snippet") && currentResult != nil {
126+ currentResult.Snippet = getTextContent(n)
127+ }
86128 }
87− for c := n.FirstChild; c != nil && len(results) < maxResults; c = c.NextSibling {
129+ for c := n.FirstChild; c != nil; c = c.NextSibling {
130+ if len(results) >= maxResults {
131+ return
132+ }
88133 traverse(c)
89134 }
90135 }
91136
92137 traverse(doc)
138+
139+ if currentResult != nil && currentResult.Link != "" && len(results) < maxResults {
140+ currentResult.Position = len(results) + 1
141+ results = append(results, *currentResult)
142+ }
143+
93144 return results, nil
94145}
95146
96−// hasClass checks if an HTML node has a specific class.
97147func hasClass(n *html.Node, class string) bool {
98148 for _, attr := range n.Attr {
99149 if attr.Key == "class" {
100− return slices.Contains(strings.Fields(attr.Val), class)
101− }
102− }
103− return false
104−}
105−
106−// extractResult extracts a search result from a result div node.
107−func extractResult(n *html.Node) *SearchResult {
108− result := &SearchResult{}
109−
110− var traverse func(*html.Node)
111− traverse = func(node *html.Node) {
112− if node.Type == html.ElementNode {
113− // Look for title link.
114− if node.Data == "a" && hasClass(node, "result__a") {
115− result.Title = getTextContent(node)
116− for _, attr := range node.Attr {
117− if attr.Key == "href" {
118− result.Link = cleanDuckDuckGoURL(attr.Val)
119− break
120− }
121− }
122− }
123− // Look for snippet.
124− if node.Data == "a" && hasClass(node, "result__snippet") {
125− result.Snippet = getTextContent(node)
150+ if slices.Contains(strings.Fields(attr.Val), class) {
151+ return true
126152 }
127153 }
128− for c := node.FirstChild; c != nil; c = c.NextSibling {
129− traverse(c)
130− }
131154 }
132−
133− traverse(n)
134− return result
155+ return false
135156}
136157
137−// getTextContent extracts all text content from a node and its children.
138158func getTextContent(n *html.Node) string {
139159 var text strings.Builder
140160 var traverse func(*html.Node)
141−
142161 traverse = func(node *html.Node) {
143162 if node.Type == html.TextNode {
144163 text.WriteString(node.Data)
@@ -147,22 +166,18 @@ func getTextContent(n *html.Node) string {
147166 traverse(c)
148167 }
149168 }
150−
151169 traverse(n)
152170 return strings.TrimSpace(text.String())
153171}
154172
155−// cleanDuckDuckGoURL extracts the actual URL from DuckDuckGo's redirect URL.
156173func cleanDuckDuckGoURL(rawURL string) string {
157174 if strings.HasPrefix(rawURL, "//duckduckgo.com/l/?uddg=") {
158− // Extract the actual URL from the redirect.
159175 if idx := strings.Index(rawURL, "uddg="); idx != -1 {
160176 encoded := rawURL[idx+5:]
161177 if ampIdx := strings.Index(encoded, "&"); ampIdx != -1 {
162178 encoded = encoded[:ampIdx]
163179 }
164− decoded, err := url.QueryUnescape(encoded)
165− if err == nil {
180+ if decoded, err := url.QueryUnescape(encoded); err == nil {
166181 return decoded
167182 }
168183 }
@@ -170,20 +185,35 @@ func cleanDuckDuckGoURL(rawURL string) string {
170185 return rawURL
171186}
172187
173−// formatSearchResults formats search results for LLM consumption.
174188func formatSearchResults(results []SearchResult) string {
175189 if len(results) == 0 {
176− return "No results were found for your search query. This could be due to DuckDuckGo's bot detection or the query returned no matches. Please try rephrasing your search or try again in a few minutes."
190+ return "No results found. Try rephrasing your search."
177191 }
178192
179193 var sb strings.Builder
180194 sb.WriteString(fmt.Sprintf("Found %d search results:\n\n", len(results)))
181−
182195 for _, result := range results {
183196 sb.WriteString(fmt.Sprintf("%d. %s\n", result.Position, result.Title))
184197 sb.WriteString(fmt.Sprintf(" URL: %s\n", result.Link))
185198 sb.WriteString(fmt.Sprintf(" Summary: %s\n\n", result.Snippet))
186199 }
187−
188200 return sb.String()
189201}
202+
203+var (
204+ lastSearchMu sync.Mutex
205+ lastSearchTime time.Time
206+)
207+
208+// maybeDelaySearch adds a random delay if the last search was recent.
209+func maybeDelaySearch() {
210+ lastSearchMu.Lock()
211+ defer lastSearchMu.Unlock()
212+
213+ minGap := time.Duration(500+rand.IntN(1500)) * time.Millisecond
214+ elapsed := time.Since(lastSearchTime)
215+ if elapsed < minGap {
216+ time.Sleep(minGap - elapsed)
217+ }
218+ lastSearchTime = time.Now()
219+}