git.iko.soy/crush
on main help sign in
feat: add web search (#1565)
db22f2fKujtim Hoxha
go.mod+1−1
compare with main
unifiedsplit
@@ -55,6 +55,7 @@ require (
55 github.com/tidwall/sjson v1.2.5
55 github.com/tidwall/sjson v1.2.5
56 github.com/zeebo/xxh3 v1.0.2
56 github.com/zeebo/xxh3 v1.0.2
57 golang.org/x/mod v0.30.0
57 golang.org/x/mod v0.30.0
58+ golang.org/x/net v0.47.0
58 golang.org/x/sync v0.18.0
59 golang.org/x/sync v0.18.0
59 golang.org/x/text v0.31.0
60 golang.org/x/text v0.31.0
60 gopkg.in/natefinch/lumberjack.v2 v2.2.1
61 gopkg.in/natefinch/lumberjack.v2 v2.2.1
@@ -164,7 +165,6 @@ require (
164 golang.org/x/crypto v0.45.0 // indirect
165 golang.org/x/crypto v0.45.0 // indirect
165 golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect
166 golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect
166 golang.org/x/image v0.27.0 // indirect
167 golang.org/x/image v0.27.0 // indirect
167− golang.org/x/net v0.47.0 // indirect
168 golang.org/x/oauth2 v0.33.0 // indirect
168 golang.org/x/oauth2 v0.33.0 // indirect
169 golang.org/x/sys v0.38.0 // indirect
169 golang.org/x/sys v0.38.0 // indirect
170 golang.org/x/term v0.37.0 // indirect
170 golang.org/x/term v0.37.0 // indirect
internal/agent/agentic_fetch_tool.go+35−21
@@ -27,10 +27,6 @@ type agenticFetchValidationResult struct {
27
27
28// validateAgenticFetchParams validates the tool call parameters and extracts required context values.
28// validateAgenticFetchParams validates the tool call parameters and extracts required context values.
29func validateAgenticFetchParams(ctx context.Context, params tools.AgenticFetchParams) (agenticFetchValidationResult, error) {
29func validateAgenticFetchParams(ctx context.Context, params tools.AgenticFetchParams) (agenticFetchValidationResult, error) {
30− if params.URL == "" {
31− return agenticFetchValidationResult{}, errors.New("url is required")
32− }
33−
34 if params.Prompt == "" {
30 if params.Prompt == "" {
35 return agenticFetchValidationResult{}, errors.New("prompt is required")
31 return agenticFetchValidationResult{}, errors.New("prompt is required")
36 }
32 }
@@ -75,6 +71,14 @@ func (c *coordinator) agenticFetchTool(_ context.Context, client *http.Client) (
75 return fantasy.NewTextErrorResponse(err.Error()), nil
71 return fantasy.NewTextErrorResponse(err.Error()), nil
76 }
72 }
77
73
74+ // Determine description based on mode.
75+ var description string
76+ if params.URL != "" {
77+ description = fmt.Sprintf("Fetch and analyze content from URL: %s", params.URL)
78+ } else {
79+ description = "Search the web and analyze results"
80+ }
81+
78 p := c.permissions.Request(
82 p := c.permissions.Request(
79 permission.CreatePermissionRequest{
83 permission.CreatePermissionRequest{
80 SessionID: validationResult.SessionID,
84 SessionID: validationResult.SessionID,
@@ -82,7 +86,7 @@ func (c *coordinator) agenticFetchTool(_ context.Context, client *http.Client) (
82 ToolCallID: call.ID,
86 ToolCallID: call.ID,
83 ToolName: tools.AgenticFetchToolName,
87 ToolName: tools.AgenticFetchToolName,
84 Action: "fetch",
88 Action: "fetch",
85− Description: fmt.Sprintf("Fetch and analyze content from URL: %s", params.URL),
89+ Description: description,
86 Params: tools.AgenticFetchPermissionsParams(params),
90 Params: tools.AgenticFetchPermissionsParams(params),
87 },
91 },
88 )
92 )
@@ -91,36 +95,43 @@ func (c *coordinator) agenticFetchTool(_ context.Context, client *http.Client) (
91 return fantasy.ToolResponse{}, permission.ErrorPermissionDenied
95 return fantasy.ToolResponse{}, permission.ErrorPermissionDenied
92 }
96 }
93
97
94− content, err := tools.FetchURLAndConvert(ctx, client, params.URL)
95− if err != nil {
96− return fantasy.NewTextErrorResponse(fmt.Sprintf("Failed to fetch URL: %s", err)), nil
97− }
98−
99 tmpDir, err := os.MkdirTemp(c.cfg.Options.DataDirectory, "crush-fetch-*")
98 tmpDir, err := os.MkdirTemp(c.cfg.Options.DataDirectory, "crush-fetch-*")
100 if err != nil {
99 if err != nil {
101 return fantasy.NewTextErrorResponse(fmt.Sprintf("Failed to create temporary directory: %s", err)), nil
100 return fantasy.NewTextErrorResponse(fmt.Sprintf("Failed to create temporary directory: %s", err)), nil
102 }
101 }
103 defer os.RemoveAll(tmpDir)
102 defer os.RemoveAll(tmpDir)
104
103
105− hasLargeContent := len(content) > tools.LargeContentThreshold
106 var fullPrompt string
104 var fullPrompt string
107
105
108− if hasLargeContent {
106+ if params.URL != "" {
109− tempFile, err := os.CreateTemp(tmpDir, "page-*.md")
107+ // URL mode: fetch the URL content first.
108+ content, err := tools.FetchURLAndConvert(ctx, client, params.URL)
110 if err != nil {
109 if err != nil {
111− return fantasy.NewTextErrorResponse(fmt.Sprintf("Failed to create temporary file: %s", err)), nil
110+ return fantasy.NewTextErrorResponse(fmt.Sprintf("Failed to fetch URL: %s", err)), nil
112 }
111 }
113− tempFilePath := tempFile.Name()
114
112
113+ hasLargeContent := len(content) > tools.LargeContentThreshold
114+
115+ if hasLargeContent {
116+ tempFile, err := os.CreateTemp(tmpDir, "page-*.md")
117+ if err != nil {
118+ return fantasy.NewTextErrorResponse(fmt.Sprintf("Failed to create temporary file: %s", err)), nil
119+ }
120+ tempFilePath := tempFile.Name()
121+
115− if _, err := tempFile.WriteString(content); err != nil {
122+ if _, err := tempFile.WriteString(content); err != nil {
123+ tempFile.Close()
124+ return fantasy.NewTextErrorResponse(fmt.Sprintf("Failed to write content to file: %s", err)), nil
125+ }
116 tempFile.Close()
126 tempFile.Close()
117− return fantasy.NewTextErrorResponse(fmt.Sprintf("Failed to write content to file: %s", err)), nil
118− }
119− tempFile.Close()
120
127
121− fullPrompt = fmt.Sprintf("%s\n\nThe web page from %s has been saved to: %s\n\nUse the view and grep tools to analyze this file and extract the requested information.", params.Prompt, params.URL, tempFilePath)
128+ fullPrompt = fmt.Sprintf("%s\n\nThe web page from %s has been saved to: %s\n\nUse the view and grep tools to analyze this file and extract the requested information.", params.Prompt, params.URL, tempFilePath)
129+ } else {
130+ fullPrompt = fmt.Sprintf("%s\n\nWeb page URL: %s\n\n<webpage_content>\n%s\n</webpage_content>", params.Prompt, params.URL, content)
131+ }
122 } else {
132 } else {
123− fullPrompt = fmt.Sprintf("%s\n\nWeb page URL: %s\n\n<webpage_content>\n%s\n</webpage_content>", params.Prompt, params.URL, content)
133+ // Search mode: let the sub-agent search and fetch as needed.
134+ fullPrompt = fmt.Sprintf("%s\n\nUse the web_search tool to find relevant information. Break down the question into smaller, focused searches if needed. After searching, use web_fetch to get detailed content from the most relevant results.", params.Prompt)
124 }
135 }
125
136
126 promptOpts := []prompt.Option{
137 promptOpts := []prompt.Option{
@@ -148,10 +159,13 @@ func (c *coordinator) agenticFetchTool(_ context.Context, client *http.Client) (
148 }
159 }
149
160
150 webFetchTool := tools.NewWebFetchTool(tmpDir, client)
161 webFetchTool := tools.NewWebFetchTool(tmpDir, client)
162+ webSearchTool := tools.NewWebSearchTool(client)
151 fetchTools := []fantasy.AgentTool{
163 fetchTools := []fantasy.AgentTool{
152 webFetchTool,
164 webFetchTool,
165+ webSearchTool,
153 tools.NewGlobTool(tmpDir),
166 tools.NewGlobTool(tmpDir),
154 tools.NewGrepTool(tmpDir),
167 tools.NewGrepTool(tmpDir),
168+ tools.NewSourcegraphTool(client),
155 tools.NewViewTool(c.lspClients, c.permissions, tmpDir),
169 tools.NewViewTool(c.lspClients, c.permissions, tmpDir),
156 }
170 }
157
171
internal/agent/templates/agentic_fetch.md+36−23
@@ -1,12 +1,12 @@
1−Fetches content from a specified URL and processes it using an AI model to extract information or answer questions.
1+Fetches content from a URL or searches the web, then processes it using an AI model to extract information or answer questions.
2
2
3<when_to_use>
3<when_to_use>
4Use this tool when you need to:
4Use this tool when you need to:
5+- Search the web for information (omit the url parameter)
5−- Extract specific information from a webpage (e.g., "get pricing info")
6+- Extract specific information from a webpage (provide a url)
6−- Answer questions about web content (e.g., "what does this article say about X?")
7+- Answer questions about web content
7- Summarize or analyze web pages
8- Summarize or analyze web pages
8−- Find specific data within large pages
9+- Research topics by searching and following links
9−- Interpret or process web content with AI
10
10
11DO NOT use this tool when:
11DO NOT use this tool when:
12- You just need raw content without analysis (use fetch instead - faster and cheaper)
12- You just need raw content without analysis (use fetch instead - faster and cheaper)
@@ -15,37 +15,50 @@ DO NOT use this tool when:
15</when_to_use>
15</when_to_use>
16
16
17<usage>
17<usage>
18−- Takes a URL and a prompt as input
18+- Provide a prompt describing what information you want to find or extract (required)
19−- Fetches the URL content, converts HTML to markdown
19+- Optionally provide a URL to fetch and analyze specific content
20−- Processes the content with the prompt using a small, fast model
20+- If no URL is provided, the agent will search the web to find relevant information
21+- The tool spawns a sub-agent with web_search, web_fetch, and analysis tools
21−- Returns the model's response about the content
22+- Returns the agent's response about the content
22−- Use this tool when you need to retrieve and analyze web content
23</usage>
23</usage>
24
24
25−<usage_notes>
25+<parameters>
26+- prompt: What information you want to find or extract (required)
27+- url: The URL to fetch content from (optional - if not provided, agent will search the web)
28+</parameters>
26
29
30+<usage_notes>
27- IMPORTANT: If an MCP-provided web fetch tool is available, prefer using that tool instead of this one, as it may have fewer restrictions. All MCP-provided tools start with "mcp_".
31- IMPORTANT: If an MCP-provided web fetch tool is available, prefer using that tool instead of this one, as it may have fewer restrictions. All MCP-provided tools start with "mcp_".
28−- The URL must be a fully-formed valid URL
29−- HTTP URLs will be automatically upgraded to HTTPS
32+- When using URL mode: The URL must be a fully-formed valid URL. HTTP URLs will be automatically upgraded to HTTPS.
30−- The prompt should describe what information you want to extract from the page
33+- When searching: Just provide the prompt describing what you want to find - the agent will search and fetch relevant pages.
34+- The sub-agent can perform multiple searches and fetch multiple pages to gather comprehensive information.
31−- This tool is read-only and does not modify any files
35+- This tool is read-only and does not modify any files.
32−- Results will be summarized if the content is very large
36+- Results will be summarized if the content is very large.
33−- For very large pages, the content will be saved to a temporary file and the agent will have access to grep/view tools to analyze it
34−- When a URL redirects to a different host, the tool will inform you and provide the redirect URL. You should then make a new fetch request with the redirect URL to fetch the content.
35−- This tool uses AI processing and costs more tokens than the simple fetch tool
37+- This tool uses AI processing and costs more tokens than the simple fetch tool.
36− </usage_notes>
38+</usage_notes>
37
39
38<limitations>
40<limitations>
39−- Max response size: 5MB
41+- Max response size: 5MB per page
40- Only supports HTTP and HTTPS protocols
42- Only supports HTTP and HTTPS protocols
41- Cannot handle authentication or cookies
43- Cannot handle authentication or cookies
42- Some websites may block automated requests
44- Some websites may block automated requests
43- Uses additional tokens for AI processing
45- Uses additional tokens for AI processing
46+- Search results depend on DuckDuckGo availability
44</limitations>
47</limitations>
45
48
46<tips>
49<tips>
47−- Be specific in your prompt about what information you want to extract
50+- Be specific in your prompt about what information you want
51+- For research tasks, omit the URL and let the agent search and follow relevant links
48- For complex pages, ask the agent to focus on specific sections
52- For complex pages, ask the agent to focus on specific sections
49−- The agent has access to grep and view tools when analyzing large pages
53+- The agent has access to web_search, web_fetch, grep, and view tools
50- If you just need raw content, use the fetch tool instead to save tokens
54- If you just need raw content, use the fetch tool instead to save tokens
51</tips>
55</tips>
56+
57+<examples>
58+Search for information:
59+- prompt: "What are the main new features in the latest Python release?"
60+
61+Fetch and analyze a URL:
62+- url: "https://docs.python.org/3/whatsnew/3.12.html"
63+- prompt: "Summarize the key changes in Python 3.12"
64+</examples>
internal/agent/templates/agentic_fetch_prompt.md.tpl+41−10
@@ -1,18 +1,38 @@
1−You are a web content analysis agent for Crush. Your task is to analyze web page content and extract the information requested by the user.
1+You are a web content analysis agent for Crush. Your task is to analyze web content, search results, or web pages to extract the information requested by the user.
2
2
3<rules>
3<rules>
4−1. You should be concise and direct in your responses
4+1. Be concise and direct in your responses
52. Focus only on the information requested in the user's prompt
52. Focus only on the information requested in the user's prompt
63. If the content is provided in a file path, use the grep and view tools to efficiently search through it
63. If the content is provided in a file path, use the grep and view tools to efficiently search through it
7−4. When relevant, quote specific sections from the page to support your answer
7+4. When relevant, quote specific sections from the content to support your answer
85. If the requested information is not found, clearly state that
85. If the requested information is not found, clearly state that
96. Any file paths you use MUST be absolute
96. Any file paths you use MUST be absolute
10−7. **IMPORTANT**: If you need information from a linked page to answer the question, use the web_fetch tool to follow that link
10+7. **IMPORTANT**: If you need information from a linked page or search result, use the web_fetch tool to get that content
11+8. **IMPORTANT**: If you need to search for more information, use the web_search tool
11−8. After fetching a link, analyze the content yourself to extract what's needed
12+9. After fetching a link, analyze the content yourself to extract what's needed
12−9. Don't hesitate to follow multiple links if necessary to get complete information
13+10. Don't hesitate to follow multiple links or perform multiple searches if necessary to get complete information
13−10. **CRITICAL**: At the end of your response, include a "Sources" section listing ALL URLs that were useful in answering the question
14+11. **CRITICAL**: At the end of your response, include a "Sources" section listing ALL URLs that were useful in answering the question
14</rules>
15</rules>
15
16
17+<search_strategy>
18+When searching for information:
19+
20+1. **Break down complex questions** - If the user's question has multiple parts, search for each part separately
21+2. **Use specific, targeted queries** - Prefer multiple small searches over one broad search
22+ - Bad: "Python 3.12 new features performance improvements async changes"
23+ - Good: First "Python 3.12 new features", then "Python 3.12 performance improvements", then "Python 3.12 async changes"
24+3. **Iterate and refine** - If initial results aren't helpful, try different search terms or more specific queries
25+4. **Search for different aspects** - For comprehensive answers, search for different angles of the topic
26+5. **Follow up on promising results** - When you find a good source, fetch it and look for links to related information
27+
28+Example workflow for "What are the pros and cons of using Rust vs Go for web services?":
29+- Search 1: "Rust web services advantages"
30+- Search 2: "Go web services advantages"
31+- Search 3: "Rust vs Go performance comparison"
32+- Search 4: "Rust vs Go developer experience"
33+- Then fetch the most relevant results from each search
34+</search_strategy>
35+
16<response_format>
36<response_format>
17Your response should be structured as follows:
37Your response should be structured as follows:
18
38
@@ -24,7 +44,7 @@ Your response should be structured as follows:
24- [URL 3 that was useful]
44- [URL 3 that was useful]
25...
45...
26
46
27−Only include URLs that actually contributed information to your answer. The main URL is always included. Add any additional URLs you fetched that provided relevant information.
47+Only include URLs that actually contributed information to your answer. Include the main URL or search results that were helpful. Add any additional URLs you fetched that provided relevant information.
28</response_format>
48</response_format>
29
49
30<env>
50<env>
@@ -33,9 +53,20 @@ Platform: {{.Platform}}
33Today's date: {{.Date}}
53Today's date: {{.Date}}
34</env>
54</env>
35
55
56+<web_search_tool>
57+You have access to a web_search tool that allows you to search the web:
58+- Provide a search query and optionally max_results (default: 10)
59+- The tool returns search results with titles, URLs, and snippets
60+- After getting search results, use web_fetch to get full content from relevant URLs
61+- **Prefer multiple focused searches over single broad searches**
62+- Keep queries short and specific (3-6 words is often ideal)
63+- If results aren't relevant, try rephrasing with different keywords
64+- Don't be afraid to do 3-5+ searches to thoroughly answer a complex question
65+</web_search_tool>
66+
36<web_fetch_tool>
67<web_fetch_tool>
37−You have access to a web_fetch tool that allows you to fetch additional web pages:
68+You have access to a web_fetch tool that allows you to fetch web pages:
38−- Use it when you need to follow links from the current page
69+- Use it when you need to follow links from search results or the current page
39- Provide just the URL (no prompt parameter)
70- Provide just the URL (no prompt parameter)
40- The tool will fetch and return the content (or save to a file if large)
71- The tool will fetch and return the content (or save to a file if large)
41- YOU must then analyze that content to answer the user's question
72- YOU must then analyze that content to answer the user's question
internal/agent/tools/fetch_helpers.go+84−6
@@ -8,12 +8,17 @@ import (
8 "fmt"
8 "fmt"
9 "io"
9 "io"
10 "net/http"
10 "net/http"
11+ "regexp"
11 "strings"
12 "strings"
12 "unicode/utf8"
13 "unicode/utf8"
13
14
14 md "github.com/JohannesKaufmann/html-to-markdown"
15 md "github.com/JohannesKaufmann/html-to-markdown"
16+ "golang.org/x/net/html"
15)
17)
16
18
19+// BrowserUserAgent is a realistic browser User-Agent for better compatibility.
20+const BrowserUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
21+
17// FetchURLAndConvert fetches a URL and converts HTML content to markdown.
22// FetchURLAndConvert fetches a URL and converts HTML content to markdown.
18func FetchURLAndConvert(ctx context.Context, client *http.Client, url string) (string, error) {
23func FetchURLAndConvert(ctx context.Context, client *http.Client, url string) (string, error) {
19 req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
24 req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
@@ -21,7 +26,10 @@ func FetchURLAndConvert(ctx context.Context, client *http.Client, url string) (s
21 return "", fmt.Errorf("failed to create request: %w", err)
26 return "", fmt.Errorf("failed to create request: %w", err)
22 }
27 }
23
28
29+ // Use realistic browser headers for better compatibility.
24− req.Header.Set("User-Agent", "crush/1.0")
30+ req.Header.Set("User-Agent", BrowserUserAgent)
31+ req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
32+ req.Header.Set("Accept-Language", "en-US,en;q=0.5")
25
33
26 resp, err := client.Do(req)
34 resp, err := client.Do(req)
27 if err != nil {
35 if err != nil {
@@ -49,11 +57,13 @@ func FetchURLAndConvert(ctx context.Context, client *http.Client, url string) (s
49
57
50 // Convert HTML to markdown for better AI processing.
58 // Convert HTML to markdown for better AI processing.
51 if strings.Contains(contentType, "text/html") {
59 if strings.Contains(contentType, "text/html") {
60+ // Remove noisy elements before conversion.
61+ cleanedHTML := removeNoisyElements(content)
52− markdown, err := ConvertHTMLToMarkdown(content)
62+ markdown, err := ConvertHTMLToMarkdown(cleanedHTML)
53 if err != nil {
63 if err != nil {
54 return "", fmt.Errorf("failed to convert HTML to markdown: %w", err)
64 return "", fmt.Errorf("failed to convert HTML to markdown: %w", err)
55 }
65 }
56− content = markdown
66+ content = cleanupMarkdown(markdown)
57 } else if strings.Contains(contentType, "application/json") || strings.Contains(contentType, "text/json") {
67 } else if strings.Contains(contentType, "application/json") || strings.Contains(contentType, "text/json") {
58 // Format JSON for better readability.
68 // Format JSON for better readability.
59 formatted, err := FormatJSON(content)
69 formatted, err := FormatJSON(content)
@@ -66,11 +76,79 @@ func FetchURLAndConvert(ctx context.Context, client *http.Client, url string) (s
66 return content, nil
76 return content, nil
67}
77}
68
78
79+// removeNoisyElements removes script, style, nav, header, footer, and other
80+// noisy elements from HTML to improve content extraction.
81+func removeNoisyElements(htmlContent string) string {
82+ doc, err := html.Parse(strings.NewReader(htmlContent))
83+ if err != nil {
84+ // If parsing fails, return original content.
85+ return htmlContent
86+ }
87+
88+ // Elements to remove entirely.
89+ noisyTags := map[string]bool{
90+ "script": true,
91+ "style": true,
92+ "nav": true,
93+ "header": true,
94+ "footer": true,
95+ "aside": true,
96+ "noscript": true,
97+ "iframe": true,
98+ "svg": true,
99+ }
100+
101+ var removeNodes func(*html.Node)
102+ removeNodes = func(n *html.Node) {
103+ var toRemove []*html.Node
104+
105+ for c := n.FirstChild; c != nil; c = c.NextSibling {
106+ if c.Type == html.ElementNode && noisyTags[c.Data] {
107+ toRemove = append(toRemove, c)
108+ } else {
109+ removeNodes(c)
110+ }
111+ }
112+
113+ for _, node := range toRemove {
114+ n.RemoveChild(node)
115+ }
116+ }
117+
118+ removeNodes(doc)
119+
120+ var buf bytes.Buffer
121+ if err := html.Render(&buf, doc); err != nil {
122+ return htmlContent
123+ }
124+
125+ return buf.String()
126+}
127+
128+// cleanupMarkdown removes excessive whitespace and blank lines from markdown.
129+func cleanupMarkdown(content string) string {
130+ // Collapse multiple blank lines into at most two.
131+ multipleNewlines := regexp.MustCompile(`\n{3,}`)
132+ content = multipleNewlines.ReplaceAllString(content, "\n\n")
133+
134+ // Remove trailing whitespace from each line.
135+ lines := strings.Split(content, "\n")
136+ for i, line := range lines {
137+ lines[i] = strings.TrimRight(line, " \t")
138+ }
139+ content = strings.Join(lines, "\n")
140+
141+ // Trim leading/trailing whitespace.
142+ content = strings.TrimSpace(content)
143+
144+ return content
145+}
146+
69// ConvertHTMLToMarkdown converts HTML content to markdown format.
147// ConvertHTMLToMarkdown converts HTML content to markdown format.
70−func ConvertHTMLToMarkdown(html string) (string, error) {
148+func ConvertHTMLToMarkdown(htmlContent string) (string, error) {
71 converter := md.NewConverter("", true, nil)
149 converter := md.NewConverter("", true, nil)
72
150
73− markdown, err := converter.ConvertString(html)
151+ markdown, err := converter.ConvertString(htmlContent)
74 if err != nil {
152 if err != nil {
75 return "", err
153 return "", err
76 }
154 }
@@ -80,7 +158,7 @@ func ConvertHTMLToMarkdown(html string) (string, error) {
80
158
81// FormatJSON formats JSON content with proper indentation.
159// FormatJSON formats JSON content with proper indentation.
82func FormatJSON(content string) (string, error) {
160func FormatJSON(content string) (string, error) {
83− var data interface{}
161+ var data any
84 if err := json.Unmarshal([]byte(content), &data); err != nil {
162 if err := json.Unmarshal([]byte(content), &data); err != nil {
85 return "", err
163 return "", err
86 }
164 }
internal/agent/tools/fetch_types.go+12−3
@@ -6,18 +6,21 @@ const AgenticFetchToolName = "agentic_fetch"
6// WebFetchToolName is the name of the web_fetch tool.
6// WebFetchToolName is the name of the web_fetch tool.
7const WebFetchToolName = "web_fetch"
7const WebFetchToolName = "web_fetch"
8
8
9+// WebSearchToolName is the name of the web_search tool for sub-agents.
10+const WebSearchToolName = "web_search"
11+
9// LargeContentThreshold is the size threshold for saving content to a file.
12// LargeContentThreshold is the size threshold for saving content to a file.
10const LargeContentThreshold = 50000 // 50KB
13const LargeContentThreshold = 50000 // 50KB
11
14
12// AgenticFetchParams defines the parameters for the agentic fetch tool.
15// AgenticFetchParams defines the parameters for the agentic fetch tool.
13type AgenticFetchParams struct {
16type AgenticFetchParams struct {
14− URL string `json:"url" description:"The URL to fetch content from"`
17+ URL string `json:"url,omitempty" description:"The URL to fetch content from (optional - if not provided, the agent will search the web)"`
15− Prompt string `json:"prompt" description:"The prompt to run on the fetched content"`
18+ Prompt string `json:"prompt" description:"The prompt describing what information to find or extract"`
16}
19}
17
20
18// AgenticFetchPermissionsParams defines the permission parameters for the agentic fetch tool.
21// AgenticFetchPermissionsParams defines the permission parameters for the agentic fetch tool.
19type AgenticFetchPermissionsParams struct {
22type AgenticFetchPermissionsParams struct {
20− URL string `json:"url"`
23+ URL string `json:"url,omitempty"`
21 Prompt string `json:"prompt"`
24 Prompt string `json:"prompt"`
22}
25}
23
26
@@ -26,6 +29,12 @@ type WebFetchParams struct {
26 URL string `json:"url" description:"The URL to fetch content from"`
29 URL string `json:"url" description:"The URL to fetch content from"`
27}
30}
28
31
32+// WebSearchParams defines the parameters for the web_search tool.
33+type WebSearchParams struct {
34+ Query string `json:"query" description:"The search query to find information on the web"`
35+ MaxResults int `json:"max_results,omitempty" description:"Maximum number of results to return (default: 10, max: 20)"`
36+}
37+
29// FetchParams defines the parameters for the simple fetch tool.
38// FetchParams defines the parameters for the simple fetch tool.
30type FetchParams struct {
39type FetchParams struct {
31 URL string `json:"url" description:"The URL to fetch content from"`
40 URL string `json:"url" description:"The URL to fetch content from"`
internal/agent/tools/search.gonew file+183
@@ -0,0 +1,183 @@
1+package tools
2+
3+import (
4+ "context"
5+ "fmt"
6+ "io"
7+ "net/http"
8+ "net/url"
9+ "slices"
10+ "strings"
11+
12+ "golang.org/x/net/html"
13+)
14+
15+// SearchResult represents a single search result from DuckDuckGo.
16+type SearchResult struct {
17+ Title string
18+ Link string
19+ Snippet string
20+ Position int
21+}
22+
23+// searchDuckDuckGo performs a web search using DuckDuckGo's HTML endpoint.
24+func searchDuckDuckGo(ctx context.Context, client *http.Client, query string, maxResults int) ([]SearchResult, error) {
25+ if maxResults <= 0 {
26+ maxResults = 10
27+ }
28+
29+ formData := url.Values{}
30+ formData.Set("q", query)
31+ formData.Set("b", "")
32+ formData.Set("kl", "")
33+
34+ req, err := http.NewRequestWithContext(ctx, "POST", "https://html.duckduckgo.com/html", strings.NewReader(formData.Encode()))
35+ if err != nil {
36+ return nil, fmt.Errorf("failed to create request: %w", err)
37+ }
38+
39+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
40+ req.Header.Set("User-Agent", BrowserUserAgent)
41+
42+ resp, err := client.Do(req)
43+ if err != nil {
44+ return nil, fmt.Errorf("failed to execute search: %w", err)
45+ }
46+ defer resp.Body.Close()
47+
48+ if resp.StatusCode != http.StatusOK {
49+ return nil, fmt.Errorf("search failed with status code: %d", resp.StatusCode)
50+ }
51+
52+ body, err := io.ReadAll(resp.Body)
53+ if err != nil {
54+ return nil, fmt.Errorf("failed to read response: %w", err)
55+ }
56+
57+ return parseSearchResults(string(body), maxResults)
58+}
59+
60+// parseSearchResults extracts search results from DuckDuckGo HTML response.
61+func parseSearchResults(htmlContent string, maxResults int) ([]SearchResult, error) {
62+ doc, err := html.Parse(strings.NewReader(htmlContent))
63+ if err != nil {
64+ return nil, fmt.Errorf("failed to parse HTML: %w", err)
65+ }
66+
67+ var results []SearchResult
68+ var traverse func(*html.Node)
69+
70+ traverse = func(n *html.Node) {
71+ if n.Type == html.ElementNode && n.Data == "div" && hasClass(n, "result") {
72+ result := extractResult(n)
73+ if result != nil && result.Link != "" && !strings.Contains(result.Link, "y.js") {
74+ result.Position = len(results) + 1
75+ results = append(results, *result)
76+ if len(results) >= maxResults {
77+ return
78+ }
79+ }
80+ }
81+ for c := n.FirstChild; c != nil && len(results) < maxResults; c = c.NextSibling {
82+ traverse(c)
83+ }
84+ }
85+
86+ traverse(doc)
87+ return results, nil
88+}
89+
90+// hasClass checks if an HTML node has a specific class.
91+func hasClass(n *html.Node, class string) bool {
92+ for _, attr := range n.Attr {
93+ if attr.Key == "class" {
94+ return slices.Contains(strings.Fields(attr.Val), class)
95+ }
96+ }
97+ return false
98+}
99+
100+// extractResult extracts a search result from a result div node.
101+func extractResult(n *html.Node) *SearchResult {
102+ result := &SearchResult{}
103+
104+ var traverse func(*html.Node)
105+ traverse = func(node *html.Node) {
106+ if node.Type == html.ElementNode {
107+ // Look for title link.
108+ if node.Data == "a" && hasClass(node, "result__a") {
109+ result.Title = getTextContent(node)
110+ for _, attr := range node.Attr {
111+ if attr.Key == "href" {
112+ result.Link = cleanDuckDuckGoURL(attr.Val)
113+ break
114+ }
115+ }
116+ }
117+ // Look for snippet.
118+ if node.Data == "a" && hasClass(node, "result__snippet") {
119+ result.Snippet = getTextContent(node)
120+ }
121+ }
122+ for c := node.FirstChild; c != nil; c = c.NextSibling {
123+ traverse(c)
124+ }
125+ }
126+
127+ traverse(n)
128+ return result
129+}
130+
131+// getTextContent extracts all text content from a node and its children.
132+func getTextContent(n *html.Node) string {
133+ var text strings.Builder
134+ var traverse func(*html.Node)
135+
136+ traverse = func(node *html.Node) {
137+ if node.Type == html.TextNode {
138+ text.WriteString(node.Data)
139+ }
140+ for c := node.FirstChild; c != nil; c = c.NextSibling {
141+ traverse(c)
142+ }
143+ }
144+
145+ traverse(n)
146+ return strings.TrimSpace(text.String())
147+}
148+
149+// cleanDuckDuckGoURL extracts the actual URL from DuckDuckGo's redirect URL.
150+func cleanDuckDuckGoURL(rawURL string) string {
151+ if strings.HasPrefix(rawURL, "//duckduckgo.com/l/?uddg=") {
152+ // Extract the actual URL from the redirect.
153+ if idx := strings.Index(rawURL, "uddg="); idx != -1 {
154+ encoded := rawURL[idx+5:]
155+ if ampIdx := strings.Index(encoded, "&"); ampIdx != -1 {
156+ encoded = encoded[:ampIdx]
157+ }
158+ decoded, err := url.QueryUnescape(encoded)
159+ if err == nil {
160+ return decoded
161+ }
162+ }
163+ }
164+ return rawURL
165+}
166+
167+// formatSearchResults formats search results for LLM consumption.
168+func formatSearchResults(results []SearchResult) string {
169+ if len(results) == 0 {
170+ return "No results were found for your search query. This could be due to DuckDuckGo's bot detection or the query returned no matches. Please try rephrasing your search or try again in a few minutes."
171+ }
172+
173+ var sb strings.Builder
174+ sb.WriteString(fmt.Sprintf("Found %d search results:\n\n", len(results)))
175+
176+ for _, result := range results {
177+ sb.WriteString(fmt.Sprintf("%d. %s\n", result.Position, result.Title))
178+ sb.WriteString(fmt.Sprintf(" URL: %s\n", result.Link))
179+ sb.WriteString(fmt.Sprintf(" Summary: %s\n\n", result.Snippet))
180+ }
181+
182+ return sb.String()
183+}
internal/agent/tools/web_search.gonew file+51
@@ -0,0 +1,51 @@
1+package tools
2+
3+import (
4+ "context"
5+ _ "embed"
6+ "net/http"
7+ "time"
8+
9+ "charm.land/fantasy"
10+)
11+
12+//go:embed web_search.md
13+var webSearchToolDescription []byte
14+
15+// NewWebSearchTool creates a web search tool for sub-agents (no permissions needed).
16+func NewWebSearchTool(client *http.Client) fantasy.AgentTool {
17+ if client == nil {
18+ client = &http.Client{
19+ Timeout: 30 * time.Second,
20+ Transport: &http.Transport{
21+ MaxIdleConns: 100,
22+ MaxIdleConnsPerHost: 10,
23+ IdleConnTimeout: 90 * time.Second,
24+ },
25+ }
26+ }
27+
28+ return fantasy.NewParallelAgentTool(
29+ WebSearchToolName,
30+ string(webSearchToolDescription),
31+ func(ctx context.Context, params WebSearchParams, call fantasy.ToolCall) (fantasy.ToolResponse, error) {
32+ if params.Query == "" {
33+ return fantasy.NewTextErrorResponse("query is required"), nil
34+ }
35+
36+ maxResults := params.MaxResults
37+ if maxResults <= 0 {
38+ maxResults = 10
39+ }
40+ if maxResults > 20 {
41+ maxResults = 20
42+ }
43+
44+ results, err := searchDuckDuckGo(ctx, client, params.Query, maxResults)
45+ if err != nil {
46+ return fantasy.NewTextErrorResponse("Failed to search: " + err.Error()), nil
47+ }
48+
49+ return fantasy.NewTextResponse(formatSearchResults(results)), nil
50+ })
51+}
internal/agent/tools/web_search.mdnew file+18
@@ -0,0 +1,18 @@
1+Searches the web using DuckDuckGo and returns search results.
2+
3+<usage>
4+- Provide a search query to find information on the web
5+- Returns a list of search results with titles, URLs, and snippets
6+- Use this to find relevant web pages, then use web_fetch to get full content
7+</usage>
8+
9+<parameters>
10+- query: The search query string (required)
11+- max_results: Maximum number of results to return (default: 10, max: 20)
12+</parameters>
13+
14+<tips>
15+- Use specific, targeted search queries for better results
16+- After getting results, use web_fetch to get the full content of relevant pages
17+- Combine multiple searches to gather comprehensive information
18+</tips>
internal/tui/components/chat/messages/renderer.go+34−5
@@ -191,6 +191,7 @@ func init() {
191 registry.register(tools.FetchToolName, func() renderer { return simpleFetchRenderer{} })
191 registry.register(tools.FetchToolName, func() renderer { return simpleFetchRenderer{} })
192 registry.register(tools.AgenticFetchToolName, func() renderer { return agenticFetchRenderer{} })
192 registry.register(tools.AgenticFetchToolName, func() renderer { return agenticFetchRenderer{} })
193 registry.register(tools.WebFetchToolName, func() renderer { return webFetchRenderer{} })
193 registry.register(tools.WebFetchToolName, func() renderer { return webFetchRenderer{} })
194+ registry.register(tools.WebSearchToolName, func() renderer { return webSearchRenderer{} })
194 registry.register(tools.GlobToolName, func() renderer { return globRenderer{} })
195 registry.register(tools.GlobToolName, func() renderer { return globRenderer{} })
195 registry.register(tools.GrepToolName, func() renderer { return grepRenderer{} })
196 registry.register(tools.GrepToolName, func() renderer { return grepRenderer{} })
196 registry.register(tools.LSToolName, func() renderer { return lsRenderer{} })
197 registry.register(tools.LSToolName, func() renderer { return lsRenderer{} })
@@ -636,15 +637,17 @@ type agenticFetchRenderer struct {
636 baseRenderer
637 baseRenderer
637}
638}
638
639
639−// Render displays the fetched URL with prompt parameter and nested tool calls
640+// Render displays the fetched URL or web search with prompt parameter and nested tool calls
640func (fr agenticFetchRenderer) Render(v *toolCallCmp) string {
641func (fr agenticFetchRenderer) Render(v *toolCallCmp) string {
641 t := styles.CurrentTheme()
642 t := styles.CurrentTheme()
642 var params tools.AgenticFetchParams
643 var params tools.AgenticFetchParams
643 var args []string
644 var args []string
644 if err := fr.unmarshalParams(v.call.Input, &params); err == nil {
645 if err := fr.unmarshalParams(v.call.Input, &params); err == nil {
646+ if params.URL != "" {
645− args = newParamBuilder().
647+ args = newParamBuilder().
646− addMain(params.URL).
648+ addMain(params.URL).
647− build()
649+ build()
650+ }
648 }
651 }
649
652
650 prompt := params.Prompt
653 prompt := params.Prompt
@@ -731,6 +734,30 @@ func (wfr webFetchRenderer) Render(v *toolCallCmp) string {
731 })
734 })
732}
735}
733
736
737+// -----------------------------------------------------------------------------
738+// Web search renderer
739+// -----------------------------------------------------------------------------
740+
741+// webSearchRenderer handles web search with query display
742+type webSearchRenderer struct {
743+ baseRenderer
744+}
745+
746+// Render displays a compact view of web_search with just the query
747+func (wsr webSearchRenderer) Render(v *toolCallCmp) string {
748+ var params tools.WebSearchParams
749+ var args []string
750+ if err := wsr.unmarshalParams(v.call.Input, &params); err == nil {
751+ args = newParamBuilder().
752+ addMain(params.Query).
753+ build()
754+ }
755+
756+ return wsr.renderWithParams(v, "Search", args, func() string {
757+ return renderMarkdownContent(v, v.result.Content)
758+ })
759+}
760+
734// -----------------------------------------------------------------------------
761// -----------------------------------------------------------------------------
735// Download renderer
762// Download renderer
736// -----------------------------------------------------------------------------
763// -----------------------------------------------------------------------------
@@ -1260,7 +1287,9 @@ func prettifyToolName(name string) string {
1260 case tools.AgenticFetchToolName:
1287 case tools.AgenticFetchToolName:
1261 return "Agentic Fetch"
1288 return "Agentic Fetch"
1262 case tools.WebFetchToolName:
1289 case tools.WebFetchToolName:
1263− return "Fetching"
1290+ return "Fetch"
1291+ case tools.WebSearchToolName:
1292+ return "Search"
1264 case tools.GlobToolName:
1293 case tools.GlobToolName:
1265 return "Glob"
1294 return "Glob"
1266 case tools.GrepToolName:
1295 case tools.GrepToolName:
internal/tui/components/chat/messages/tool.go+3−1
@@ -301,7 +301,9 @@ func (m *toolCallCmp) formatParametersForCopy() string {
301 var params tools.AgenticFetchParams
301 var params tools.AgenticFetchParams
302 if json.Unmarshal([]byte(m.call.Input), &params) == nil {
302 if json.Unmarshal([]byte(m.call.Input), &params) == nil {
303 var parts []string
303 var parts []string
304+ if params.URL != "" {
304− parts = append(parts, fmt.Sprintf("**URL:** %s", params.URL))
305+ parts = append(parts, fmt.Sprintf("**URL:** %s", params.URL))
306+ }
305 if params.Prompt != "" {
307 if params.Prompt != "" {
306 parts = append(parts, fmt.Sprintf("**Prompt:** %s", params.Prompt))
308 parts = append(parts, fmt.Sprintf("**Prompt:** %s", params.Prompt))
307 }
309 }
internal/tui/components/dialogs/permissions/permissions.go+7−2
@@ -389,7 +389,7 @@ func (p *permissionDialogCmp) renderHeader() string {
389 case tools.AgenticFetchToolName:
389 case tools.AgenticFetchToolName:
390 headerParts = append(headerParts,
390 headerParts = append(headerParts,
391 baseStyle.Render(strings.Repeat(" ", p.width)),
391 baseStyle.Render(strings.Repeat(" ", p.width)),
392− t.S().Muted.Width(p.width).Bold(true).Render("URL"),
392+ t.S().Muted.Width(p.width).Bold(true).Render("Web"),
393 )
393 )
394 case tools.ViewToolName:
394 case tools.ViewToolName:
395 params := p.permission.Params.(tools.ViewPermissionsParams)
395 params := p.permission.Params.(tools.ViewPermissionsParams)
@@ -605,7 +605,12 @@ func (p *permissionDialogCmp) generateAgenticFetchContent() string {
605 t := styles.CurrentTheme()
605 t := styles.CurrentTheme()
606 baseStyle := t.S().Base.Background(t.BgSubtle)
606 baseStyle := t.S().Base.Background(t.BgSubtle)
607 if pr, ok := p.permission.Params.(tools.AgenticFetchPermissionsParams); ok {
607 if pr, ok := p.permission.Params.(tools.AgenticFetchPermissionsParams); ok {
608+ var content string
609+ if pr.URL != "" {
608− content := fmt.Sprintf("URL: %s\n\nPrompt: %s", pr.URL, pr.Prompt)
610+ content = fmt.Sprintf("URL: %s\n\nPrompt: %s", pr.URL, pr.Prompt)
611+ } else {
612+ content = fmt.Sprintf("Prompt: %s", pr.Prompt)
613+ }
609 finalContent := baseStyle.
614 finalContent := baseStyle.
610 Padding(1, 2).
615 Padding(1, 2).
611 Width(p.contentViewPort.Width()).
616 Width(p.contentViewPort.Width()).