git.iko.soy/crush
on main help sign in
fix: fix fetch and view tool
a6298d6Kujtim Hoxha
internal/llm/tools/fetch.go+33−9
compare with main
unifiedsplit
@@ -8,6 +8,7 @@ import (
8 "net/http"
8 "net/http"
9 "strings"
9 "strings"
10 "time"
10 "time"
11+ "unicode/utf8"
11
12
12 md "github.com/JohannesKaufmann/html-to-markdown"
13 md "github.com/JohannesKaufmann/html-to-markdown"
13 "github.com/PuerkitoBio/goquery"
14 "github.com/PuerkitoBio/goquery"
@@ -182,6 +183,11 @@ func (t *fetchTool) Run(ctx context.Context, call ToolCall) (ToolResponse, error
182 }
183 }
183
184
184 content := string(body)
185 content := string(body)
186+
187+ isValidUt8 := utf8.ValidString(content)
188+ if !isValidUt8 {
189+ return NewTextErrorResponse("Response content is not valid UTF-8"), nil
190+ }
185 contentType := resp.Header.Get("Content-Type")
191 contentType := resp.Header.Get("Content-Type")
186
192
187 switch format {
193 switch format {
@@ -191,9 +197,8 @@ func (t *fetchTool) Run(ctx context.Context, call ToolCall) (ToolResponse, error
191 if err != nil {
197 if err != nil {
192 return NewTextErrorResponse("Failed to extract text from HTML: " + err.Error()), nil
198 return NewTextErrorResponse("Failed to extract text from HTML: " + err.Error()), nil
193 }
199 }
194− return NewTextResponse(text), nil
200+ content = text
195 }
201 }
196− return NewTextResponse(content), nil
197
202
198 case "markdown":
203 case "markdown":
199 if strings.Contains(contentType, "text/html") {
204 if strings.Contains(contentType, "text/html") {
@@ -201,17 +206,36 @@ func (t *fetchTool) Run(ctx context.Context, call ToolCall) (ToolResponse, error
201 if err != nil {
206 if err != nil {
202 return NewTextErrorResponse("Failed to convert HTML to Markdown: " + err.Error()), nil
207 return NewTextErrorResponse("Failed to convert HTML to Markdown: " + err.Error()), nil
203 }
208 }
204− return NewTextResponse(markdown), nil
209+ content = markdown
205 }
210 }
206
211
207− return NewTextResponse("```\n" + content + "\n```"), nil
212+ content = "```\n" + content + "\n```"
208
213
209 case "html":
214 case "html":
210− return NewTextResponse(content), nil
215+ // return only the body of the HTML document
211−
216+ if strings.Contains(contentType, "text/html") {
212− default:
217+ doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
213− return NewTextResponse(content), nil
218+ if err != nil {
219+ return NewTextErrorResponse("Failed to parse HTML: " + err.Error()), nil
220+ }
221+ body, err := doc.Find("body").Html()
222+ if err != nil {
223+ return NewTextErrorResponse("Failed to extract body from HTML: " + err.Error()), nil
224+ }
225+ if body == "" {
226+ return NewTextErrorResponse("No body content found in HTML"), nil
227+ }
228+ content = "<html>\n<body>\n" + body + "\n</body>\n</html>"
229+ }
230+ }
231+ // calculate byte size of content
232+ contentSize := int64(len(content))
233+ if contentSize > MaxReadSize {
234+ content = content[:MaxReadSize]
235+ content += fmt.Sprintf("\n\n[Content truncated to %d bytes]", MaxReadSize)
214 }
236 }
237+
238+ return NewTextResponse(content), nil
215}
239}
216
240
217func extractTextFromHTML(html string) (string, error) {
241func extractTextFromHTML(html string) (string, error) {
@@ -220,7 +244,7 @@ func extractTextFromHTML(html string) (string, error) {
220 return "", err
244 return "", err
221 }
245 }
222
246
223− text := doc.Text()
247+ text := doc.Find("body").Text()
224 text = strings.Join(strings.Fields(text), " ")
248 text = strings.Join(strings.Fields(text), " ")
225
249
226 return text, nil
250 return text, nil
internal/llm/tools/view.go+6−1
@@ -9,6 +9,7 @@ import (
9 "os"
9 "os"
10 "path/filepath"
10 "path/filepath"
11 "strings"
11 "strings"
12+ "unicode/utf8"
12
13
13 "github.com/charmbracelet/crush/internal/lsp"
14 "github.com/charmbracelet/crush/internal/lsp"
14)
15)
@@ -173,11 +174,15 @@ func (v *viewTool) Run(ctx context.Context, call ToolCall) (ToolResponse, error)
173 isImage, imageType := isImageFile(filePath)
174 isImage, imageType := isImageFile(filePath)
174 // TODO: handle images
175 // TODO: handle images
175 if isImage {
176 if isImage {
176− return NewTextErrorResponse(fmt.Sprintf("This is an image file of type: %s\nUse a different tool to process images", imageType)), nil
177+ return NewTextErrorResponse(fmt.Sprintf("This is an image file of type: %s\n", imageType)), nil
177 }
178 }
178
179
179 // Read the file content
180 // Read the file content
180 content, lineCount, err := readTextFile(filePath, params.Offset, params.Limit)
181 content, lineCount, err := readTextFile(filePath, params.Offset, params.Limit)
182+ isValidUt8 := utf8.ValidString(content)
183+ if !isValidUt8 {
184+ return NewTextErrorResponse("File content is not valid UTF-8"), nil
185+ }
181 if err != nil {
186 if err != nil {
182 return ToolResponse{}, fmt.Errorf("error reading file: %w", err)
187 return ToolResponse{}, fmt.Errorf("error reading file: %w", err)
183 }
188 }