git.iko.soy/crush
on main help sign in
fix(agent): prevent session bricking when non-vision models receive tool result media
d3d6804Kieran Klukas
internal/agent/agent.go+17
compare with main
unifiedsplit
@@ -2100,6 +2100,8 @@ func (a *sessionAgent) workaroundProviderMediaLimitations(messages []fantasy.Mes
2100 return messages
2100 return messages
2101 }
2101 }
2102
2102
2103+ supportsImages := largeModel.CatwalkCfg.SupportsImages
2104+
2103 convertedMessages := make([]fantasy.Message, 0, len(messages))
2105 convertedMessages := make([]fantasy.Message, 0, len(messages))
2104
2106
2105 for _, msg := range messages {
2107 for _, msg := range messages {
@@ -2119,6 +2121,21 @@ func (a *sessionAgent) workaroundProviderMediaLimitations(messages []fantasy.Mes
2119 }
2121 }
2120
2122
2121 if media, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentMedia](toolResult.Output); ok {
2123 if media, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentMedia](toolResult.Output); ok {
2124+ if !supportsImages {
2125+ // Model cannot process images. Replace with a text
2126+ // placeholder and skip creating a synthetic user
2127+ // message with FilePart, which would brick the
2128+ // session on text-only models.
2129+ textParts = append(textParts, fantasy.ToolResultPart{
2130+ ToolCallID: toolResult.ToolCallID,
2131+ Output: fantasy.ToolResultOutputContentText{
2132+ Text: "[Image/media content not supported by this model]",
2133+ },
2134+ ProviderOptions: toolResult.ProviderOptions,
2135+ })
2136+ continue
2137+ }
2138+
2122 decoded, err := base64.StdEncoding.DecodeString(media.Data)
2139 decoded, err := base64.StdEncoding.DecodeString(media.Data)
2123 if err != nil {
2140 if err != nil {
2124 slog.Warn("Failed to decode media data", "error", err)
2141 slog.Warn("Failed to decode media data", "error", err)
internal/agent/agent_test.go+143
@@ -1,6 +1,7 @@
1package agent
1package agent
2
2
3import (
3import (
4+ "encoding/base64"
4 "fmt"
5 "fmt"
5 "log/slog"
6 "log/slog"
6 "os"
7 "os"
@@ -10,9 +11,11 @@ import (
10 "testing"
11 "testing"
11 "time"
12 "time"
12
13
14+ "charm.land/catwalk/pkg/catwalk"
13 "charm.land/fantasy"
15 "charm.land/fantasy"
14 "charm.land/x/vcr"
16 "charm.land/x/vcr"
15 "github.com/charmbracelet/crush/internal/agent/tools"
17 "github.com/charmbracelet/crush/internal/agent/tools"
18+ "github.com/charmbracelet/crush/internal/config"
16 "github.com/charmbracelet/crush/internal/message"
19 "github.com/charmbracelet/crush/internal/message"
17 "github.com/charmbracelet/crush/internal/session"
20 "github.com/charmbracelet/crush/internal/session"
18 "github.com/stretchr/testify/assert"
21 "github.com/stretchr/testify/assert"
@@ -840,6 +843,146 @@ func TestPreparePrompt_OrphanedToolUseMixed(t *testing.T) {
840 require.Equal(t, 1, syntheticCount, "expected exactly one synthetic result for the orphaned call")
843 require.Equal(t, 1, syntheticCount, "expected exactly one synthetic result for the orphaned call")
841}
844}
842
845
846+func TestWorkaroundProviderMediaLimitations_TextOnlyModel(t *testing.T) {
847+ env := testEnv(t)
848+ sa := testSessionAgent(env, nil, nil, "test prompt")
849+ agent := sa.(*sessionAgent)
850+
851+ pngBase64 := base64.StdEncoding.EncodeToString([]byte("fake-png-data"))
852+
853+ messages := []fantasy.Message{
854+ {
855+ Role: fantasy.MessageRoleTool,
856+ Content: []fantasy.MessagePart{
857+ fantasy.ToolResultPart{
858+ ToolCallID: "call_1",
859+ Output: fantasy.ToolResultOutputContentMedia{
860+ Data: pngBase64,
861+ MediaType: "image/png",
862+ },
863+ },
864+ },
865+ },
866+ }
867+
868+ // Non-Anthropic provider, no image support — should replace media with
869+ // a text placeholder and not create a synthetic user message.
870+ largeModel := Model{
871+ ModelCfg: config.SelectedModel{Provider: "openai"},
872+ CatwalkCfg: catwalk.Model{
873+ SupportsImages: false,
874+ },
875+ }
876+
877+ result := agent.workaroundProviderMediaLimitations(messages, largeModel)
878+
879+ // Should produce exactly one message: the tool message with a text
880+ // placeholder. No synthetic user message with FilePart.
881+ require.Len(t, result, 1)
882+ require.Equal(t, fantasy.MessageRoleTool, result[0].Role)
883+
884+ tr, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](result[0].Content[0])
885+ require.True(t, ok)
886+ _, ok = fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentText](tr.Output)
887+ require.True(t, ok)
888+}
889+
890+func TestWorkaroundProviderMediaLimitations_VisionModel(t *testing.T) {
891+ env := testEnv(t)
892+ sa := testSessionAgent(env, nil, nil, "test prompt")
893+ agent := sa.(*sessionAgent)
894+
895+ pngBase64 := base64.StdEncoding.EncodeToString([]byte("fake-png-data"))
896+
897+ messages := []fantasy.Message{
898+ {
899+ Role: fantasy.MessageRoleTool,
900+ Content: []fantasy.MessagePart{
901+ fantasy.ToolResultPart{
902+ ToolCallID: "call_1",
903+ Output: fantasy.ToolResultOutputContentMedia{
904+ Data: pngBase64,
905+ MediaType: "image/png",
906+ },
907+ },
908+ },
909+ },
910+ }
911+
912+ // Non-Anthropic provider, image support — should create a synthetic
913+ // user message with FilePart.
914+ largeModel := Model{
915+ ModelCfg: config.SelectedModel{Provider: "openai"},
916+ CatwalkCfg: catwalk.Model{
917+ SupportsImages: true,
918+ },
919+ }
920+
921+ result := agent.workaroundProviderMediaLimitations(messages, largeModel)
922+
923+ // Should produce two messages: tool message with placeholder text,
924+ // and synthetic user message with FilePart.
925+ require.Len(t, result, 2)
926+ require.Equal(t, fantasy.MessageRoleTool, result[0].Role)
927+ require.Equal(t, fantasy.MessageRoleUser, result[1].Role)
928+
929+ // The tool message should have text placeholder.
930+ tr, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](result[0].Content[0])
931+ require.True(t, ok)
932+ textOutput, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentText](tr.Output)
933+ require.True(t, ok)
934+ require.Contains(t, textOutput.Text, "see attached file")
935+
936+ // The synthetic user message should contain a TextPart and a FilePart.
937+ require.Len(t, result[1].Content, 2)
938+ file, ok := fantasy.AsMessagePart[fantasy.FilePart](result[1].Content[1])
939+ require.True(t, ok)
940+ require.Equal(t, "image/png", file.MediaType)
941+}
942+
943+func TestWorkaroundProviderMediaLimitations_AnthropicProvider(t *testing.T) {
944+ env := testEnv(t)
945+ sa := testSessionAgent(env, nil, nil, "test prompt")
946+ agent := sa.(*sessionAgent)
947+
948+ pngBase64 := base64.StdEncoding.EncodeToString([]byte("fake-png-data"))
949+
950+ messages := []fantasy.Message{
951+ {
952+ Role: fantasy.MessageRoleTool,
953+ Content: []fantasy.MessagePart{
954+ fantasy.ToolResultPart{
955+ ToolCallID: "call_1",
956+ Output: fantasy.ToolResultOutputContentMedia{
957+ Data: pngBase64,
958+ MediaType: "image/png",
959+ },
960+ },
961+ },
962+ },
963+ }
964+
965+ // Anthropic provider — should return messages unchanged regardless of
966+ // SupportsImages, since Anthropic handles media in tool results natively.
967+ largeModel := Model{
968+ ModelCfg: config.SelectedModel{Provider: string(catwalk.InferenceProviderAnthropic)},
969+ CatwalkCfg: catwalk.Model{
970+ SupportsImages: true,
971+ },
972+ }
973+
974+ result := agent.workaroundProviderMediaLimitations(messages, largeModel)
975+ require.Len(t, result, 1)
976+ require.Equal(t, fantasy.MessageRoleTool, result[0].Role)
977+
978+ // The media should still be in the tool result, untouched.
979+ tr, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](result[0].Content[0])
980+ require.True(t, ok)
981+ media, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentMedia](tr.Output)
982+ require.True(t, ok)
983+ require.Equal(t, "image/png", media.MediaType)
984+}
985+
843func TestProviderRetryLogFields(t *testing.T) {
986func TestProviderRetryLogFields(t *testing.T) {
844 t.Run("nil provider error", func(t *testing.T) {
987 t.Run("nil provider error", func(t *testing.T) {
845 fields := providerRetryLogFields(nil, 2*time.Second)
988 fields := providerRetryLogFields(nil, 2*time.Second)