git.iko.soy/crush
on main help sign in
chore: proper caching for anthropic models with openrouter
c35e700Kujtim Hoxha
internal/llm/provider/openai.go+42−3
compare with main
unifiedsplit
@@ -7,6 +7,7 @@ import (
7 "fmt"
7 "fmt"
8 "io"
8 "io"
9 "log/slog"
9 "log/slog"
10+ "strings"
10 "time"
11 "time"
11
12
12 "github.com/charmbracelet/catwalk/pkg/catwalk"
13 "github.com/charmbracelet/catwalk/pkg/catwalk"
@@ -56,14 +57,33 @@ func createOpenAIClient(opts providerClientOptions) openai.Client {
56}
57}
57
58
58func (o *openaiClient) convertMessages(messages []message.Message) (openaiMessages []openai.ChatCompletionMessageParamUnion) {
59func (o *openaiClient) convertMessages(messages []message.Message) (openaiMessages []openai.ChatCompletionMessageParamUnion) {
60+ isAnthropicModel := o.providerOptions.config.ID == "openrouter" && strings.HasPrefix(o.Model().ID, "anthropic/")
59 // Add system message first
61 // Add system message first
60 systemMessage := o.providerOptions.systemMessage
62 systemMessage := o.providerOptions.systemMessage
61 if o.providerOptions.systemPromptPrefix != "" {
63 if o.providerOptions.systemPromptPrefix != "" {
62 systemMessage = o.providerOptions.systemPromptPrefix + "\n" + systemMessage
64 systemMessage = o.providerOptions.systemPromptPrefix + "\n" + systemMessage
63 }
65 }
64− openaiMessages = append(openaiMessages, openai.SystemMessage(systemMessage))
65
66
67+ systemTextBlock := openai.ChatCompletionContentPartTextParam{Text: systemMessage}
68+ if isAnthropicModel && !o.providerOptions.disableCache {
69+ systemTextBlock.SetExtraFields(
70+ map[string]any{
71+ "cache_control": map[string]string{
72+ "type": "ephemeral",
73+ },
74+ },
75+ )
76+ }
77+ var content []openai.ChatCompletionContentPartTextParam
78+ content = append(content, systemTextBlock)
79+ system := openai.SystemMessage(content)
80+ openaiMessages = append(openaiMessages, system)
81+
66− for _, msg := range messages {
82+ for i, msg := range messages {
83+ cache := false
84+ if i > len(messages)-3 {
85+ cache = true
86+ }
67 switch msg.Role {
87 switch msg.Role {
68 case message.User:
88 case message.User:
69 var content []openai.ChatCompletionContentPartUnionParam
89 var content []openai.ChatCompletionContentPartUnionParam
@@ -75,6 +95,13 @@ func (o *openaiClient) convertMessages(messages []message.Message) (openaiMessag
75
95
76 content = append(content, openai.ChatCompletionContentPartUnionParam{OfImageURL: &imageBlock})
96 content = append(content, openai.ChatCompletionContentPartUnionParam{OfImageURL: &imageBlock})
77 }
97 }
98+ if cache && !o.providerOptions.disableCache && isAnthropicModel {
99+ textBlock.SetExtraFields(map[string]any{
100+ "cache_control": map[string]string{
101+ "type": "ephemeral",
102+ },
103+ })
104+ }
78
105
79 openaiMessages = append(openaiMessages, openai.UserMessage(content))
106 openaiMessages = append(openaiMessages, openai.UserMessage(content))
80
107
@@ -86,8 +113,20 @@ func (o *openaiClient) convertMessages(messages []message.Message) (openaiMessag
86 hasContent := false
113 hasContent := false
87 if msg.Content().String() != "" {
114 if msg.Content().String() != "" {
88 hasContent = true
115 hasContent = true
116+ textBlock := openai.ChatCompletionContentPartTextParam{Text: msg.Content().String()}
117+ if cache && !o.providerOptions.disableCache && isAnthropicModel {
118+ textBlock.SetExtraFields(map[string]any{
119+ "cache_control": map[string]string{
120+ "type": "ephemeral",
121+ },
122+ })
123+ }
89 assistantMsg.Content = openai.ChatCompletionAssistantMessageParamContentUnion{
124 assistantMsg.Content = openai.ChatCompletionAssistantMessageParamContentUnion{
90− OfString: openai.String(msg.Content().String()),
125+ OfArrayOfContentParts: []openai.ChatCompletionAssistantMessageParamContentArrayOfContentPartUnion{
126+ {
127+ OfText: &textBlock,
128+ },
129+ },
91 }
130 }
92 }
131 }
93
132