git.iko.soy/crush
on main help sign in
chore: enable thinking for anthropic models
19f1a2dKujtim Hoxha
internal/config/config.go+5−2
compare with main
unifiedsplit
@@ -159,10 +159,13 @@ type Options struct {
159type PreferredModel struct {
159type PreferredModel struct {
160 ModelID string `json:"model_id"`
160 ModelID string `json:"model_id"`
161 Provider provider.InferenceProvider `json:"provider"`
161 Provider provider.InferenceProvider `json:"provider"`
162− // Overrides the default reasoning effort for this model
162+ // ReasoningEffort overrides the default reasoning effort for this model
163 ReasoningEffort string `json:"reasoning_effort,omitempty"`
163 ReasoningEffort string `json:"reasoning_effort,omitempty"`
164− // Overrides the default max tokens for this model
164+ // MaxTokens overrides the default max tokens for this model
165 MaxTokens int64 `json:"max_tokens,omitempty"`
165 MaxTokens int64 `json:"max_tokens,omitempty"`
166+
167+ // Think indicates if the model should think, only applicable for anthropic reasoning models
168+ Think bool `json:"think,omitempty"`
166}
169}
167
170
168type PreferredModels struct {
171type PreferredModels struct {
internal/llm/provider/anthropic.go+7−23
@@ -6,7 +6,6 @@ import (
6 "errors"
6 "errors"
7 "fmt"
7 "fmt"
8 "io"
8 "io"
9− "strings"
10 "time"
9 "time"
11
10
12 "github.com/anthropics/anthropic-sdk-go"
11 "github.com/anthropics/anthropic-sdk-go"
@@ -150,28 +149,18 @@ func (a *anthropicClient) finishReason(reason string) message.FinishReason {
150func (a *anthropicClient) preparedMessages(messages []anthropic.MessageParam, tools []anthropic.ToolUnionParam) anthropic.MessageNewParams {
149func (a *anthropicClient) preparedMessages(messages []anthropic.MessageParam, tools []anthropic.ToolUnionParam) anthropic.MessageNewParams {
151 model := a.providerOptions.model(a.providerOptions.modelType)
150 model := a.providerOptions.model(a.providerOptions.modelType)
152 var thinkingParam anthropic.ThinkingConfigParamUnion
151 var thinkingParam anthropic.ThinkingConfigParamUnion
153− // TODO: Implement a proper thinking function
154− // lastMessage := messages[len(messages)-1]
155− // isUser := lastMessage.Role == anthropic.MessageParamRoleUser
156− // messageContent := ""
157− temperature := anthropic.Float(0)
158− // if isUser {
159− // for _, m := range lastMessage.Content {
160− // if m.OfText != nil && m.OfText.Text != "" {
161− // messageContent = m.OfText.Text
162− // }
163− // }
164− // if messageContent != "" && a.shouldThink != nil && a.options.shouldThink(messageContent) {
165− // thinkingParam = anthropic.ThinkingConfigParamOfEnabled(int64(float64(a.providerOptions.maxTokens) * 0.8))
166− // temperature = anthropic.Float(1)
167− // }
168− // }
169−
170 cfg := config.Get()
152 cfg := config.Get()
171 modelConfig := cfg.Models.Large
153 modelConfig := cfg.Models.Large
172 if a.providerOptions.modelType == config.SmallModel {
154 if a.providerOptions.modelType == config.SmallModel {
173 modelConfig = cfg.Models.Small
155 modelConfig = cfg.Models.Small
174 }
156 }
157+ temperature := anthropic.Float(0)
158+
159+ if a.Model().CanReason && modelConfig.Think {
160+ thinkingParam = anthropic.ThinkingConfigParamOfEnabled(int64(float64(a.providerOptions.maxTokens) * 0.8))
161+ temperature = anthropic.Float(1)
162+ }
163+
175 maxTokens := model.DefaultMaxTokens
164 maxTokens := model.DefaultMaxTokens
176 if modelConfig.MaxTokens > 0 {
165 if modelConfig.MaxTokens > 0 {
177 maxTokens = modelConfig.MaxTokens
166 maxTokens = modelConfig.MaxTokens
@@ -456,8 +445,3 @@ func (a *anthropicClient) usage(msg anthropic.Message) TokenUsage {
456func (a *anthropicClient) Model() config.Model {
445func (a *anthropicClient) Model() config.Model {
457 return a.providerOptions.model(a.providerOptions.modelType)
446 return a.providerOptions.model(a.providerOptions.modelType)
458}
447}
459−
460−// TODO: check if we need
461−func DefaultShouldThinkFn(s string) bool {
462− return strings.Contains(strings.ToLower(s), "think")
463−}