6bb85dc · crush · git.iko.soy

git.iko.soy/crush

on main help sign in

chore: support provider extra body and provider options

6bb85dcKujtim Hoxha1 year ago

internal/agent/agent_tool.go+1−1

compare with main

unifiedsplit

@@ -79,7 +79,7 @@ func (c *coordinator) agentTool(ctx context.Context) (fantasy.AgentTool, error)

79 SessionID: session.ID,

80 Prompt: params.Prompt,

81 MaxOutputTokens: maxTokens,

82− ProviderOptions: getProviderOptions(model, providerCfg.Type),

82+ ProviderOptions: getProviderOptions(model, providerCfg),

83 Temperature: model.ModelCfg.Temperature,

84 TopP: model.ModelCfg.TopP,

85 TopK: model.ModelCfg.TopK,

internal/agent/coordinator.go+22−8

@@ -32,6 +32,7 @@ import (

32 "charm.land/fantasy/providers/openai"

33 "charm.land/fantasy/providers/openaicompat"

34 "charm.land/fantasy/providers/openrouter"

35+ openaisdk "github.com/openai/openai-go/v2/option"

35 "github.com/qjebbs/go-jsons"

36 "github.com/qjebbs/go-jsons"

36)

37)

@@ -118,7 +119,7 @@ func (c *coordinator) Run(ctx context.Context, sessionID string, prompt string,

118 return nil, errors.New("model provider not configured")

119 return nil, errors.New("model provider not configured")

119 }

120 }

120

121

121− mergedOptions, temp, topP, topK, freqPenalty, presPenalty := mergeCallOptions(model, providerCfg.Type)

122+ mergedOptions, temp, topP, topK, freqPenalty, presPenalty := mergeCallOptions(model, providerCfg)

122

123

123 return c.currentAgent.Run(ctx, SessionAgentCall{

124 return c.currentAgent.Run(ctx, SessionAgentCall{

124 SessionID: sessionID,

125 SessionID: sessionID,

@@ -134,10 +135,11 @@ func (c *coordinator) Run(ctx context.Context, sessionID string, prompt string,

134 })

135 })

135}

136}

136

137

137−func getProviderOptions(model Model, tp catwalk.Type) fantasy.ProviderOptions {

138+func getProviderOptions(model Model, providerCfg config.ProviderConfig) fantasy.ProviderOptions {

138 options := fantasy.ProviderOptions{}

139 options := fantasy.ProviderOptions{}

139

140

140 cfgOpts := []byte("{}")

141 cfgOpts := []byte("{}")

142+ providerCfgOpts := []byte("{}")

141 catwalkOpts := []byte("{}")

143 catwalkOpts := []byte("{}")

142

144

143 if model.ModelCfg.ProviderOptions != nil {

145 if model.ModelCfg.ProviderOptions != nil {

@@ -147,6 +149,13 @@ func getProviderOptions(model Model, tp catwalk.Type) fantasy.ProviderOptions {

147 }

149 }

148 }

150 }

149

151

152+ if providerCfg.ProviderOptions != nil {

153+ data, err := json.Marshal(providerCfg.ProviderOptions)

154+ if err == nil {

155+ providerCfgOpts = data

156+ }

157+ }

158+

150 if model.CatwalkCfg.Options.ProviderOptions != nil {

159 if model.CatwalkCfg.Options.ProviderOptions != nil {

151 data, err := json.Marshal(model.CatwalkCfg.Options.ProviderOptions)

160 data, err := json.Marshal(model.CatwalkCfg.Options.ProviderOptions)

152 if err == nil {

161 if err == nil {

@@ -156,6 +165,7 @@ func getProviderOptions(model Model, tp catwalk.Type) fantasy.ProviderOptions {

156

165

157 readers := []io.Reader{

166 readers := []io.Reader{

158 bytes.NewReader(catwalkOpts),

167 bytes.NewReader(catwalkOpts),

168+ bytes.NewReader(providerCfgOpts),

159 bytes.NewReader(cfgOpts),

169 bytes.NewReader(cfgOpts),

160 }

170 }

161

171

@@ -173,7 +183,7 @@ func getProviderOptions(model Model, tp catwalk.Type) fantasy.ProviderOptions {

173 return options

183 return options

174 }

184 }

175

185

176− switch tp {

186+ switch providerCfg.Type {

177 case openai.Name:

187 case openai.Name:

178 _, hasReasoningEffort := mergedOptions["reasoning_effort"]

188 _, hasReasoningEffort := mergedOptions["reasoning_effort"]

179 if !hasReasoningEffort && model.ModelCfg.ReasoningEffort != "" {

189 if !hasReasoningEffort && model.ModelCfg.ReasoningEffort != "" {

@@ -255,8 +265,8 @@ func getProviderOptions(model Model, tp catwalk.Type) fantasy.ProviderOptions {

255 return options

265 return options

256}

266}

257

267

258−func mergeCallOptions(model Model, tp catwalk.Type) (fantasy.ProviderOptions, *float64, *float64, *int64, *float64, *float64) {

268+func mergeCallOptions(model Model, cfg config.ProviderConfig) (fantasy.ProviderOptions, *float64, *float64, *int64, *float64, *float64) {

259− modelOptions := getProviderOptions(model, tp)

269+ modelOptions := getProviderOptions(model, cfg)

260 temp := cmp.Or(model.ModelCfg.Temperature, model.CatwalkCfg.Options.Temperature)

270 temp := cmp.Or(model.ModelCfg.Temperature, model.CatwalkCfg.Options.Temperature)

261 topP := cmp.Or(model.ModelCfg.TopP, model.CatwalkCfg.Options.TopP)

271 topP := cmp.Or(model.ModelCfg.TopP, model.CatwalkCfg.Options.TopP)

262 topK := cmp.Or(model.ModelCfg.TopK, model.CatwalkCfg.Options.TopK)

272 topK := cmp.Or(model.ModelCfg.TopK, model.CatwalkCfg.Options.TopK)

@@ -516,7 +526,7 @@ func (c *coordinator) buildOpenrouterProvider(_, apiKey string, headers map[stri

516 return openrouter.New(opts...)

526 return openrouter.New(opts...)

517}

527}

518

528

519−func (c *coordinator) buildOpenaiCompatProvider(baseURL, apiKey string, headers map[string]string) (fantasy.Provider, error) {

529+func (c *coordinator) buildOpenaiCompatProvider(baseURL, apiKey string, headers map[string]string, extraBody map[string]any) (fantasy.Provider, error) {

520 opts := []openaicompat.Option{

530 opts := []openaicompat.Option{

521 openaicompat.WithBaseURL(baseURL),

531 openaicompat.WithBaseURL(baseURL),

522 openaicompat.WithAPIKey(apiKey),

532 openaicompat.WithAPIKey(apiKey),

@@ -529,6 +539,10 @@ func (c *coordinator) buildOpenaiCompatProvider(baseURL, apiKey string, headers

529 opts = append(opts, openaicompat.WithHeaders(headers))

539 opts = append(opts, openaicompat.WithHeaders(headers))

530 }

540 }

531

541

542+ for extraKey, extraValue := range extraBody {

543+ opts = append(opts, openaicompat.WithSDKOptions(openaisdk.WithJSONSet(extraKey, extraValue)))

544+ }

545+

532 return openaicompat.New(opts...)

546 return openaicompat.New(opts...)

533}

547}

534

548

@@ -646,7 +660,7 @@ func (c *coordinator) buildProvider(providerCfg config.ProviderConfig, model con

646 case "google-vertex", "vertexai":

660 case "google-vertex", "vertexai":

647 return c.buildGoogleVertexProvider(headers, providerCfg.ExtraParams)

661 return c.buildGoogleVertexProvider(headers, providerCfg.ExtraParams)

648 case openaicompat.Name:

662 case openaicompat.Name:

649− return c.buildOpenaiCompatProvider(baseURL, apiKey, headers)

663+ return c.buildOpenaiCompatProvider(baseURL, apiKey, headers, providerCfg.ExtraBody)

650 default:

664 default:

651 return nil, fmt.Errorf("provider type not supported: %q", providerCfg.Type)

665 return nil, fmt.Errorf("provider type not supported: %q", providerCfg.Type)

652 }

666 }

@@ -717,5 +731,5 @@ func (c *coordinator) Summarize(ctx context.Context, sessionID string) error {

717 if !ok {

731 if !ok {

718 return errors.New("model provider not configured")

732 return errors.New("model provider not configured")

719 }

733 }

720− return c.currentAgent.Summarize(ctx, sessionID, getProviderOptions(c.currentAgent.Model(), providerCfg.Type))

734+ return c.currentAgent.Summarize(ctx, sessionID, getProviderOptions(c.currentAgent.Model(), providerCfg))

721}

735}

internal/config/config.go+3−1

@@ -99,7 +99,9 @@ type ProviderConfig struct {

99 // Extra headers to send with each request to the provider.

100 ExtraHeaders map[string]string `json:"extra_headers,omitempty" jsonschema:"description=Additional HTTP headers to send with requests"`

101 // Extra body

102− ExtraBody map[string]any `json:"extra_body,omitempty" jsonschema:"description=Additional fields to include in request bodies"`

102+ ExtraBody map[string]any `json:"extra_body,omitempty" jsonschema:"description=Additional fields to include in request bodies, only works with openai-compatible providers"`

103+

104+ ProviderOptions map[string]any `json:"provider_options,omitempty" jsonschema:"description=Additional provider-specific options for this provider"`

103

105

104 // Used to pass extra parameters to the provider.

106 // Used to pass extra parameters to the provider.

105 ExtraParams map[string]string `json:"-"`

107 ExtraParams map[string]string `json:"-"`