git.iko.soy/crush
on main help sign in
Merge pull request #357 from charmbracelet/fix-openrouter
dfe5ffcKujtim Hoxha
internal/llm/agent/agent.go+6
compare with main
unifiedsplit
@@ -420,6 +420,12 @@ func (a *agent) processGeneration(ctx context.Context, sessionID, content string
420 msgHistory = append(msgHistory, agentMessage, *toolResults)
420 msgHistory = append(msgHistory, agentMessage, *toolResults)
421 continue
421 continue
422 }
422 }
423+ if agentMessage.FinishReason() == "" {
424+ // Kujtim: could not track down where this is happening but this means its cancelled
425+ agentMessage.AddFinish(message.FinishReasonCanceled, "Request cancelled", "")
426+ _ = a.messages.Update(context.Background(), agentMessage)
427+ return a.err(ErrRequestCancelled)
428+ }
423 return AgentEvent{
429 return AgentEvent{
424 Type: AgentEventTypeResponse,
430 Type: AgentEventTypeResponse,
425 Message: agentMessage,
431 Message: agentMessage,
internal/llm/provider/openai.go+56−19
@@ -319,6 +319,10 @@ func (o *openaiClient) stream(ctx context.Context, messages []message.Message, t
319 go func() {
319 go func() {
320 for {
320 for {
321 attempts++
321 attempts++
322+ // Kujtim: fixes an issue with anthropig models on openrouter
323+ if len(params.Tools) == 0 {
324+ params.Tools = nil
325+ }
322 openaiStream := o.client.Chat.Completions.NewStreaming(
326 openaiStream := o.client.Chat.Completions.NewStreaming(
323 ctx,
327 ctx,
324 params,
328 params,
@@ -331,8 +335,14 @@ func (o *openaiClient) stream(ctx context.Context, messages []message.Message, t
331 var currentToolCallID string
335 var currentToolCallID string
332 var currentToolCall openai.ChatCompletionMessageToolCall
336 var currentToolCall openai.ChatCompletionMessageToolCall
333 var msgToolCalls []openai.ChatCompletionMessageToolCall
337 var msgToolCalls []openai.ChatCompletionMessageToolCall
338+ currentToolIndex := 0
334 for openaiStream.Next() {
339 for openaiStream.Next() {
335 chunk := openaiStream.Current()
340 chunk := openaiStream.Current()
341+ // Kujtim: this is an issue with openrouter qwen, its sending -1 for the tool index
342+ if len(chunk.Choices) > 0 && len(chunk.Choices[0].Delta.ToolCalls) > 0 && chunk.Choices[0].Delta.ToolCalls[0].Index == -1 {
343+ chunk.Choices[0].Delta.ToolCalls[0].Index = int64(currentToolIndex)
344+ currentToolIndex++
345+ }
336 acc.AddChunk(chunk)
346 acc.AddChunk(chunk)
337 // This fixes multiple tool calls for some providers
347 // This fixes multiple tool calls for some providers
338 for _, choice := range chunk.Choices {
348 for _, choice := range chunk.Choices {
@@ -348,6 +358,14 @@ func (o *openaiClient) stream(ctx context.Context, messages []message.Message, t
348 if currentToolCallID == "" {
358 if currentToolCallID == "" {
349 if toolCall.ID != "" {
359 if toolCall.ID != "" {
350 currentToolCallID = toolCall.ID
360 currentToolCallID = toolCall.ID
361+ eventChan <- ProviderEvent{
362+ Type: EventToolUseStart,
363+ ToolCall: &message.ToolCall{
364+ ID: toolCall.ID,
365+ Name: toolCall.Function.Name,
366+ Finished: false,
367+ },
368+ }
351 currentToolCall = openai.ChatCompletionMessageToolCall{
369 currentToolCall = openai.ChatCompletionMessageToolCall{
352 ID: toolCall.ID,
370 ID: toolCall.ID,
353 Type: "function",
371 Type: "function",
@@ -359,13 +377,21 @@ func (o *openaiClient) stream(ctx context.Context, messages []message.Message, t
359 }
377 }
360 } else {
378 } else {
361 // Delta tool use
379 // Delta tool use
362− if toolCall.ID == "" {
380+ if toolCall.ID == "" || toolCall.ID == currentToolCallID {
363 currentToolCall.Function.Arguments += toolCall.Function.Arguments
381 currentToolCall.Function.Arguments += toolCall.Function.Arguments
364 } else {
382 } else {
365 // Detect new tool use
383 // Detect new tool use
366 if toolCall.ID != currentToolCallID {
384 if toolCall.ID != currentToolCallID {
367 msgToolCalls = append(msgToolCalls, currentToolCall)
385 msgToolCalls = append(msgToolCalls, currentToolCall)
368 currentToolCallID = toolCall.ID
386 currentToolCallID = toolCall.ID
387+ eventChan <- ProviderEvent{
388+ Type: EventToolUseStart,
389+ ToolCall: &message.ToolCall{
390+ ID: toolCall.ID,
391+ Name: toolCall.Function.Name,
392+ Finished: false,
393+ },
394+ }
369 currentToolCall = openai.ChatCompletionMessageToolCall{
395 currentToolCall = openai.ChatCompletionMessageToolCall{
370 ID: toolCall.ID,
396 ID: toolCall.ID,
371 Type: "function",
397 Type: "function",
@@ -378,7 +404,8 @@ func (o *openaiClient) stream(ctx context.Context, messages []message.Message, t
378 }
404 }
379 }
405 }
380 }
406 }
407+ // Kujtim: some models send finish stop even for tool calls
381− if choice.FinishReason == "tool_calls" {
408+ if choice.FinishReason == "tool_calls" || (choice.FinishReason == "stop" && currentToolCallID != "") {
382 msgToolCalls = append(msgToolCalls, currentToolCall)
409 msgToolCalls = append(msgToolCalls, currentToolCall)
383 if len(acc.Choices) > 0 {
410 if len(acc.Choices) > 0 {
384 acc.Choices[0].Message.ToolCalls = msgToolCalls
411 acc.Choices[0].Message.ToolCalls = msgToolCalls
@@ -461,31 +488,41 @@ func (o *openaiClient) stream(ctx context.Context, messages []message.Message, t
461}
488}
462
489
463func (o *openaiClient) shouldRetry(attempts int, err error) (bool, int64, error) {
490func (o *openaiClient) shouldRetry(attempts int, err error) (bool, int64, error) {
464− var apiErr *openai.Error
465− if !errors.As(err, &apiErr) {
466− return false, 0, err
467− }
468−
469 if attempts > maxRetries {
491 if attempts > maxRetries {
470 return false, 0, fmt.Errorf("maximum retry attempts reached for rate limit: %d retries", maxRetries)
492 return false, 0, fmt.Errorf("maximum retry attempts reached for rate limit: %d retries", maxRetries)
471 }
493 }
494+ if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
495+ return false, 0, err
496+ }
497+ var apiErr *openai.Error
498+ retryMs := 0
499+ retryAfterValues := []string{}
500+ if errors.As(err, &apiErr) {
501+ // Check for token expiration (401 Unauthorized)
502+ if apiErr.StatusCode == 401 {
503+ o.providerOptions.apiKey, err = config.Get().Resolve(o.providerOptions.config.APIKey)
504+ if err != nil {
505+ return false, 0, fmt.Errorf("failed to resolve API key: %w", err)
506+ }
507+ o.client = createOpenAIClient(o.providerOptions)
508+ return true, 0, nil
509+ }
472
510
473− // Check for token expiration (401 Unauthorized)
474− if apiErr.StatusCode == 401 {
511+ if apiErr.StatusCode != 429 && apiErr.StatusCode != 500 {
475− o.providerOptions.apiKey, err = config.Get().Resolve(o.providerOptions.config.APIKey)
476− if err != nil {
477− return false, 0, fmt.Errorf("failed to resolve API key: %w", err)
512+ return false, 0, err
478 }
513 }
479− o.client = createOpenAIClient(o.providerOptions)
480− return true, 0, nil
481− }
482
514
483− if apiErr.StatusCode != 429 && apiErr.StatusCode != 500 {
515+ retryAfterValues = apiErr.Response.Header.Values("Retry-After")
484− return false, 0, err
485 }
516 }
486
517
487− retryMs := 0
518+ if apiErr != nil {
488− retryAfterValues := apiErr.Response.Header.Values("Retry-After")
519+ slog.Warn("OpenAI API error", "status_code", apiErr.StatusCode, "message", apiErr.Message, "type", apiErr.Type)
520+ if len(retryAfterValues) > 0 {
521+ slog.Warn("Retry-After header", "values", retryAfterValues)
522+ }
523+ } else {
524+ slog.Warn("OpenAI API error", "error", err.Error())
525+ }
489
526
490 backoffMs := 2000 * (1 << (attempts - 1))
527 backoffMs := 2000 * (1 << (attempts - 1))
491 jitterMs := int(float64(backoffMs) * 0.2)
528 jitterMs := int(float64(backoffMs) * 0.2)