7303158 · crush · git.iko.soy

git.iko.soy/crush

on main help sign in

feat: auto-discover models from openai-compat providers

7303158Kieran Klukas5 months ago

internal/agent/coordinator.go+15

compare with main

unifiedsplit

@@ -23,6 +23,7 @@ import (

23 "github.com/charmbracelet/crush/internal/agent/prompt"

24 "github.com/charmbracelet/crush/internal/agent/tools"

25 "github.com/charmbracelet/crush/internal/config"

26+ "github.com/charmbracelet/crush/internal/discover"

26 "github.com/charmbracelet/crush/internal/event"

27 "github.com/charmbracelet/crush/internal/event"

27 "github.com/charmbracelet/crush/internal/filetracker"

28 "github.com/charmbracelet/crush/internal/filetracker"

28 "github.com/charmbracelet/crush/internal/history"

29 "github.com/charmbracelet/crush/internal/history"

@@ -511,6 +512,15 @@ func getProviderOptions(model Model, providerCfg config.ProviderConfig) fantasy.

511 if err == nil {

512 if err == nil {

512 options[openaicompat.Name] = parsed

513 options[openaicompat.Name] = parsed

513 }

514 }

515+ default:

516+ // Known custom providers (litellm, ollama, omlx) are

517+ // openai-compat under the hood.

518+ if discover.IsKnownCustomProvider(string(providerCfg.Type)) {

519+ parsed, err := openaicompat.ParseOptions(mergedOptions)

520+ if err == nil {

521+ options[openaicompat.Name] = parsed

522+ }

523+ }

514 }

524 }

515

525

516 return options

526 return options

@@ -1030,6 +1040,11 @@ func (c *coordinator) buildProvider(providerCfg config.ProviderConfig, model con

1030 }

1040 }

1031 return c.buildOpenaiCompatProvider(baseURL, apiKey, headers, providerCfg.ExtraBody, providerCfg.ID, isSubAgent)

1041 return c.buildOpenaiCompatProvider(baseURL, apiKey, headers, providerCfg.ExtraBody, providerCfg.ID, isSubAgent)

1032 default:

1042 default:

1043+ // Known custom providers (litellm, ollama, omlx) are

1044+ // openai-compat under the hood.

1045+ if discover.IsKnownCustomProvider(string(providerCfg.Type)) {

1046+ return c.buildOpenaiCompatProvider(baseURL, apiKey, headers, providerCfg.ExtraBody, providerCfg.ID, isSubAgent)

1047+ }

1033 return nil, fmt.Errorf("provider type not supported: %q", providerCfg.Type)

1048 return nil, fmt.Errorf("provider type not supported: %q", providerCfg.Type)

1034 }

1049 }

1035}

1050}

internal/config/config.go+7

@@ -133,6 +133,13 @@ type ProviderConfig struct {

133 // Skip cost accumulation for this provider when using subscription or flat rate billing.

134 FlatRate bool `json:"flat_rate,omitempty" jsonschema:"description=Flat-rate mode for this provider"`

135

136+ // AutoDiscoverModels controls model discovery via /v1/models endpoint.

137+ // When Models is empty and this is nil or true, Crush auto-discovers

138+ // models. When true and Models is non-empty, discovered models are

139+ // merged in (user-specified models take precedence). When false,

140+ // only explicitly listed models are used.

141+ AutoDiscoverModels *bool `json:"discover_models,omitempty" jsonschema:"description=Auto-discover models from /v1/models endpoint. When true with existing models they are merged (yours win),default=true"`

142+

136 // The provider models

143 // The provider models

137 Models []catwalk.Model `json:"models,omitempty" jsonschema:"description=List of models available from this provider"`

144 Models []catwalk.Model `json:"models,omitempty" jsonschema:"description=List of models available from this provider"`

138}

145}

internal/config/load.go+77−6

@@ -15,11 +15,14 @@ import (

15 "slices"

16 "strconv"

17 "strings"

18+ "sync"

18 "testing"

19 "testing"

20+ "time"

20 "charm.land/catwalk/pkg/catwalk"

22 "charm.land/catwalk/pkg/catwalk"

21 "github.com/charmbracelet/crush/internal/agent/hyper"

23 "github.com/charmbracelet/crush/internal/agent/hyper"

22 "github.com/charmbracelet/crush/internal/csync"

24 "github.com/charmbracelet/crush/internal/csync"

25+ "github.com/charmbracelet/crush/internal/discover"

23 "github.com/charmbracelet/crush/internal/env"

26 "github.com/charmbracelet/crush/internal/env"

24 "github.com/charmbracelet/crush/internal/filepathext"

27 "github.com/charmbracelet/crush/internal/filepathext"

25 "github.com/charmbracelet/crush/internal/fsext"

28 "github.com/charmbracelet/crush/internal/fsext"

@@ -113,7 +116,7 @@ func Load(workingDir, dataDir string, debug bool) (*ConfigStore, error) {

113 store.reloadMu.Lock()

116 store.reloadMu.Lock()

114 defer store.reloadMu.Unlock()

117 defer store.reloadMu.Unlock()

115

118

116− if err := cfg.configureProviders(store, env, valueResolver, store.knownProviders); err != nil {

119+ if err := cfg.configureProviders(context.Background(), store, env, valueResolver, store.knownProviders); err != nil {

117 return nil, fmt.Errorf("failed to configure providers: %w", err)

120 return nil, fmt.Errorf("failed to configure providers: %w", err)

118 }

121 }

119

122

@@ -171,7 +174,7 @@ func PushPopCrushEnv() func() {

171 return restore

174 return restore

172}

175}

173

176

174−func (c *Config) configureProviders(store *ConfigStore, env env.Env, resolver VariableResolver, knownProviders []catwalk.Provider) error {

177+func (c *Config) configureProviders(ctx context.Context, store *ConfigStore, env env.Env, resolver VariableResolver, knownProviders []catwalk.Provider) error {

175 knownProviderNames := make(map[string]bool)

178 knownProviderNames := make(map[string]bool)

176 restore := PushPopCrushEnv()

179 restore := PushPopCrushEnv()

177 defer restore()

180 defer restore()

@@ -340,18 +343,69 @@ func (c *Config) configureProviders(store *ConfigStore, env env.Env, resolver Va

340 c.Providers.Set(string(p.ID), prepared)

343 c.Providers.Set(string(p.ID), prepared)

341 }

344 }

342

345

346+ // Discover models concurrently for custom providers that need it.

347+ // A provider needs discovery when discover_models is explicitly true,

348+ // or when the models list is empty (auto-trigger, unless opted out).

349+ type discoveryResult struct {

350+ models []catwalk.Model

351+ err error

352+ }

353+

354+ discoveryResults := make(map[string]discoveryResult)

355+ var mu sync.Mutex

356+ var wg sync.WaitGroup

357+

358+ discoverCtx, discoverCancel := context.WithTimeout(ctx, 3*time.Second)

359+ for id, pc := range c.Providers.Seq2() {

360+ if knownProviderNames[id] {

361+ continue

362+ }

363+ if pc.Disable || pc.BaseURL == "" {

364+ continue

365+ }

366+ wantsDiscovery := pc.AutoDiscoverModels != nil && *pc.AutoDiscoverModels

367+ autoTrigger := len(pc.Models) == 0 && (pc.AutoDiscoverModels == nil || *pc.AutoDiscoverModels)

368+ if !wantsDiscovery && !autoTrigger {

369+ continue

370+ }

371+ providerID := cmp.Or(pc.ID, id)

372+ cfg := discover.Config{

373+ ID: providerID,

374+ BaseURL: pc.BaseURL,

375+ APIKey: pc.APIKey,

376+ ExtraHeaders: pc.ExtraHeaders,

377+ ExistingModels: pc.Models,

378+ }

379+ providerType := cmp.Or(pc.Type, catwalk.TypeOpenAICompat)

380+ wg.Go(func() {

381+ models, err := discover.DiscoverModels(discoverCtx, cfg, resolver)

382+ if err == nil && len(models) > 0 {

383+ if enricher := discover.GetEnricher(string(providerType)); enricher != nil {

384+ models, _ = enricher.EnrichModels(discoverCtx, cfg, resolver, models)

385+ }

386+ }

387+ mu.Lock()

388+ discoveryResults[id] = discoveryResult{models: models, err: err}

389+ mu.Unlock()

390+ })

391+ }

392+ wg.Wait()

393+ discoverCancel()

394+

343− // validate the custom providers

395+ // Validate the custom providers.

344 for id, providerConfig := range c.Providers.Seq2() {

396 for id, providerConfig := range c.Providers.Seq2() {

345 if knownProviderNames[id] {

397 if knownProviderNames[id] {

346 continue

398 continue

347 }

399 }

348

400

349− // Make sure the provider ID is set

401+ // Make sure the provider ID is set.

350 providerConfig.ID = id

402 providerConfig.ID = id

351 providerConfig.Name = cmp.Or(providerConfig.Name, id) // Use ID as name if not set

403 providerConfig.Name = cmp.Or(providerConfig.Name, id) // Use ID as name if not set

352− // default to OpenAI if not set

404+ // Default to OpenAI if not set.

353 providerConfig.Type = cmp.Or(providerConfig.Type, catwalk.TypeOpenAICompat)

405 providerConfig.Type = cmp.Or(providerConfig.Type, catwalk.TypeOpenAICompat)

354− if !slices.Contains(catwalk.KnownProviderTypes(), providerConfig.Type) && providerConfig.Type != hyper.Name {

406+ if !slices.Contains(catwalk.KnownProviderTypes(), providerConfig.Type) &&

407+ providerConfig.Type != hyper.Name &&

408+ !discover.IsKnownCustomProvider(string(providerConfig.Type)) {

355 slog.Warn("Skipping custom provider due to unsupported provider type", "provider", id)

409 slog.Warn("Skipping custom provider due to unsupported provider type", "provider", id)

356 c.Providers.Del(id)

410 c.Providers.Del(id)

357 continue

411 continue

@@ -370,11 +424,28 @@ func (c *Config) configureProviders(store *ConfigStore, env env.Env, resolver Va

370 c.Providers.Del(id)

424 c.Providers.Del(id)

371 continue

425 continue

372 }

426 }

427+

428+ // Apply discovery results if available.

429+ if result, ok := discoveryResults[id]; ok {

430+ if result.err != nil {

431+ slog.Warn("Model discovery failed", "provider", id, "error", result.err)

432+ if len(providerConfig.Models) == 0 {

433+ slog.Warn("Skipping provider with no models after failed discovery", "provider", id)

434+ c.Providers.Del(id)

435+ continue

436+ }

437+ } else if len(result.models) > 0 {

438+ providerConfig.Models = result.models

439+ slog.Info("Discovered models for provider", "provider", id, "count", len(result.models))

440+ }

441+ }

442+

373 if len(providerConfig.Models) == 0 {

443 if len(providerConfig.Models) == 0 {

374 slog.Warn("Skipping custom provider because the provider has no models", "provider", id)

444 slog.Warn("Skipping custom provider because the provider has no models", "provider", id)

375 c.Providers.Del(id)

445 c.Providers.Del(id)

376 continue

446 continue

377 }

447 }

448+

378 apiKey, err := resolver.ResolveValue(providerConfig.APIKey)

449 apiKey, err := resolver.ResolveValue(providerConfig.APIKey)

379 if apiKey == "" || err != nil {

450 if apiKey == "" || err != nil {

380 slog.Warn("Provider is missing API key, this might be OK for local providers", "provider", id)

451 slog.Warn("Provider is missing API key, this might be OK for local providers", "provider", id)

internal/config/load_test.go+180−49

@@ -1,8 +1,11 @@

1package config

3import (

4+ "context"

4 "io"

5 "io"

5 "log/slog"

6 "log/slog"

7+ "net/http"

8+ "net/http/httptest"

6 "os"

9 "os"

7 "os/exec"

10 "os/exec"

8 "path/filepath"

11 "path/filepath"

@@ -317,7 +320,7 @@ func TestConfig_configureProviders(t *testing.T) {

317 "OPENAI_API_KEY": "test-key",

320 "OPENAI_API_KEY": "test-key",

318 })

321 })

319 resolver := NewShellVariableResolver(env)

322 resolver := NewShellVariableResolver(env)

320− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

323+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

321 require.NoError(t, err)

324 require.NoError(t, err)

322 require.Equal(t, 1, cfg.Providers.Len())

325 require.Equal(t, 1, cfg.Providers.Len())

323

326

@@ -360,7 +363,7 @@ func TestConfig_configureProvidersWithOverride(t *testing.T) {

360 "OPENAI_API_KEY": "test-key",

363 "OPENAI_API_KEY": "test-key",

361 })

364 })

362 resolver := NewShellVariableResolver(env)

365 resolver := NewShellVariableResolver(env)

363− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

366+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

364 require.NoError(t, err)

367 require.NoError(t, err)

365 require.Equal(t, 1, cfg.Providers.Len())

368 require.Equal(t, 1, cfg.Providers.Len())

366

369

@@ -402,7 +405,7 @@ func TestConfig_configureProvidersWithNewProvider(t *testing.T) {

402 "OPENAI_API_KEY": "test-key",

405 "OPENAI_API_KEY": "test-key",

403 })

406 })

404 resolver := NewShellVariableResolver(env)

407 resolver := NewShellVariableResolver(env)

405− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

408+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

406 require.NoError(t, err)

409 require.NoError(t, err)

407 // Should be to because of the env variable

410 // Should be to because of the env variable

408 require.Equal(t, cfg.Providers.Len(), 2)

411 require.Equal(t, cfg.Providers.Len(), 2)

@@ -438,7 +441,7 @@ func TestConfig_configureProvidersBedrockWithCredentials(t *testing.T) {

438 "AWS_SECRET_ACCESS_KEY": "test-secret-key",

441 "AWS_SECRET_ACCESS_KEY": "test-secret-key",

439 })

442 })

440 resolver := NewShellVariableResolver(env)

443 resolver := NewShellVariableResolver(env)

441− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

444+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

442 require.NoError(t, err)

445 require.NoError(t, err)

443 require.Equal(t, cfg.Providers.Len(), 1)

446 require.Equal(t, cfg.Providers.Len(), 1)

444

447

@@ -464,7 +467,7 @@ func TestConfig_configureProvidersBedrockWithoutCredentials(t *testing.T) {

464 cfg.setDefaults("/tmp", "")

467 cfg.setDefaults("/tmp", "")

465 env := env.NewFromMap(map[string]string{})

468 env := env.NewFromMap(map[string]string{})

466 resolver := NewShellVariableResolver(env)

469 resolver := NewShellVariableResolver(env)

467− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

470+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

468 require.NoError(t, err)

471 require.NoError(t, err)

469 // Provider should not be configured without credentials

472 // Provider should not be configured without credentials

470 require.Equal(t, cfg.Providers.Len(), 0)

473 require.Equal(t, cfg.Providers.Len(), 0)

@@ -489,7 +492,7 @@ func TestConfig_configureProvidersVertexAIWithCredentials(t *testing.T) {

489 "VERTEXAI_LOCATION": "us-central1",

492 "VERTEXAI_LOCATION": "us-central1",

490 })

493 })

491 resolver := NewShellVariableResolver(env)

494 resolver := NewShellVariableResolver(env)

492− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

495+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

493 require.NoError(t, err)

496 require.NoError(t, err)

494 require.Equal(t, cfg.Providers.Len(), 1)

497 require.Equal(t, cfg.Providers.Len(), 1)

495

498

@@ -521,7 +524,7 @@ func TestConfig_configureProvidersVertexAIWithoutCredentials(t *testing.T) {

521 "GOOGLE_CLOUD_LOCATION": "us-central1",

524 "GOOGLE_CLOUD_LOCATION": "us-central1",

522 })

525 })

523 resolver := NewShellVariableResolver(env)

526 resolver := NewShellVariableResolver(env)

524− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

527+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

525 require.NoError(t, err)

528 require.NoError(t, err)

526 // Provider should not be configured without proper credentials

529 // Provider should not be configured without proper credentials

527 require.Equal(t, cfg.Providers.Len(), 0)

530 require.Equal(t, cfg.Providers.Len(), 0)

@@ -546,7 +549,7 @@ func TestConfig_configureProvidersVertexAIMissingProject(t *testing.T) {

546 "GOOGLE_CLOUD_LOCATION": "us-central1",

549 "GOOGLE_CLOUD_LOCATION": "us-central1",

547 })

550 })

548 resolver := NewShellVariableResolver(env)

551 resolver := NewShellVariableResolver(env)

549− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

552+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

550 require.NoError(t, err)

553 require.NoError(t, err)

551 // Provider should not be configured without project

554 // Provider should not be configured without project

552 require.Equal(t, cfg.Providers.Len(), 0)

555 require.Equal(t, cfg.Providers.Len(), 0)

@@ -570,7 +573,7 @@ func TestConfig_configureProvidersSetProviderID(t *testing.T) {

570 "OPENAI_API_KEY": "test-key",

573 "OPENAI_API_KEY": "test-key",

571 })

574 })

572 resolver := NewShellVariableResolver(env)

575 resolver := NewShellVariableResolver(env)

573− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

576+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

574 require.NoError(t, err)

577 require.NoError(t, err)

575 require.Equal(t, cfg.Providers.Len(), 1)

578 require.Equal(t, cfg.Providers.Len(), 1)

576

579

@@ -761,7 +764,7 @@ func TestConfig_configureProvidersWithDisabledProvider(t *testing.T) {

761 "OPENAI_API_KEY": "test-key",

764 "OPENAI_API_KEY": "test-key",

762 })

765 })

763 resolver := NewShellVariableResolver(env)

766 resolver := NewShellVariableResolver(env)

764− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

767+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

765 require.NoError(t, err)

768 require.NoError(t, err)

766

769

767 require.Equal(t, cfg.Providers.Len(), 1)

770 require.Equal(t, cfg.Providers.Len(), 1)

@@ -789,7 +792,7 @@ func TestConfig_configureProvidersCustomProviderValidation(t *testing.T) {

789

792

790 env := env.NewFromMap(map[string]string{})

793 env := env.NewFromMap(map[string]string{})

791 resolver := NewShellVariableResolver(env)

794 resolver := NewShellVariableResolver(env)

792− err := cfg.configureProviders(testStore(cfg), env, resolver, []catwalk.Provider{})

795+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, []catwalk.Provider{})

793 require.NoError(t, err)

796 require.NoError(t, err)

794

797

795 require.Equal(t, cfg.Providers.Len(), 1)

798 require.Equal(t, cfg.Providers.Len(), 1)

@@ -812,7 +815,7 @@ func TestConfig_configureProvidersCustomProviderValidation(t *testing.T) {

812

815

813 env := env.NewFromMap(map[string]string{})

816 env := env.NewFromMap(map[string]string{})

814 resolver := NewShellVariableResolver(env)

817 resolver := NewShellVariableResolver(env)

815− err := cfg.configureProviders(testStore(cfg), env, resolver, []catwalk.Provider{})

818+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, []catwalk.Provider{})

816 require.NoError(t, err)

819 require.NoError(t, err)

817

820

818 require.Equal(t, cfg.Providers.Len(), 0)

821 require.Equal(t, cfg.Providers.Len(), 0)

@@ -820,7 +823,7 @@ func TestConfig_configureProvidersCustomProviderValidation(t *testing.T) {

820 require.False(t, exists)

823 require.False(t, exists)

821 })

824 })

822

825

823− t.Run("custom provider with no models is removed", func(t *testing.T) {

826+ t.Run("custom provider with no models attempts discovery and is removed on failure", func(t *testing.T) {

824 cfg := &Config{

827 cfg := &Config{

825 Providers: csync.NewMapFrom(map[string]ProviderConfig{

828 Providers: csync.NewMapFrom(map[string]ProviderConfig{

826 "custom": {

829 "custom": {

@@ -834,14 +837,142 @@ func TestConfig_configureProvidersCustomProviderValidation(t *testing.T) {

834

837

835 env := env.NewFromMap(map[string]string{})

838 env := env.NewFromMap(map[string]string{})

836 resolver := NewShellVariableResolver(env)

839 resolver := NewShellVariableResolver(env)

837− err := cfg.configureProviders(testStore(cfg), env, resolver, []catwalk.Provider{})

840+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, []catwalk.Provider{})

838 require.NoError(t, err)

841 require.NoError(t, err)

839

842

843+ // Discovery fails (unreachable URL) so provider is removed.

840− require.Equal(t, cfg.Providers.Len(), 0)

844+ require.Equal(t, 0, cfg.Providers.Len())

845+ _, exists := cfg.Providers.Get("custom")

846+ require.False(t, exists)

847+ })

848+

849+ t.Run("custom provider with no models and discover_models:false is removed", func(t *testing.T) {

850+ discoverFalse := false

851+ cfg := &Config{

852+ Providers: csync.NewMapFrom(map[string]ProviderConfig{

853+ "custom": {

854+ APIKey: "test-key",

855+ BaseURL: "https://api.custom.com/v1",

856+ Models: []catwalk.Model{},

857+ AutoDiscoverModels: &discoverFalse,

858+ },

859+ }),

860+ }

861+ cfg.setDefaults("/tmp", "")

862+

863+ env := env.NewFromMap(map[string]string{})

864+ resolver := NewShellVariableResolver(env)

865+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, []catwalk.Provider{})

866+ require.NoError(t, err)

867+

868+ require.Equal(t, 0, cfg.Providers.Len())

841 _, exists := cfg.Providers.Get("custom")

869 _, exists := cfg.Providers.Get("custom")

842 require.False(t, exists)

870 require.False(t, exists)

843 })

871 })

844

872

873+ t.Run("custom provider with models and discover_models:true merges discovered models", func(t *testing.T) {

874+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

875+ w.Header().Set("Content-Type", "application/json")

876+ _, _ = w.Write([]byte(`{"data": [

877+ {"id": "existing-model", "object": "model"},

878+ {"id": "discovered-model", "object": "model"}

879+ ]}`))

880+ }))

881+ defer server.Close()

882+

883+ discoverTrue := true

884+ cfg := &Config{

885+ Providers: csync.NewMapFrom(map[string]ProviderConfig{

886+ "custom": {

887+ APIKey: "test-key",

888+ BaseURL: server.URL + "/v1",

889+ Models: []catwalk.Model{

890+ {ID: "existing-model", Name: "My Custom Name", ContextWindow: 200000},

891+ },

892+ AutoDiscoverModels: &discoverTrue,

893+ },

894+ }),

895+ }

896+ cfg.setDefaults("/tmp", "")

897+

898+ env := env.NewFromMap(map[string]string{})

899+ resolver := NewShellVariableResolver(env)

900+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, []catwalk.Provider{})

901+ require.NoError(t, err)

902+

903+ require.Equal(t, 1, cfg.Providers.Len())

904+ p, exists := cfg.Providers.Get("custom")

905+ require.True(t, exists)

906+ require.Len(t, p.Models, 2)

907+

908+ // User-specified model keeps its custom fields.

909+ require.Equal(t, "existing-model", p.Models[0].ID)

910+ require.Equal(t, "My Custom Name", p.Models[0].Name)

911+ require.Equal(t, int64(200000), p.Models[0].ContextWindow)

912+

913+ // Discovered model is appended.

914+ require.Equal(t, "discovered-model", p.Models[1].ID)

915+ })

916+

917+ t.Run("custom provider with models and no discover_models uses only listed models", func(t *testing.T) {

918+ cfg := &Config{

919+ Providers: csync.NewMapFrom(map[string]ProviderConfig{

920+ "custom": {

921+ APIKey: "test-key",

922+ BaseURL: "https://api.custom.com/v1",

923+ Models: []catwalk.Model{

924+ {ID: "my-model", Name: "My Model"},

925+ },

926+ },

927+ }),

928+ }

929+ cfg.setDefaults("/tmp", "")

930+

931+ env := env.NewFromMap(map[string]string{})

932+ resolver := NewShellVariableResolver(env)

933+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, []catwalk.Provider{})

934+ require.NoError(t, err)

935+

936+ require.Equal(t, 1, cfg.Providers.Len())

937+ p, exists := cfg.Providers.Get("custom")

938+ require.True(t, exists)

939+ require.Len(t, p.Models, 1)

940+ require.Equal(t, "my-model", p.Models[0].ID)

941+ })

942+

943+ t.Run("custom provider with no models auto-discovers successfully", func(t *testing.T) {

944+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

945+ w.Header().Set("Content-Type", "application/json")

946+ _, _ = w.Write([]byte(`{"data": [

947+ {"id": "auto-model-a", "object": "model"},

948+ {"id": "auto-model-b", "object": "model"}

949+ ]}`))

950+ }))

951+ defer server.Close()

952+

953+ cfg := &Config{

954+ Providers: csync.NewMapFrom(map[string]ProviderConfig{

955+ "custom": {

956+ APIKey: "test-key",

957+ BaseURL: server.URL + "/v1",

958+ },

959+ }),

960+ }

961+ cfg.setDefaults("/tmp", "")

962+

963+ env := env.NewFromMap(map[string]string{})

964+ resolver := NewShellVariableResolver(env)

965+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, []catwalk.Provider{})

966+ require.NoError(t, err)

967+

968+ require.Equal(t, 1, cfg.Providers.Len())

969+ p, exists := cfg.Providers.Get("custom")

970+ require.True(t, exists)

971+ require.Len(t, p.Models, 2)

972+ require.Equal(t, "auto-model-a", p.Models[0].ID)

973+ require.Equal(t, "auto-model-b", p.Models[1].ID)

974+ })

975+

845 t.Run("custom provider with unsupported type is removed", func(t *testing.T) {

976 t.Run("custom provider with unsupported type is removed", func(t *testing.T) {

846 cfg := &Config{

977 cfg := &Config{

847 Providers: csync.NewMapFrom(map[string]ProviderConfig{

978 Providers: csync.NewMapFrom(map[string]ProviderConfig{

@@ -859,7 +990,7 @@ func TestConfig_configureProvidersCustomProviderValidation(t *testing.T) {

859

990

860 env := env.NewFromMap(map[string]string{})

991 env := env.NewFromMap(map[string]string{})

861 resolver := NewShellVariableResolver(env)

992 resolver := NewShellVariableResolver(env)

862− err := cfg.configureProviders(testStore(cfg), env, resolver, []catwalk.Provider{})

993+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, []catwalk.Provider{})

863 require.NoError(t, err)

994 require.NoError(t, err)

864

995

865 require.Equal(t, cfg.Providers.Len(), 0)

996 require.Equal(t, cfg.Providers.Len(), 0)

@@ -884,7 +1015,7 @@ func TestConfig_configureProvidersCustomProviderValidation(t *testing.T) {

884

1015

885 env := env.NewFromMap(map[string]string{})

1016 env := env.NewFromMap(map[string]string{})

886 resolver := NewShellVariableResolver(env)

1017 resolver := NewShellVariableResolver(env)

887− err := cfg.configureProviders(testStore(cfg), env, resolver, []catwalk.Provider{})

1018+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, []catwalk.Provider{})

888 require.NoError(t, err)

1019 require.NoError(t, err)

889

1020

890 require.Equal(t, cfg.Providers.Len(), 1)

1021 require.Equal(t, cfg.Providers.Len(), 1)

@@ -912,7 +1043,7 @@ func TestConfig_configureProvidersCustomProviderValidation(t *testing.T) {

912

1043

913 env := env.NewFromMap(map[string]string{})

1044 env := env.NewFromMap(map[string]string{})

914 resolver := NewShellVariableResolver(env)

1045 resolver := NewShellVariableResolver(env)

915− err := cfg.configureProviders(testStore(cfg), env, resolver, []catwalk.Provider{})

1046+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, []catwalk.Provider{})

916 require.NoError(t, err)

1047 require.NoError(t, err)

917

1048

918 require.Equal(t, cfg.Providers.Len(), 1)

1049 require.Equal(t, cfg.Providers.Len(), 1)

@@ -942,7 +1073,7 @@ func TestConfig_configureProvidersCustomProviderValidation(t *testing.T) {

942

1073

943 env := env.NewFromMap(map[string]string{})

1074 env := env.NewFromMap(map[string]string{})

944 resolver := NewShellVariableResolver(env)

1075 resolver := NewShellVariableResolver(env)

945− err := cfg.configureProviders(testStore(cfg), env, resolver, []catwalk.Provider{})

1076+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, []catwalk.Provider{})

946 require.NoError(t, err)

1077 require.NoError(t, err)

947

1078

948 require.Equal(t, cfg.Providers.Len(), 0)

1079 require.Equal(t, cfg.Providers.Len(), 0)

@@ -977,7 +1108,7 @@ func TestConfig_configureProvidersEnhancedCredentialValidation(t *testing.T) {

977 "GOOGLE_GENAI_USE_VERTEXAI": "false",

1108 "GOOGLE_GENAI_USE_VERTEXAI": "false",

978 })

1109 })

979 resolver := NewShellVariableResolver(env)

1110 resolver := NewShellVariableResolver(env)

980− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

1111+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

981 require.NoError(t, err)

1112 require.NoError(t, err)

982

1113

983 require.Equal(t, cfg.Providers.Len(), 0)

1114 require.Equal(t, cfg.Providers.Len(), 0)

@@ -1008,7 +1139,7 @@ func TestConfig_configureProvidersEnhancedCredentialValidation(t *testing.T) {

1008

1139

1009 env := env.NewFromMap(map[string]string{})

1140 env := env.NewFromMap(map[string]string{})

1010 resolver := NewShellVariableResolver(env)

1141 resolver := NewShellVariableResolver(env)

1011− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

1142+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

1012 require.NoError(t, err)

1143 require.NoError(t, err)

1013

1144

1014 require.Equal(t, cfg.Providers.Len(), 0)

1145 require.Equal(t, cfg.Providers.Len(), 0)

@@ -1039,7 +1170,7 @@ func TestConfig_configureProvidersEnhancedCredentialValidation(t *testing.T) {

1039

1170

1040 env := env.NewFromMap(map[string]string{})

1171 env := env.NewFromMap(map[string]string{})

1041 resolver := NewShellVariableResolver(env)

1172 resolver := NewShellVariableResolver(env)

1042− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

1173+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

1043 require.NoError(t, err)

1174 require.NoError(t, err)

1044

1175

1045 require.Equal(t, cfg.Providers.Len(), 0)

1176 require.Equal(t, cfg.Providers.Len(), 0)

@@ -1072,7 +1203,7 @@ func TestConfig_configureProvidersEnhancedCredentialValidation(t *testing.T) {

1072 "OPENAI_API_KEY": "test-key",

1203 "OPENAI_API_KEY": "test-key",

1073 })

1204 })

1074 resolver := NewShellVariableResolver(env)

1205 resolver := NewShellVariableResolver(env)

1075− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

1206+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

1076 require.NoError(t, err)

1207 require.NoError(t, err)

1077

1208

1078 require.Equal(t, cfg.Providers.Len(), 1)

1209 require.Equal(t, cfg.Providers.Len(), 1)

@@ -1106,7 +1237,7 @@ func TestConfig_defaultModelSelection(t *testing.T) {

1106 cfg.setDefaults("/tmp", "")

1237 cfg.setDefaults("/tmp", "")

1107 env := env.NewFromMap(map[string]string{})

1238 env := env.NewFromMap(map[string]string{})

1108 resolver := NewShellVariableResolver(env)

1239 resolver := NewShellVariableResolver(env)

1109− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

1240+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

1110 require.NoError(t, err)

1241 require.NoError(t, err)

1111

1242

1112 large, small, err := cfg.defaultModelSelection(knownProviders)

1243 large, small, err := cfg.defaultModelSelection(knownProviders)

@@ -1142,7 +1273,7 @@ func TestConfig_defaultModelSelection(t *testing.T) {

1142 cfg.setDefaults("/tmp", "")

1273 cfg.setDefaults("/tmp", "")

1143 env := env.NewFromMap(map[string]string{})

1274 env := env.NewFromMap(map[string]string{})

1144 resolver := NewShellVariableResolver(env)

1275 resolver := NewShellVariableResolver(env)

1145− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

1276+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

1146 require.NoError(t, err)

1277 require.NoError(t, err)

1147

1278

1148 _, _, err = cfg.defaultModelSelection(knownProviders)

1279 _, _, err = cfg.defaultModelSelection(knownProviders)

@@ -1172,7 +1303,7 @@ func TestConfig_defaultModelSelection(t *testing.T) {

1172 cfg.setDefaults("/tmp", "")

1303 cfg.setDefaults("/tmp", "")

1173 env := env.NewFromMap(map[string]string{})

1304 env := env.NewFromMap(map[string]string{})

1174 resolver := NewShellVariableResolver(env)

1305 resolver := NewShellVariableResolver(env)

1175− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

1306+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

1176 require.NoError(t, err)

1307 require.NoError(t, err)

1177 _, _, err = cfg.defaultModelSelection(knownProviders)

1308 _, _, err = cfg.defaultModelSelection(knownProviders)

1178 require.NoError(t, err)

1309 require.NoError(t, err)

@@ -1215,7 +1346,7 @@ func TestConfig_defaultModelSelection(t *testing.T) {

1215 cfg.setDefaults("/tmp", "")

1346 cfg.setDefaults("/tmp", "")

1216 env := env.NewFromMap(map[string]string{})

1347 env := env.NewFromMap(map[string]string{})

1217 resolver := NewShellVariableResolver(env)

1348 resolver := NewShellVariableResolver(env)

1218− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

1349+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

1219 require.NoError(t, err)

1350 require.NoError(t, err)

1220 large, small, err := cfg.defaultModelSelection(knownProviders)

1351 large, small, err := cfg.defaultModelSelection(knownProviders)

1221 require.NoError(t, err)

1352 require.NoError(t, err)

@@ -1259,7 +1390,7 @@ func TestConfig_defaultModelSelection(t *testing.T) {

1259 cfg.setDefaults("/tmp", "")

1390 cfg.setDefaults("/tmp", "")

1260 env := env.NewFromMap(map[string]string{})

1391 env := env.NewFromMap(map[string]string{})

1261 resolver := NewShellVariableResolver(env)

1392 resolver := NewShellVariableResolver(env)

1262− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

1393+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

1263 require.NoError(t, err)

1394 require.NoError(t, err)

1264 _, _, err = cfg.defaultModelSelection(knownProviders)

1395 _, _, err = cfg.defaultModelSelection(knownProviders)

1265 require.Error(t, err)

1396 require.Error(t, err)

@@ -1301,7 +1432,7 @@ func TestConfig_defaultModelSelection(t *testing.T) {

1301 cfg.setDefaults("/tmp", "")

1432 cfg.setDefaults("/tmp", "")

1302 env := env.NewFromMap(map[string]string{})

1433 env := env.NewFromMap(map[string]string{})

1303 resolver := NewShellVariableResolver(env)

1434 resolver := NewShellVariableResolver(env)

1304− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

1435+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

1305 require.NoError(t, err)

1436 require.NoError(t, err)

1306 large, small, err := cfg.defaultModelSelection(knownProviders)

1437 large, small, err := cfg.defaultModelSelection(knownProviders)

1307 require.NoError(t, err)

1438 require.NoError(t, err)

@@ -1346,7 +1477,7 @@ func TestConfig_configureProvidersDisableDefaultProviders(t *testing.T) {

1346 "OPENAI_API_KEY": "test-key",

1477 "OPENAI_API_KEY": "test-key",

1347 })

1478 })

1348 resolver := NewShellVariableResolver(env)

1479 resolver := NewShellVariableResolver(env)

1349− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

1480+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

1350 require.ErrorContains(t, err, "no custom providers")

1481 require.ErrorContains(t, err, "no custom providers")

1351

1482

1352 // openai should NOT be present because it lacks base_url and models.

1483 // openai should NOT be present because it lacks base_url and models.

@@ -1389,7 +1520,7 @@ func TestConfig_configureProvidersDisableDefaultProviders(t *testing.T) {

1389 "OPENAI_API_KEY": "test-key",

1520 "OPENAI_API_KEY": "test-key",

1390 })

1521 })

1391 resolver := NewShellVariableResolver(env)

1522 resolver := NewShellVariableResolver(env)

1392− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

1523+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

1393 require.NoError(t, err)

1524 require.NoError(t, err)

1394

1525

1395 // Only fully specified provider should be present.

1526 // Only fully specified provider should be present.

@@ -1443,7 +1574,7 @@ func TestConfig_configureProvidersDisableDefaultProviders(t *testing.T) {

1443 "ANTHROPIC_API_KEY": "test-key",

1574 "ANTHROPIC_API_KEY": "test-key",

1444 })

1575 })

1445 resolver := NewShellVariableResolver(env)

1576 resolver := NewShellVariableResolver(env)

1446− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

1577+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

1447 require.NoError(t, err)

1578 require.NoError(t, err)

1448

1579

1449 // Both providers should be present.

1580 // Both providers should be present.

@@ -1454,7 +1585,7 @@ func TestConfig_configureProvidersDisableDefaultProviders(t *testing.T) {

1454 require.True(t, exists, "anthropic should be present")

1585 require.True(t, exists, "anthropic should be present")

1455 })

1586 })

1456

1587

1457− t.Run("when enabled, provider missing models is rejected", func(t *testing.T) {

1588+ t.Run("when enabled, provider missing models attempts discovery but still triggers no-custom-providers error", func(t *testing.T) {

1458 cfg := &Config{

1589 cfg := &Config{

1459 Options: &Options{

1590 Options: &Options{

1460 DisableDefaultProviders: true,

1591 DisableDefaultProviders: true,

@@ -1471,10 +1602,10 @@ func TestConfig_configureProvidersDisableDefaultProviders(t *testing.T) {

1471

1602

1472 env := env.NewFromMap(map[string]string{})

1603 env := env.NewFromMap(map[string]string{})

1473 resolver := NewShellVariableResolver(env)

1604 resolver := NewShellVariableResolver(env)

1474− err := cfg.configureProviders(testStore(cfg), env, resolver, []catwalk.Provider{})

1605+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, []catwalk.Provider{})

1475 require.ErrorContains(t, err, "no custom providers")

1606 require.ErrorContains(t, err, "no custom providers")

1476

1607

1477− // Provider should be rejected for missing models.

1608+ // Discovery fails (unreachable URL) so provider is removed.

1478 require.Equal(t, 0, cfg.Providers.Len())

1609 require.Equal(t, 0, cfg.Providers.Len())

1479 })

1610 })

1480

1611

@@ -1495,7 +1626,7 @@ func TestConfig_configureProvidersDisableDefaultProviders(t *testing.T) {

1495

1626

1496 env := env.NewFromMap(map[string]string{})

1627 env := env.NewFromMap(map[string]string{})

1497 resolver := NewShellVariableResolver(env)

1628 resolver := NewShellVariableResolver(env)

1498− err := cfg.configureProviders(testStore(cfg), env, resolver, []catwalk.Provider{})

1629+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, []catwalk.Provider{})

1499 require.ErrorContains(t, err, "no custom providers")

1630 require.ErrorContains(t, err, "no custom providers")

1500

1631

1501 // Provider should be rejected for missing base_url.

1632 // Provider should be rejected for missing base_url.

@@ -1553,7 +1684,7 @@ func TestConfig_configureSelectedModels(t *testing.T) {

1553 store := &ConfigStore{config: cfg, globalDataPath: globalPath}

1684 store := &ConfigStore{config: cfg, globalDataPath: globalPath}

1554 env := env.NewFromMap(map[string]string{})

1685 env := env.NewFromMap(map[string]string{})

1555 resolver := NewShellVariableResolver(env)

1686 resolver := NewShellVariableResolver(env)

1556− err := cfg.configureProviders(store, env, resolver, knownProviders)

1687+ err := cfg.configureProviders(context.Background(), store, env, resolver, knownProviders)

1557 require.NoError(t, err)

1688 require.NoError(t, err)

1558

1689

1559 err = configureSelectedModels(store, knownProviders, false)

1690 err = configureSelectedModels(store, knownProviders, false)

@@ -1603,7 +1734,7 @@ func TestConfig_configureSelectedModels(t *testing.T) {

1603 cfg.setDefaults("/tmp", "")

1734 cfg.setDefaults("/tmp", "")

1604 env := env.NewFromMap(map[string]string{})

1735 env := env.NewFromMap(map[string]string{})

1605 resolver := NewShellVariableResolver(env)

1736 resolver := NewShellVariableResolver(env)

1606− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

1737+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

1607 require.NoError(t, err)

1738 require.NoError(t, err)

1608

1739

1609 err = configureSelectedModels(testStore(cfg), knownProviders, true)

1740 err = configureSelectedModels(testStore(cfg), knownProviders, true)

@@ -1665,7 +1796,7 @@ func TestConfig_configureSelectedModels(t *testing.T) {

1665 cfg.setDefaults("/tmp", "")

1796 cfg.setDefaults("/tmp", "")

1666 env := env.NewFromMap(map[string]string{})

1797 env := env.NewFromMap(map[string]string{})

1667 resolver := NewShellVariableResolver(env)

1798 resolver := NewShellVariableResolver(env)

1668− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

1799+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

1669 require.NoError(t, err)

1800 require.NoError(t, err)

1670

1801

1671 err = configureSelectedModels(testStore(cfg), knownProviders, true)

1802 err = configureSelectedModels(testStore(cfg), knownProviders, true)

@@ -1710,7 +1841,7 @@ func TestConfig_configureSelectedModels(t *testing.T) {

1710 cfg.setDefaults("/tmp", "")

1841 cfg.setDefaults("/tmp", "")

1711 env := env.NewFromMap(map[string]string{})

1842 env := env.NewFromMap(map[string]string{})

1712 resolver := NewShellVariableResolver(env)

1843 resolver := NewShellVariableResolver(env)

1713− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

1844+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

1714 require.NoError(t, err)

1845 require.NoError(t, err)

1715

1846

1716 err = configureSelectedModels(testStore(cfg), knownProviders, true)

1847 err = configureSelectedModels(testStore(cfg), knownProviders, true)

@@ -1749,7 +1880,7 @@ func TestConfig_configureProviders_HyperAPIKeyFromEnv(t *testing.T) {

1749 "HYPER_API_KEY": "env-api-key",

1880 "HYPER_API_KEY": "env-api-key",

1750 })

1881 })

1751 resolver := NewShellVariableResolver(env)

1882 resolver := NewShellVariableResolver(env)

1752− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

1883+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

1753 require.NoError(t, err)

1884 require.NoError(t, err)

1754 require.Equal(t, 1, cfg.Providers.Len())

1885 require.Equal(t, 1, cfg.Providers.Len())

1755

1886

@@ -1796,7 +1927,7 @@ func TestConfig_configureProviders_HyperAPIKeyFromConfigOverrides(t *testing.T)

1796 "HYPER_API_KEY": "env-api-key",

1927 "HYPER_API_KEY": "env-api-key",

1797 })

1928 })

1798 resolver := NewShellVariableResolver(env)

1929 resolver := NewShellVariableResolver(env)

1799− err := cfg.configureProviders(testStore(cfg), env, resolver, knownProviders)

1930+ err := cfg.configureProviders(context.Background(), testStore(cfg), env, resolver, knownProviders)

1800 require.NoError(t, err)

1931 require.NoError(t, err)

1801 require.Equal(t, 1, cfg.Providers.Len())

1932 require.Equal(t, 1, cfg.Providers.Len())

1802

1933

@@ -1839,7 +1970,7 @@ func TestConfig_configureProviders_ProviderHeaderResolveError(t *testing.T) {

1839 })

1970 })

1840 resolver := NewShellVariableResolver(testEnv)

1971 resolver := NewShellVariableResolver(testEnv)

1841

1972

1842− err := cfg.configureProviders(testStore(cfg), testEnv, resolver, knownProviders)

1973+ err := cfg.configureProviders(context.Background(), testStore(cfg), testEnv, resolver, knownProviders)

1843 require.Error(t, err, "failing $(cmd) in a header must fail the provider load")

1974 require.Error(t, err, "failing $(cmd) in a header must fail the provider load")

1844 require.Contains(t, err.Error(), "X-Broken", "error must name the offending header")

1975 require.Contains(t, err.Error(), "X-Broken", "error must name the offending header")

1845}

1976}

@@ -1871,7 +2002,7 @@ func TestConfig_configureProviders_CatwalkDefaultWithUnsetVarLoads(t *testing.T)

1871 })

2002 })

1872 resolver := NewShellVariableResolver(testEnv)

2003 resolver := NewShellVariableResolver(testEnv)

1873

2004

1874− err := cfg.configureProviders(testStore(cfg), testEnv, resolver, knownProviders)

2005+ err := cfg.configureProviders(context.Background(), testStore(cfg), testEnv, resolver, knownProviders)

1875 require.NoError(t, err, "optional env-gated header must not fail the load")

2006 require.NoError(t, err, "optional env-gated header must not fail the load")

1876

2007

1877 pc, ok := cfg.Providers.Get("openai")

2008 pc, ok := cfg.Providers.Get("openai")

@@ -1907,7 +2038,7 @@ func TestConfig_configureProviders_LiteralEmptyHeaderDropped(t *testing.T) {

1907 })

2038 })

1908 resolver := NewShellVariableResolver(testEnv)

2039 resolver := NewShellVariableResolver(testEnv)

1909

2040

1910− err := cfg.configureProviders(testStore(cfg), testEnv, resolver, []catwalk.Provider{})

2041+ err := cfg.configureProviders(context.Background(), testStore(cfg), testEnv, resolver, []catwalk.Provider{})

1911 require.NoError(t, err)

2042 require.NoError(t, err)

1912

2043

1913 pc, ok := cfg.Providers.Get("my-llm")

2044 pc, ok := cfg.Providers.Get("my-llm")

@@ -1944,7 +2075,7 @@ func TestConfig_configureProviders_EchoEmptyHeaderDropped(t *testing.T) {

1944 })

2075 })

1945 resolver := NewShellVariableResolver(testEnv)

2076 resolver := NewShellVariableResolver(testEnv)

1946

2077

1947− err := cfg.configureProviders(testStore(cfg), testEnv, resolver, knownProviders)

2078+ err := cfg.configureProviders(context.Background(), testStore(cfg), testEnv, resolver, knownProviders)

1948 require.NoError(t, err)

2079 require.NoError(t, err)

1949

2080

1950 pc, ok := cfg.Providers.Get("openai")

2081 pc, ok := cfg.Providers.Get("openai")

@@ -1992,7 +2123,7 @@ func TestConfig_configureProviders_UnsetAPIKeySkipsProvider(t *testing.T) {

1992 })

2123 })

1993 resolver := NewShellVariableResolver(testEnv)

2124 resolver := NewShellVariableResolver(testEnv)

1994

2125

1995− err := cfg.configureProviders(testStore(cfg), testEnv, resolver, knownProviders)

2126+ err := cfg.configureProviders(context.Background(), testStore(cfg), testEnv, resolver, knownProviders)

1996 require.NoError(t, err, "skip path must not surface as a load error")

2127 require.NoError(t, err, "skip path must not surface as a load error")

1997

2128

1998 require.Equal(t, 0, cfg.Providers.Len(), "provider with unset API key must be skipped")

2129 require.Equal(t, 0, cfg.Providers.Len(), "provider with unset API key must be skipped")

@@ -2030,7 +2161,7 @@ func TestConfig_configureProviders_FailingAPIKeyCmdSkipsProvider(t *testing.T) {

2030 })

2161 })

2031 resolver := NewShellVariableResolver(testEnv)

2162 resolver := NewShellVariableResolver(testEnv)

2032

2163

2033− err := cfg.configureProviders(testStore(cfg), testEnv, resolver, knownProviders)

2164+ err := cfg.configureProviders(context.Background(), testStore(cfg), testEnv, resolver, knownProviders)

2034 require.NoError(t, err, "failing $(cmd) in API key must skip provider, not fail load")

2165 require.NoError(t, err, "failing $(cmd) in API key must skip provider, not fail load")

2035

2166

2036 require.Equal(t, 0, cfg.Providers.Len(), "provider with failing $(cmd) API key must be skipped")

2167 require.Equal(t, 0, cfg.Providers.Len(), "provider with failing $(cmd) API key must be skipped")

@@ -2067,7 +2198,7 @@ func TestConfig_configureProviders_UnsetAzureEndpointSkipsProvider(t *testing.T)

2067 })

2198 })

2068 resolver := NewShellVariableResolver(testEnv)

2199 resolver := NewShellVariableResolver(testEnv)

2069

2200

2070− err := cfg.configureProviders(testStore(cfg), testEnv, resolver, knownProviders)

2201+ err := cfg.configureProviders(context.Background(), testStore(cfg), testEnv, resolver, knownProviders)

2071 require.NoError(t, err)

2202 require.NoError(t, err)

2072

2203

2073 require.Equal(t, 0, cfg.Providers.Len(), "azure provider with unset endpoint must be skipped")

2204 require.Equal(t, 0, cfg.Providers.Len(), "azure provider with unset endpoint must be skipped")

internal/config/store.go+1−1

@@ -783,7 +783,7 @@ func (s *ConfigStore) reloadFromDiskLocked(ctx context.Context) error {

783 return fmt.Errorf("failed to load providers during reload: %w", err)

784 }

785

786− if err := cfg.configureProviders(s, env, resolver, providers); err != nil {

786+ if err := cfg.configureProviders(ctx, s, env, resolver, providers); err != nil {

787 return fmt.Errorf("failed to configure providers during reload: %w", err)

788 }

789

internal/discover/discover.gonew file+135

@@ -0,0 +1,135 @@

1+package discover

3+import (

4+ "bytes"

5+ "context"

6+ "encoding/json"

7+ "fmt"

8+ "net/http"

9+ "strings"

10+ "time"

11+

12+ "charm.land/catwalk/pkg/catwalk"

13+)

14+

15+// httpClient is shared across all discovery and enrichment calls. It

16+// has a reasonable timeout so individual requests cannot block forever

17+// even if the caller forgets to set a context deadline.

18+var httpClient = &http.Client{Timeout: 10 * time.Second}

19+

20+// doRequest builds and executes an authenticated HTTP request using the

21+// shared client. It resolves variable references in the base URL, API

22+// key, and extra headers via the provided Resolver. The path is joined

23+// to the base URL with proper slash handling.

24+func doRequest(ctx context.Context, method, baseURL, path, apiKey string, extraHeaders map[string]string, resolver Resolver, body any) (*http.Response, error) {

25+ resolvedBase, _ := resolver.ResolveValue(baseURL)

26+ resolvedKey, _ := resolver.ResolveValue(apiKey)

27+

28+ url := strings.TrimRight(resolvedBase, "/") + "/" + strings.TrimLeft(path, "/")

29+

30+ var reqBody *bytes.Reader

31+ if body != nil {

32+ data, err := json.Marshal(body)

33+ if err != nil {

34+ return nil, fmt.Errorf("marshaling request body: %w", err)

35+ }

36+ reqBody = bytes.NewReader(data)

37+ }

38+

39+ var req *http.Request

40+ var err error

41+ if reqBody != nil {

42+ req, err = http.NewRequestWithContext(ctx, method, url, reqBody)

43+ } else {

44+ req, err = http.NewRequestWithContext(ctx, method, url, nil)

45+ }

46+ if err != nil {

47+ return nil, err

48+ }

49+

50+ if reqBody != nil {

51+ req.Header.Set("Content-Type", "application/json")

52+ }

53+ if resolvedKey != "" {

54+ req.Header.Set("Authorization", "Bearer "+resolvedKey)

55+ }

56+ for k, v := range extraHeaders {

57+ resolved, err := resolver.ResolveValue(v)

58+ if err != nil || resolved == "" {

59+ continue

60+ }

61+ req.Header.Set(k, resolved)

62+ }

63+

64+ return httpClient.Do(req)

65+}

66+

67+// Config holds the provider configuration needed for model discovery.

68+type Config struct {

69+ ID string

70+ BaseURL string

71+ APIKey string

72+ ExtraHeaders map[string]string

73+ // Existing models from config — IDs present in this list are skipped

74+ // during discovery (user-specified models win).

75+ ExistingModels []catwalk.Model

76+}

77+

78+// Resolver resolves variable references (e.g. $ENV_VAR) in config values.

79+type Resolver interface {

80+ ResolveValue(val string) (string, error)

81+}

82+

83+type modelsResponse struct {

84+ Data []struct {

85+ ID string `json:"id"`

86+ Object string `json:"object"`

87+ Created int64 `json:"created"`

88+ OwnedBy string `json:"owned_by"`

89+ } `json:"data"`

90+}

91+

92+// DiscoverModels fetches available models from the provider's /models endpoint.

93+// It uses the provided context for cancellation and timeout; callers should set

94+// a deadline (e.g. context.WithTimeout) to avoid blocking indefinitely.

95+// Models whose IDs already appear in cfg.ExistingModels are skipped —

96+// user-specified models take precedence.

97+func DiscoverModels(ctx context.Context, cfg Config, resolver Resolver) ([]catwalk.Model, error) {

98+ resp, err := doRequest(ctx, http.MethodGet, cfg.BaseURL, "/models", cfg.APIKey, cfg.ExtraHeaders, resolver, nil)

99+ if err != nil {

100+ return nil, fmt.Errorf("discover models for provider %s: %w", cfg.ID, err)

101+ }

102+ defer resp.Body.Close()

103+

104+ if resp.StatusCode != http.StatusOK {

105+ return nil, fmt.Errorf("discover models for provider %s: %s", cfg.ID, resp.Status)

106+ }

107+

108+ var modelsResp modelsResponse

109+ if err := json.NewDecoder(resp.Body).Decode(&modelsResp); err != nil {

110+ return nil, fmt.Errorf("discover models for provider %s: %w", cfg.ID, err)

111+ }

112+

113+ // Build set of existing model IDs to skip.

114+ existing := make(map[string]struct{}, len(cfg.ExistingModels))

115+ for _, m := range cfg.ExistingModels {

116+ existing[m.ID] = struct{}{}

117+ }

118+

119+ // Start with user-specified models.

120+ result := make([]catwalk.Model, len(cfg.ExistingModels))

121+ copy(result, cfg.ExistingModels)

122+

123+ // Append discovered models not already in the list.

124+ for _, e := range modelsResp.Data {

125+ if _, ok := existing[e.ID]; ok {

126+ continue

127+ }

128+ result = append(result, catwalk.Model{

129+ ID: e.ID,

130+ Name: e.ID,

131+ })

132+ }

133+

134+ return result, nil

135+}

internal/discover/discover_test.gonew file+202

@@ -0,0 +1,202 @@

1+package discover

3+import (

4+ "context"

5+ "net/http"

6+ "net/http/httptest"

7+ "testing"

9+ "charm.land/catwalk/pkg/catwalk"

10+ "github.com/stretchr/testify/require"

11+)

12+

13+func TestDiscoverModels(t *testing.T) {

14+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

15+ require.Equal(t, "/v1/models", r.URL.Path)

16+ require.Equal(t, "Bearer test-key", r.Header.Get("Authorization"))

17+

18+ w.Header().Set("Content-Type", "application/json")

19+ _, _ = w.Write([]byte(`{

20+ "object": "list",

21+ "data": [

22+ {"id": "model-a", "object": "model", "owned_by": "org"},

23+ {"id": "model-b", "object": "model", "owned_by": "org"}

24+ ]

25+ }`))

26+ }))

27+ defer server.Close()

28+

29+ cfg := Config{

30+ ID: "test",

31+ BaseURL: server.URL + "/v1",

32+ APIKey: "test-key",

33+ }

34+

35+ models, err := DiscoverModels(context.Background(), cfg, &mockResolver{})

36+ require.NoError(t, err)

37+ require.Len(t, models, 2)

38+ require.Equal(t, "model-a", models[0].ID)

39+ require.Equal(t, "model-b", models[1].ID)

40+}

41+

42+func TestDiscoverModels_ExistingModelsWin(t *testing.T) {

43+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

44+ w.Header().Set("Content-Type", "application/json")

45+ _, _ = w.Write([]byte(`{

46+ "data": [

47+ {"id": "model-a", "object": "model"},

48+ {"id": "model-b", "object": "model"}

49+ ]

50+ }`))

51+ }))

52+ defer server.Close()

53+

54+ cfg := Config{

55+ ID: "test",

56+ BaseURL: server.URL + "/v1",

57+ APIKey: "test-key",

58+ ExistingModels: []catwalk.Model{

59+ {ID: "model-a", Name: "My Custom Name", ContextWindow: 200000, CanReason: true},

60+ },

61+ }

62+

63+ models, err := DiscoverModels(context.Background(), cfg, &mockResolver{})

64+ require.NoError(t, err)

65+ require.Len(t, models, 2)

66+

67+ require.Equal(t, "model-a", models[0].ID)

68+ require.Equal(t, "My Custom Name", models[0].Name)

69+ require.Equal(t, int64(200000), models[0].ContextWindow)

70+ require.True(t, models[0].CanReason)

71+

72+ require.Equal(t, "model-b", models[1].ID)

73+ require.Equal(t, "model-b", models[1].Name)

74+}

75+

76+func TestDiscoverModels_HTTPError(t *testing.T) {

77+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

78+ w.WriteHeader(http.StatusInternalServerError)

79+ }))

80+ defer server.Close()

81+

82+ cfg := Config{

83+ ID: "test",

84+ BaseURL: server.URL + "/v1",

85+ APIKey: "test-key",

86+ }

87+

88+ models, err := DiscoverModels(context.Background(), cfg, &mockResolver{})

89+ require.Error(t, err)

90+ require.Nil(t, models)

91+}

92+

93+func TestDiscoverModels_ExtraHeaders(t *testing.T) {

94+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

95+ require.Equal(t, "custom-value", r.Header.Get("X-Custom"))

96+ w.Header().Set("Content-Type", "application/json")

97+ _, _ = w.Write([]byte(`{"data": [{"id": "m1", "object": "model"}]}`))

98+ }))

99+ defer server.Close()

100+

101+ cfg := Config{

102+ ID: "test",

103+ BaseURL: server.URL + "/v1",

104+ APIKey: "test-key",

105+ ExtraHeaders: map[string]string{

106+ "X-Custom": "custom-value",

107+ },

108+ }

109+

110+ models, err := DiscoverModels(context.Background(), cfg, &mockResolver{})

111+ require.NoError(t, err)

112+ require.Len(t, models, 1)

113+}

114+

115+type mockResolver struct{}

116+

117+func (m *mockResolver) ResolveValue(val string) (string, error) { return val, nil }

118+

119+type envResolver struct {

120+ env map[string]string

121+}

122+

123+func (e *envResolver) ResolveValue(val string) (string, error) {

124+ if v, ok := e.env[val]; ok {

125+ return v, nil

126+ }

127+ return val, nil

128+}

129+

130+func TestDiscoverModels_ResolvesShellVariables(t *testing.T) {

131+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

132+ require.Equal(t, "Bearer resolved-key", r.Header.Get("Authorization"))

133+ require.Equal(t, "resolved-header", r.Header.Get("X-Custom"))

134+ w.Header().Set("Content-Type", "application/json")

135+ _, _ = w.Write([]byte(`{"data": [{"id": "m1", "object": "model"}]}`))

136+ }))

137+ defer server.Close()

138+

139+ cfg := Config{

140+ ID: "test",

141+ BaseURL: server.URL + "/v1",

142+ APIKey: "$MY_API_KEY",

143+ ExtraHeaders: map[string]string{

144+ "X-Custom": "$MY_HEADER",

145+ },

146+ }

147+

148+ resolver := &envResolver{env: map[string]string{

149+ "$MY_API_KEY": "resolved-key",

150+ "$MY_HEADER": "resolved-header",

151+ }}

152+

153+ models, err := DiscoverModels(context.Background(), cfg, resolver)

154+ require.NoError(t, err)

155+ require.Len(t, models, 1)

156+}

157+

158+func TestDiscoverModels_SkipsEmptyExtraHeaders(t *testing.T) {

159+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

160+ require.Equal(t, "Bearer test-key", r.Header.Get("Authorization"))

161+ require.Empty(t, r.Header.Get("X-Empty"))

162+ w.Header().Set("Content-Type", "application/json")

163+ _, _ = w.Write([]byte(`{"data": [{"id": "m1", "object": "model"}]}`))

164+ }))

165+ defer server.Close()

166+

167+ cfg := Config{

168+ ID: "test",

169+ BaseURL: server.URL + "/v1",

170+ APIKey: "test-key",

171+ ExtraHeaders: map[string]string{

172+ "X-Empty": "$UNSET_VAR",

173+ },

174+ }

175+

176+ resolver := &envResolver{env: map[string]string{

177+ "$UNSET_VAR": "",

178+ }}

179+

180+ models, err := DiscoverModels(context.Background(), cfg, resolver)

181+ require.NoError(t, err)

182+ require.Len(t, models, 1)

183+}

184+

185+func TestDiscoverModels_NoAuthWhenNoAPIKey(t *testing.T) {

186+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

187+ require.Empty(t, r.Header.Get("Authorization"))

188+ w.Header().Set("Content-Type", "application/json")

189+ _, _ = w.Write([]byte(`{"data": [{"id": "m1", "object": "model"}]}`))

190+ }))

191+ defer server.Close()

192+

193+ cfg := Config{

194+ ID: "test",

195+ BaseURL: server.URL + "/v1",

196+ APIKey: "",

197+ }

198+

199+ models, err := DiscoverModels(context.Background(), cfg, &mockResolver{})

200+ require.NoError(t, err)

201+ require.Len(t, models, 1)

202+}

internal/discover/enricher.gonew file+44

@@ -0,0 +1,44 @@

1+package discover

3+import (

4+ "context"

6+ "charm.land/catwalk/pkg/catwalk"

7+)

9+// Enricher fills in model metadata (context window, max tokens, pricing,

10+// etc.) for discovered models. Providers that expose richer endpoints

11+// beyond /v1/models can register an Enricher to populate fields that the

12+// standard listing endpoint omits.

13+type Enricher interface {

14+ // EnrichModels takes a slice of bare discovered models and returns

15+ // them with metadata populated. Implementations should preserve

16+ // existing non-zero fields (user overrides take precedence).

17+ EnrichModels(ctx context.Context, cfg Config, resolver Resolver, models []catwalk.Model) ([]catwalk.Model, error)

18+}

19+

20+// enrichers maps provider type strings to their enrichment

21+// implementations. Each enricher self-registers via init() so that

22+// adding a new provider requires only a new file — no changes to

23+// load.go or any existing enricher.

24+var enrichers = map[string]Enricher{}

25+

26+// RegisterEnricher registers an Enricher for the given provider type.

27+// Called from init() in each enricher implementation file.

28+func RegisterEnricher(providerType string, e Enricher) {

29+ enrichers[providerType] = e

30+}

31+

32+// GetEnricher returns the Enricher for the given provider type, or nil

33+// if no enricher is registered.

34+func GetEnricher(providerType string) Enricher {

35+ return enrichers[providerType]

36+}

37+

38+// IsKnownCustomProvider reports whether the given provider type has a

39+// registered enricher. This signals that the provider type is

40+// recognized and speaks OpenAI-compat protocol, without exposing the

41+// enricher itself to callers that only need the compatibility check.

42+func IsKnownCustomProvider(providerType string) bool {

43+ return enrichers[providerType] != nil

44+}