git.iko.soy/crush
on main help sign in
feat(providers): add llamacpp enricher
c1a4822Kieran Klukas
README.md+17−2
compare with main
unifiedsplit
@@ -771,8 +771,9 @@ To add specific models to the configuration, configure as such:
771### Local Models
771### Local Models
772
772
773Crush can auto-discovers models from local providers. Add a custom provider
773Crush can auto-discovers models from local providers. Add a custom provider
774−with `type` set to `omlx`, `lmstudio`, `litellm`, or `ollama` and leave out
774+with `type` set to `llamacpp`, `omlx`, `lmstudio`, `litellm`, or `ollama`
775−the models list. Crush will populate the model list automatically.
775+and leave out the models list. Crush will populate the model list
776+automatically.
776
777
777```json
778```json
778{
779{
@@ -786,6 +787,20 @@ the models list. Crush will populate the model list automatically.
786}
787}
787```
788```
788
789
790+For llama.cpp (`llama-server`), point at the server's base URL:
791+
792+```json
793+{
794+ "providers": {
795+ "llamacpp": {
796+ "name": "llama.cpp",
797+ "base_url": "http://localhost:2222",
798+ "type": "llamacpp"
799+ }
800+ }
801+}
802+```
803+
789#### Manual Model Configuration
804#### Manual Model Configuration
790
805
791You can still list models explicitly. User-defined models always take
806You can still list models explicitly. User-defined models always take
internal/discover/llamacpp.gonew file+83
@@ -0,0 +1,83 @@
1+package discover
2+
3+import (
4+ "context"
5+ "encoding/json"
6+ "net/http"
7+
8+ "charm.land/catwalk/pkg/catwalk"
9+)
10+
11+func init() {
12+ RegisterEnricher("llamacpp", &llamacppEnricher{})
13+}
14+
15+// llamacppModelsResponse mirrors the response from llama-server's
16+// GET /v1/models endpoint. Unlike the standard OpenAI listing,
17+// llama-server embeds a meta block with model architecture details.
18+type llamacppModelsResponse struct {
19+ Data []llamacppModelEntry `json:"data"`
20+}
21+
22+// llamacppModelEntry is a single entry from /v1/models.
23+type llamacppModelEntry struct {
24+ ID string `json:"id"`
25+ Meta llamacppMeta `json:"meta"`
26+}
27+
28+// llamacppMeta holds per-model architecture metadata exposed by
29+// llama-server in the /v1/models response.
30+type llamacppMeta struct {
31+ NCtx int64 `json:"n_ctx"`
32+ NCtxTrain int64 `json:"n_ctx_train"`
33+ NParams int64 `json:"n_params"`
34+ Size int64 `json:"size"`
35+}
36+
37+// llamacppEnricher fetches model metadata from llama-server's
38+// /v1/models endpoint and populates context window on discovered
39+// models. It prefers n_ctx (the configured runtime context) and
40+// falls back to n_ctx_train (the model's trained maximum).
41+type llamacppEnricher struct{}
42+
43+func (e *llamacppEnricher) EnrichModels(ctx context.Context, cfg Config, resolver Resolver, models []catwalk.Model) ([]catwalk.Model, error) {
44+ resp, err := doRequest(ctx, http.MethodGet, cfg.BaseURL, "/v1/models", cfg.APIKey, cfg.ExtraHeaders, resolver, nil)
45+ if err != nil {
46+ return models, nil
47+ }
48+ defer resp.Body.Close()
49+
50+ if resp.StatusCode != http.StatusOK {
51+ return models, nil
52+ }
53+
54+ var modelsResp llamacppModelsResponse
55+ if err := json.NewDecoder(resp.Body).Decode(&modelsResp); err != nil {
56+ return models, nil
57+ }
58+
59+ // Index by ID for O(1) lookup.
60+ metaByID := make(map[string]llamacppMeta, len(modelsResp.Data))
61+ for _, m := range modelsResp.Data {
62+ metaByID[m.ID] = m.Meta
63+ }
64+
65+ for i := range models {
66+ meta, ok := metaByID[models[i].ID]
67+ if !ok {
68+ continue
69+ }
70+
71+ // Context window: prefer configured n_ctx, fall back to
72+ // the model's trained maximum.
73+ if models[i].ContextWindow == 0 {
74+ if meta.NCtx > 0 {
75+ models[i].ContextWindow = meta.NCtx
76+ } else if meta.NCtxTrain > 0 {
77+ models[i].ContextWindow = meta.NCtxTrain
78+ }
79+ }
80+ }
81+
82+ return models, nil
83+}
internal/discover/llamacpp_test.gonew file+135
@@ -0,0 +1,135 @@
1+package discover
2+
3+import (
4+ "context"
5+ "encoding/json"
6+ "net/http"
7+ "net/http/httptest"
8+ "testing"
9+
10+ "charm.land/catwalk/pkg/catwalk"
11+ "github.com/stretchr/testify/require"
12+)
13+
14+func TestLlamacppEnricher(t *testing.T) {
15+ t.Parallel()
16+
17+ t.Run("populates context window from meta.n_ctx", func(t *testing.T) {
18+ t.Parallel()
19+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
20+ require.Equal(t, "/v1/models", r.URL.Path)
21+ w.Header().Set("Content-Type", "application/json")
22+ json.NewEncoder(w).Encode(llamacppModelsResponse{
23+ Data: []llamacppModelEntry{
24+ {
25+ ID: "ggml-org/Qwen2.5-Coder-1.5B-Instruct-Q8_0-GGUF",
26+ Meta: llamacppMeta{NCtx: 8192, NCtxTrain: 32768},
27+ },
28+ {
29+ ID: "ggml-org/Llama-3.2-3B-Instruct-Q8_0-GGUF",
30+ Meta: llamacppMeta{NCtx: 4096, NCtxTrain: 131072},
31+ },
32+ },
33+ })
34+ }))
35+ defer srv.Close()
36+
37+ cfg := Config{ID: "test-llamacpp", BaseURL: srv.URL}
38+ models := []catwalk.Model{
39+ {ID: "ggml-org/Qwen2.5-Coder-1.5B-Instruct-Q8_0-GGUF"},
40+ {ID: "ggml-org/Llama-3.2-3B-Instruct-Q8_0-GGUF"},
41+ {ID: "unknown-model"},
42+ }
43+
44+ e := &llamacppEnricher{}
45+ result, err := e.EnrichModels(context.Background(), cfg, &mockResolver{}, models)
46+ require.NoError(t, err)
47+ require.Len(t, result, 3)
48+ require.Equal(t, int64(8192), result[0].ContextWindow)
49+ require.Equal(t, int64(4096), result[1].ContextWindow)
50+ require.Equal(t, int64(0), result[2].ContextWindow)
51+ })
52+
53+ t.Run("falls back to n_ctx_train when n_ctx is zero", func(t *testing.T) {
54+ t.Parallel()
55+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
56+ w.Header().Set("Content-Type", "application/json")
57+ json.NewEncoder(w).Encode(llamacppModelsResponse{
58+ Data: []llamacppModelEntry{
59+ {
60+ ID: "m1",
61+ Meta: llamacppMeta{NCtx: 0, NCtxTrain: 131072},
62+ },
63+ },
64+ })
65+ }))
66+ defer srv.Close()
67+
68+ cfg := Config{ID: "test-llamacpp", BaseURL: srv.URL}
69+ models := []catwalk.Model{{ID: "m1"}}
70+
71+ e := &llamacppEnricher{}
72+ result, err := e.EnrichModels(context.Background(), cfg, &mockResolver{}, models)
73+ require.NoError(t, err)
74+ require.Equal(t, int64(131072), result[0].ContextWindow)
75+ })
76+
77+ t.Run("preserves existing non-zero context window", func(t *testing.T) {
78+ t.Parallel()
79+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
80+ w.Header().Set("Content-Type", "application/json")
81+ json.NewEncoder(w).Encode(llamacppModelsResponse{
82+ Data: []llamacppModelEntry{
83+ {
84+ ID: "m1",
85+ Meta: llamacppMeta{NCtx: 8192, NCtxTrain: 32768},
86+ },
87+ },
88+ })
89+ }))
90+ defer srv.Close()
91+
92+ cfg := Config{ID: "test-llamacpp", BaseURL: srv.URL}
93+ models := []catwalk.Model{{ID: "m1", ContextWindow: 65536}}
94+
95+ e := &llamacppEnricher{}
96+ result, err := e.EnrichModels(context.Background(), cfg, &mockResolver{}, models)
97+ require.NoError(t, err)
98+ require.Equal(t, int64(65536), result[0].ContextWindow)
99+ })
100+
101+ t.Run("returns models unchanged on HTTP error", func(t *testing.T) {
102+ t.Parallel()
103+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
104+ w.WriteHeader(http.StatusServiceUnavailable)
105+ }))
106+ defer srv.Close()
107+
108+ cfg := Config{ID: "test-llamacpp", BaseURL: srv.URL}
109+ models := []catwalk.Model{{ID: "m1"}}
110+
111+ e := &llamacppEnricher{}
112+ result, err := e.EnrichModels(context.Background(), cfg, &mockResolver{}, models)
113+ require.NoError(t, err)
114+ require.Len(t, result, 1)
115+ require.Equal(t, int64(0), result[0].ContextWindow)
116+ })
117+
118+ t.Run("returns models unchanged on invalid JSON", func(t *testing.T) {
119+ t.Parallel()
120+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
121+ w.Header().Set("Content-Type", "application/json")
122+ w.Write([]byte("not json"))
123+ }))
124+ defer srv.Close()
125+
126+ cfg := Config{ID: "test-llamacpp", BaseURL: srv.URL}
127+ models := []catwalk.Model{{ID: "m1"}}
128+
129+ e := &llamacppEnricher{}
130+ result, err := e.EnrichModels(context.Background(), cfg, &mockResolver{}, models)
131+ require.NoError(t, err)
132+ require.Len(t, result, 1)
133+ require.Equal(t, int64(0), result[0].ContextWindow)
134+ })
135+}
internal/skills/builtin/crush-config/SKILL.md+1−1
@@ -130,7 +130,7 @@ reviewed.
130}
130}
131```
131```
132
132
133−- `type` (required): `openai`, `openai-compat`, or `anthropic`
133+- `type` (required): `openai`, `openai-compat`, `anthropic`, or a local provider type (`llamacpp`, `omlx`, `lmstudio`, `litellm`, `ollama`)
134- `api_key`, `base_url`, `api_endpoint`, and `extra_headers` are shell-expanded (see [Shell Expansion](#shell-expansion)).
134- `api_key`, `base_url`, `api_endpoint`, and `extra_headers` are shell-expanded (see [Shell Expansion](#shell-expansion)).
135- `extra_body` is a JSON passthrough and is **not** expanded.
135- `extra_body` is a JSON passthrough and is **not** expanded.
136- Additional fields: `disable`, `system_prompt_prefix`, `extra_headers`, `extra_body`, `provider_options`.
136- Additional fields: `disable`, `system_prompt_prefix`, `extra_headers`, `extra_body`, `provider_options`.