ea86101 · crush · git.iko.soy

git.iko.soy/crush

on main help sign in

feat(hyper): use openai-compatible endpoint for hyper (#2640)

ea86101Andrey Nering6 months ago

internal/agent/agent.go+4−2

compare with main

unifiedsplit

@@ -15,6 +15,7 @@ import (

15 "errors"

16 "fmt"

17 "log/slog"

18+ "net/http"

18 "os"

19 "os"

19 "regexp"

20 "regexp"

20 "strconv"

21 "strconv"

@@ -461,6 +462,7 @@ func (a *sessionAgent) Run(ctx context.Context, call SessionAgentCall) (*fantasy

461 a.eventPromptResponded(call.SessionID, time.Since(startTime).Truncate(time.Second))

462 a.eventPromptResponded(call.SessionID, time.Since(startTime).Truncate(time.Second))

462

463

463 if err != nil {

464 if err != nil {

465+ isHyper := largeModel.ModelCfg.Provider == hyper.Name

464 isCancelErr := errors.Is(err, context.Canceled)

466 isCancelErr := errors.Is(err, context.Canceled)

465 isPermissionErr := errors.Is(err, permission.ErrorPermissionDenied)

467 isPermissionErr := errors.Is(err, permission.ErrorPermissionDenied)

466 if currentAssistant == nil {

468 if currentAssistant == nil {

@@ -532,7 +534,7 @@ func (a *sessionAgent) Run(ctx context.Context, call SessionAgentCall) (*fantasy

532 currentAssistant.AddFinish(message.FinishReasonCanceled, "User canceled request", "")

534 currentAssistant.AddFinish(message.FinishReasonCanceled, "User canceled request", "")

533 } else if isPermissionErr {

535 } else if isPermissionErr {

534 currentAssistant.AddFinish(message.FinishReasonPermissionDenied, "User denied permission", "")

536 currentAssistant.AddFinish(message.FinishReasonPermissionDenied, "User denied permission", "")

535− } else if errors.Is(err, hyper.ErrUnauthorized) {

537+ } else if isHyper && errors.As(err, &providerErr) && providerErr.StatusCode == http.StatusUnauthorized {

536 currentAssistant.AddFinish(message.FinishReasonError, "Unauthorized", `Please re-authenticate with Hyper. You can also run "crush auth" to re-authenticate.`)

538 currentAssistant.AddFinish(message.FinishReasonError, "Unauthorized", `Please re-authenticate with Hyper. You can also run "crush auth" to re-authenticate.`)

537 if a.notify != nil {

539 if a.notify != nil {

538 a.notify.Publish(pubsub.CreatedEvent, notify.Notification{

540 a.notify.Publish(pubsub.CreatedEvent, notify.Notification{

@@ -542,7 +544,7 @@ func (a *sessionAgent) Run(ctx context.Context, call SessionAgentCall) (*fantasy

542 ProviderID: largeModel.ModelCfg.Provider,

544 ProviderID: largeModel.ModelCfg.Provider,

543 })

545 })

544 }

546 }

545− } else if errors.Is(err, hyper.ErrNoCredits) {

547+ } else if isHyper && errors.As(err, &providerErr) && providerErr.StatusCode == http.StatusPaymentRequired {

546 url := hyper.BaseURL()

548 url := hyper.BaseURL()

547 link := linkStyle.Hyperlink(url, "id=hyper").Render(url)

549 link := linkStyle.Hyperlink(url, "id=hyper").Render(url)

548 currentAssistant.AddFinish(message.FinishReasonError, "No credits", "You're out of credits. Add more at "+link)

550 currentAssistant.AddFinish(message.FinishReasonError, "No credits", "You're out of credits. Add more at "+link)

internal/agent/coordinator.go+8−17

@@ -23,6 +23,7 @@ import (

23 "github.com/charmbracelet/crush/internal/agent/prompt"

24 "github.com/charmbracelet/crush/internal/agent/tools"

25 "github.com/charmbracelet/crush/internal/config"

26+ "github.com/charmbracelet/crush/internal/event"

26 "github.com/charmbracelet/crush/internal/filetracker"

27 "github.com/charmbracelet/crush/internal/filetracker"

27 "github.com/charmbracelet/crush/internal/history"

28 "github.com/charmbracelet/crush/internal/history"

28 "github.com/charmbracelet/crush/internal/home"

29 "github.com/charmbracelet/crush/internal/home"

@@ -797,17 +798,6 @@ func (c *coordinator) buildGoogleVertexProvider(headers map[string]string, optio

797 return google.New(opts...)

798 return google.New(opts...)

798}

799}

799

800

800−func (c *coordinator) buildHyperProvider(apiKey string) (fantasy.Provider, error) {

801− opts := []hyper.Option{

802− hyper.WithAPIKey(apiKey),

803− }

804− if c.cfg.Config().Options.Debug {

805− httpClient := log.NewHTTPClient()

806− opts = append(opts, hyper.WithHTTPClient(httpClient))

807− }

808− return hyper.New(opts...)

809−}

810−

811func (c *coordinator) isAnthropicThinking(model config.SelectedModel) bool {

801func (c *coordinator) isAnthropicThinking(model config.SelectedModel) bool {

812 if model.Think {

802 if model.Think {

813 return true

803 return true

@@ -851,16 +841,18 @@ func (c *coordinator) buildProvider(providerCfg config.ProviderConfig, model con

851 return c.buildGoogleProvider(baseURL, apiKey, headers)

841 return c.buildGoogleProvider(baseURL, apiKey, headers)

852 case "google-vertex":

842 case "google-vertex":

853 return c.buildGoogleVertexProvider(headers, providerCfg.ExtraParams)

843 return c.buildGoogleVertexProvider(headers, providerCfg.ExtraParams)

854− case openaicompat.Name:

844+ case openaicompat.Name, hyper.Name:

845+ switch providerCfg.ID {

846+ case hyper.Name:

847+ baseURL = hyper.BaseURL() + "/v1"

848+ headers["x-crush-id"] = event.GetID()

855− if providerCfg.ID == string(catwalk.InferenceProviderZAI) {

849+ case string(catwalk.InferenceProviderZAI):

856 if providerCfg.ExtraBody == nil {

850 if providerCfg.ExtraBody == nil {

857 providerCfg.ExtraBody = map[string]any{}

851 providerCfg.ExtraBody = map[string]any{}

858 }

852 }

859 providerCfg.ExtraBody["tool_stream"] = true

853 providerCfg.ExtraBody["tool_stream"] = true

860 }

854 }

861 return c.buildOpenaiCompatProvider(baseURL, apiKey, headers, providerCfg.ExtraBody, providerCfg.ID, isSubAgent)

855 return c.buildOpenaiCompatProvider(baseURL, apiKey, headers, providerCfg.ExtraBody, providerCfg.ID, isSubAgent)

862− case hyper.Name:

863− return c.buildHyperProvider(apiKey)

864 default:

856 default:

865 return nil, fmt.Errorf("provider type not supported: %q", providerCfg.Type)

857 return nil, fmt.Errorf("provider type not supported: %q", providerCfg.Type)

866 }

858 }

@@ -940,8 +932,7 @@ func (c *coordinator) Summarize(ctx context.Context, sessionID string) error {

940

932

941func (c *coordinator) isUnauthorized(err error) bool {

933func (c *coordinator) isUnauthorized(err error) bool {

942 var providerErr *fantasy.ProviderError

934 var providerErr *fantasy.ProviderError

943− return (errors.As(err, &providerErr) && providerErr.StatusCode == http.StatusUnauthorized) ||

935+ return errors.As(err, &providerErr) && providerErr.StatusCode == http.StatusUnauthorized

944− errors.Is(err, hyper.ErrUnauthorized)

945}

936}

946

937

947func (c *coordinator) refreshOAuth2Token(ctx context.Context, providerCfg config.ProviderConfig) error {

938func (c *coordinator) refreshOAuth2Token(ctx context.Context, providerCfg config.ProviderConfig) error {

internal/agent/hyper/provider.go−285

@@ -2,29 +2,15 @@

2package hyper

4import (

5− "bufio"

6− "bytes"

7 "cmp"

5 "cmp"

8− "context"

9 _ "embed"

6 _ "embed"

10 "encoding/json"

7 "encoding/json"

11− "errors"

12− "fmt"

13− "io"

14 "log/slog"

8 "log/slog"

15− "maps"

16− "net/http"

17− "net/url"

18 "os"

9 "os"

19 "strconv"

10 "strconv"

20− "strings"

21 "sync"

11 "sync"

22− "time"

24 "charm.land/catwalk/pkg/catwalk"

13 "charm.land/catwalk/pkg/catwalk"

25− "charm.land/fantasy"

26− "charm.land/fantasy/object"

27− "github.com/charmbracelet/crush/internal/event"

28)

14)

30//go:generate wget -O provider.json https://hyper.charm.land/api/v1/provider

16//go:generate wget -O provider.json https://hyper.charm.land/api/v1/provider

@@ -68,274 +54,3 @@ const (

68var BaseURL = sync.OnceValue(func() string {

54var BaseURL = sync.OnceValue(func() string {

69 return cmp.Or(os.Getenv("HYPER_URL"), defaultBaseURL)

55 return cmp.Or(os.Getenv("HYPER_URL"), defaultBaseURL)

70})

56})

71−

72−var (

73− ErrNoCredits = errors.New("you're out of credits")

74− ErrUnauthorized = errors.New("unauthorized")

75−)

76−

77−type options struct {

78− baseURL string

79− apiKey string

80− name string

81− headers map[string]string

82− client *http.Client

83−}

84−

85−// Option configures the proxy provider.

86−type Option = func(*options)

87−

88−// New creates a new proxy provider.

89−func New(opts ...Option) (fantasy.Provider, error) {

90− o := options{

91− baseURL: BaseURL() + "/api/v1/fantasy",

92− name: Name,

93− headers: map[string]string{

94− "x-crush-id": event.GetID(),

95− },

96− client: &http.Client{Timeout: 0}, // stream-safe

97− }

98− for _, opt := range opts {

99− opt(&o)

100− }

101− return &provider{options: o}, nil

102−}

103−

104−// WithBaseURL sets the proxy base URL (e.g. http://localhost:8080).

105−func WithBaseURL(url string) Option { return func(o *options) { o.baseURL = url } }

106−

107−// WithName sets the provider name.

108−func WithName(name string) Option { return func(o *options) { o.name = name } }

109−

110−// WithHeaders sets extra headers sent to the proxy.

111−func WithHeaders(headers map[string]string) Option {

112− return func(o *options) {

113− maps.Copy(o.headers, headers)

114− }

115−}

116−

117−// WithHTTPClient sets custom HTTP client.

118−func WithHTTPClient(c *http.Client) Option { return func(o *options) { o.client = c } }

119−

120−// WithAPIKey sets the API key.

121−func WithAPIKey(key string) Option {

122− return func(o *options) {

123− o.apiKey = key

124− }

125−}

126−

127−type provider struct{ options options }

128−

129−func (p *provider) Name() string { return p.options.name }

130−

131−// LanguageModel implements fantasy.Provider.

132−func (p *provider) LanguageModel(_ context.Context, modelID string) (fantasy.LanguageModel, error) {

133− if modelID == "" {

134− return nil, errors.New("missing model id")

135− }

136− return &languageModel{modelID: modelID, provider: p.options.name, opts: p.options}, nil

137−}

138−

139−type languageModel struct {

140− provider string

141− modelID string

142− opts options

143−}

144−

145−// GenerateObject implements fantasy.LanguageModel.

146−func (m *languageModel) GenerateObject(ctx context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {

147− return object.GenerateWithTool(ctx, m, call)

148−}

149−

150−// StreamObject implements fantasy.LanguageModel.

151−func (m *languageModel) StreamObject(ctx context.Context, call fantasy.ObjectCall) (fantasy.ObjectStreamResponse, error) {

152− return object.StreamWithTool(ctx, m, call)

153−}

154−

155−func (m *languageModel) Provider() string { return m.provider }

156−func (m *languageModel) Model() string { return m.modelID }

157−

158−// Generate implements fantasy.LanguageModel by calling the proxy JSON endpoint.

159−func (m *languageModel) Generate(ctx context.Context, call fantasy.Call) (*fantasy.Response, error) {

160− resp, err := m.doRequest(ctx, false, call)

161− if err != nil {

162− return nil, err

163− }

164− defer func() { _ = resp.Body.Close() }()

165− if resp.StatusCode < 200 || resp.StatusCode >= 300 {

166− b, _ := ioReadAllLimit(resp.Body, 64*1024)

167− return nil, fmt.Errorf("proxy generate error: %s", strings.TrimSpace(string(b)))

168− }

169− var out fantasy.Response

170− if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {

171− return nil, err

172− }

173− return &out, nil

174−}

175−

176−// Stream implements fantasy.LanguageModel using SSE from the proxy.

177−func (m *languageModel) Stream(ctx context.Context, call fantasy.Call) (fantasy.StreamResponse, error) {

178− // Prefer explicit /stream endpoint

179− resp, err := m.doRequest(ctx, true, call)

180− if err != nil {

181− return nil, err

182− }

183− switch resp.StatusCode {

184− case http.StatusTooManyRequests:

185− _ = resp.Body.Close()

186− return nil, toProviderError(resp, retryAfter(resp))

187− case http.StatusUnauthorized:

188− _ = resp.Body.Close()

189− return nil, ErrUnauthorized

190− case http.StatusPaymentRequired:

191− _ = resp.Body.Close()

192− return nil, ErrNoCredits

193− }

194−

195− if resp.StatusCode < 200 || resp.StatusCode >= 300 {

196− defer func() { _ = resp.Body.Close() }()

197− b, _ := ioReadAllLimit(resp.Body, 64*1024)

198− return nil, &fantasy.ProviderError{

199− Title: "Stream Error",

200− Message: strings.TrimSpace(string(b)),

201− StatusCode: resp.StatusCode,

202− }

203− }

204−

205− return func(yield func(fantasy.StreamPart) bool) {

206− defer func() { _ = resp.Body.Close() }()

207− scanner := bufio.NewScanner(resp.Body)

208− buf := make([]byte, 0, 64*1024)

209− scanner.Buffer(buf, 4*1024*1024)

210−

211− var (

212− event string

213− dataBuf bytes.Buffer

214− sawFinish bool

215− dispatch = func() bool {

216− if dataBuf.Len() == 0 || event == "" {

217− dataBuf.Reset()

218− event = ""

219− return true

220− }

221− var part fantasy.StreamPart

222− if err := json.Unmarshal(dataBuf.Bytes(), &part); err != nil {

223− return yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeError, Error: err})

224− }

225− if part.Type == fantasy.StreamPartTypeFinish {

226− sawFinish = true

227− }

228− ok := yield(part)

229− dataBuf.Reset()

230− event = ""

231− return ok

232− }

233− )

234−

235− for scanner.Scan() {

236− line := scanner.Text()

237− if line == "" { // event boundary

238− if !dispatch() {

239− return

240− }

241− continue

242− }

243− if strings.HasPrefix(line, ":") { // comment / ping

244− continue

245− }

246− if strings.HasPrefix(line, "event: ") {

247− event = strings.TrimSpace(line[len("event: "):])

248− continue

249− }

250− if strings.HasPrefix(line, "data: ") {

251− if dataBuf.Len() > 0 {

252− dataBuf.WriteByte('\n')

253− }

254− dataBuf.WriteString(line[len("data: "):])

255− continue

256− }

257− }

258− if err := scanner.Err(); err != nil {

259− if sawFinish && (errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)) {

260− // If we already saw an explicit finish event, treat cancellation as a no-op.

261− } else {

262− _ = yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeError, Error: err})

263− return

264− }

265− }

266− if err := ctx.Err(); err != nil && !sawFinish {

267− _ = yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeError, Error: err})

268− return

269− }

270− // flush any pending data

271− _ = dispatch()

272− if !sawFinish {

273− _ = yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeFinish})

274− }

275− }, nil

276−}

277−

278−func (m *languageModel) doRequest(ctx context.Context, stream bool, call fantasy.Call) (*http.Response, error) {

279− addr, err := url.Parse(m.opts.baseURL)

280− if err != nil {

281− return nil, err

282− }

283− addr = addr.JoinPath(m.modelID)

284− if stream {

285− addr = addr.JoinPath("stream")

286− } else {

287− addr = addr.JoinPath("generate")

288− }

289−

290− body, err := json.Marshal(call)

291− if err != nil {

292− return nil, err

293− }

294−

295− req, err := http.NewRequestWithContext(ctx, http.MethodPost, addr.String(), bytes.NewReader(body))

296− if err != nil {

297− return nil, err

298− }

299− req.Header.Set("Content-Type", "application/json")

300− if stream {

301− req.Header.Set("Accept", "text/event-stream")

302− } else {

303− req.Header.Set("Accept", "application/json")

304− }

305− for k, v := range m.opts.headers {

306− req.Header.Set(k, v)

307− }

308−

309− if m.opts.apiKey != "" {

310− req.Header.Set("Authorization", m.opts.apiKey)

311− }

312− return m.opts.client.Do(req)

313−}

314−

315−// ioReadAllLimit reads up to n bytes.

316−func ioReadAllLimit(r io.Reader, n int64) ([]byte, error) {

317− var b bytes.Buffer

318− if n <= 0 {

319− n = 1 << 20

320− }

321− lr := &io.LimitedReader{R: r, N: n}

322− _, err := b.ReadFrom(lr)

323− return b.Bytes(), err

324−}

325−

326−func toProviderError(resp *http.Response, message string) error {

327− return &fantasy.ProviderError{

328− Title: fantasy.ErrorTitleForStatusCode(resp.StatusCode),

329− Message: message,

330− StatusCode: resp.StatusCode,

331− }

332−}

333−

334−func retryAfter(resp *http.Response) string {

335− after, err := strconv.Atoi(resp.Header.Get("Retry-After"))

336− if err == nil && after > 0 {

337− d := time.Duration(after) * time.Second

338− return "Try again in " + d.String()

339− }

340− return "Try again later"

341−}