4097732 · crush · git.iko.soy

git.iko.soy/crush

on main help sign in

perf: improve startup, specifically provider updates (#1577)

4097732Carlos Alexandro Becker1 year ago

go.mod+2−2

compare with main

unifiedsplit

@@ -1,6 +1,6 @@

1module github.com/charmbracelet/crush

3−go 1.25.0

3+go 1.25.5

5require (

6 charm.land/bubbles/v2 v2.0.0-rc.1

@@ -16,7 +16,7 @@ require (

16 github.com/aymanbagabas/go-udiff v0.3.1

17 github.com/bmatcuk/doublestar/v4 v4.9.1

18 github.com/charlievieth/fastwalk v1.0.14

19− github.com/charmbracelet/catwalk v0.9.5

19+ github.com/charmbracelet/catwalk v0.9.7-0.20251208190755-350e2a004c74

20 github.com/charmbracelet/colorprofile v0.3.3

21 github.com/charmbracelet/fang v0.4.4

22 github.com/charmbracelet/glamour/v2 v2.0.0-20251106195642-800eb8175930

go.sum+2−2

@@ -88,8 +88,8 @@ github.com/charlievieth/fastwalk v1.0.14 h1:3Eh5uaFGwHZd8EGwTjJnSpBkfwfsak9h6ICg

88github.com/charlievieth/fastwalk v1.0.14/go.mod h1:diVcUreiU1aQ4/Wu3NbxxH4/KYdKpLDojrQ1Bb2KgNY=

89github.com/charmbracelet/anthropic-sdk-go v0.0.0-20251024181547-21d6f3d9a904 h1:rwLdEpG9wE6kL69KkEKDiWprO8pQOZHZXeod6+9K+mw=

90github.com/charmbracelet/anthropic-sdk-go v0.0.0-20251024181547-21d6f3d9a904/go.mod h1:8TIYxZxsuCqqeJ0lga/b91tBwrbjoHDC66Sq5t8N2R4=

91−github.com/charmbracelet/catwalk v0.9.5 h1:QLqajLJfjGTVh2MIVIdAhww2XvPklxmu+p0Z4wrT7KU=

91+github.com/charmbracelet/catwalk v0.9.7-0.20251208190755-350e2a004c74 h1:pJI8ivIgSWOeCNnZFXeZzL6px1vZVS3LU4Cqk3Gx37I=

92−github.com/charmbracelet/catwalk v0.9.5/go.mod h1:ReU4SdrLfe63jkEjWMdX2wlZMV3k9r11oQAmzN0m+KY=

92+github.com/charmbracelet/catwalk v0.9.7-0.20251208190755-350e2a004c74/go.mod h1:ReU4SdrLfe63jkEjWMdX2wlZMV3k9r11oQAmzN0m+KY=

93github.com/charmbracelet/colorprofile v0.3.3 h1:DjJzJtLP6/NZ8p7Cgjno0CKGr7wwRJGxWUwh2IyhfAI=

94github.com/charmbracelet/colorprofile v0.3.3/go.mod h1:nB1FugsAbzq284eJcjfah2nhdSLppN2NqvfotkfRYP4=

95github.com/charmbracelet/fang v0.4.4 h1:G4qKxF6or/eTPgmAolwPuRNyuci3hTUGGX1rj1YkHJY=

internal/cmd/update_providers.go+3−3

@@ -31,12 +31,12 @@ crush update-providers embedded

31 // NOTE(@andreynering): We want to skip logging output do stdout here.

32 slog.SetDefault(slog.New(slog.DiscardHandler))

34− var pathOrUrl string

34+ var pathOrURL string

35 if len(args) > 0 {

36− pathOrUrl = args[0]

36+ pathOrURL = args[0]

37 }

39− if err := config.UpdateProviders(pathOrUrl); err != nil {

39+ if err := config.UpdateProviders(pathOrURL); err != nil {

40 return err

41 }

internal/config/provider.go+56−53

@@ -2,7 +2,9 @@ package config

3import (

4 "cmp"

5+ "context"

5 "encoding/json"

6 "encoding/json"

7+ "errors"

6 "fmt"

8 "fmt"

7 "log/slog"

9 "log/slog"

8 "os"

10 "os"

@@ -17,7 +19,7 @@ import (

17)

19)

19type ProviderClient interface {

21type ProviderClient interface {

20− GetProviders() ([]catwalk.Provider, error)

22+ GetProviders(context.Context, string) ([]catwalk.Provider, error)

21}

23}

23var (

25var (

@@ -53,7 +55,7 @@ func saveProvidersInCache(path string, providers []catwalk.Provider) error {

53 return fmt.Errorf("failed to create directory for provider cache: %w", err)

55 return fmt.Errorf("failed to create directory for provider cache: %w", err)

54 }

56 }

56− data, err := json.MarshalIndent(providers, "", " ")

58+ data, err := json.Marshal(providers)

57 if err != nil {

59 if err != nil {

58 return fmt.Errorf("failed to marshal provider data: %w", err)

60 return fmt.Errorf("failed to marshal provider data: %w", err)

59 }

61 }

@@ -64,34 +66,35 @@ func saveProvidersInCache(path string, providers []catwalk.Provider) error {

64 return nil

66 return nil

65}

67}

67−func loadProvidersFromCache(path string) ([]catwalk.Provider, error) {

69+func loadProvidersFromCache(path string) ([]catwalk.Provider, string, error) {

68 data, err := os.ReadFile(path)

70 data, err := os.ReadFile(path)

69 if err != nil {

71 if err != nil {

70− return nil, fmt.Errorf("failed to read provider cache file: %w", err)

72+ return nil, "", fmt.Errorf("failed to read provider cache file: %w", err)

71 }

73 }

73 var providers []catwalk.Provider

75 var providers []catwalk.Provider

74 if err := json.Unmarshal(data, &providers); err != nil {

76 if err := json.Unmarshal(data, &providers); err != nil {

75− return nil, fmt.Errorf("failed to unmarshal provider data from cache: %w", err)

77+ return nil, "", fmt.Errorf("failed to unmarshal provider data from cache: %w", err)

76 }

78 }

79+

77− return providers, nil

80+ return providers, catwalk.Etag(data), nil

78}

81}

80−func UpdateProviders(pathOrUrl string) error {

83+func UpdateProviders(pathOrURL string) error {

81 var providers []catwalk.Provider

84 var providers []catwalk.Provider

82− pathOrUrl = cmp.Or(pathOrUrl, os.Getenv("CATWALK_URL"), defaultCatwalkURL)

85+ pathOrURL = cmp.Or(pathOrURL, os.Getenv("CATWALK_URL"), defaultCatwalkURL)

84 switch {

87 switch {

85− case pathOrUrl == "embedded":

88+ case pathOrURL == "embedded":

86 providers = embedded.GetAll()

89 providers = embedded.GetAll()

87− case strings.HasPrefix(pathOrUrl, "http://") || strings.HasPrefix(pathOrUrl, "https://"):

90+ case strings.HasPrefix(pathOrURL, "http://") || strings.HasPrefix(pathOrURL, "https://"):

88 var err error

91 var err error

89− providers, err = catwalk.NewWithURL(pathOrUrl).GetProviders()

92+ providers, err = catwalk.NewWithURL(pathOrURL).GetProviders(context.Background(), "")

90 if err != nil {

93 if err != nil {

91 return fmt.Errorf("failed to fetch providers from Catwalk: %w", err)

94 return fmt.Errorf("failed to fetch providers from Catwalk: %w", err)

92 }

95 }

93 default:

96 default:

94− content, err := os.ReadFile(pathOrUrl)

97+ content, err := os.ReadFile(pathOrURL)

95 if err != nil {

98 if err != nil {

96 return fmt.Errorf("failed to read file: %w", err)

99 return fmt.Errorf("failed to read file: %w", err)

97 }

100 }

@@ -108,61 +111,61 @@ func UpdateProviders(pathOrUrl string) error {

108 return fmt.Errorf("failed to save providers to cache: %w", err)

111 return fmt.Errorf("failed to save providers to cache: %w", err)

109 }

112 }

110

113

111− slog.Info("Providers updated successfully", "count", len(providers), "from", pathOrUrl, "to", cachePath)

114+ slog.Info("Providers updated successfully", "count", len(providers), "from", pathOrURL, "to", cachePath)

112 return nil

115 return nil

113}

116}

114

117

118+// Providers returns the list of providers, taking into account cached results

119+// and whether or not auto update is enabled.

120+//

121+// It will:

122+// 1. if auto update is disabled, it'll return the embedded providers at the

123+// time of release.

124+// 2. load the cached providers

125+// 3. try to get the fresh list of providers, and return either this new list,

126+// the cached list, or the embedded list if all others fail.

115func Providers(cfg *Config) ([]catwalk.Provider, error) {

127func Providers(cfg *Config) ([]catwalk.Provider, error) {

116 providerOnce.Do(func() {

128 providerOnce.Do(func() {

117 catwalkURL := cmp.Or(os.Getenv("CATWALK_URL"), defaultCatwalkURL)

129 catwalkURL := cmp.Or(os.Getenv("CATWALK_URL"), defaultCatwalkURL)

118 client := catwalk.NewWithURL(catwalkURL)

130 client := catwalk.NewWithURL(catwalkURL)

119 path := providerCacheFileData()

131 path := providerCacheFileData()

120

132

121− autoUpdateDisabled := cfg.Options.DisableProviderAutoUpdate

133+ if cfg.Options.DisableProviderAutoUpdate {

134+ slog.Info("Using embedded Catwalk providers")

122− providerList, providerErr = loadProviders(autoUpdateDisabled, client, path)

135+ providerList, providerErr = embedded.GetAll(), nil

123− })

136+ return

124− return providerList, providerErr

125−}

126−

127−func loadProviders(autoUpdateDisabled bool, client ProviderClient, path string) ([]catwalk.Provider, error) {

128− catwalkGetAndSave := func() ([]catwalk.Provider, error) {

129− providers, err := client.GetProviders()

130− if err != nil {

131− return nil, fmt.Errorf("failed to fetch providers from catwalk: %w", err)

132 }

137 }

133− if len(providers) == 0 {

134− return nil, fmt.Errorf("empty providers list from catwalk")

135− }

136− if err := saveProvidersInCache(path, providers); err != nil {

137− return nil, err

138− }

139− return providers, nil

140− }

141−

142− switch {

143− case autoUpdateDisabled:

144− slog.Warn("Providers auto-update is disabled")

145

138

146− if _, err := os.Stat(path); err == nil {

139+ cached, etag, cachedErr := loadProvidersFromCache(path)

147− slog.Warn("Using locally cached providers")

140+ if len(cached) == 0 || cachedErr != nil {

148− return loadProvidersFromCache(path)

141+ // if cached file is empty, default to embedded providers

142+ cached = embedded.GetAll()

149 }

143 }

150

144

151− slog.Warn("Saving embedded providers to cache")

145+ providerList, providerErr = loadProviders(client, etag, path)

152− providers := embedded.GetAll()

146+ if errors.Is(providerErr, catwalk.ErrNotModified) {

153− if err := saveProvidersInCache(path, providers); err != nil {

147+ slog.Info("Catwalk providers not modified")

154− return nil, err

148+ providerList, providerErr = cached, nil

155 }

149 }

156− return providers, nil

150+ })

157−

151+ if providerErr != nil {

158− default:

152+ catwalkURL := fmt.Sprintf("%s/v2/providers", cmp.Or(os.Getenv("CATWALK_URL"), defaultCatwalkURL))

159− slog.Info("Fetching providers from Catwalk.", "path", path)

153+ return nil, fmt.Errorf("Crush was unable to fetch an updated list of providers from %s. Consider setting CRUSH_DISABLE_PROVIDER_AUTO_UPDATE=1 to use the embedded providers bundled at the time of this Crush release. You can also update providers manually. For more info see crush update-providers --help.\n\nCause: %w", catwalkURL, providerErr) //nolint:staticcheck

154+ }

155+ return providerList, nil

156+}

160

157

161− providers, err := catwalkGetAndSave()

158+func loadProviders(client ProviderClient, etag, path string) ([]catwalk.Provider, error) {

159+ slog.Info("Fetching providers from Catwalk.", "path", path)

160+ providers, err := client.GetProviders(context.Background(), etag)

162− if err != nil {

161+ if err != nil {

163− catwalkUrl := fmt.Sprintf("%s/v2/providers", cmp.Or(os.Getenv("CATWALK_URL"), defaultCatwalkURL))

162+ return nil, fmt.Errorf("failed to fetch providers from catwalk: %w", err)

164− return nil, fmt.Errorf("Crush was unable to fetch an updated list of providers from %s. Consider setting CRUSH_DISABLE_PROVIDER_AUTO_UPDATE=1 to use the embedded providers bundled at the time of this Crush release. You can also update providers manually. For more info see crush update-providers --help. %w", catwalkUrl, err) //nolint:staticcheck

165− }

163+ }

166− return providers, nil

164+ if len(providers) == 0 {

165+ return nil, errors.New("empty providers list from catwalk")

167 }

166 }

167+ if err := saveProvidersInCache(path, providers); err != nil {

168+ return nil, err

169+ }

170+ return providers, nil

168}

171}

internal/config/provider_empty_test.go+7−23

@@ -1,8 +1,7 @@

1package config

3import (

4− "encoding/json"

4+ "context"

5− "os"

6 "testing"

5 "testing"

8 "github.com/charmbracelet/catwalk/pkg/catwalk"

7 "github.com/charmbracelet/catwalk/pkg/catwalk"

@@ -11,37 +10,22 @@ import (

12type emptyProviderClient struct{}

11type emptyProviderClient struct{}

14−func (m *emptyProviderClient) GetProviders() ([]catwalk.Provider, error) {

13+func (m *emptyProviderClient) GetProviders(context.Context, string) ([]catwalk.Provider, error) {

15 return []catwalk.Provider{}, nil

14 return []catwalk.Provider{}, nil

16}

15}

17+// TestProvider_loadProvidersEmptyResult tests that loadProviders returns an

18+// error when the client returns an empty list. This ensures we don't cache

19+// empty provider lists.

18func TestProvider_loadProvidersEmptyResult(t *testing.T) {

20func TestProvider_loadProvidersEmptyResult(t *testing.T) {

19 client := &emptyProviderClient{}

21 client := &emptyProviderClient{}

20 tmpPath := t.TempDir() + "/providers.json"

22 tmpPath := t.TempDir() + "/providers.json"

22− providers, err := loadProviders(false, client, tmpPath)

24+ providers, err := loadProviders(client, "", tmpPath)

23− require.Contains(t, err.Error(), "Crush was unable to fetch an updated list of providers")

25+ require.Contains(t, err.Error(), "empty providers list from catwalk")

24 require.Empty(t, providers)

26 require.Empty(t, providers)

25 require.Len(t, providers, 0)

27 require.Len(t, providers, 0)

27 // Check that no cache file was created for empty results

29 // Check that no cache file was created for empty results

28 require.NoFileExists(t, tmpPath, "Cache file should not exist for empty results")

30 require.NoFileExists(t, tmpPath, "Cache file should not exist for empty results")

29}

31}

30−

31−func TestProvider_loadProvidersEmptyCache(t *testing.T) {

32− client := &mockProviderClient{shouldFail: false}

33− tmpPath := t.TempDir() + "/providers.json"

34−

35− // Create an empty cache file

36− emptyProviders := []catwalk.Provider{}

37− data, err := json.Marshal(emptyProviders)

38− require.NoError(t, err)

39− require.NoError(t, os.WriteFile(tmpPath, data, 0o644))

40−

41− // Should refresh and get real providers instead of using empty cache

42− providers, err := loadProviders(false, client, tmpPath)

43− require.NoError(t, err)

44− require.NotNil(t, providers)

45− require.Len(t, providers, 1)

46− require.Equal(t, "Mock", providers[0].Name)

47−}

internal/config/provider_test.go+91−20

@@ -1,9 +1,11 @@

1package config

3import (

4+ "context"

4 "encoding/json"

5 "encoding/json"

5 "errors"

6 "errors"

6 "os"

7 "os"

8+ "sync"

7 "testing"

9 "testing"

9 "github.com/charmbracelet/catwalk/pkg/catwalk"

11 "github.com/charmbracelet/catwalk/pkg/catwalk"

@@ -11,10 +13,14 @@ import (

11)

13)

13type mockProviderClient struct {

15type mockProviderClient struct {

14− shouldFail bool

16+ shouldFail bool

17+ shouldReturnErr error

15}

18}

17−func (m *mockProviderClient) GetProviders() ([]catwalk.Provider, error) {

20+func (m *mockProviderClient) GetProviders(context.Context, string) ([]catwalk.Provider, error) {

21+ if m.shouldReturnErr != nil {

22+ return nil, m.shouldReturnErr

23+ }

18 if m.shouldFail {

24 if m.shouldFail {

19 return nil, errors.New("failed to load providers")

25 return nil, errors.New("failed to load providers")

20 }

26 }

@@ -25,10 +31,16 @@ func (m *mockProviderClient) GetProviders() ([]catwalk.Provider, error) {

25 }, nil

31 }, nil

26}

32}

34+func resetProviderState() {

35+ providerOnce = sync.Once{}

36+ providerList = nil

37+ providerErr = nil

38+}

39+

28func TestProvider_loadProvidersNoIssues(t *testing.T) {

40func TestProvider_loadProvidersNoIssues(t *testing.T) {

29 client := &mockProviderClient{shouldFail: false}

41 client := &mockProviderClient{shouldFail: false}

30 tmpPath := t.TempDir() + "/providers.json"

42 tmpPath := t.TempDir() + "/providers.json"

31− providers, err := loadProviders(false, client, tmpPath)

43+ providers, err := loadProviders(client, "", tmpPath)

32 require.NoError(t, err)

44 require.NoError(t, err)

33 require.NotNil(t, providers)

45 require.NotNil(t, providers)

34 require.Len(t, providers, 1)

46 require.Len(t, providers, 1)

@@ -39,35 +51,94 @@ func TestProvider_loadProvidersNoIssues(t *testing.T) {

39 require.False(t, fileInfo.IsDir(), "Expected a file, not a directory")

51 require.False(t, fileInfo.IsDir(), "Expected a file, not a directory")

40}

52}

42−func TestProvider_loadProvidersWithIssues(t *testing.T) {

54+func TestProvider_DisableAutoUpdate(t *testing.T) {

43− client := &mockProviderClient{shouldFail: true}

55+ t.Setenv("XDG_DATA_HOME", t.TempDir())

44− tmpPath := t.TempDir() + "/providers.json"

56+

45− // store providers to a temporary file

57+ resetProviderState()

46− oldProviders := []catwalk.Provider{

58+ defer resetProviderState()

47− {

59+

48− Name: "OldProvider",

60+ cfg := &Config{

61+ Options: &Options{

62+ DisableProviderAutoUpdate: true,

49 },

63 },

50 }

64 }

51− data, err := json.Marshal(oldProviders)

52− if err != nil {

53− t.Fatalf("Failed to marshal old providers: %v", err)

54− }

56− err = os.WriteFile(tmpPath, data, 0o644)

66+ providers, err := Providers(cfg)

57− if err != nil {

67+ require.NoError(t, err)

58− t.Fatalf("Failed to write old providers to file: %v", err)

68+ require.NotNil(t, providers)

69+ require.Greater(t, len(providers), 5, "Expected embedded providers")

70+}

71+

72+func TestProvider_WithValidCache(t *testing.T) {

73+ tmpDir := t.TempDir()

74+ t.Setenv("XDG_DATA_HOME", tmpDir)

75+

76+ resetProviderState()

77+ defer resetProviderState()

78+

79+ cachePath := tmpDir + "/crush/providers.json"

80+ require.NoError(t, os.MkdirAll(tmpDir+"/crush", 0o755))

81+ cachedProviders := []catwalk.Provider{

82+ {Name: "Cached"},

59 }

83 }

84+ data, err := json.Marshal(cachedProviders)

85+ require.NoError(t, err)

86+ require.NoError(t, os.WriteFile(cachePath, data, 0o644))

87+

88+ mockClient := &mockProviderClient{shouldFail: false}

89+

60− providers, err := loadProviders(true, client, tmpPath)

90+ providers, err := loadProviders(mockClient, "", cachePath)

61 require.NoError(t, err)

91 require.NoError(t, err)

62 require.NotNil(t, providers)

92 require.NotNil(t, providers)

63 require.Len(t, providers, 1)

93 require.Len(t, providers, 1)

64− require.Equal(t, "OldProvider", providers[0].Name, "Expected to keep old provider when loading fails")

94+ require.Equal(t, "Mock", providers[0].Name, "Expected fresh provider from fetch")

95+}

96+

97+func TestProvider_NotModifiedUsesCached(t *testing.T) {

98+ tmpDir := t.TempDir()

99+ t.Setenv("XDG_DATA_HOME", tmpDir)

100+

101+ resetProviderState()

102+ defer resetProviderState()

103+

104+ cachePath := tmpDir + "/crush/providers.json"

105+ require.NoError(t, os.MkdirAll(tmpDir+"/crush", 0o755))

106+ cachedProviders := []catwalk.Provider{

107+ {Name: "Cached"},

108+ }

109+ data, err := json.Marshal(cachedProviders)

110+ require.NoError(t, err)

111+ require.NoError(t, os.WriteFile(cachePath, data, 0o644))

112+

113+ mockClient := &mockProviderClient{shouldReturnErr: catwalk.ErrNotModified}

114+ providers, err := loadProviders(mockClient, "", cachePath)

115+ require.ErrorIs(t, err, catwalk.ErrNotModified)

116+ require.Nil(t, providers)

117+}

118+

119+func TestProvider_EmptyCacheDefaultsToEmbedded(t *testing.T) {

120+ tmpDir := t.TempDir()

121+ t.Setenv("XDG_DATA_HOME", tmpDir)

122+

123+ resetProviderState()

124+ defer resetProviderState()

125+

126+ cachePath := tmpDir + "/crush/providers.json"

127+ require.NoError(t, os.MkdirAll(tmpDir+"/crush", 0o755))

128+ emptyProviders := []catwalk.Provider{}

129+ data, err := json.Marshal(emptyProviders)

130+ require.NoError(t, err)

131+ require.NoError(t, os.WriteFile(cachePath, data, 0o644))

132+

133+ cached, _, err := loadProvidersFromCache(cachePath)

134+ require.NoError(t, err)

135+ require.Empty(t, cached, "Expected empty cache")

65}

136}

137

67func TestProvider_loadProvidersWithIssuesAndNoCache(t *testing.T) {

138func TestProvider_loadProvidersWithIssuesAndNoCache(t *testing.T) {

68 client := &mockProviderClient{shouldFail: true}

139 client := &mockProviderClient{shouldFail: true}

69 tmpPath := t.TempDir() + "/providers.json"

140 tmpPath := t.TempDir() + "/providers.json"

70− providers, err := loadProviders(false, client, tmpPath)

141+ providers, err := loadProviders(client, "", tmpPath)

71 require.Error(t, err)

142 require.Error(t, err)

72 require.Nil(t, providers, "Expected nil providers when loading fails and no cache exists")

143 require.Nil(t, providers, "Expected nil providers when loading fails and no cache exists")

73}

144}