@@ -2,7 +2,9 @@ package config
5 "encoding/json"
6 "encoding/json"
@@ -17,7 +19,7 @@ import (
19type ProviderClient interface {
21type ProviderClient interface {
20− GetProviders() ([]catwalk.Provider, error)
22+ GetProviders(context.Context, string) ([]catwalk.Provider, error)
@@ -53,7 +55,7 @@ func saveProvidersInCache(path string, providers []catwalk.Provider) error {
53 return fmt.Errorf("failed to create directory for provider cache: %w", err)
55 return fmt.Errorf("failed to create directory for provider cache: %w", err)
56− data, err := json.MarshalIndent(providers, "", " ")
58+ data, err := json.Marshal(providers)
57 if err != nil {
59 if err != nil {
58 return fmt.Errorf("failed to marshal provider data: %w", err)
60 return fmt.Errorf("failed to marshal provider data: %w", err)
@@ -64,34 +66,35 @@ func saveProvidersInCache(path string, providers []catwalk.Provider) error {
64 return nil
66 return nil
67−func loadProvidersFromCache(path string) ([]catwalk.Provider, error) {
69+func loadProvidersFromCache(path string) ([]catwalk.Provider, string, error) {
68 data, err := os.ReadFile(path)
70 data, err := os.ReadFile(path)
69 if err != nil {
71 if err != nil {
70− return nil, fmt.Errorf("failed to read provider cache file: %w", err)
72+ return nil, "", fmt.Errorf("failed to read provider cache file: %w", err)
73 var providers []catwalk.Provider
75 var providers []catwalk.Provider
74 if err := json.Unmarshal(data, &providers); err != nil {
76 if err := json.Unmarshal(data, &providers); err != nil {
75− return nil, fmt.Errorf("failed to unmarshal provider data from cache: %w", err)
77+ return nil, "", fmt.Errorf("failed to unmarshal provider data from cache: %w", err)
77− return providers, nil
80+ return providers, catwalk.Etag(data), nil
80−func UpdateProviders(pathOrUrl string) error {
83+func UpdateProviders(pathOrURL string) error {
81 var providers []catwalk.Provider
84 var providers []catwalk.Provider
82− pathOrUrl = cmp.Or(pathOrUrl, os.Getenv("CATWALK_URL"), defaultCatwalkURL)
85+ pathOrURL = cmp.Or(pathOrURL, os.Getenv("CATWALK_URL"), defaultCatwalkURL)
85− case pathOrUrl == "embedded":
88+ case pathOrURL == "embedded":
86 providers = embedded.GetAll()
89 providers = embedded.GetAll()
87− case strings.HasPrefix(pathOrUrl, "http://") || strings.HasPrefix(pathOrUrl, "https://"):
90+ case strings.HasPrefix(pathOrURL, "http://") || strings.HasPrefix(pathOrURL, "https://"):
88 var err error
91 var err error
89− providers, err = catwalk.NewWithURL(pathOrUrl).GetProviders()
92+ providers, err = catwalk.NewWithURL(pathOrURL).GetProviders(context.Background(), "")
90 if err != nil {
93 if err != nil {
91 return fmt.Errorf("failed to fetch providers from Catwalk: %w", err)
94 return fmt.Errorf("failed to fetch providers from Catwalk: %w", err)
94− content, err := os.ReadFile(pathOrUrl)
97+ content, err := os.ReadFile(pathOrURL)
95 if err != nil {
98 if err != nil {
96 return fmt.Errorf("failed to read file: %w", err)
99 return fmt.Errorf("failed to read file: %w", err)
@@ -108,61 +111,61 @@ func UpdateProviders(pathOrUrl string) error {
108 return fmt.Errorf("failed to save providers to cache: %w", err)
111 return fmt.Errorf("failed to save providers to cache: %w", err)
111− slog.Info("Providers updated successfully", "count", len(providers), "from", pathOrUrl, "to", cachePath)
114+ slog.Info("Providers updated successfully", "count", len(providers), "from", pathOrURL, "to", cachePath)
112 return nil
115 return nil
118+// Providers returns the list of providers, taking into account cached results
119+// and whether or not auto update is enabled.
122+// 1. if auto update is disabled, it'll return the embedded providers at the
124+// 2. load the cached providers
125+// 3. try to get the fresh list of providers, and return either this new list,
126+// the cached list, or the embedded list if all others fail.
115func Providers(cfg *Config) ([]catwalk.Provider, error) {
127func Providers(cfg *Config) ([]catwalk.Provider, error) {
116 providerOnce.Do(func() {
128 providerOnce.Do(func() {
117 catwalkURL := cmp.Or(os.Getenv("CATWALK_URL"), defaultCatwalkURL)
129 catwalkURL := cmp.Or(os.Getenv("CATWALK_URL"), defaultCatwalkURL)
118 client := catwalk.NewWithURL(catwalkURL)
130 client := catwalk.NewWithURL(catwalkURL)
119 path := providerCacheFileData()
131 path := providerCacheFileData()
121− autoUpdateDisabled := cfg.Options.DisableProviderAutoUpdate
133+ if cfg.Options.DisableProviderAutoUpdate {
134+ slog.Info("Using embedded Catwalk providers")
122− providerList, providerErr = loadProviders(autoUpdateDisabled, client, path)
135+ providerList, providerErr = embedded.GetAll(), nil
124− return providerList, providerErr
127−func loadProviders(autoUpdateDisabled bool, client ProviderClient, path string) ([]catwalk.Provider, error) {
128− catwalkGetAndSave := func() ([]catwalk.Provider, error) {
129− providers, err := client.GetProviders()
131− return nil, fmt.Errorf("failed to fetch providers from catwalk: %w", err)
133− if len(providers) == 0 {
134− return nil, fmt.Errorf("empty providers list from catwalk")
136− if err := saveProvidersInCache(path, providers); err != nil {
139− return providers, nil
143− case autoUpdateDisabled:
144− slog.Warn("Providers auto-update is disabled")
146− if _, err := os.Stat(path); err == nil {
139+ cached, etag, cachedErr := loadProvidersFromCache(path)
147− slog.Warn("Using locally cached providers")
140+ if len(cached) == 0 || cachedErr != nil {
148− return loadProvidersFromCache(path)
141+ // if cached file is empty, default to embedded providers
142+ cached = embedded.GetAll()
151− slog.Warn("Saving embedded providers to cache")
145+ providerList, providerErr = loadProviders(client, etag, path)
152− providers := embedded.GetAll()
146+ if errors.Is(providerErr, catwalk.ErrNotModified) {
153− if err := saveProvidersInCache(path, providers); err != nil {
147+ slog.Info("Catwalk providers not modified")
154− return nil, err
148+ providerList, providerErr = cached, nil
156− return providers, nil
150+ })
157−
151+ if providerErr != nil {
158− default:
152+ catwalkURL := fmt.Sprintf("%s/v2/providers", cmp.Or(os.Getenv("CATWALK_URL"), defaultCatwalkURL))
159− slog.Info("Fetching providers from Catwalk.", "path", path)
153+ return nil, fmt.Errorf("Crush was unable to fetch an updated list of providers from %s. Consider setting CRUSH_DISABLE_PROVIDER_AUTO_UPDATE=1 to use the embedded providers bundled at the time of this Crush release. You can also update providers manually. For more info see crush update-providers --help.\n\nCause: %w", catwalkURL, providerErr) //nolint:staticcheck
155+ return providerList, nil
161− providers, err := catwalkGetAndSave()
158+func loadProviders(client ProviderClient, etag, path string) ([]catwalk.Provider, error) {
159+ slog.Info("Fetching providers from Catwalk.", "path", path)
160+ providers, err := client.GetProviders(context.Background(), etag)
162− if err != nil {
161+ if err != nil {
163− catwalkUrl := fmt.Sprintf("%s/v2/providers", cmp.Or(os.Getenv("CATWALK_URL"), defaultCatwalkURL))
162+ return nil, fmt.Errorf("failed to fetch providers from catwalk: %w", err)
164− return nil, fmt.Errorf("Crush was unable to fetch an updated list of providers from %s. Consider setting CRUSH_DISABLE_PROVIDER_AUTO_UPDATE=1 to use the embedded providers bundled at the time of this Crush release. You can also update providers manually. For more info see crush update-providers --help. %w", catwalkUrl, err) //nolint:staticcheck
166− return providers, nil
164+ if len(providers) == 0 {
165+ return nil, errors.New("empty providers list from catwalk")
167+ if err := saveProvidersInCache(path, providers); err != nil {
170+ return providers, nil