// Package yamlstore contains an implementation of config.Store that // uses YAML files for persistence. package yamlstore import ( "context" "fmt" "os" "path/filepath" "github.com/neilotoole/sq/cli/buildinfo" "github.com/neilotoole/sq/cli/config" "github.com/neilotoole/sq/libsq/core/errz" "github.com/neilotoole/sq/libsq/core/ioz" "github.com/neilotoole/sq/libsq/core/ioz/lockfile" "github.com/neilotoole/sq/libsq/core/lg" "github.com/neilotoole/sq/libsq/core/lg/lga" "github.com/neilotoole/sq/libsq/core/options" "github.com/neilotoole/sq/libsq/source" ) // Origin of the config path. // See Store.PathOrigin. const ( originFlag = "flag" originEnv = "env" originDefault = "default" ) var _ config.Store = (*Store)(nil) // Store provides persistence of config via YAML file. // It implements config.Store. type Store struct { // If HookLoad is non-nil, it is invoked by Load // on Path's bytes before the YAML is unmarshalled. // This allows expansion of variables etc. HookLoad func(data []byte) ([]byte, error) // UpgradeRegistry holds upgrade funcs for upgrading the config file. UpgradeRegistry UpgradeRegistry // OptionsRegistry holds options. OptionsRegistry *options.Registry // Path is the location of the config file Path string // PathOrigin is one of "flag", "env", or "default". PathOrigin string // ExtPaths holds locations of potential ext config, both dirs and files (with suffix ".sq.yml") ExtPaths []string } // Lockfile implements Store.Lockfile. func (fs *Store) Lockfile() (lockfile.Lockfile, error) { fp := filepath.Join(filepath.Dir(fs.Path), "config.pid.lock") fp, err := filepath.Abs(fp) if err != nil { return "", errz.Wrap(err, "failed to get abs path for lockfile") } return lockfile.Lockfile(fp), nil } // String returns a log/debug-friendly representation. func (fs *Store) String() string { return fmt.Sprintf("config via %s: %v", fs.PathOrigin, fs.Path) } // Location implements Store. It returns the location of the config dir. func (fs *Store) Location() string { return filepath.Dir(fs.Path) } // Load reads config from disk. It implements Store. func (fs *Store) Load(ctx context.Context) (*config.Config, error) { log := lg.FromContext(ctx) log.Debug("Loading config from file", lga.Path, fs.Path) if fs.UpgradeRegistry != nil { mightNeedUpgrade, _, err := checkNeedsUpgrade(ctx, fs.Path) if err != nil { return nil, errz.Wrapf(err, "config: %s", fs.Path) } if mightNeedUpgrade { // The config might need to be upgraded. But, there's an edge case // where another process might upgrade the config file before we // get a chance to do so. So, we acquire the config lock, and // then check again if it still needs upgrade. unlock, err := fs.acquireLock(ctx) if err != nil { return nil, err } defer unlock() // Lock is acquired; check again if config needs upgrade. var foundVers string mightNeedUpgrade, foundVers, err = checkNeedsUpgrade(ctx, fs.Path) if err != nil { return nil, errz.Wrapf(err, "config: %s", fs.Path) } if mightNeedUpgrade { log.Info("Upgrade config?", lga.From, foundVers, lga.To, buildinfo.Version) if _, err = fs.doUpgrade(ctx, foundVers, buildinfo.Version); err != nil { return nil, err } // We do a cycle of loading and saving the config after the upgrade, // because the upgrade may have written YAML via a map, which // doesn't preserve order. Loading and saving should fix that. cfg, err := fs.doLoad(ctx) if err != nil { return nil, errz.Wrapf(err, "config: %s: load failed after config upgrade", fs.Path) } if err = fs.Save(ctx, cfg); err != nil { return nil, errz.Wrapf(err, "config: %s: save failed after config upgrade", fs.Path) } } } } return fs.doLoad(ctx) } func (fs *Store) doLoad(ctx context.Context) (*config.Config, error) { bytes, err := os.ReadFile(fs.Path) if err != nil { return nil, errz.Wrapf(err, "config: failed to load file: %s", fs.Path) } loadHookFn := fs.HookLoad if loadHookFn != nil { bytes, err = loadHookFn(bytes) if err != nil { return nil, err } } cfg := config.New() if err = ioz.UnmarshallYAML(bytes, cfg); err != nil { return nil, errz.Wrapf(err, "config: %s: failed to unmarshal config YAML", fs.Path) } if cfg.Version == "" { cfg.Version = buildinfo.Version } if cfg.Options == nil { cfg.Options = options.Options{} } if cfg.Options, err = fs.OptionsRegistry.Process(cfg.Options); err != nil { return nil, errz.Wrapf(err, "config: %s", fs.Path) } if cfg.Collection == nil { cfg.Collection = &source.Collection{} } repaired, err := source.VerifyIntegrity(cfg.Collection) if err != nil { if repaired { // The config was repaired. Save the changes. err = errz.Combine(err, fs.Save(ctx, cfg)) } return nil, errz.Wrapf(err, "config: %s", fs.Path) } if err = fs.loadExt(cfg); err != nil { return nil, err } return cfg, nil } // Save writes config to disk. It implements Store. func (fs *Store) Save(ctx context.Context, cfg *config.Config) error { if fs == nil { return errz.New("config file store is nil") } if err := canonicalizeConfig(fs.OptionsRegistry, cfg); err != nil { return err } data, err := ioz.MarshalYAML(cfg) if err != nil { return err } return fs.write(ctx, data) } // Write writes the config bytes to disk. func (fs *Store) write(ctx context.Context, data []byte) error { // It's possible that the parent dir of fs.Path doesn't exist. if err := ioz.RequireDir(filepath.Dir(fs.Path)); err != nil { return errz.Wrapf(err, "failed to make parent dir of config file: %s", filepath.Dir(fs.Path)) } // FIXME: Store.Save should do a two-step atomic write of the file. if err := os.WriteFile(fs.Path, data, ioz.RWPerms); err != nil { return errz.Wrap(err, "failed to save config file") } lg.FromContext(ctx).Info("Wrote config file", lga.Path, fs.Path) return nil } // Exists returns true if the backing file can be accessed, false if it doesn't // exist or on any error. func (fs *Store) Exists() bool { _, err := os.Stat(fs.Path) return err == nil } // acquireLock acquires the config lock, and returns an unlock func. func (fs *Store) acquireLock(ctx context.Context) (unlock func(), err error) { lock, err := fs.Lockfile() if err != nil { return nil, errz.Wrap(err, "failed to get config lock") } // We use the default timeout because config isn't loaded yet, // so we don't know what the value is. lockTimeout := config.OptConfigLockTimeout.Default() if err = lock.Lock(ctx, lockTimeout); err != nil { return nil, errz.Wrap(err, "acquire config lock") } return func() { lg.WarnIfFuncError(lg.FromContext(ctx), "Release config lock", lock.Unlock) }, nil } // canonicalizeConfig checks cfg's validity, and patches cfg to the canonical // form,cfg's validity. For example, an unknown or nil value in an // options.Options is deleted. func canonicalizeConfig(optsReg *options.Registry, cfg *config.Config) error { var err error if err = config.Valid(cfg); err != nil { return err } cfg.Options, err = optsReg.Process(cfg.Options) if err != nil { return errz.Wrapf(err, "processing 'config.options'") } cfg.Options = options.DeleteNil(cfg.Options) if cfg.Collection == nil { return nil } if err = cfg.Collection.Visit(func(src *source.Source) error { if src.Options, err = optsReg.Process(src.Options); err != nil { return errz.Wrapf(err, "processing source options for %s", src.Handle) } src.Options = options.DeleteNil(src.Options) return nil }); err != nil { return err } return config.Valid(cfg) }