sq/cli/config/yamlstore/yamlstore.go

272 lines
7.5 KiB
Go
Raw Normal View History

// Package yamlstore contains an implementation of config.Store that
// uses YAML files for persistence.
package yamlstore
import (
"context"
"fmt"
"os"
"path/filepath"
"github.com/neilotoole/sq/cli/buildinfo"
"github.com/neilotoole/sq/cli/config"
"github.com/neilotoole/sq/libsq/core/errz"
"github.com/neilotoole/sq/libsq/core/ioz"
"github.com/neilotoole/sq/libsq/core/ioz/lockfile"
"github.com/neilotoole/sq/libsq/core/lg"
"github.com/neilotoole/sq/libsq/core/lg/lga"
"github.com/neilotoole/sq/libsq/core/options"
"github.com/neilotoole/sq/libsq/source"
)
// Origin of the config path.
// See Store.PathOrigin.
const (
originFlag = "flag"
originEnv = "env"
originDefault = "default"
)
var _ config.Store = (*Store)(nil)
// Store provides persistence of config via YAML file.
// It implements config.Store.
type Store struct {
// If HookLoad is non-nil, it is invoked by Load
// on Path's bytes before the YAML is unmarshalled.
// This allows expansion of variables etc.
HookLoad func(data []byte) ([]byte, error)
// UpgradeRegistry holds upgrade funcs for upgrading the config file.
UpgradeRegistry UpgradeRegistry
// OptionsRegistry holds options.
OptionsRegistry *options.Registry
// Path is the location of the config file
Path string
// PathOrigin is one of "flag", "env", or "default".
PathOrigin string
// ExtPaths holds locations of potential ext config, both dirs and files (with suffix ".sq.yml")
ExtPaths []string
}
// Lockfile implements Store.Lockfile.
func (fs *Store) Lockfile() (lockfile.Lockfile, error) {
fp := filepath.Join(filepath.Dir(fs.Path), "config.pid.lock")
fp, err := filepath.Abs(fp)
if err != nil {
return "", errz.Wrap(err, "failed to get abs path for lockfile")
}
return lockfile.Lockfile(fp), nil
}
// String returns a log/debug-friendly representation.
func (fs *Store) String() string {
return fmt.Sprintf("config via %s: %v", fs.PathOrigin, fs.Path)
}
// Location implements Store. It returns the location of the config dir.
func (fs *Store) Location() string {
return filepath.Dir(fs.Path)
}
// Load reads config from disk. It implements Store.
func (fs *Store) Load(ctx context.Context) (*config.Config, error) {
log := lg.FromContext(ctx)
log.Debug("Loading config from file", lga.Path, fs.Path)
if fs.UpgradeRegistry != nil {
mightNeedUpgrade, _, err := checkNeedsUpgrade(ctx, fs.Path)
if err != nil {
return nil, errz.Wrapf(err, "config: %s", fs.Path)
}
if mightNeedUpgrade {
// The config might need to be upgraded. But, there's an edge case
// where another process might upgrade the config file before we
// get a chance to do so. So, we acquire the config lock, and
// then check again if it still needs upgrade.
unlock, err := fs.acquireLock(ctx)
if err != nil {
return nil, err
}
defer unlock()
// Lock is acquired; check again if config needs upgrade.
var foundVers string
mightNeedUpgrade, foundVers, err = checkNeedsUpgrade(ctx, fs.Path)
if err != nil {
return nil, errz.Wrapf(err, "config: %s", fs.Path)
}
if mightNeedUpgrade {
log.Info("Upgrade config?", lga.From, foundVers, lga.To, buildinfo.Version)
if _, err = fs.doUpgrade(ctx, foundVers, buildinfo.Version); err != nil {
return nil, err
}
// We do a cycle of loading and saving the config after the upgrade,
// because the upgrade may have written YAML via a map, which
// doesn't preserve order. Loading and saving should fix that.
cfg, err := fs.doLoad(ctx)
if err != nil {
return nil, errz.Wrapf(err, "config: %s: load failed after config upgrade", fs.Path)
}
if err = fs.Save(ctx, cfg); err != nil {
return nil, errz.Wrapf(err, "config: %s: save failed after config upgrade", fs.Path)
}
}
}
}
return fs.doLoad(ctx)
}
func (fs *Store) doLoad(ctx context.Context) (*config.Config, error) {
bytes, err := os.ReadFile(fs.Path)
if err != nil {
return nil, errz.Wrapf(err, "config: failed to load file: %s", fs.Path)
}
loadHookFn := fs.HookLoad
if loadHookFn != nil {
bytes, err = loadHookFn(bytes)
if err != nil {
return nil, err
}
}
cfg := config.New()
if err = ioz.UnmarshallYAML(bytes, cfg); err != nil {
return nil, errz.Wrapf(err, "config: %s: failed to unmarshal config YAML", fs.Path)
}
if cfg.Version == "" {
cfg.Version = buildinfo.Version
}
if cfg.Options == nil {
cfg.Options = options.Options{}
}
if cfg.Options, err = fs.OptionsRegistry.Process(cfg.Options); err != nil {
return nil, errz.Wrapf(err, "config: %s", fs.Path)
}
if cfg.Collection == nil {
cfg.Collection = &source.Collection{}
}
repaired, err := source.VerifyIntegrity(cfg.Collection)
if err != nil {
if repaired {
// The config was repaired. Save the changes.
err = errz.Combine(err, fs.Save(ctx, cfg))
}
return nil, errz.Wrapf(err, "config: %s", fs.Path)
}
if err = fs.loadExt(cfg); err != nil {
return nil, err
}
return cfg, nil
}
// Save writes config to disk. It implements Store.
func (fs *Store) Save(ctx context.Context, cfg *config.Config) error {
if fs == nil {
return errz.New("config file store is nil")
}
if err := canonicalizeConfig(fs.OptionsRegistry, cfg); err != nil {
return err
}
data, err := ioz.MarshalYAML(cfg)
if err != nil {
return err
}
return fs.write(ctx, data)
}
// Write writes the config bytes to disk.
func (fs *Store) write(ctx context.Context, data []byte) error {
// It's possible that the parent dir of fs.Path doesn't exist.
if err := ioz.RequireDir(filepath.Dir(fs.Path)); err != nil {
return errz.Wrapf(err, "failed to make parent dir of config file: %s", filepath.Dir(fs.Path))
}
// FIXME: Store.Save should do a two-step atomic write of the file.
if err := os.WriteFile(fs.Path, data, ioz.RWPerms); err != nil {
return errz.Wrap(err, "failed to save config file")
}
lg.FromContext(ctx).Info("Wrote config file", lga.Path, fs.Path)
return nil
}
// Exists returns true if the backing file can be accessed, false if it doesn't
// exist or on any error.
func (fs *Store) Exists() bool {
_, err := os.Stat(fs.Path)
return err == nil
}
// acquireLock acquires the config lock, and returns an unlock func.
func (fs *Store) acquireLock(ctx context.Context) (unlock func(), err error) {
lock, err := fs.Lockfile()
if err != nil {
return nil, errz.Wrap(err, "failed to get config lock")
}
// We use the default timeout because config isn't loaded yet,
// so we don't know what the value is.
lockTimeout := config.OptConfigLockTimeout.Default()
if err = lock.Lock(ctx, lockTimeout); err != nil {
return nil, errz.Wrap(err, "acquire config lock")
}
return func() {
lg.WarnIfFuncError(lg.FromContext(ctx), "Release config lock", lock.Unlock)
}, nil
}
// canonicalizeConfig checks cfg's validity, and patches cfg to the canonical
// form,cfg's validity. For example, an unknown or nil value in an
// options.Options is deleted.
func canonicalizeConfig(optsReg *options.Registry, cfg *config.Config) error {
var err error
if err = config.Valid(cfg); err != nil {
return err
}
cfg.Options, err = optsReg.Process(cfg.Options)
if err != nil {
return errz.Wrapf(err, "processing 'config.options'")
}
cfg.Options = options.DeleteNil(cfg.Options)
if cfg.Collection == nil {
return nil
}
if err = cfg.Collection.Visit(func(src *source.Source) error {
if src.Options, err = optsReg.Process(src.Options); err != nil {
return errz.Wrapf(err, "processing source options for %s", src.Handle)
}
src.Options = options.DeleteNil(src.Options)
return nil
}); err != nil {
return err
}
return config.Valid(cfg)
}