2023-04-26 18:16:42 +03:00
|
|
|
// Package yamlstore contains an implementation of config.Store that
|
|
|
|
// uses YAML files for persistence.
|
|
|
|
package yamlstore
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"fmt"
|
|
|
|
"os"
|
|
|
|
"path/filepath"
|
|
|
|
|
|
|
|
"github.com/neilotoole/sq/cli/buildinfo"
|
|
|
|
"github.com/neilotoole/sq/cli/config"
|
|
|
|
"github.com/neilotoole/sq/libsq/core/errz"
|
|
|
|
"github.com/neilotoole/sq/libsq/core/ioz"
|
2024-01-15 04:45:34 +03:00
|
|
|
"github.com/neilotoole/sq/libsq/core/ioz/lockfile"
|
2023-04-26 18:16:42 +03:00
|
|
|
"github.com/neilotoole/sq/libsq/core/lg"
|
|
|
|
"github.com/neilotoole/sq/libsq/core/lg/lga"
|
|
|
|
"github.com/neilotoole/sq/libsq/core/options"
|
|
|
|
"github.com/neilotoole/sq/libsq/source"
|
|
|
|
)
|
|
|
|
|
|
|
|
// Origin of the config path.
|
|
|
|
// See Store.PathOrigin.
|
|
|
|
const (
|
|
|
|
originFlag = "flag"
|
|
|
|
originEnv = "env"
|
|
|
|
originDefault = "default"
|
|
|
|
)
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
var _ config.Store = (*Store)(nil)
|
|
|
|
|
2023-04-26 18:16:42 +03:00
|
|
|
// Store provides persistence of config via YAML file.
|
|
|
|
// It implements config.Store.
|
|
|
|
type Store struct {
|
|
|
|
// If HookLoad is non-nil, it is invoked by Load
|
|
|
|
// on Path's bytes before the YAML is unmarshalled.
|
|
|
|
// This allows expansion of variables etc.
|
|
|
|
HookLoad func(data []byte) ([]byte, error)
|
|
|
|
|
|
|
|
// UpgradeRegistry holds upgrade funcs for upgrading the config file.
|
|
|
|
UpgradeRegistry UpgradeRegistry
|
2023-05-01 06:59:34 +03:00
|
|
|
|
|
|
|
// OptionsRegistry holds options.
|
|
|
|
OptionsRegistry *options.Registry
|
2024-01-27 10:11:24 +03:00
|
|
|
// Path is the location of the config file
|
|
|
|
Path string
|
|
|
|
|
|
|
|
// PathOrigin is one of "flag", "env", or "default".
|
|
|
|
PathOrigin string
|
|
|
|
|
|
|
|
// ExtPaths holds locations of potential ext config, both dirs and files (with suffix ".sq.yml")
|
|
|
|
ExtPaths []string
|
2023-04-26 18:16:42 +03:00
|
|
|
}
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
// Lockfile implements Store.Lockfile.
|
|
|
|
func (fs *Store) Lockfile() (lockfile.Lockfile, error) {
|
|
|
|
fp := filepath.Join(filepath.Dir(fs.Path), "config.pid.lock")
|
|
|
|
fp, err := filepath.Abs(fp)
|
|
|
|
if err != nil {
|
|
|
|
return "", errz.Wrap(err, "failed to get abs path for lockfile")
|
|
|
|
}
|
|
|
|
return lockfile.Lockfile(fp), nil
|
|
|
|
}
|
|
|
|
|
2023-04-26 18:16:42 +03:00
|
|
|
// String returns a log/debug-friendly representation.
|
|
|
|
func (fs *Store) String() string {
|
|
|
|
return fmt.Sprintf("config via %s: %v", fs.PathOrigin, fs.Path)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Location implements Store. It returns the location of the config dir.
|
|
|
|
func (fs *Store) Location() string {
|
|
|
|
return filepath.Dir(fs.Path)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Load reads config from disk. It implements Store.
|
2023-05-01 06:59:34 +03:00
|
|
|
func (fs *Store) Load(ctx context.Context) (*config.Config, error) {
|
2023-05-03 15:36:10 +03:00
|
|
|
log := lg.FromContext(ctx)
|
2023-04-26 18:16:42 +03:00
|
|
|
log.Debug("Loading config from file", lga.Path, fs.Path)
|
|
|
|
|
|
|
|
if fs.UpgradeRegistry != nil {
|
2024-01-15 04:45:34 +03:00
|
|
|
mightNeedUpgrade, _, err := checkNeedsUpgrade(ctx, fs.Path)
|
2023-04-26 18:16:42 +03:00
|
|
|
if err != nil {
|
|
|
|
return nil, errz.Wrapf(err, "config: %s", fs.Path)
|
|
|
|
}
|
|
|
|
|
|
|
|
if mightNeedUpgrade {
|
2024-01-15 04:45:34 +03:00
|
|
|
// The config might need to be upgraded. But, there's an edge case
|
|
|
|
// where another process might upgrade the config file before we
|
|
|
|
// get a chance to do so. So, we acquire the config lock, and
|
|
|
|
// then check again if it still needs upgrade.
|
|
|
|
unlock, err := fs.acquireLock(ctx)
|
|
|
|
if err != nil {
|
2023-04-26 18:16:42 +03:00
|
|
|
return nil, err
|
|
|
|
}
|
2024-01-15 04:45:34 +03:00
|
|
|
defer unlock()
|
2023-04-26 18:16:42 +03:00
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
// Lock is acquired; check again if config needs upgrade.
|
|
|
|
var foundVers string
|
|
|
|
mightNeedUpgrade, foundVers, err = checkNeedsUpgrade(ctx, fs.Path)
|
2023-04-26 18:16:42 +03:00
|
|
|
if err != nil {
|
2024-01-15 04:45:34 +03:00
|
|
|
return nil, errz.Wrapf(err, "config: %s", fs.Path)
|
2023-04-26 18:16:42 +03:00
|
|
|
}
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
if mightNeedUpgrade {
|
|
|
|
log.Info("Upgrade config?", lga.From, foundVers, lga.To, buildinfo.Version)
|
|
|
|
if _, err = fs.doUpgrade(ctx, foundVers, buildinfo.Version); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// We do a cycle of loading and saving the config after the upgrade,
|
|
|
|
// because the upgrade may have written YAML via a map, which
|
|
|
|
// doesn't preserve order. Loading and saving should fix that.
|
|
|
|
cfg, err := fs.doLoad(ctx)
|
|
|
|
if err != nil {
|
|
|
|
return nil, errz.Wrapf(err, "config: %s: load failed after config upgrade", fs.Path)
|
|
|
|
}
|
|
|
|
|
|
|
|
if err = fs.Save(ctx, cfg); err != nil {
|
|
|
|
return nil, errz.Wrapf(err, "config: %s: save failed after config upgrade", fs.Path)
|
|
|
|
}
|
2023-04-26 18:16:42 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-05-01 06:59:34 +03:00
|
|
|
return fs.doLoad(ctx)
|
2023-04-26 18:16:42 +03:00
|
|
|
}
|
|
|
|
|
2023-05-01 06:59:34 +03:00
|
|
|
func (fs *Store) doLoad(ctx context.Context) (*config.Config, error) {
|
2023-04-26 18:16:42 +03:00
|
|
|
bytes, err := os.ReadFile(fs.Path)
|
|
|
|
if err != nil {
|
|
|
|
return nil, errz.Wrapf(err, "config: failed to load file: %s", fs.Path)
|
|
|
|
}
|
|
|
|
|
|
|
|
loadHookFn := fs.HookLoad
|
|
|
|
if loadHookFn != nil {
|
|
|
|
bytes, err = loadHookFn(bytes)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
cfg := config.New()
|
2023-11-19 03:05:48 +03:00
|
|
|
if err = ioz.UnmarshallYAML(bytes, cfg); err != nil {
|
2023-04-26 18:16:42 +03:00
|
|
|
return nil, errz.Wrapf(err, "config: %s: failed to unmarshal config YAML", fs.Path)
|
|
|
|
}
|
|
|
|
|
|
|
|
if cfg.Version == "" {
|
|
|
|
cfg.Version = buildinfo.Version
|
|
|
|
}
|
|
|
|
|
|
|
|
if cfg.Options == nil {
|
|
|
|
cfg.Options = options.Options{}
|
|
|
|
}
|
|
|
|
|
2023-11-19 03:05:48 +03:00
|
|
|
if cfg.Options, err = fs.OptionsRegistry.Process(cfg.Options); err != nil {
|
2023-05-05 17:32:50 +03:00
|
|
|
return nil, errz.Wrapf(err, "config: %s", fs.Path)
|
2023-04-26 18:16:42 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
if cfg.Collection == nil {
|
|
|
|
cfg.Collection = &source.Collection{}
|
|
|
|
}
|
|
|
|
|
|
|
|
repaired, err := source.VerifyIntegrity(cfg.Collection)
|
|
|
|
if err != nil {
|
|
|
|
if repaired {
|
|
|
|
// The config was repaired. Save the changes.
|
|
|
|
err = errz.Combine(err, fs.Save(ctx, cfg))
|
|
|
|
}
|
|
|
|
return nil, errz.Wrapf(err, "config: %s", fs.Path)
|
|
|
|
}
|
|
|
|
|
2023-11-19 03:05:48 +03:00
|
|
|
if err = fs.loadExt(cfg); err != nil {
|
2023-04-26 18:16:42 +03:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return cfg, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Save writes config to disk. It implements Store.
|
2024-01-15 04:45:34 +03:00
|
|
|
func (fs *Store) Save(ctx context.Context, cfg *config.Config) error {
|
2023-04-26 18:16:42 +03:00
|
|
|
if fs == nil {
|
|
|
|
return errz.New("config file store is nil")
|
|
|
|
}
|
|
|
|
|
2023-05-01 06:59:34 +03:00
|
|
|
if err := canonicalizeConfig(fs.OptionsRegistry, cfg); err != nil {
|
2023-04-26 18:16:42 +03:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
data, err := ioz.MarshalYAML(cfg)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
return fs.write(ctx, data)
|
2023-04-26 18:16:42 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
// Write writes the config bytes to disk.
|
2024-01-15 04:45:34 +03:00
|
|
|
func (fs *Store) write(ctx context.Context, data []byte) error {
|
2023-04-26 18:16:42 +03:00
|
|
|
// It's possible that the parent dir of fs.Path doesn't exist.
|
2024-01-15 04:45:34 +03:00
|
|
|
if err := ioz.RequireDir(filepath.Dir(fs.Path)); err != nil {
|
|
|
|
return errz.Wrapf(err, "failed to make parent dir of config file: %s", filepath.Dir(fs.Path))
|
2023-04-26 18:16:42 +03:00
|
|
|
}
|
|
|
|
|
2024-02-09 19:08:39 +03:00
|
|
|
// FIXME: Store.Save should do a two-step atomic write of the file.
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
if err := os.WriteFile(fs.Path, data, ioz.RWPerms); err != nil {
|
2023-04-26 18:16:42 +03:00
|
|
|
return errz.Wrap(err, "failed to save config file")
|
|
|
|
}
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
lg.FromContext(ctx).Info("Wrote config file", lga.Path, fs.Path)
|
2023-04-26 18:16:42 +03:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2024-01-30 08:03:03 +03:00
|
|
|
// Exists returns true if the backing file can be accessed, false if it doesn't
|
2023-04-26 18:16:42 +03:00
|
|
|
// exist or on any error.
|
2024-01-30 08:03:03 +03:00
|
|
|
func (fs *Store) Exists() bool {
|
2023-04-26 18:16:42 +03:00
|
|
|
_, err := os.Stat(fs.Path)
|
|
|
|
return err == nil
|
|
|
|
}
|
2023-05-01 06:59:34 +03:00
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
// acquireLock acquires the config lock, and returns an unlock func.
|
|
|
|
func (fs *Store) acquireLock(ctx context.Context) (unlock func(), err error) {
|
|
|
|
lock, err := fs.Lockfile()
|
|
|
|
if err != nil {
|
|
|
|
return nil, errz.Wrap(err, "failed to get config lock")
|
|
|
|
}
|
|
|
|
|
|
|
|
// We use the default timeout because config isn't loaded yet,
|
|
|
|
// so we don't know what the value is.
|
|
|
|
lockTimeout := config.OptConfigLockTimeout.Default()
|
|
|
|
if err = lock.Lock(ctx, lockTimeout); err != nil {
|
|
|
|
return nil, errz.Wrap(err, "acquire config lock")
|
|
|
|
}
|
|
|
|
|
|
|
|
return func() {
|
|
|
|
lg.WarnIfFuncError(lg.FromContext(ctx), "Release config lock", lock.Unlock)
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
2023-05-01 06:59:34 +03:00
|
|
|
// canonicalizeConfig checks cfg's validity, and patches cfg to the canonical
|
|
|
|
// form,cfg's validity. For example, an unknown or nil value in an
|
|
|
|
// options.Options is deleted.
|
|
|
|
func canonicalizeConfig(optsReg *options.Registry, cfg *config.Config) error {
|
|
|
|
var err error
|
|
|
|
if err = config.Valid(cfg); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
cfg.Options, err = optsReg.Process(cfg.Options)
|
|
|
|
if err != nil {
|
|
|
|
return errz.Wrapf(err, "processing 'config.options'")
|
|
|
|
}
|
|
|
|
|
|
|
|
cfg.Options = options.DeleteNil(cfg.Options)
|
|
|
|
if cfg.Collection == nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
if err = cfg.Collection.Visit(func(src *source.Source) error {
|
|
|
|
if src.Options, err = optsReg.Process(src.Options); err != nil {
|
|
|
|
return errz.Wrapf(err, "processing source options for %s", src.Handle)
|
|
|
|
}
|
|
|
|
|
|
|
|
src.Options = options.DeleteNil(src.Options)
|
|
|
|
return nil
|
|
|
|
}); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return config.Valid(cfg)
|
|
|
|
}
|