From f0aa65791b1e64afdea70389b22cc6a0118dc6aa Mon Sep 17 00:00:00 2001 From: Neil O'Toole Date: Fri, 5 May 2023 08:32:50 -0600 Subject: [PATCH] #199: More config/options work (#215) * CHANGELOG text clarification * Dialing in config/options * Yet more dialing in of config/options * Refactor output writers * YAML output for more commands --- CHANGELOG.md | 31 +- cli/cli_test.go | 1 + cli/cmd_add.go | 2 + cli/cmd_config_get.go | 11 +- cli/cmd_config_set.go | 63 +++- cli/cmd_driver.go | 8 +- cli/cmd_group.go | 2 + cli/cmd_inspect.go | 1 + cli/cmd_list.go | 2 + cli/cmd_mv.go | 6 + cli/cmd_ping.go | 2 + cli/cmd_remove.go | 2 + cli/cmd_root.go | 3 + cli/cmd_src.go | 2 + cli/cmd_tbl.go | 1 + cli/cmd_version.go | 1 + cli/complete.go | 30 ++ cli/config/config.go | 9 +- cli/config/yamlstore/testdata/good.06.sq.yml | 9 + cli/config/yamlstore/testdata/good.07.sq.yml | 7 + cli/config/yamlstore/yamlstore.go | 2 +- cli/config/yamlstore/yamlstore_test.go | 8 - cli/flag/flag.go | 24 +- cli/flags.go | 28 ++ cli/internal_test.go | 2 +- cli/logging.go | 338 +++++++++++++++++- cli/options.go | 3 + cli/output/jsonw/README.md | 4 +- cli/output/jsonw/configw.go | 56 --- cli/output/jsonw/configwriter.go | 94 +++++ cli/output/jsonw/internal/jcolorenc/json.go | 4 +- cli/output/jsonw/jsonw.go | 29 ++ .../{jsonwriter_test.go => jsonw_test.go} | 0 .../jsonw/{jsonwriter.go => recordwriter.go} | 20 -- cli/output/jsonw/sourcewriter.go | 6 +- cli/output/outputx/outputx.go | 33 ++ .../tablew/{configw.go => configwriter.go} | 29 +- .../tablew/{versionw.go => versionwriter.go} | 0 cli/output/writers.go | 8 +- cli/output/yamlw/configw.go | 59 --- cli/output/yamlw/configwriter.go | 98 +++++ .../yamlw/{mdwriter.go => metadatawriter.go} | 6 +- cli/output/yamlw/sourcewriter.go | 123 +++++++ cli/output/yamlw/yamlw.go | 2 +- cli/runcontext.go | 9 +- cli/{ioutilz.go => term.go} | 0 cli/writers.go | 6 +- libsq/core/lg/lga/lga.go | 2 + libsq/core/lg/slogbuf/slogbuf.go | 5 + libsq/core/lg/userlogdir/userlogdir.go | 60 ++++ libsq/source/collection.go | 8 +- 51 files changed, 1040 insertions(+), 219 deletions(-) create mode 100644 cli/config/yamlstore/testdata/good.06.sq.yml create mode 100644 cli/config/yamlstore/testdata/good.07.sq.yml delete mode 100644 cli/output/jsonw/configw.go create mode 100644 cli/output/jsonw/configwriter.go create mode 100644 cli/output/jsonw/jsonw.go rename cli/output/jsonw/{jsonwriter_test.go => jsonw_test.go} (100%) rename cli/output/jsonw/{jsonwriter.go => recordwriter.go} (95%) create mode 100644 cli/output/outputx/outputx.go rename cli/output/tablew/{configw.go => configwriter.go} (83%) rename cli/output/tablew/{versionw.go => versionwriter.go} (100%) delete mode 100644 cli/output/yamlw/configw.go create mode 100644 cli/output/yamlw/configwriter.go rename cli/output/yamlw/{mdwriter.go => metadatawriter.go} (90%) create mode 100644 cli/output/yamlw/sourcewriter.go rename cli/{ioutilz.go => term.go} (100%) create mode 100644 libsq/core/lg/userlogdir/userlogdir.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e1ddf935..e4b8ab80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Breaking changes are annotated with ☢️. - ## Upcoming -This release completely overhauls `sq`'s config mechanism. There are a -handful of minor breaking changes ☢️. +This release significantly overhauls `sq`'s config mechanism ([#199]). There +are several minor breaking changes ☢️. ### Added @@ -22,16 +21,34 @@ handful of minor breaking changes ☢️. - `sq config location` prints the location of the config dir. - `--config` flag is now honored globally. - Many more knobs are exposed in config. +- Added flags `--log`, `--log.file` and `--log.level`. + - These values can also be set in config via `sq config edit` or `sq config set log.level DEBUG` etc. + - And they can also be set via envars, e.g. + ```shell + export SQ_LOG=true + export SQ_LOG_FILE=/var/log/sq.log + export SQ_LOG_LEVEL=WARN + ``` +- Several more commands support YAML output: + - [`sq group`](https://sq.io/docs/cmd/group) + - [`sq ls`](https://sq.io/docs/cmd/ls) + - [`sq mv`](https://sq.io/docs/cmd/mv) + - [`sq rm`](https://sq.io/docs/cmd/rm) + - [`sq src`](https://sq.io/docs/cmd/src) + ### Changed +- The structure of `sq`'s config file (`sq.yml`) has changed. The config + file is automatically upgraded when using the new version. - ☢️ Envar `SQ_CONFIG` replaces `SQ_CONFIGDIR`. +- ☢️ Envar `SQ_LOG_FILE` replaces `SQ_LOGFILE`. - ☢️ Format flag `--table` is renamed to `--text`. This is changed because while the output is mostly in table format, sometimes it's just plain text. Thus `table` was not quite accurate. - ☢️ The flag to explicitly specify a driver when piping input to `sq` has been - renamed from `--driver` to `--ingest.driver`. This change is made to align - the naming of all the ingest options and reduce ambiguity. + renamed from `--driver` to `--ingest.driver`. This change aligns + the naming of the ingest options and reduces ambiguity. ```shell # previously $ cat mystery.data | sq --driver=csv '.data' @@ -40,7 +57,7 @@ handful of minor breaking changes ☢️. $ cat mystery.data | sq --ingest.driver=csv '.data' ``` - ☢️ `sq add` no longer has the generic `--opts x=y` mechanism. This flag was - ambiguous and confusing. Instead use explicit option flags. + ambiguous and confusing. Instead, use explicit option flags. ```shell # previously $ sq add ./actor.csv --opts=header=false @@ -59,7 +76,6 @@ handful of minor breaking changes ☢️. $ sq add ./actor.csv -n @actor ``` - ## [v0.33.0] - 2023-04-15 The headline feature is [source groups](https://sq.io/docs/source#groups). @@ -396,6 +412,7 @@ make working with lots of sources much easier. [#189]: https://github.com/neilotoole/sq/issues/189 [#191]: https://github.com/neilotoole/sq/issues/191 [#192]: https://github.com/neilotoole/sq/issues/192 +[#199]: https://github.com/neilotoole/sq/issues/199 [v0.15.2]: https://github.com/neilotoole/sq/releases/tag/v0.15.2 [v0.15.3]: https://github.com/neilotoole/sq/compare/v0.15.2...v0.15.3 diff --git a/cli/cli_test.go b/cli/cli_test.go index 272e377f..7848b15e 100644 --- a/cli/cli_test.go +++ b/cli/cli_test.go @@ -38,6 +38,7 @@ func TestSmoke(t *testing.T) { {a: []string{"inspect"}, errBecause: "no active data source"}, {a: []string{"inspect", "--help"}}, {a: []string{"version"}}, + {a: []string{"version", "--help"}}, {a: []string{"--version"}}, {a: []string{"help"}}, {a: []string{"--help"}}, diff --git a/cli/cmd_add.go b/cli/cmd_add.go index 8e451cff..6f018dca 100644 --- a/cli/cmd_add.go +++ b/cli/cmd_add.go @@ -138,6 +138,8 @@ More examples: } cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage) + cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage) + cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage) cmd.Flags().StringP(flag.AddDriver, flag.AddDriverShort, "", flag.AddDriverUsage) panicOn(cmd.RegisterFlagCompletionFunc(flag.AddDriver, completeDriverType)) diff --git a/cli/cmd_config_get.go b/cli/cmd_config_get.go index 293be09a..df884981 100644 --- a/cli/cmd_config_get.go +++ b/cli/cmd_config_get.go @@ -40,6 +40,7 @@ Use the --verbose flag (in text output format) to see all options.`, cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage) cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage) + cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage) cmd.Flags().String(flag.ConfigSrc, "", flag.ConfigSrcUsage) panicOn(cmd.RegisterFlagCompletionFunc(flag.ConfigSrc, completeHandle(1))) @@ -87,13 +88,5 @@ func execConfigGet(cmd *cobra.Command, args []string) error { return errz.Errorf("invalid option key: %s", args[0]) } - // A bit of a hack... create a new registry with just the desired opt. - reg2 := &options.Registry{} - reg2.Add(opt) - o2 := options.Options{} - if v, ok := o[opt.Key()]; ok { - o2[opt.Key()] = v - } - - return rc.writers.configw.Options(reg2, o2) + return rc.writers.configw.Opt(o, opt) } diff --git a/cli/cmd_config_set.go b/cli/cmd_config_set.go index 907da6a6..edc3d63f 100644 --- a/cli/cmd_config_set.go +++ b/cli/cmd_config_set.go @@ -1,9 +1,10 @@ package cli import ( + "fmt" + "github.com/neilotoole/sq/cli/flag" "github.com/neilotoole/sq/libsq/core/errz" - "github.com/neilotoole/sq/libsq/core/lg" "github.com/neilotoole/sq/libsq/core/lg/lga" "github.com/neilotoole/sq/libsq/core/options" "github.com/neilotoole/sq/libsq/source" @@ -14,30 +15,38 @@ func newConfigSetCmd() *cobra.Command { cmd := &cobra.Command{ Use: "set", RunE: execConfigSet, - Args: cobra.ExactArgs(2), + Args: cobra.RangeArgs(1, 2), ValidArgsFunction: completeConfigSet, Short: "Set config value", - Long: `Set config value globally, or for a specific source. + Long: `Set base config value, or set value for a specific source. Use "sq config get -v" to see available options.`, - Example: ` # Set default output format + Example: ` # Set base output format $ sq config set format json - # Set default max DB connections + # Set base max DB connections $ sq config set conn.max-open 10 # Set max DB connections for source @sakila - $ sq config set --src @sakila conn.max-open 50`, + $ sq config set --src @sakila conn.max-open 50 + + # Delete an option (resets to default value) + $ sq config set -D conn.max-open`, } cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage) + cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage) cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage) cmd.Flags().String(flag.ConfigSrc, "", flag.ConfigSrcUsage) panicOn(cmd.RegisterFlagCompletionFunc(flag.ConfigSrc, completeHandle(1))) + + cmd.Flags().BoolP(flag.ConfigDelete, flag.ConfigDeleteShort, false, flag.ConfigDeleteUsage) + return cmd } func execConfigSet(cmd *cobra.Command, args []string) error { + log := logFrom(cmd) rc, ctx := RunContextFrom(cmd.Context()), cmd.Context() o := rc.Config.Options @@ -66,6 +75,25 @@ func execConfigSet(cmd *cobra.Command, args []string) error { o = src.Options } + if cmdFlagChanged(cmd, flag.ConfigDelete) { + if len(args) > 1 { + return errz.Errorf("accepts 1 arg when used with --%s flag", flag.ConfigDelete) + } + + delete(o, opt.Key()) + if src == nil { + log.Info("Unset base config value", lga.Key, opt.Key()) + } else { + log.Info("Unset source config value", lga.Src, src, lga.Key, opt.Key()) + } + + if err := rc.ConfigStore.Save(ctx, rc.Config); err != nil { + return err + } + + return rc.writers.configw.UnsetOption(opt) + } + o2 := options.Options{} o2[opt.Key()] = args[1] var err error @@ -79,20 +107,35 @@ func execConfigSet(cmd *cobra.Command, args []string) error { return err } - if src != nil { - lg.FromContext(ctx).Info("Set default config value", lga.Val, o) + if src == nil { + log.Info( + "Set base config value", + lga.Key, opt.Key(), + lga.Val, o[opt.Key()], + ) } else { - lg.FromContext(ctx).Info("Set source config value", lga.Src, src, lga.Val, o) + log.Info( + "Set source config value", + lga.Key, opt.Key(), + lga.Src, src, + lga.Val, o, + ) } - return rc.writers.configw.SetOption(rc.OptionsRegistry, o, opt) + return rc.writers.configw.SetOption(o, opt) } func completeConfigSet(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { switch len(args) { case 0: return completeOptKey(cmd, args, toComplete) + case 1: + if cmdFlagChanged(cmd, flag.ConfigDelete) { + logFrom(cmd).Warn(fmt.Sprintf("No 2nd arg when using --%s flag", flag.ConfigDelete)) + return nil, cobra.ShellCompDirectiveError + } + return completeOptValue(cmd, args, toComplete) default: // Maximum of two args diff --git a/cli/cmd_driver.go b/cli/cmd_driver.go index 6d6ed095..d2a7a783 100644 --- a/cli/cmd_driver.go +++ b/cli/cmd_driver.go @@ -8,7 +8,8 @@ import ( func newDriverCmd() *cobra.Command { cmd := &cobra.Command{ Use: "driver", - Short: "List or manage drivers", + Short: "Manage drivers", + Long: "Manage drivers.", RunE: func(cmd *cobra.Command, args []string) error { return cmd.Help() }, @@ -23,14 +24,15 @@ func newDriverCmd() *cobra.Command { func newDriverListCmd() *cobra.Command { cmd := &cobra.Command{ Use: "ls", - Short: "List available drivers", - Long: "List available drivers.", + Short: "List installed drivers", + Long: "List installed drivers.", Args: cobra.NoArgs, RunE: execDriverList, } cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage) cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage) + cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage) return cmd } diff --git a/cli/cmd_group.go b/cli/cmd_group.go index c6052c98..7a16a5c0 100644 --- a/cli/cmd_group.go +++ b/cli/cmd_group.go @@ -34,6 +34,8 @@ Use 'sq ls -g' to list groups.`, } cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage) + cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage) + cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage) return cmd } diff --git a/cli/cmd_inspect.go b/cli/cmd_inspect.go index 5023f362..25ea020d 100644 --- a/cli/cmd_inspect.go +++ b/cli/cmd_inspect.go @@ -45,6 +45,7 @@ If @HANDLE is not provided, the active data source is assumed.`, } cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage) + cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage) cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage) return cmd diff --git a/cli/cmd_list.go b/cli/cmd_list.go index ed70577d..153ab3b4 100644 --- a/cli/cmd_list.go +++ b/cli/cmd_list.go @@ -40,6 +40,8 @@ any further descendants. } cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage) + cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage) + cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage) cmd.Flags().BoolP(flag.ListGroup, flag.ListGroupShort, false, flag.ListGroupUsage) return cmd diff --git a/cli/cmd_mv.go b/cli/cmd_mv.go index 8bf1ea78..b05772ac 100644 --- a/cli/cmd_mv.go +++ b/cli/cmd_mv.go @@ -3,6 +3,8 @@ package cli import ( "strings" + "github.com/neilotoole/sq/cli/flag" + "github.com/neilotoole/sq/libsq/core/stringz" "golang.org/x/exp/slices" @@ -41,6 +43,10 @@ source handles are files, and groups are directories.`, $ sq mv production prod`, } + cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage) + cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage) + cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage) + return cmd } diff --git a/cli/cmd_ping.go b/cli/cmd_ping.go index 5bb456bd..12020483 100644 --- a/cli/cmd_ping.go +++ b/cli/cmd_ping.go @@ -65,6 +65,8 @@ The exit code is 1 if ping fails for any of the sources.`, cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage) cmd.Flags().BoolP(flag.CSV, flag.CSVShort, false, flag.CSVUsage) cmd.Flags().BoolP(flag.TSV, flag.TSVShort, false, flag.TSVUsage) + cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage) + cmd.Flags().Duration(flag.PingTimeout, time.Second*10, flag.PingTimeoutUsage) return cmd } diff --git a/cli/cmd_remove.go b/cli/cmd_remove.go index a024b7c6..969f5372 100644 --- a/cli/cmd_remove.go +++ b/cli/cmd_remove.go @@ -33,6 +33,8 @@ may have changed, if that source or group was removed.`, } cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage) + cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage) + cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage) return cmd } diff --git a/cli/cmd_root.go b/cli/cmd_root.go index ff23dac1..58ba29a5 100644 --- a/cli/cmd_root.go +++ b/cli/cmd_root.go @@ -98,5 +98,8 @@ See docs and more: https://sq.io`, cmd.PersistentFlags().BoolP(flag.Monochrome, flag.MonochromeShort, false, flag.MonochromeUsage) cmd.PersistentFlags().BoolP(flag.Verbose, flag.VerboseShort, false, flag.VerboseUsage) cmd.PersistentFlags().String(flag.Config, "", flag.ConfigUsage) + cmd.PersistentFlags().Bool(flag.LogEnabled, false, flag.LogEnabledUsage) + cmd.PersistentFlags().String(flag.LogFile, "", flag.LogFileUsage) + cmd.PersistentFlags().String(flag.LogLevel, "", flag.LogLevelUsage) return cmd } diff --git a/cli/cmd_src.go b/cli/cmd_src.go index db4f9ee7..ae347373 100644 --- a/cli/cmd_src.go +++ b/cli/cmd_src.go @@ -22,6 +22,8 @@ source. Otherwise, set @HANDLE as the active data source.`, } cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage) + cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage) + cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage) return cmd } diff --git a/cli/cmd_tbl.go b/cli/cmd_tbl.go index 6b877025..aa73254e 100644 --- a/cli/cmd_tbl.go +++ b/cli/cmd_tbl.go @@ -55,6 +55,7 @@ func newTblCopyCmd() *cobra.Command { } cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage) + cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage) cmd.Flags().Bool(flag.TblData, true, flag.TblDataUsage) return cmd diff --git a/cli/cmd_version.go b/cli/cmd_version.go index 6f05edad..4282faa9 100644 --- a/cli/cmd_version.go +++ b/cli/cmd_version.go @@ -51,6 +51,7 @@ Before upgrading, check the changelog: https://sq.io/changelog`, } cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage) + cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage) return cmd } diff --git a/cli/complete.go b/cli/complete.go index db72f003..535d8443 100644 --- a/cli/complete.go +++ b/cli/complete.go @@ -73,6 +73,11 @@ func completeHandle(max int) completionFunc { slices.Sort(handles) // REVISIT: what's the logic for sorting or not? handles, _ = lo.Difference(handles, args) + + if rc.Config.Collection.Active() != nil { + handles = append([]string{source.ActiveHandle}, handles...) + } + return handles, cobra.ShellCompDirectiveNoFileComp } } @@ -173,6 +178,22 @@ func completeOptKey(cmd *cobra.Command, _ []string, toComplete string) ([]string keys = lo.Map(opts, func(item options.Opt, index int) string { return item.Key() }) + + if cmdFlagChanged(cmd, flag.ConfigDelete) { + if len(src.Options) == 0 { + // Nothing to delete + return nil, cobra.ShellCompDirectiveError + } + + // There are options to delete + return src.Options.Keys(), cobra.ShellCompDirectiveDefault + } + } + + if cmdFlagChanged(cmd, flag.ConfigDelete) { + // At this stage, we have to offer all opts, because the user + // input could become: $ sq config set -D ingest.header --src @csv + return rc.OptionsRegistry.Keys(), cobra.ShellCompDirectiveDefault } keys = lo.Filter(keys, func(item string, index int) bool { @@ -203,6 +224,14 @@ func completeOptValue(cmd *cobra.Command, args []string, toComplete string) ([]s var a []string switch opt.(type) { + case options.String: + if opt.Key() == OptLogFile.Key() { + // We return the default directive, so that the shell will offer + // regular ol' file completion. + return a, cobra.ShellCompDirectiveDefault + } + case LogLevelOpt: + a = []string{"debug", "DEBUG", "info", "INFO", "warn", "WARN", "error", "ERROR"} case FormatOpt: a = stringz.Strings(format.All()) case options.Bool: @@ -394,6 +423,7 @@ func (c *handleTableCompleter) completeHandle(ctx context.Context, rc *RunContex } handles := rc.Config.Collection.Handles() + handles = append([]string{source.ActiveHandle}, handles...) // Else, we're dealing with just a handle so far var matchingHandles []string for _, handle := range handles { diff --git a/cli/config/config.go b/cli/config/config.go index 26e637e6..ded1b6ab 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -14,7 +14,14 @@ import ( ) const ( - EnvarLogPath = "SQ_LOGFILE" + // EnvarLogPath is the log file path. + EnvarLogPath = "SQ_LOG_FILE" + + // EnvarLogLevel is the log level. It maps to a slog.Level. + EnvarLogLevel = "SQ_LOG_LEVEL" + + // EnvarLogEnabled turns logging on or off. + EnvarLogEnabled = "SQ_LOG" // EnvarConfigDir is the legacy envar for config location. // Instead use EnvarConfig. diff --git a/cli/config/yamlstore/testdata/good.06.sq.yml b/cli/config/yamlstore/testdata/good.06.sq.yml new file mode 100644 index 00000000..680b5461 --- /dev/null +++ b/cli/config/yamlstore/testdata/good.06.sq.yml @@ -0,0 +1,9 @@ +# This file has log.level in lowercase and empty log.file +config.version: v0.34.0 +options: + format: table + header: false + log.level: debug + log.file: + + diff --git a/cli/config/yamlstore/testdata/good.07.sq.yml b/cli/config/yamlstore/testdata/good.07.sq.yml new file mode 100644 index 00000000..786203b6 --- /dev/null +++ b/cli/config/yamlstore/testdata/good.07.sq.yml @@ -0,0 +1,7 @@ +# This file has custom log.level "off" (not an actual defined slog.Level). +config.version: v0.34.0 +options: + format: table + header: false + + diff --git a/cli/config/yamlstore/yamlstore.go b/cli/config/yamlstore/yamlstore.go index 94a22166..eaf91eec 100644 --- a/cli/config/yamlstore/yamlstore.go +++ b/cli/config/yamlstore/yamlstore.go @@ -124,7 +124,7 @@ func (fs *Store) doLoad(ctx context.Context) (*config.Config, error) { cfg.Options, err = fs.OptionsRegistry.Process(cfg.Options) if err != nil { - return nil, err + return nil, errz.Wrapf(err, "config: %s", fs.Path) } if cfg.Collection == nil { diff --git a/cli/config/yamlstore/yamlstore_test.go b/cli/config/yamlstore/yamlstore_test.go index 2f63076b..6a9d8860 100644 --- a/cli/config/yamlstore/yamlstore_test.go +++ b/cli/config/yamlstore/yamlstore_test.go @@ -18,17 +18,13 @@ import ( ) func TestFileStore_Nil_Save(t *testing.T) { - t.Parallel() - var f *yamlstore.Store - // noinspection GoNilness err := f.Save(context.Background(), config.New()) require.Error(t, err) } func TestFileStore_LoadSaveLoad(t *testing.T) { - t.Parallel() ctx := context.Background() const wantVers = `v0.34.0` @@ -72,8 +68,6 @@ var hookExpand = func(data []byte) ([]byte, error) { } func TestFileStore_Load(t *testing.T) { - t.Parallel() - optsReg := &options.Registry{} cli.RegisterDefaultOpts(optsReg) @@ -92,8 +86,6 @@ func TestFileStore_Load(t *testing.T) { for _, match := range good { match := match t.Run(tutil.Name(match), func(t *testing.T) { - t.Parallel() - fs.Path = match _, err = fs.Load(context.Background()) require.NoError(t, err, match) diff --git a/cli/flag/flag.go b/cli/flag/flag.go index c949862d..e91b8fda 100644 --- a/cli/flag/flag.go +++ b/cli/flag/flag.go @@ -85,13 +85,6 @@ const ( SQLQuery = "query" SQLQueryUsage = "Execute the SQL as a query (as opposed to statement)" - // SrcOptions is deprecated. - // - // //Deprecated: Use specific options like flag.IngestHeader. - // FIXME: get rid of SrcOptions. - SrcOptions = "opts" - SrcOptionsUsage = "Driver-dependent data source options" - TSV = "tsv" TSVShort = "T" TSVUsage = "Output TSV" @@ -135,7 +128,7 @@ const ( ConfigUsage = "Load config from here" IngestHeader = "ingest.header" - IngestHeaderUsage = "Treat first row of import data as header" + IngestHeaderUsage = "Treat first row of ingest data as header" CSVEmptyAsNull = "driver.csv.empty-as-null" CSVEmptyAsNullUsage = "Treat empty CSV fields as null" @@ -144,7 +137,16 @@ const ( CSVDelimUsage = "CSV delimiter: one of comma, space, pipe, tab, colon, semi, period" CSVDelimDefault = "comma" - ConfigSetDelete = "delete" - ConfigSetDeleteShort = "D" - ConfigSetDeleteUsage = "Unset this option" + ConfigDelete = "delete" + ConfigDeleteShort = "D" + ConfigDeleteUsage = "Reset this option to default value" + + LogEnabled = "log" + LogEnabledUsage = "Enable logging" + + LogFile = "log.file" + LogFileUsage = "Path to log file; empty disables logging" + + LogLevel = "log.level" + LogLevelUsage = "Log level: one of DEBUG, INFO, WARN, ERROR" ) diff --git a/cli/flags.go b/cli/flags.go index ca684fa0..2134e6b7 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -1,7 +1,11 @@ package cli import ( + "io" + + "github.com/neilotoole/sq/libsq/core/errz" "github.com/spf13/cobra" + "github.com/spf13/pflag" ) // cmdFlagChanged returns true if cmd is non-nil and @@ -33,3 +37,27 @@ func cmdFlagTrue(cmd *cobra.Command, name string) bool { return b } + +// getBootstrapFlagValue parses osArgs looking for flg. The flag is always +// treated as string. This function exists because some components such +// as logging and config interrogate flags before cobra has loaded. +func getBootstrapFlagValue(flg, flgShort, flgUsage string, osArgs []string) (val string, ok bool, err error) { + fs := pflag.NewFlagSet("bootstrap", pflag.ContinueOnError) + fs.ParseErrorsWhitelist.UnknownFlags = true + fs.SetOutput(io.Discard) + + _ = fs.StringP(flg, flgShort, "", flgUsage) + if err = fs.Parse(osArgs); err != nil { + return "", false, errz.Err(err) + } + + if !fs.Changed(flg) { + return "", false, nil + } + + if val, err = fs.GetString(flg); err != nil { + return "", false, errz.Err(err) + } + + return val, true, nil +} diff --git a/cli/internal_test.go b/cli/internal_test.go index 2612c460..ee395482 100644 --- a/cli/internal_test.go +++ b/cli/internal_test.go @@ -119,5 +119,5 @@ func TestRegisterDefaultOpts(t *testing.T) { log.Debug("options.Registry (after)", "reg", reg) keys := reg.Keys() - require.Len(t, keys, 19) + require.Len(t, keys, 22) } diff --git a/cli/logging.go b/cli/logging.go index ea18d1c7..8f5a7a8c 100644 --- a/cli/logging.go +++ b/cli/logging.go @@ -7,6 +7,13 @@ import ( "path/filepath" "strings" + "github.com/neilotoole/sq/libsq/core/stringz" + + "github.com/neilotoole/sq/libsq/core/options" + + "github.com/neilotoole/sq/cli/flag" + "github.com/neilotoole/sq/libsq/core/lg/userlogdir" + "github.com/neilotoole/sq/cli/config" "github.com/neilotoole/sq/libsq/core/errz" "github.com/neilotoole/sq/libsq/core/lg" @@ -15,19 +22,50 @@ import ( "golang.org/x/exp/slog" ) +var ( + OptLogEnabled = options.NewBool( + "log", + false, + "Enable logging.", + ) + + OptLogFile = options.NewString( + "log.file", + getDefaultLogFilePath(), + `Path to log file. Empty value disables logging.`) + + OptLogLevel = NewLogLevelOpt( + "log.level", + slog.LevelDebug, + `Log level, one of DEBUG, INFO, WARN, ERROR.`) +) + // defaultLogging returns a *slog.Logger, its slog.Handler, and // possibly a *cleanup.Cleanup, which the caller is responsible // for invoking at the appropriate time. If an error is returned, the // other returned values will be nil. If logging is not enabled, -// all returned values will be nil. -func defaultLogging(ctx context.Context) (log *slog.Logger, h slog.Handler, closer func() error, err error) { - logFilePath, ok := os.LookupEnv(config.EnvarLogPath) - if !ok || strings.TrimSpace(logFilePath) == "" { - lg.FromContext(ctx).Debug("Logging: not enabled via envar", lga.Key, config.EnvarLogPath) - return lg.Discard(), nil, nil, nil +// the returned values will also be nil. +func defaultLogging(ctx context.Context, osArgs []string, cfg *config.Config, +) (log *slog.Logger, h slog.Handler, closer func() error, err error) { + bootLog := lg.FromContext(ctx) + + enabled := getLogEnabled(ctx, osArgs, cfg) + if !enabled { + return nil, nil, nil, nil } - lg.FromContext(ctx).Debug("Logging: enabled via envar", lga.Key, config.EnvarLogPath, lga.Val, logFilePath) + // First, get the log file path. It can come from flag, envar, or config. + logFilePath := strings.TrimSpace(getLogFilePath(ctx, osArgs, cfg)) + if logFilePath == "" { + bootLog.Debug("Logging: not enabled (log file path not set)") + return nil, nil, nil, nil + } + + lvl := getLogLevel(ctx, osArgs, cfg) + + // Allow for $HOME/sq.log etc. + logFilePath = os.ExpandEnv(logFilePath) + bootLog.Debug("Logging: enabled", lga.Path, logFilePath) // Let's try to create the dir holding the logfile... if it already exists, // then os.MkdirAll will just no-op @@ -43,19 +81,19 @@ func defaultLogging(ctx context.Context) (log *slog.Logger, h slog.Handler, clos } closer = logFile.Close - h = newJSONHandler(logFile) + h = newJSONHandler(logFile, lvl) return slog.New(h), h, closer, nil } func stderrLogger() (*slog.Logger, slog.Handler) { - h := newJSONHandler(os.Stderr) + h := newJSONHandler(os.Stderr, slog.LevelDebug) return slog.New(h), h } -func newJSONHandler(w io.Writer) slog.Handler { +func newJSONHandler(w io.Writer, lvl slog.Leveler) slog.Handler { return slog.HandlerOptions{ AddSource: true, - Level: slog.LevelDebug, + Level: lvl, ReplaceAttr: slogReplaceAttrs, }.NewJSONHandler(w) } @@ -105,3 +143,281 @@ func logFrom(cmd *cobra.Command) *slog.Logger { return log } + +// getLogEnabled determines if logging is enabled based on flags, envars, or config. +// Any error is logged to the ctx logger. +func getLogEnabled(ctx context.Context, osArgs []string, cfg *config.Config) bool { + bootLog := lg.FromContext(ctx) + var enabled bool + + val, ok, err := getBootstrapFlagValue(flag.LogEnabled, "", flag.LogEnabledUsage, osArgs) + if err != nil { + bootLog.Error("Reading log 'enabled' from flag", lga.Flag, flag.LogEnabled, lga.Err, err) + } + if ok { + bootLog.Debug("Using log 'enabled' specified via flag", lga.Flag, flag.LogEnabled, lga.Val, val) + + enabled, err = stringz.ParseBool(val) + if err != nil { + bootLog.Error( + "Reading bool flag", + lga.Flag, flag.LogEnabled, + lga.Val, val, + ) + // When in doubt, enable logging? + return true + } + + return enabled + } + + val, ok = os.LookupEnv(config.EnvarLogEnabled) + if ok { + bootLog.Debug("Using log 'enabled' specified via envar", + lga.Env, config.EnvarLogEnabled, + lga.Val, val, + ) + + enabled, err = stringz.ParseBool(val) + if err != nil { + bootLog.Error( + "Reading bool envar", + lga.Env, config.EnvarLogEnabled, + lga.Val, val, + ) + + // When in doubt, enable logging? + return true + } + + return enabled + } + + var o options.Options + if cfg != nil { + o = cfg.Options + } + + enabled = OptLogEnabled.Get(o) + bootLog.Debug("Using log 'enabled' specified via config", lga.Key, OptLogEnabled.Key(), lga.Val, enabled) + return enabled +} + +// getLogLevel gets the log level, based on flags, envars, or config. +// Any error is logged to the ctx logger. +func getLogLevel(ctx context.Context, osArgs []string, cfg *config.Config) slog.Level { + bootLog := lg.FromContext(ctx) + + val, ok, err := getBootstrapFlagValue(flag.LogLevel, "", flag.LogLevelUsage, osArgs) + if err != nil { + bootLog.Error("Reading log level from flag", lga.Flag, flag.LogLevel, lga.Err, err) + } + if ok { + bootLog.Debug("Using log level specified via flag", lga.Flag, flag.LogLevel, lga.Val, val) + + lvl := new(slog.Level) + if err = lvl.UnmarshalText([]byte(val)); err != nil { + bootLog.Error("Invalid log level specified via flag", + lga.Flag, flag.LogLevel, + lga.Val, val, + lga.Err, err) + } else { + return *lvl + } + } + + val, ok = os.LookupEnv(config.EnvarLogLevel) + if ok { + bootLog.Debug("Using log level specified via envar", + lga.Env, config.EnvarLogLevel, + lga.Val, val) + + lvl := new(slog.Level) + if err = lvl.UnmarshalText([]byte(val)); err != nil { + bootLog.Error("Invalid log level specified by envar", + lga.Env, config.EnvarLogLevel, + lga.Val, val, + lga.Err, err) + } else { + return *lvl + } + } + + var o options.Options + if cfg != nil { + o = cfg.Options + } + + lvl := OptLogLevel.Get(o) + bootLog.Debug("Using log level specified via config", lga.Key, OptLogLevel.Key(), lga.Val, lvl) + return lvl +} + +// getLogFilePath gets the log file path, based on flags, envars, or config. +// If a log file is not specified (and thus logging is disabled), empty string +// is returned. +func getLogFilePath(ctx context.Context, osArgs []string, cfg *config.Config) string { + bootLog := lg.FromContext(ctx) + + fp, ok, err := getBootstrapFlagValue(flag.LogFile, "", flag.LogFileUsage, osArgs) + if err != nil { + bootLog.Error("Reading log file from flag", lga.Flag, flag.LogFile, lga.Err, err) + } + if ok { + bootLog.Debug("Log file specified via flag", lga.Flag, flag.LogFile, lga.Path, fp) + return fp + } + + fp, ok = os.LookupEnv(config.EnvarLogPath) + if ok { + bootLog.Debug("Log file specified via envar", lga.Env, config.EnvarLogPath, lga.Path, fp) + return fp + } + + var o options.Options + if cfg != nil { + o = cfg.Options + } + + fp = OptLogFile.Get(o) + bootLog = bootLog.With(lga.Key, OptLogFile.Key(), lga.Path, fp) + + if !o.IsSet(OptLogFile) { + bootLog.Debug("Log file not explicitly set in config; using default") + return fp + } + + if fp == "" { + bootLog.Debug(`Log file explicitly set to "" in config; logging disabled`) + } + + bootLog.Debug("Log file specified via config") + return fp +} + +// getDefaultLogFilePath returns the OS-dependent log file path, +// or an empty string if it can't be determined. The file (and its +// parent dir) may not exist. +func getDefaultLogFilePath() string { + p, err := userlogdir.UserLogDir() + if err != nil { + return "" + } + + return filepath.Join(p, "sq", "sq.log") +} + +var _ options.Opt = LogLevelOpt{} + +// NewLogLevelOpt returns a new LogLevelOpt instance. +func NewLogLevelOpt(key string, defaultVal slog.Level, comment string, tags ...string) LogLevelOpt { + return LogLevelOpt{key: key, defaultVal: defaultVal, comment: comment, tags: tags} +} + +// LogLevelOpt is an options.Opt for slog.Level. +type LogLevelOpt struct { + key string + comment string + defaultVal slog.Level + tags []string +} + +// Comment implements options.Opt. +func (op LogLevelOpt) Comment() string { + return op.comment +} + +// Tags implements options.Opt. +func (op LogLevelOpt) Tags() []string { + return op.tags +} + +// Key implements options.Opt. +func (op LogLevelOpt) Key() string { + return op.key +} + +// String implements options.Opt. +func (op LogLevelOpt) String() string { + return op.key +} + +// IsSet implements options.Opt. +func (op LogLevelOpt) IsSet(o options.Options) bool { + if o == nil { + return false + } + + return o.IsSet(op) +} + +// Process implements options.Processor. It converts matching +// string values in o into slog.Level. If no match found, +// the input arg is returned unchanged. Otherwise, a clone is +// returned. +func (op LogLevelOpt) Process(o options.Options) (options.Options, error) { + if o == nil { + return nil, nil + } + + v, ok := o[op.key] + if !ok || v == nil { + return o, nil + } + + // v should be a string + switch x := v.(type) { + case string: + // continue below + case int: + v = slog.Level(x) + // continue below + case slog.Level: + return o, nil + default: + return nil, errz.Errorf("option {%s} should be {%T} or {%T} but got {%T}: %v", + op.key, slog.LevelDebug, "", x, x) + } + + var s string + s, ok = v.(string) + if !ok { + return nil, errz.Errorf("option {%s} should be {%T} but got {%T}: %v", + op.key, s, v, v) + } + + var lvl slog.Level + if err := lvl.UnmarshalText([]byte(s)); err != nil { + return nil, errz.Wrapf(err, "option {%s} is not a valid {%T}", op.key, lvl) + } + + o = o.Clone() + o[op.key] = lvl + return o, nil +} + +// GetAny implements options.Opt. +func (op LogLevelOpt) GetAny(o options.Options) any { + return op.Get(o) +} + +// Get returns op's value in o. If o is nil, or no value +// is set, op's default value is returned. +func (op LogLevelOpt) Get(o options.Options) slog.Level { + if o == nil { + return op.defaultVal + } + + v, ok := o[op.key] + if !ok { + return op.defaultVal + } + + var lvl slog.Level + lvl, ok = v.(slog.Level) + if !ok { + return op.defaultVal + } + + return lvl +} diff --git a/cli/options.go b/cli/options.go index 01e9e7af..28dccab0 100644 --- a/cli/options.go +++ b/cli/options.go @@ -136,6 +136,9 @@ func RegisterDefaultOpts(reg *options.Registry) { OptPretty, OptPingTimeout, OptShellCompletionTimeout, + OptLogEnabled, + OptLogFile, + OptLogLevel, driver.OptConnMaxOpen, driver.OptConnMaxIdle, driver.OptConnMaxIdleTime, diff --git a/cli/output/jsonw/README.md b/cli/output/jsonw/README.md index 085c7cb6..648e2144 100644 --- a/cli/output/jsonw/README.md +++ b/cli/output/jsonw/README.md @@ -8,7 +8,9 @@ Note that there are three implementations of `output.RecordWriter`. - `NewArrayRecordWriter` outputs each record on its own line as an element of a JSON array. - `NewObjectRecordWriter` outputs each record as a JSON object on its own line. -These `RecordWriter`s correspond to the `--json`, `--jsona`, and `--jsonl` flags (where `jsonl` means "JSON Lines"). There are also other writer implementations, such as an `output.ErrorWriter` and an `output.MetadataWriter`. +These `RecordWriter`s correspond to the `--json`, `--jsona`, and `--jsonl` flags +(where `jsonl` means "JSON Lines"). There are also other writer implementations, +such as an `output.ErrorWriter` and an `output.MetadataWriter`. #### Standard JSON `--json`: diff --git a/cli/output/jsonw/configw.go b/cli/output/jsonw/configw.go deleted file mode 100644 index ed167801..00000000 --- a/cli/output/jsonw/configw.go +++ /dev/null @@ -1,56 +0,0 @@ -package jsonw - -import ( - "io" - - "github.com/neilotoole/sq/libsq/core/options" - - "github.com/neilotoole/sq/cli/output" -) - -var _ output.ConfigWriter = (*configWriter)(nil) - -// configWriter implements output.ConfigWriter. -type configWriter struct { - out io.Writer - pr *output.Printing -} - -// NewConfigWriter returns a new output.ConfigWriter. -func NewConfigWriter(out io.Writer, pr *output.Printing) output.ConfigWriter { - return &configWriter{out: out, pr: pr} -} - -// Location implements output.ConfigWriter. -func (w *configWriter) Location(loc, origin string) error { - type cfgInfo struct { - Location string `json:"location"` - Origin string `json:"origin,omitempty"` - } - - c := cfgInfo{ - Location: loc, - Origin: origin, - } - - return writeJSON(w.out, w.pr, c) -} - -// Options implements output.ConfigWriter. -func (w *configWriter) Options(_ *options.Registry, o options.Options) error { - if len(o) == 0 { - return nil - } - - return writeJSON(w.out, w.pr, o) -} - -// SetOption implements output.ConfigWriter. -func (w *configWriter) SetOption(_ *options.Registry, o options.Options, opt options.Opt) error { - if !w.pr.Verbose { - return nil - } - - o = options.Effective(o, opt) - return writeJSON(w.out, w.pr, o) -} diff --git a/cli/output/jsonw/configwriter.go b/cli/output/jsonw/configwriter.go new file mode 100644 index 00000000..5deea18d --- /dev/null +++ b/cli/output/jsonw/configwriter.go @@ -0,0 +1,94 @@ +package jsonw + +import ( + "io" + + "github.com/neilotoole/sq/cli/output/outputx" + "github.com/neilotoole/sq/libsq/core/options" + + "github.com/neilotoole/sq/cli/output" +) + +var _ output.ConfigWriter = (*configWriter)(nil) + +// configWriter implements output.ConfigWriter. +type configWriter struct { + out io.Writer + pr *output.Printing +} + +// NewConfigWriter returns a new output.ConfigWriter. +func NewConfigWriter(out io.Writer, pr *output.Printing) output.ConfigWriter { + return &configWriter{out: out, pr: pr} +} + +// Location implements output.ConfigWriter. +func (w *configWriter) Location(loc, origin string) error { + type cfgInfo struct { + Location string `json:"location"` + Origin string `json:"origin,omitempty"` + } + + c := cfgInfo{ + Location: loc, + Origin: origin, + } + + return writeJSON(w.out, w.pr, c) +} + +// Opt implements output.ConfigWriter. +func (w *configWriter) Opt(o options.Options, opt options.Opt) error { + if o == nil || opt == nil { + return nil + } + + o2 := options.Options{opt.Key(): o[opt.Key()]} + + if !w.pr.Verbose { + return writeJSON(w.out, w.pr, o2) + } + + vo := outputx.NewVerboseOpt(opt, o2) + return writeJSON(w.out, w.pr, vo) +} + +// Options implements output.ConfigWriter. +func (w *configWriter) Options(reg *options.Registry, o options.Options) error { + if len(o) == 0 { + return nil + } + + if !w.pr.Verbose { + return writeJSON(w.out, w.pr, o) + } + + opts := reg.Opts() + m := map[string]outputx.VerboseOpt{} + for _, opt := range opts { + m[opt.Key()] = outputx.NewVerboseOpt(opt, o) + } + + return writeJSON(w.out, w.pr, m) +} + +// SetOption implements output.ConfigWriter. +func (w *configWriter) SetOption(o options.Options, opt options.Opt) error { + if !w.pr.Verbose { + return nil + } + + vo := outputx.NewVerboseOpt(opt, o) + return writeJSON(w.out, w.pr, vo) +} + +// UnsetOption implements output.ConfigWriter. +func (w *configWriter) UnsetOption(opt options.Opt) error { + if !w.pr.Verbose { + return nil + } + + o := options.Options{opt.Key(): opt.GetAny(nil)} + vo := outputx.NewVerboseOpt(opt, o) + return writeJSON(w.out, w.pr, vo) +} diff --git a/cli/output/jsonw/internal/jcolorenc/json.go b/cli/output/jsonw/internal/jcolorenc/json.go index 83557abe..08deff6e 100644 --- a/cli/output/jsonw/internal/jcolorenc/json.go +++ b/cli/output/jsonw/internal/jcolorenc/json.go @@ -372,9 +372,7 @@ func (dec *Decoder) InputOffset() int64 { // Encoder is documented at https://golang.org/pkg/encoding/json/#Encoder type Encoder struct { - writer io.Writer - // prefix string - // indent string + writer io.Writer buffer *bytes.Buffer err error flags AppendFlags diff --git a/cli/output/jsonw/jsonw.go b/cli/output/jsonw/jsonw.go new file mode 100644 index 00000000..01262749 --- /dev/null +++ b/cli/output/jsonw/jsonw.go @@ -0,0 +1,29 @@ +// Package jsonw implements output writers for JSON. +package jsonw + +import ( + "io" + + "github.com/neilotoole/sq/cli/output" + "github.com/neilotoole/sq/cli/output/jsonw/internal" + jcolorenc "github.com/neilotoole/sq/cli/output/jsonw/internal/jcolorenc" + "github.com/neilotoole/sq/libsq/core/errz" +) + +// writeJSON prints a JSON representation of v to out, using specs +// from pr. +func writeJSON(out io.Writer, pr *output.Printing, v any) error { + enc := jcolorenc.NewEncoder(out) + enc.SetColors(internal.NewColors(pr)) + enc.SetEscapeHTML(false) + if pr.Pretty { + enc.SetIndent("", pr.Indent) + } + + err := enc.Encode(v) + if err != nil { + return errz.Err(err) + } + + return nil +} diff --git a/cli/output/jsonw/jsonwriter_test.go b/cli/output/jsonw/jsonw_test.go similarity index 100% rename from cli/output/jsonw/jsonwriter_test.go rename to cli/output/jsonw/jsonw_test.go diff --git a/cli/output/jsonw/jsonwriter.go b/cli/output/jsonw/recordwriter.go similarity index 95% rename from cli/output/jsonw/jsonwriter.go rename to cli/output/jsonw/recordwriter.go index 5564fa93..7d2f1c7c 100644 --- a/cli/output/jsonw/jsonwriter.go +++ b/cli/output/jsonw/recordwriter.go @@ -1,4 +1,3 @@ -// Package jsonw implements output writers for JSON. package jsonw import ( @@ -8,29 +7,10 @@ import ( "github.com/neilotoole/sq/cli/output" "github.com/neilotoole/sq/cli/output/jsonw/internal" - jcolorenc "github.com/neilotoole/sq/cli/output/jsonw/internal/jcolorenc" "github.com/neilotoole/sq/libsq/core/errz" "github.com/neilotoole/sq/libsq/core/sqlz" ) -// writeJSON prints a JSON representation of v to out, using specs -// from pr. -func writeJSON(out io.Writer, pr *output.Printing, v any) error { - enc := jcolorenc.NewEncoder(out) - enc.SetColors(internal.NewColors(pr)) - enc.SetEscapeHTML(false) - if pr.Pretty { - enc.SetIndent("", pr.Indent) - } - - err := enc.Encode(v) - if err != nil { - return errz.Err(err) - } - - return nil -} - // NewStdRecordWriter returns a record writer that outputs each // record as a JSON object that is an element of JSON array. This is // to say, standard JSON. For example: diff --git a/cli/output/jsonw/sourcewriter.go b/cli/output/jsonw/sourcewriter.go index 760d0c68..7f4a6393 100644 --- a/cli/output/jsonw/sourcewriter.go +++ b/cli/output/jsonw/sourcewriter.go @@ -99,7 +99,7 @@ func (w *sourceWriter) Group(group *source.Group) error { return nil } - group.RedactLocations() + source.RedactGroup(group) return writeJSON(w.out, w.pr, group) } @@ -109,12 +109,12 @@ func (w *sourceWriter) SetActiveGroup(group *source.Group) error { return nil } - group.RedactLocations() + source.RedactGroup(group) return writeJSON(w.out, w.pr, group) } // Groups implements output.SourceWriter. func (w *sourceWriter) Groups(tree *source.Group) error { - tree.RedactLocations() + source.RedactGroup(tree) return writeJSON(w.out, w.pr, tree) } diff --git a/cli/output/outputx/outputx.go b/cli/output/outputx/outputx.go new file mode 100644 index 00000000..7c29a58b --- /dev/null +++ b/cli/output/outputx/outputx.go @@ -0,0 +1,33 @@ +// Package outputx contains extensions to pkg output, and helpers +// for implementing output writers. +package outputx + +import ( + "reflect" + + "github.com/neilotoole/sq/libsq/core/options" +) + +// VerboseOpt is a verbose realization of an options.Opt value. +type VerboseOpt struct { + Key string `json:"key"` + Type string `json:"type"` + IsSet bool `json:"is_set"` + DefaultValue any `json:"default_value"` + Value any `json:"value"` + Comment string `json:"comment"` +} + +// NewVerboseOpt returns a VerboseOpt built from opt and o. +func NewVerboseOpt(opt options.Opt, o options.Options) VerboseOpt { + v := VerboseOpt{ + Key: opt.Key(), + DefaultValue: opt.GetAny(nil), + IsSet: o.IsSet(opt), + Comment: opt.Comment(), + Value: opt.GetAny(o), + Type: reflect.TypeOf(opt.GetAny(nil)).String(), + } + + return v +} diff --git a/cli/output/tablew/configw.go b/cli/output/tablew/configwriter.go similarity index 83% rename from cli/output/tablew/configw.go rename to cli/output/tablew/configwriter.go index aef2ee0a..d31b8225 100644 --- a/cli/output/tablew/configw.go +++ b/cli/output/tablew/configwriter.go @@ -38,6 +38,18 @@ func (w *configWriter) Location(path, origin string) error { return nil } +// Opt implements output.ConfigWriter. +func (w *configWriter) Opt(o options.Options, opt options.Opt) error { + if o == nil || opt == nil { + return nil + } + + o2 := options.Options{opt.Key(): o[opt.Key()]} + reg2 := &options.Registry{} + reg2.Add(opt) + return w.Options(reg2, o2) +} + // Options implements output.ConfigWriter. func (w *configWriter) Options(reg *options.Registry, o options.Options) error { if o == nil { @@ -134,7 +146,7 @@ func (w *configWriter) doPrintOptions(reg *options.Registry, o options.Options, } // SetOption implements output.ConfigWriter. -func (w *configWriter) SetOption(_ *options.Registry, o options.Options, opt options.Opt) error { +func (w *configWriter) SetOption(o options.Options, opt options.Opt) error { if !w.tbl.pr.Verbose { // No output unless verbose return nil @@ -151,6 +163,21 @@ func (w *configWriter) SetOption(_ *options.Registry, o options.Options, opt opt return nil } +// UnsetOption implements output.ConfigWriter. +func (w *configWriter) UnsetOption(opt options.Opt) error { + if !w.tbl.pr.Verbose { + // No output unless verbose + return nil + } + + reg := &options.Registry{} + reg.Add(opt) + o := options.Options{} + + w.doPrintOptions(reg, o, true) + return nil +} + func getOptColor(pr *output.Printing, opt options.Opt) *color.Color { if opt == nil { return pr.Null diff --git a/cli/output/tablew/versionw.go b/cli/output/tablew/versionwriter.go similarity index 100% rename from cli/output/tablew/versionw.go rename to cli/output/tablew/versionwriter.go diff --git a/cli/output/writers.go b/cli/output/writers.go index 48694a9b..afc915f8 100644 --- a/cli/output/writers.go +++ b/cli/output/writers.go @@ -111,9 +111,15 @@ type ConfigWriter interface { // of "flag", "env", "default". Location(loc, origin string) error + // Opt prints a single options.Opt. + Opt(o options.Options, opt options.Opt) error + // Options prints config options. Options(reg *options.Registry, o options.Options) error // SetOption is called when an option is set. - SetOption(reg *options.Registry, o options.Options, opt options.Opt) error + SetOption(o options.Options, opt options.Opt) error + + // UnsetOption is called when an option is unset. + UnsetOption(opt options.Opt) error } diff --git a/cli/output/yamlw/configw.go b/cli/output/yamlw/configw.go deleted file mode 100644 index 5d99be7d..00000000 --- a/cli/output/yamlw/configw.go +++ /dev/null @@ -1,59 +0,0 @@ -package yamlw - -import ( - "io" - - "github.com/neilotoole/sq/libsq/core/options" - - "github.com/goccy/go-yaml/printer" - - "github.com/neilotoole/sq/cli/output" -) - -var _ output.ConfigWriter = (*configWriter)(nil) - -// configWriter implements output.ConfigWriter. -type configWriter struct { - p printer.Printer - out io.Writer - pr *output.Printing -} - -// NewConfigWriter returns a new output.ConfigWriter. -func NewConfigWriter(out io.Writer, pr *output.Printing) output.ConfigWriter { - return &configWriter{out: out, pr: pr, p: newPrinter(pr)} -} - -// Location implements output.ConfigWriter. -func (w *configWriter) Location(loc, origin string) error { - type cfgInfo struct { - Location string `json:"location"` - Origin string `json:"origin,omitempty"` - } - - c := cfgInfo{ - Location: loc, - Origin: origin, - } - - return writeYAML(w.p, w.out, c) -} - -// Options implements output.ConfigWriter. -func (w *configWriter) Options(_ *options.Registry, o options.Options) error { - if len(o) == 0 { - return nil - } - - return writeYAML(w.p, w.out, o) -} - -// SetOption implements output.ConfigWriter. -func (w *configWriter) SetOption(_ *options.Registry, o options.Options, opt options.Opt) error { - if !w.pr.Verbose { - return nil - } - - o = options.Effective(o, opt) - return w.Options(nil, o) -} diff --git a/cli/output/yamlw/configwriter.go b/cli/output/yamlw/configwriter.go new file mode 100644 index 00000000..6f08d577 --- /dev/null +++ b/cli/output/yamlw/configwriter.go @@ -0,0 +1,98 @@ +package yamlw + +import ( + "io" + + "github.com/neilotoole/sq/cli/output/outputx" + + "github.com/neilotoole/sq/libsq/core/options" + + "github.com/goccy/go-yaml/printer" + + "github.com/neilotoole/sq/cli/output" +) + +var _ output.ConfigWriter = (*configWriter)(nil) + +// configWriter implements output.ConfigWriter. +type configWriter struct { + p printer.Printer + out io.Writer + pr *output.Printing +} + +// NewConfigWriter returns a new output.ConfigWriter. +func NewConfigWriter(out io.Writer, pr *output.Printing) output.ConfigWriter { + return &configWriter{out: out, pr: pr, p: newPrinter(pr)} +} + +// Location implements output.ConfigWriter. +func (w *configWriter) Location(loc, origin string) error { + type cfgInfo struct { + Location string `json:"location"` + Origin string `json:"origin,omitempty"` + } + + c := cfgInfo{ + Location: loc, + Origin: origin, + } + + return writeYAML(w.out, w.p, c) +} + +// Opt implements output.ConfigWriter. +func (w *configWriter) Opt(o options.Options, opt options.Opt) error { + if o == nil || opt == nil { + return nil + } + + o2 := options.Options{opt.Key(): o[opt.Key()]} + + if !w.pr.Verbose { + return writeYAML(w.out, w.p, o2) + } + + vo := outputx.NewVerboseOpt(opt, o2) + return writeYAML(w.out, w.p, vo) +} + +// Options implements output.ConfigWriter. +func (w *configWriter) Options(reg *options.Registry, o options.Options) error { + if len(o) == 0 { + return nil + } + + if !w.pr.Verbose { + return writeYAML(w.out, w.p, o) + } + + opts := reg.Opts() + m := map[string]outputx.VerboseOpt{} + for _, opt := range opts { + m[opt.Key()] = outputx.NewVerboseOpt(opt, o) + } + + return writeYAML(w.out, w.p, m) +} + +// SetOption implements output.ConfigWriter. +func (w *configWriter) SetOption(o options.Options, opt options.Opt) error { + if !w.pr.Verbose { + return nil + } + + vo := outputx.NewVerboseOpt(opt, o) + return writeYAML(w.out, w.p, vo) +} + +// UnsetOption implements output.ConfigWriter. +func (w *configWriter) UnsetOption(opt options.Opt) error { + if !w.pr.Verbose { + return nil + } + + o := options.Options{opt.Key(): opt.GetAny(nil)} + vo := outputx.NewVerboseOpt(opt, o) + return writeYAML(w.out, w.p, vo) +} diff --git a/cli/output/yamlw/mdwriter.go b/cli/output/yamlw/metadatawriter.go similarity index 90% rename from cli/output/yamlw/mdwriter.go rename to cli/output/yamlw/metadatawriter.go index 47334234..a1eb41b0 100644 --- a/cli/output/yamlw/mdwriter.go +++ b/cli/output/yamlw/metadatawriter.go @@ -25,12 +25,12 @@ func NewMetadataWriter(out io.Writer, pr *output.Printing) output.MetadataWriter // DriverMetadata implements output.MetadataWriter. func (w *mdWriter) DriverMetadata(md []driver.Metadata) error { - return writeYAML(w.yp, w.out, md) + return writeYAML(w.out, w.yp, md) } // TableMetadata implements output.MetadataWriter. func (w *mdWriter) TableMetadata(md *source.TableMetadata) error { - return writeYAML(w.yp, w.out, md) + return writeYAML(w.out, w.yp, md) } // SourceMetadata implements output.MetadataWriter. @@ -38,5 +38,5 @@ func (w *mdWriter) SourceMetadata(md *source.Metadata) error { md2 := *md // Shallow copy is fine md2.Location = source.RedactLocation(md2.Location) - return writeYAML(w.yp, w.out, &md2) + return writeYAML(w.out, w.yp, &md2) } diff --git a/cli/output/yamlw/sourcewriter.go b/cli/output/yamlw/sourcewriter.go new file mode 100644 index 00000000..c5cf6af6 --- /dev/null +++ b/cli/output/yamlw/sourcewriter.go @@ -0,0 +1,123 @@ +package yamlw + +import ( + "io" + + "github.com/goccy/go-yaml/printer" + + "golang.org/x/exp/slices" + + "github.com/neilotoole/sq/cli/output" + "github.com/neilotoole/sq/libsq/source" +) + +var _ output.SourceWriter = (*sourceWriter)(nil) + +type sourceWriter struct { + p printer.Printer + out io.Writer + pr *output.Printing +} + +// NewSourceWriter returns a source writer that outputs source +// details in text table format. +func NewSourceWriter(out io.Writer, pr *output.Printing) output.SourceWriter { + return &sourceWriter{out: out, pr: pr, p: newPrinter(pr)} +} + +// Collection implements output.SourceWriter. +func (w *sourceWriter) Collection(coll *source.Collection) error { + if coll == nil { + return nil + } + + // This is a bit hacky. Basically we want to YAML-print coll.Data(). + // But, we want to do it just for the active group. + // So, our hack is that we clone the coll, and remove any + // sources that are not in the active group. + // + // This whole function, including what it outputs, should be revisited. + coll = coll.Clone() + group := coll.ActiveGroup() + + // We store the active src handle + activeHandle := coll.ActiveHandle() + + handles, err := coll.HandlesInGroup(group) + if err != nil { + return err + } + + srcs := coll.Sources() + for _, src := range srcs { + if !slices.Contains(handles, src.Handle) { + if err = coll.Remove(src.Handle); err != nil { + // Should never happen + return err + } + } + } + + srcs = coll.Sources() + for i := range srcs { + srcs[i].Location = srcs[i].RedactedLocation() + } + + // HACK: we set the activeHandle back, even though that + // active source may have been removed (because it is not in + // the active group). This whole thing is a mess. + _, _ = coll.SetActive(activeHandle, true) + + return writeYAML(w.out, w.p, coll.Data()) +} + +// Source implements output.SourceWriter. +func (w *sourceWriter) Source(_ *source.Collection, src *source.Source) error { + if src == nil { + return nil + } + + src = src.Clone() + src.Location = src.RedactedLocation() + return writeYAML(w.out, w.p, src) +} + +// Removed implements output.SourceWriter. +func (w *sourceWriter) Removed(srcs ...*source.Source) error { + if !w.pr.Verbose || len(srcs) == 0 { + return nil + } + + srcs2 := make([]*source.Source, len(srcs)) + for i := range srcs { + srcs2[i] = srcs[i].Clone() + srcs2[i].Location = srcs2[i].RedactedLocation() + } + return writeYAML(w.out, w.p, srcs2) +} + +// Group implements output.SourceWriter. +func (w *sourceWriter) Group(group *source.Group) error { + if group == nil { + return nil + } + + source.RedactGroup(group) + return writeYAML(w.out, w.p, group) +} + +// SetActiveGroup implements output.SourceWriter. +func (w *sourceWriter) SetActiveGroup(group *source.Group) error { + if group == nil { + return nil + } + + source.RedactGroup(group) + return writeYAML(w.out, w.p, group) +} + +// Groups implements output.SourceWriter. +func (w *sourceWriter) Groups(tree *source.Group) error { + source.RedactGroup(tree) + return writeYAML(w.out, w.p, tree) +} diff --git a/cli/output/yamlw/yamlw.go b/cli/output/yamlw/yamlw.go index 587b4d9b..580dab4f 100644 --- a/cli/output/yamlw/yamlw.go +++ b/cli/output/yamlw/yamlw.go @@ -16,7 +16,7 @@ import ( // writeYAML prints a YAML representation of v to out, using specs // from pr. -func writeYAML(p printer.Printer, out io.Writer, v any) error { +func writeYAML(out io.Writer, p printer.Printer, v any) error { b, err := goccy.Marshal(v) if err != nil { return errz.Err(err) diff --git a/cli/runcontext.go b/cli/runcontext.go index 149e1de7..0f5b3752 100644 --- a/cli/runcontext.go +++ b/cli/runcontext.go @@ -147,7 +147,7 @@ func newDefaultRunContext(ctx context.Context, rc.Config, rc.ConfigStore, configErr = yamlstore.Load(ctx, args, rc.OptionsRegistry, upgrades) - log, logHandler, logCloser, logErr := defaultLogging(ctx) + log, logHandler, logCloser, logErr := defaultLogging(ctx, args, rc.Config) rc.clnup = cleanup.New().AddE(logCloser) if logErr != nil { stderrLog, h := stderrLogger() @@ -160,10 +160,13 @@ func newDefaultRunContext(ctx context.Context, return rc, log, err } } - if log != nil { - log = log.With(lga.Pid, os.Getpid()) + + if log == nil { + log = lg.Discard() } + log = log.With(lga.Pid, os.Getpid()) + if rc.Config == nil { rc.Config = config.New() } diff --git a/cli/ioutilz.go b/cli/term.go similarity index 100% rename from cli/ioutilz.go rename to cli/term.go diff --git a/cli/writers.go b/cli/writers.go index 5fea1743..bd5929a5 100644 --- a/cli/writers.go +++ b/cli/writers.go @@ -165,6 +165,7 @@ func newWriters(cmd *cobra.Command, opts options.Options, out, errOut io.Writer, case format.YAML: w.configw = yamlw.NewConfigWriter(out2, pr) w.metaw = yamlw.NewMetadataWriter(out2, pr) + w.srcw = yamlw.NewSourceWriter(out2, pr) } return w, out2, errOut2 @@ -238,11 +239,10 @@ func getPrinting(cmd *cobra.Command, opts options.Options, out, errOut io.Writer return pr, out2, errOut2 } -func getFormat(cmd *cobra.Command, defaults options.Options) format.Format { +func getFormat(cmd *cobra.Command, o options.Options) format.Format { var fm format.Format switch { - // cascade through the format flags in low-to-high order of precedence. case cmdFlagChanged(cmd, flag.TSV): fm = format.TSV case cmdFlagChanged(cmd, flag.CSV): @@ -269,7 +269,7 @@ func getFormat(cmd *cobra.Command, defaults options.Options) format.Format { fm = format.YAML default: // no format flag, use the config value - fm = OptOutputFormat.Get(defaults) + fm = OptOutputFormat.Get(o) } return fm } diff --git a/libsq/core/lg/lga/lga.go b/libsq/core/lg/lga/lga.go index e5fdcf88..dde38374 100644 --- a/libsq/core/lg/lga/lga.go +++ b/libsq/core/lg/lga/lga.go @@ -14,8 +14,10 @@ const ( Driver = "driver" DefaultTo = "default_to" Elapsed = "elapsed" + Env = "env" Err = "error" From = "from" + Flag = "flag" Handle = "handle" Index = "index" Key = "key" diff --git a/libsq/core/lg/slogbuf/slogbuf.go b/libsq/core/lg/slogbuf/slogbuf.go index 09da13d0..2bcfc8b0 100644 --- a/libsq/core/lg/slogbuf/slogbuf.go +++ b/libsq/core/lg/slogbuf/slogbuf.go @@ -63,6 +63,11 @@ func (b *Buffer) Flush(ctx context.Context, dest slog.Handler) error { for i := range b.entries { d, h, rec := dest, b.entries[i].handler, b.entries[i].record + + if !d.Enabled(ctx, rec.Level) { + continue + } + d = d.WithAttrs(h.attrs) for _, g := range h.groups { d = d.WithGroup(g) diff --git a/libsq/core/lg/userlogdir/userlogdir.go b/libsq/core/lg/userlogdir/userlogdir.go new file mode 100644 index 00000000..ead7cebf --- /dev/null +++ b/libsq/core/lg/userlogdir/userlogdir.go @@ -0,0 +1,60 @@ +// Package userlogdir has a single function, UserLogDir, that returns +// an OS-specific path for storing user logs. +package userlogdir + +import ( + "errors" + "os" + "runtime" +) + +// UserLogDir returns the default root directory to use for user-specific +// log data. Users should create their own application-specific subdirectory +// within this one and use that. +// +// On Unix systems, it returns $XDG_CACHE_HOME as specified by +// https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html if +// non-empty, else $HOME/.cache. +// On Darwin, it returns $HOME/Library/Logs. +// On Windows, it returns %LocalAppData%. +// On Plan 9, it returns $home/lib/cache. +// +// If the location cannot be determined (for example, $HOME is not defined), +// then it will return an error. +func UserLogDir() (string, error) { + var dir string + + switch runtime.GOOS { + case "windows": + dir = os.Getenv("LocalAppData") + if dir == "" { + return "", errors.New("%LocalAppData% is not defined") + } + + case "darwin", "ios": + dir = os.Getenv("HOME") + if dir == "" { + return "", errors.New("$HOME is not defined") + } + dir += "/Library/Logs" + + case "plan9": + dir = os.Getenv("home") + if dir == "" { + return "", errors.New("$home is not defined") + } + dir += "/lib/cache" + + default: // Unix + dir = os.Getenv("XDG_CACHE_HOME") + if dir == "" { + dir = os.Getenv("HOME") + if dir == "" { + return "", errors.New("neither $XDG_CACHE_HOME nor $HOME are defined") + } + dir += "/.cache" + } + } + + return dir, nil +} diff --git a/libsq/source/collection.go b/libsq/source/collection.go index a750a835..a318ba06 100644 --- a/libsq/source/collection.go +++ b/libsq/source/collection.go @@ -949,9 +949,11 @@ func (g *Group) AllSources() []*Source { return srcs } -// RedactLocations modifies g, cloning each descendant Source, and setting +// RedactGroup modifies g, cloning each descendant Source, and setting // the Source.Location field of each contained source to its redacted value. -func (g *Group) RedactLocations() { +// +// TODO: consider moving this to a function instead of a method. +func RedactGroup(g *Group) { if g == nil { return } @@ -962,7 +964,7 @@ func (g *Group) RedactLocations() { } for i := range g.Groups { - g.Groups[i].RedactLocations() + RedactGroup(g.Groups[i]) } }