#199: More config/options work (#215)

* CHANGELOG text clarification

* Dialing in config/options

* Yet more dialing in of config/options

* Refactor output writers

* YAML output for more commands

CHANGELOG.md

@@ -7,11 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 Breaking changes are annotated with ☢️.
 
-
 ## Upcoming
 
-This release completely overhauls `sq`'s config mechanism. There are a
-handful of minor breaking changes ☢️.
+This release significantly overhauls `sq`'s config mechanism ([#199]). There 
+are several minor breaking changes ☢️.
 
 ### Added
 
@@ -22,16 +21,34 @@ handful of minor breaking changes ☢️.
 - `sq config location` prints the location of the config dir.
 - `--config` flag is now honored globally.
 - Many more knobs are exposed in config.
+- Added flags `--log`, `--log.file` and `--log.level`.
+  - These values can also be set in config via `sq config edit` or `sq config set log.level DEBUG` etc.
+  - And they can also be set via envars, e.g.
+    ```shell
+    export SQ_LOG=true
+    export SQ_LOG_FILE=/var/log/sq.log
+    export SQ_LOG_LEVEL=WARN
+    ```
+- Several more commands support YAML output:
+  - [`sq group`](https://sq.io/docs/cmd/group)
+  - [`sq ls`](https://sq.io/docs/cmd/ls)
+  - [`sq mv`](https://sq.io/docs/cmd/mv)
+  - [`sq rm`](https://sq.io/docs/cmd/rm)
+  - [`sq src`](https://sq.io/docs/cmd/src)
+
 
 ### Changed
 
+- The structure of `sq`'s config file (`sq.yml`) has changed. The config
+  file is automatically upgraded when using the new version.
 - ☢️ Envar `SQ_CONFIG` replaces `SQ_CONFIGDIR`. 
+- ☢️ Envar `SQ_LOG_FILE` replaces `SQ_LOGFILE`.
 - ☢️ Format flag `--table` is renamed to `--text`. This is changed because while the
   output is mostly in table format, sometimes it's just plain text. Thus
   `table` was not quite accurate.
 - ☢️ The flag to explicitly specify a driver when piping input to `sq` has been
-  renamed from `--driver` to `--ingest.driver`. This change is made to align
-  the naming of all the ingest options and reduce ambiguity.
+  renamed from `--driver` to `--ingest.driver`. This change aligns
+  the naming of the ingest options and reduces ambiguity.
   ```shell
   # previously
   $ cat mystery.data | sq --driver=csv '.data'
@@ -40,7 +57,7 @@ handful of minor breaking changes ☢️.
   $ cat mystery.data | sq --ingest.driver=csv '.data'
   ```
 - ☢️ `sq add` no longer has the generic `--opts x=y` mechanism. This flag was
-  ambiguous and confusing. Instead use explicit option flags.
+  ambiguous and confusing. Instead, use explicit option flags.
   ```shell
   # previously
   $ sq add ./actor.csv --opts=header=false
@@ -59,7 +76,6 @@ handful of minor breaking changes ☢️.
   $ sq add ./actor.csv -n @actor
    ```
 
-
 ## [v0.33.0] - 2023-04-15
 
 The headline feature is [source groups](https://sq.io/docs/source#groups).
@@ -396,6 +412,7 @@ make working with lots of sources much easier.
 [#189]: https://github.com/neilotoole/sq/issues/189
 [#191]: https://github.com/neilotoole/sq/issues/191
 [#192]: https://github.com/neilotoole/sq/issues/192
+[#199]: https://github.com/neilotoole/sq/issues/199
 
 [v0.15.2]: https://github.com/neilotoole/sq/releases/tag/v0.15.2
 [v0.15.3]: https://github.com/neilotoole/sq/compare/v0.15.2...v0.15.3

cli/cli_test.go

@@ -38,6 +38,7 @@ func TestSmoke(t *testing.T) {
 		{a: []string{"inspect"}, errBecause: "no active data source"},
 		{a: []string{"inspect", "--help"}},
 		{a: []string{"version"}},
+		{a: []string{"version", "--help"}},
 		{a: []string{"--version"}},
 		{a: []string{"help"}},
 		{a: []string{"--help"}},

cli/cmd_add.go

@@ -138,6 +138,8 @@ More examples:
 	}
 
 	cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage)
+	cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage)
+	cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage)
 
 	cmd.Flags().StringP(flag.AddDriver, flag.AddDriverShort, "", flag.AddDriverUsage)
 	panicOn(cmd.RegisterFlagCompletionFunc(flag.AddDriver, completeDriverType))

cli/cmd_config_get.go

@@ -40,6 +40,7 @@ Use the --verbose flag (in text output format) to see all options.`,
 
 	cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage)
 	cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage)
+	cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage)
 
 	cmd.Flags().String(flag.ConfigSrc, "", flag.ConfigSrcUsage)
 	panicOn(cmd.RegisterFlagCompletionFunc(flag.ConfigSrc, completeHandle(1)))
@@ -87,13 +88,5 @@ func execConfigGet(cmd *cobra.Command, args []string) error {
 		return errz.Errorf("invalid option key: %s", args[0])
 	}
 
-	// A bit of a hack... create a new registry with just the desired opt.
-	reg2 := &options.Registry{}
-	reg2.Add(opt)
-	o2 := options.Options{}
-	if v, ok := o[opt.Key()]; ok {
-		o2[opt.Key()] = v
-	}
-
-	return rc.writers.configw.Options(reg2, o2)
+	return rc.writers.configw.Opt(o, opt)
 }

cli/cmd_config_set.go

@@ -1,9 +1,10 @@
 package cli
 
 import (
+	"fmt"
+
 	"github.com/neilotoole/sq/cli/flag"
 	"github.com/neilotoole/sq/libsq/core/errz"
-	"github.com/neilotoole/sq/libsq/core/lg"
 	"github.com/neilotoole/sq/libsq/core/lg/lga"
 	"github.com/neilotoole/sq/libsq/core/options"
 	"github.com/neilotoole/sq/libsq/source"
@@ -14,30 +15,38 @@ func newConfigSetCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:               "set",
 		RunE:              execConfigSet,
-		Args:              cobra.ExactArgs(2),
+		Args:              cobra.RangeArgs(1, 2),
 		ValidArgsFunction: completeConfigSet,
 		Short:             "Set config value",
-		Long: `Set config value globally, or for a specific source.
+		Long: `Set base config value, or set value for a specific source.
 Use "sq config get -v" to see available options.`,
-		Example: `  # Set default output format
+		Example: `  # Set base output format
   $ sq config set format json
 
-  # Set default max DB connections
+  # Set base max DB connections
   $ sq config set conn.max-open 10
 
   # Set max DB connections for source @sakila
-  $ sq config set --src @sakila conn.max-open 50`,
+  $ sq config set --src @sakila conn.max-open 50
+
+  # Delete an option (resets to default value)
+  $ sq config set -D conn.max-open`,
 	}
 
 	cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage)
+	cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage)
 	cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage)
 
 	cmd.Flags().String(flag.ConfigSrc, "", flag.ConfigSrcUsage)
 	panicOn(cmd.RegisterFlagCompletionFunc(flag.ConfigSrc, completeHandle(1)))
+
+	cmd.Flags().BoolP(flag.ConfigDelete, flag.ConfigDeleteShort, false, flag.ConfigDeleteUsage)
+
 	return cmd
 }
 
 func execConfigSet(cmd *cobra.Command, args []string) error {
+	log := logFrom(cmd)
 	rc, ctx := RunContextFrom(cmd.Context()), cmd.Context()
 
 	o := rc.Config.Options
@@ -66,6 +75,25 @@ func execConfigSet(cmd *cobra.Command, args []string) error {
 		o = src.Options
 	}
 
+	if cmdFlagChanged(cmd, flag.ConfigDelete) {
+		if len(args) > 1 {
+			return errz.Errorf("accepts 1 arg when used with --%s flag", flag.ConfigDelete)
+		}
+
+		delete(o, opt.Key())
+		if src == nil {
+			log.Info("Unset base config value", lga.Key, opt.Key())
+		} else {
+			log.Info("Unset source config value", lga.Src, src, lga.Key, opt.Key())
+		}
+
+		if err := rc.ConfigStore.Save(ctx, rc.Config); err != nil {
+			return err
+		}
+
+		return rc.writers.configw.UnsetOption(opt)
+	}
+
 	o2 := options.Options{}
 	o2[opt.Key()] = args[1]
 	var err error
@@ -79,20 +107,35 @@ func execConfigSet(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	if src != nil {
-		lg.FromContext(ctx).Info("Set default config value", lga.Val, o)
+	if src == nil {
+		log.Info(
+			"Set base config value",
+			lga.Key, opt.Key(),
+			lga.Val, o[opt.Key()],
+		)
 	} else {
-		lg.FromContext(ctx).Info("Set source config value", lga.Src, src, lga.Val, o)
+		log.Info(
+			"Set source config value",
+			lga.Key, opt.Key(),
+			lga.Src, src,
+			lga.Val, o,
+		)
 	}
 
-	return rc.writers.configw.SetOption(rc.OptionsRegistry, o, opt)
+	return rc.writers.configw.SetOption(o, opt)
 }
 
 func completeConfigSet(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 	switch len(args) {
 	case 0:
 		return completeOptKey(cmd, args, toComplete)
+
 	case 1:
+		if cmdFlagChanged(cmd, flag.ConfigDelete) {
+			logFrom(cmd).Warn(fmt.Sprintf("No 2nd arg when using --%s flag", flag.ConfigDelete))
+			return nil, cobra.ShellCompDirectiveError
+		}
+
 		return completeOptValue(cmd, args, toComplete)
 	default:
 		// Maximum of two args

cli/cmd_driver.go

@@ -8,7 +8,8 @@ import (
 func newDriverCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "driver",
-		Short: "List or manage drivers",
+		Short: "Manage drivers",
+		Long:  "Manage drivers.",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return cmd.Help()
 		},
@@ -23,14 +24,15 @@ func newDriverCmd() *cobra.Command {
 func newDriverListCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "ls",
-		Short: "List available drivers",
-		Long:  "List available drivers.",
+		Short: "List installed drivers",
+		Long:  "List installed drivers.",
 		Args:  cobra.NoArgs,
 		RunE:  execDriverList,
 	}
 
 	cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage)
 	cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage)
+	cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage)
 
 	return cmd
 }

cli/cmd_group.go

@@ -34,6 +34,8 @@ Use 'sq ls -g' to list groups.`,
 	}
 
 	cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage)
+	cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage)
+	cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage)
 
 	return cmd
 }

cli/cmd_inspect.go

@@ -45,6 +45,7 @@ If @HANDLE is not provided, the active data source is assumed.`,
 	}
 
 	cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage)
+	cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage)
 	cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage)
 
 	return cmd

cli/cmd_list.go

@@ -40,6 +40,8 @@ any further descendants.
 	}
 
 	cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage)
+	cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage)
+	cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage)
 	cmd.Flags().BoolP(flag.ListGroup, flag.ListGroupShort, false, flag.ListGroupUsage)
 
 	return cmd

cli/cmd_mv.go

@@ -3,6 +3,8 @@ package cli
 import (
 	"strings"
 
+	"github.com/neilotoole/sq/cli/flag"
+
 	"github.com/neilotoole/sq/libsq/core/stringz"
 
 	"golang.org/x/exp/slices"
@@ -41,6 +43,10 @@ source handles are files, and groups are directories.`,
   $ sq mv production prod`,
 	}
 
+	cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage)
+	cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage)
+	cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage)
+
 	return cmd
 }
 

cli/cmd_ping.go

@@ -65,6 +65,8 @@ The exit code is 1 if ping fails for any of the sources.`,
 	cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage)
 	cmd.Flags().BoolP(flag.CSV, flag.CSVShort, false, flag.CSVUsage)
 	cmd.Flags().BoolP(flag.TSV, flag.TSVShort, false, flag.TSVUsage)
+	cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage)
+
 	cmd.Flags().Duration(flag.PingTimeout, time.Second*10, flag.PingTimeoutUsage)
 	return cmd
 }

cli/cmd_remove.go

@@ -33,6 +33,8 @@ may have changed, if that source or group was removed.`,
 	}
 
 	cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage)
+	cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage)
+	cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage)
 	return cmd
 }
 

cli/cmd_root.go

@@ -98,5 +98,8 @@ See docs and more: https://sq.io`,
 	cmd.PersistentFlags().BoolP(flag.Monochrome, flag.MonochromeShort, false, flag.MonochromeUsage)
 	cmd.PersistentFlags().BoolP(flag.Verbose, flag.VerboseShort, false, flag.VerboseUsage)
 	cmd.PersistentFlags().String(flag.Config, "", flag.ConfigUsage)
+	cmd.PersistentFlags().Bool(flag.LogEnabled, false, flag.LogEnabledUsage)
+	cmd.PersistentFlags().String(flag.LogFile, "", flag.LogFileUsage)
+	cmd.PersistentFlags().String(flag.LogLevel, "", flag.LogLevelUsage)
 	return cmd
 }

cli/cmd_src.go

@@ -22,6 +22,8 @@ source. Otherwise, set @HANDLE as the active data source.`,
 	}
 
 	cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage)
+	cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage)
+	cmd.Flags().BoolP(flag.YAML, flag.YAMLShort, false, flag.YAMLUsage)
 
 	return cmd
 }

cli/cmd_tbl.go

@@ -55,6 +55,7 @@ func newTblCopyCmd() *cobra.Command {
 	}
 
 	cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage)
+	cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage)
 	cmd.Flags().Bool(flag.TblData, true, flag.TblDataUsage)
 
 	return cmd

cli/cmd_version.go

@@ -51,6 +51,7 @@ Before upgrading, check the changelog: https://sq.io/changelog`,
 	}
 
 	cmd.Flags().BoolP(flag.JSON, flag.JSONShort, false, flag.JSONUsage)
+	cmd.Flags().Bool(flag.Pretty, true, flag.PrettyUsage)
 
 	return cmd
 }

cli/complete.go

@@ -73,6 +73,11 @@ func completeHandle(max int) completionFunc {
 
 		slices.Sort(handles) // REVISIT: what's the logic for sorting or not?
 		handles, _ = lo.Difference(handles, args)
+
+		if rc.Config.Collection.Active() != nil {
+			handles = append([]string{source.ActiveHandle}, handles...)
+		}
+
 		return handles, cobra.ShellCompDirectiveNoFileComp
 	}
 }
@@ -173,6 +178,22 @@ func completeOptKey(cmd *cobra.Command, _ []string, toComplete string) ([]string
 		keys = lo.Map(opts, func(item options.Opt, index int) string {
 			return item.Key()
 		})
+
+		if cmdFlagChanged(cmd, flag.ConfigDelete) {
+			if len(src.Options) == 0 {
+				// Nothing to delete
+				return nil, cobra.ShellCompDirectiveError
+			}
+
+			// There are options to delete
+			return src.Options.Keys(), cobra.ShellCompDirectiveDefault
+		}
+	}
+
+	if cmdFlagChanged(cmd, flag.ConfigDelete) {
+		// At this stage, we have to offer all opts, because the user
+		// input could become: $ sq config set -D ingest.header --src @csv
+		return rc.OptionsRegistry.Keys(), cobra.ShellCompDirectiveDefault
 	}
 
 	keys = lo.Filter(keys, func(item string, index int) bool {
@@ -203,6 +224,14 @@ func completeOptValue(cmd *cobra.Command, args []string, toComplete string) ([]s
 
 	var a []string
 	switch opt.(type) {
+	case options.String:
+		if opt.Key() == OptLogFile.Key() {
+			// We return the default directive, so that the shell will offer
+			// regular ol' file completion.
+			return a, cobra.ShellCompDirectiveDefault
+		}
+	case LogLevelOpt:
+		a = []string{"debug", "DEBUG", "info", "INFO", "warn", "WARN", "error", "ERROR"}
 	case FormatOpt:
 		a = stringz.Strings(format.All())
 	case options.Bool:
@@ -394,6 +423,7 @@ func (c *handleTableCompleter) completeHandle(ctx context.Context, rc *RunContex
 	}
 
 	handles := rc.Config.Collection.Handles()
+	handles = append([]string{source.ActiveHandle}, handles...)
 	// Else, we're dealing with just a handle so far
 	var matchingHandles []string
 	for _, handle := range handles {

cli/config/config.go

@@ -14,7 +14,14 @@ import (
 )
 
 const (
-	EnvarLogPath = "SQ_LOGFILE"
+	// EnvarLogPath is the log file path.
+	EnvarLogPath = "SQ_LOG_FILE"
+
+	// EnvarLogLevel is the log level. It maps to a slog.Level.
+	EnvarLogLevel = "SQ_LOG_LEVEL"
+
+	// EnvarLogEnabled turns logging on or off.
+	EnvarLogEnabled = "SQ_LOG"
 
 	// EnvarConfigDir is the legacy envar for config location.
 	// Instead use EnvarConfig.

cli/config/yamlstore/testdata/good.06.sq.yml

@@ -0,0 +1,9 @@
+# This file has log.level in lowercase and empty log.file
+config.version: v0.34.0
+options:
+    format: table
+    header: false
+    log.level: debug
+    log.file:
+
+

cli/config/yamlstore/testdata/good.07.sq.yml

@@ -0,0 +1,7 @@
+# This file has custom log.level "off" (not an actual defined slog.Level).
+config.version: v0.34.0
+options:
+    format: table
+    header: false
+
+

cli/config/yamlstore/yamlstore.go

@@ -124,7 +124,7 @@ func (fs *Store) doLoad(ctx context.Context) (*config.Config, error) {
 
 	cfg.Options, err = fs.OptionsRegistry.Process(cfg.Options)
 	if err != nil {
-		return nil, err
+		return nil, errz.Wrapf(err, "config: %s", fs.Path)
 	}
 
 	if cfg.Collection == nil {

cli/config/yamlstore/yamlstore_test.go

@@ -18,17 +18,13 @@ import (
 )
 
 func TestFileStore_Nil_Save(t *testing.T) {
-	t.Parallel()
-
 	var f *yamlstore.Store
 
-	// noinspection GoNilness
 	err := f.Save(context.Background(), config.New())
 	require.Error(t, err)
 }
 
 func TestFileStore_LoadSaveLoad(t *testing.T) {
-	t.Parallel()
 	ctx := context.Background()
 
 	const wantVers = `v0.34.0`
@@ -72,8 +68,6 @@ var hookExpand = func(data []byte) ([]byte, error) {
 }
 
 func TestFileStore_Load(t *testing.T) {
-	t.Parallel()
-
 	optsReg := &options.Registry{}
 	cli.RegisterDefaultOpts(optsReg)
 
@@ -92,8 +86,6 @@ func TestFileStore_Load(t *testing.T) {
 	for _, match := range good {
 		match := match
 		t.Run(tutil.Name(match), func(t *testing.T) {
-			t.Parallel()
-
 			fs.Path = match
 			_, err = fs.Load(context.Background())
 			require.NoError(t, err, match)

cli/flag/flag.go

@@ -85,13 +85,6 @@ const (
 	SQLQuery      = "query"
 	SQLQueryUsage = "Execute the SQL as a query (as opposed to statement)"
 
-	// SrcOptions is deprecated.
-	//
-	// //Deprecated: Use specific options like flag.IngestHeader.
-	// FIXME: get rid of SrcOptions.
-	SrcOptions      = "opts"
-	SrcOptionsUsage = "Driver-dependent data source options"
-
 	TSV      = "tsv"
 	TSVShort = "T"
 	TSVUsage = "Output TSV"
@@ -135,7 +128,7 @@ const (
 	ConfigUsage = "Load config from here"
 
 	IngestHeader      = "ingest.header"
-	IngestHeaderUsage = "Treat first row of import data as header"
+	IngestHeaderUsage = "Treat first row of ingest data as header"
 
 	CSVEmptyAsNull      = "driver.csv.empty-as-null"
 	CSVEmptyAsNullUsage = "Treat empty CSV fields as null"
@@ -144,7 +137,16 @@ const (
 	CSVDelimUsage   = "CSV delimiter: one of comma, space, pipe, tab, colon, semi, period"
 	CSVDelimDefault = "comma"
 
-	ConfigSetDelete      = "delete"
-	ConfigSetDeleteShort = "D"
-	ConfigSetDeleteUsage = "Unset this option"
+	ConfigDelete      = "delete"
+	ConfigDeleteShort = "D"
+	ConfigDeleteUsage = "Reset this option to default value"
+
+	LogEnabled      = "log"
+	LogEnabledUsage = "Enable logging"
+
+	LogFile      = "log.file"
+	LogFileUsage = "Path to log file; empty disables logging"
+
+	LogLevel      = "log.level"
+	LogLevelUsage = "Log level: one of DEBUG, INFO, WARN, ERROR"
 )

cli/flags.go

@@ -1,7 +1,11 @@
 package cli
 
 import (
+	"io"
+
+	"github.com/neilotoole/sq/libsq/core/errz"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
 // cmdFlagChanged returns true if cmd is non-nil and
@@ -33,3 +37,27 @@ func cmdFlagTrue(cmd *cobra.Command, name string) bool {
 
 	return b
 }
+
+// getBootstrapFlagValue parses osArgs looking for flg. The flag is always
+// treated as string. This function exists because some components such
+// as logging and config interrogate flags before cobra has loaded.
+func getBootstrapFlagValue(flg, flgShort, flgUsage string, osArgs []string) (val string, ok bool, err error) {
+	fs := pflag.NewFlagSet("bootstrap", pflag.ContinueOnError)
+	fs.ParseErrorsWhitelist.UnknownFlags = true
+	fs.SetOutput(io.Discard)
+
+	_ = fs.StringP(flg, flgShort, "", flgUsage)
+	if err = fs.Parse(osArgs); err != nil {
+		return "", false, errz.Err(err)
+	}
+
+	if !fs.Changed(flg) {
+		return "", false, nil
+	}
+
+	if val, err = fs.GetString(flg); err != nil {
+		return "", false, errz.Err(err)
+	}
+
+	return val, true, nil
+}

cli/internal_test.go

@@ -119,5 +119,5 @@ func TestRegisterDefaultOpts(t *testing.T) {
 	log.Debug("options.Registry (after)", "reg", reg)
 
 	keys := reg.Keys()
-	require.Len(t, keys, 19)
+	require.Len(t, keys, 22)
 }

cli/logging.go

@@ -7,6 +7,13 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/neilotoole/sq/libsq/core/stringz"
+
+	"github.com/neilotoole/sq/libsq/core/options"
+
+	"github.com/neilotoole/sq/cli/flag"
+	"github.com/neilotoole/sq/libsq/core/lg/userlogdir"
+
 	"github.com/neilotoole/sq/cli/config"
 	"github.com/neilotoole/sq/libsq/core/errz"
 	"github.com/neilotoole/sq/libsq/core/lg"
@@ -15,19 +22,50 @@ import (
 	"golang.org/x/exp/slog"
 )
 
+var (
+	OptLogEnabled = options.NewBool(
+		"log",
+		false,
+		"Enable logging.",
+	)
+
+	OptLogFile = options.NewString(
+		"log.file",
+		getDefaultLogFilePath(),
+		`Path to log file. Empty value disables logging.`)
+
+	OptLogLevel = NewLogLevelOpt(
+		"log.level",
+		slog.LevelDebug,
+		`Log level, one of DEBUG, INFO, WARN, ERROR.`)
+)
+
 // defaultLogging returns a *slog.Logger, its slog.Handler, and
 // possibly a *cleanup.Cleanup, which the caller is responsible
 // for invoking at the appropriate time. If an error is returned, the
 // other returned values will be nil. If logging is not enabled,
-// all returned values will be nil.
-func defaultLogging(ctx context.Context) (log *slog.Logger, h slog.Handler, closer func() error, err error) {
-	logFilePath, ok := os.LookupEnv(config.EnvarLogPath)
-	if !ok || strings.TrimSpace(logFilePath) == "" {
-		lg.FromContext(ctx).Debug("Logging: not enabled via envar", lga.Key, config.EnvarLogPath)
-		return lg.Discard(), nil, nil, nil
+// the returned values will also be nil.
+func defaultLogging(ctx context.Context, osArgs []string, cfg *config.Config,
+) (log *slog.Logger, h slog.Handler, closer func() error, err error) {
+	bootLog := lg.FromContext(ctx)
+
+	enabled := getLogEnabled(ctx, osArgs, cfg)
+	if !enabled {
+		return nil, nil, nil, nil
 	}
 
-	lg.FromContext(ctx).Debug("Logging: enabled via envar", lga.Key, config.EnvarLogPath, lga.Val, logFilePath)
+	// First, get the log file path. It can come from flag, envar, or config.
+	logFilePath := strings.TrimSpace(getLogFilePath(ctx, osArgs, cfg))
+	if logFilePath == "" {
+		bootLog.Debug("Logging: not enabled (log file path not set)")
+		return nil, nil, nil, nil
+	}
+
+	lvl := getLogLevel(ctx, osArgs, cfg)
+
+	// Allow for $HOME/sq.log etc.
+	logFilePath = os.ExpandEnv(logFilePath)
+	bootLog.Debug("Logging: enabled", lga.Path, logFilePath)
 
 	// Let's try to create the dir holding the logfile... if it already exists,
 	// then os.MkdirAll will just no-op
@@ -43,19 +81,19 @@ func defaultLogging(ctx context.Context) (log *slog.Logger, h slog.Handler, clos
 	}
 	closer = logFile.Close
 
-	h = newJSONHandler(logFile)
+	h = newJSONHandler(logFile, lvl)
 	return slog.New(h), h, closer, nil
 }
 
 func stderrLogger() (*slog.Logger, slog.Handler) {
-	h := newJSONHandler(os.Stderr)
+	h := newJSONHandler(os.Stderr, slog.LevelDebug)
 	return slog.New(h), h
 }
 
-func newJSONHandler(w io.Writer) slog.Handler {
+func newJSONHandler(w io.Writer, lvl slog.Leveler) slog.Handler {
 	return slog.HandlerOptions{
 		AddSource:   true,
-		Level:       slog.LevelDebug,
+		Level:       lvl,
 		ReplaceAttr: slogReplaceAttrs,
 	}.NewJSONHandler(w)
 }
@@ -105,3 +143,281 @@ func logFrom(cmd *cobra.Command) *slog.Logger {
 
 	return log
 }
+
+// getLogEnabled determines if logging is enabled based on flags, envars, or config.
+// Any error is logged to the ctx logger.
+func getLogEnabled(ctx context.Context, osArgs []string, cfg *config.Config) bool {
+	bootLog := lg.FromContext(ctx)
+	var enabled bool
+
+	val, ok, err := getBootstrapFlagValue(flag.LogEnabled, "", flag.LogEnabledUsage, osArgs)
+	if err != nil {
+		bootLog.Error("Reading log 'enabled' from flag", lga.Flag, flag.LogEnabled, lga.Err, err)
+	}
+	if ok {
+		bootLog.Debug("Using log 'enabled' specified via flag", lga.Flag, flag.LogEnabled, lga.Val, val)
+
+		enabled, err = stringz.ParseBool(val)
+		if err != nil {
+			bootLog.Error(
+				"Reading bool flag",
+				lga.Flag, flag.LogEnabled,
+				lga.Val, val,
+			)
+			// When in doubt, enable logging?
+			return true
+		}
+
+		return enabled
+	}
+
+	val, ok = os.LookupEnv(config.EnvarLogEnabled)
+	if ok {
+		bootLog.Debug("Using log 'enabled' specified via envar",
+			lga.Env, config.EnvarLogEnabled,
+			lga.Val, val,
+		)
+
+		enabled, err = stringz.ParseBool(val)
+		if err != nil {
+			bootLog.Error(
+				"Reading bool envar",
+				lga.Env, config.EnvarLogEnabled,
+				lga.Val, val,
+			)
+
+			// When in doubt, enable logging?
+			return true
+		}
+
+		return enabled
+	}
+
+	var o options.Options
+	if cfg != nil {
+		o = cfg.Options
+	}
+
+	enabled = OptLogEnabled.Get(o)
+	bootLog.Debug("Using log 'enabled' specified via config", lga.Key, OptLogEnabled.Key(), lga.Val, enabled)
+	return enabled
+}
+
+// getLogLevel gets the log level, based on flags, envars, or config.
+// Any error is logged to the ctx logger.
+func getLogLevel(ctx context.Context, osArgs []string, cfg *config.Config) slog.Level {
+	bootLog := lg.FromContext(ctx)
+
+	val, ok, err := getBootstrapFlagValue(flag.LogLevel, "", flag.LogLevelUsage, osArgs)
+	if err != nil {
+		bootLog.Error("Reading log level from flag", lga.Flag, flag.LogLevel, lga.Err, err)
+	}
+	if ok {
+		bootLog.Debug("Using log level specified via flag", lga.Flag, flag.LogLevel, lga.Val, val)
+
+		lvl := new(slog.Level)
+		if err = lvl.UnmarshalText([]byte(val)); err != nil {
+			bootLog.Error("Invalid log level specified via flag",
+				lga.Flag, flag.LogLevel,
+				lga.Val, val,
+				lga.Err, err)
+		} else {
+			return *lvl
+		}
+	}
+
+	val, ok = os.LookupEnv(config.EnvarLogLevel)
+	if ok {
+		bootLog.Debug("Using log level specified via envar",
+			lga.Env, config.EnvarLogLevel,
+			lga.Val, val)
+
+		lvl := new(slog.Level)
+		if err = lvl.UnmarshalText([]byte(val)); err != nil {
+			bootLog.Error("Invalid log level specified by envar",
+				lga.Env, config.EnvarLogLevel,
+				lga.Val, val,
+				lga.Err, err)
+		} else {
+			return *lvl
+		}
+	}
+
+	var o options.Options
+	if cfg != nil {
+		o = cfg.Options
+	}
+
+	lvl := OptLogLevel.Get(o)
+	bootLog.Debug("Using log level specified via config", lga.Key, OptLogLevel.Key(), lga.Val, lvl)
+	return lvl
+}
+
+// getLogFilePath gets the log file path, based on flags, envars, or config.
+// If a log file is not specified (and thus logging is disabled), empty string
+// is returned.
+func getLogFilePath(ctx context.Context, osArgs []string, cfg *config.Config) string {
+	bootLog := lg.FromContext(ctx)
+
+	fp, ok, err := getBootstrapFlagValue(flag.LogFile, "", flag.LogFileUsage, osArgs)
+	if err != nil {
+		bootLog.Error("Reading log file from flag", lga.Flag, flag.LogFile, lga.Err, err)
+	}
+	if ok {
+		bootLog.Debug("Log file specified via flag", lga.Flag, flag.LogFile, lga.Path, fp)
+		return fp
+	}
+
+	fp, ok = os.LookupEnv(config.EnvarLogPath)
+	if ok {
+		bootLog.Debug("Log file specified via envar", lga.Env, config.EnvarLogPath, lga.Path, fp)
+		return fp
+	}
+
+	var o options.Options
+	if cfg != nil {
+		o = cfg.Options
+	}
+
+	fp = OptLogFile.Get(o)
+	bootLog = bootLog.With(lga.Key, OptLogFile.Key(), lga.Path, fp)
+
+	if !o.IsSet(OptLogFile) {
+		bootLog.Debug("Log file not explicitly set in config; using default")
+		return fp
+	}
+
+	if fp == "" {
+		bootLog.Debug(`Log file explicitly set to "" in config; logging disabled`)
+	}
+
+	bootLog.Debug("Log file specified via config")
+	return fp
+}
+
+// getDefaultLogFilePath returns the OS-dependent log file path,
+// or an empty string if it can't be determined. The file (and its
+// parent dir) may not exist.
+func getDefaultLogFilePath() string {
+	p, err := userlogdir.UserLogDir()
+	if err != nil {
+		return ""
+	}
+
+	return filepath.Join(p, "sq", "sq.log")
+}
+
+var _ options.Opt = LogLevelOpt{}
+
+// NewLogLevelOpt returns a new LogLevelOpt instance.
+func NewLogLevelOpt(key string, defaultVal slog.Level, comment string, tags ...string) LogLevelOpt {
+	return LogLevelOpt{key: key, defaultVal: defaultVal, comment: comment, tags: tags}
+}
+
+// LogLevelOpt is an options.Opt for slog.Level.
+type LogLevelOpt struct {
+	key        string
+	comment    string
+	defaultVal slog.Level
+	tags       []string
+}
+
+// Comment implements options.Opt.
+func (op LogLevelOpt) Comment() string {
+	return op.comment
+}
+
+// Tags implements options.Opt.
+func (op LogLevelOpt) Tags() []string {
+	return op.tags
+}
+
+// Key implements options.Opt.
+func (op LogLevelOpt) Key() string {
+	return op.key
+}
+
+// String implements options.Opt.
+func (op LogLevelOpt) String() string {
+	return op.key
+}
+
+// IsSet implements options.Opt.
+func (op LogLevelOpt) IsSet(o options.Options) bool {
+	if o == nil {
+		return false
+	}
+
+	return o.IsSet(op)
+}
+
+// Process implements options.Processor. It converts matching
+// string values in o into slog.Level. If no match found,
+// the input arg is returned unchanged. Otherwise, a clone is
+// returned.
+func (op LogLevelOpt) Process(o options.Options) (options.Options, error) {
+	if o == nil {
+		return nil, nil
+	}
+
+	v, ok := o[op.key]
+	if !ok || v == nil {
+		return o, nil
+	}
+
+	// v should be a string
+	switch x := v.(type) {
+	case string:
+	// continue below
+	case int:
+		v = slog.Level(x)
+		// continue below
+	case slog.Level:
+		return o, nil
+	default:
+		return nil, errz.Errorf("option {%s} should be {%T} or {%T} but got {%T}: %v",
+			op.key, slog.LevelDebug, "", x, x)
+	}
+
+	var s string
+	s, ok = v.(string)
+	if !ok {
+		return nil, errz.Errorf("option {%s} should be {%T} but got {%T}: %v",
+			op.key, s, v, v)
+	}
+
+	var lvl slog.Level
+	if err := lvl.UnmarshalText([]byte(s)); err != nil {
+		return nil, errz.Wrapf(err, "option {%s} is not a valid {%T}", op.key, lvl)
+	}
+
+	o = o.Clone()
+	o[op.key] = lvl
+	return o, nil
+}
+
+// GetAny implements options.Opt.
+func (op LogLevelOpt) GetAny(o options.Options) any {
+	return op.Get(o)
+}
+
+// Get returns op's value in o. If o is nil, or no value
+// is set, op's default value is returned.
+func (op LogLevelOpt) Get(o options.Options) slog.Level {
+	if o == nil {
+		return op.defaultVal
+	}
+
+	v, ok := o[op.key]
+	if !ok {
+		return op.defaultVal
+	}
+
+	var lvl slog.Level
+	lvl, ok = v.(slog.Level)
+	if !ok {
+		return op.defaultVal
+	}
+
+	return lvl
+}

cli/options.go

@@ -136,6 +136,9 @@ func RegisterDefaultOpts(reg *options.Registry) {
 		OptPretty,
 		OptPingTimeout,
 		OptShellCompletionTimeout,
+		OptLogEnabled,
+		OptLogFile,
+		OptLogLevel,
 		driver.OptConnMaxOpen,
 		driver.OptConnMaxIdle,
 		driver.OptConnMaxIdleTime,

cli/output/jsonw/README.md

@@ -8,7 +8,9 @@ Note that there are three implementations of `output.RecordWriter`.
 - `NewArrayRecordWriter` outputs each record on its own line as an element of a JSON array.
 - `NewObjectRecordWriter` outputs each record as a JSON object on its own line.
 
-These `RecordWriter`s correspond to the `--json`, `--jsona`, and `--jsonl` flags (where `jsonl` means "JSON Lines"). There are also other writer implementations, such as an `output.ErrorWriter` and an `output.MetadataWriter`.
+These `RecordWriter`s correspond to the `--json`, `--jsona`, and `--jsonl` flags
+(where `jsonl` means "JSON Lines"). There are also other writer implementations,
+such as an `output.ErrorWriter` and an `output.MetadataWriter`.
 
 
 #### Standard JSON `--json`:

cli/output/jsonw/configw.go

@@ -1,56 +0,0 @@
-package jsonw
-
-import (
-	"io"
-
-	"github.com/neilotoole/sq/libsq/core/options"
-
-	"github.com/neilotoole/sq/cli/output"
-)
-
-var _ output.ConfigWriter = (*configWriter)(nil)
-
-// configWriter implements output.ConfigWriter.
-type configWriter struct {
-	out io.Writer
-	pr  *output.Printing
-}
-
-// NewConfigWriter returns a new output.ConfigWriter.
-func NewConfigWriter(out io.Writer, pr *output.Printing) output.ConfigWriter {
-	return &configWriter{out: out, pr: pr}
-}
-
-// Location implements output.ConfigWriter.
-func (w *configWriter) Location(loc, origin string) error {
-	type cfgInfo struct {
-		Location string `json:"location"`
-		Origin   string `json:"origin,omitempty"`
-	}
-
-	c := cfgInfo{
-		Location: loc,
-		Origin:   origin,
-	}
-
-	return writeJSON(w.out, w.pr, c)
-}
-
-// Options implements output.ConfigWriter.
-func (w *configWriter) Options(_ *options.Registry, o options.Options) error {
-	if len(o) == 0 {
-		return nil
-	}
-
-	return writeJSON(w.out, w.pr, o)
-}
-
-// SetOption implements output.ConfigWriter.
-func (w *configWriter) SetOption(_ *options.Registry, o options.Options, opt options.Opt) error {
-	if !w.pr.Verbose {
-		return nil
-	}
-
-	o = options.Effective(o, opt)
-	return writeJSON(w.out, w.pr, o)
-}

cli/output/jsonw/configwriter.go

@@ -0,0 +1,94 @@
+package jsonw
+
+import (
+	"io"
+
+	"github.com/neilotoole/sq/cli/output/outputx"
+	"github.com/neilotoole/sq/libsq/core/options"
+
+	"github.com/neilotoole/sq/cli/output"
+)
+
+var _ output.ConfigWriter = (*configWriter)(nil)
+
+// configWriter implements output.ConfigWriter.
+type configWriter struct {
+	out io.Writer
+	pr  *output.Printing
+}
+
+// NewConfigWriter returns a new output.ConfigWriter.
+func NewConfigWriter(out io.Writer, pr *output.Printing) output.ConfigWriter {
+	return &configWriter{out: out, pr: pr}
+}
+
+// Location implements output.ConfigWriter.
+func (w *configWriter) Location(loc, origin string) error {
+	type cfgInfo struct {
+		Location string `json:"location"`
+		Origin   string `json:"origin,omitempty"`
+	}
+
+	c := cfgInfo{
+		Location: loc,
+		Origin:   origin,
+	}
+
+	return writeJSON(w.out, w.pr, c)
+}
+
+// Opt implements output.ConfigWriter.
+func (w *configWriter) Opt(o options.Options, opt options.Opt) error {
+	if o == nil || opt == nil {
+		return nil
+	}
+
+	o2 := options.Options{opt.Key(): o[opt.Key()]}
+
+	if !w.pr.Verbose {
+		return writeJSON(w.out, w.pr, o2)
+	}
+
+	vo := outputx.NewVerboseOpt(opt, o2)
+	return writeJSON(w.out, w.pr, vo)
+}
+
+// Options implements output.ConfigWriter.
+func (w *configWriter) Options(reg *options.Registry, o options.Options) error {
+	if len(o) == 0 {
+		return nil
+	}
+
+	if !w.pr.Verbose {
+		return writeJSON(w.out, w.pr, o)
+	}
+
+	opts := reg.Opts()
+	m := map[string]outputx.VerboseOpt{}
+	for _, opt := range opts {
+		m[opt.Key()] = outputx.NewVerboseOpt(opt, o)
+	}
+
+	return writeJSON(w.out, w.pr, m)
+}
+
+// SetOption implements output.ConfigWriter.
+func (w *configWriter) SetOption(o options.Options, opt options.Opt) error {
+	if !w.pr.Verbose {
+		return nil
+	}
+
+	vo := outputx.NewVerboseOpt(opt, o)
+	return writeJSON(w.out, w.pr, vo)
+}
+
+// UnsetOption implements output.ConfigWriter.
+func (w *configWriter) UnsetOption(opt options.Opt) error {
+	if !w.pr.Verbose {
+		return nil
+	}
+
+	o := options.Options{opt.Key(): opt.GetAny(nil)}
+	vo := outputx.NewVerboseOpt(opt, o)
+	return writeJSON(w.out, w.pr, vo)
+}

cli/output/jsonw/internal/jcolorenc/json.go

@@ -372,9 +372,7 @@ func (dec *Decoder) InputOffset() int64 {
 
 // Encoder is documented at https://golang.org/pkg/encoding/json/#Encoder
 type Encoder struct {
-	writer io.Writer
-	// prefix   string
-	// indent   string
+	writer   io.Writer
 	buffer   *bytes.Buffer
 	err      error
 	flags    AppendFlags

cli/output/jsonw/jsonw.go

@@ -0,0 +1,29 @@
+// Package jsonw implements output writers for JSON.
+package jsonw
+
+import (
+	"io"
+
+	"github.com/neilotoole/sq/cli/output"
+	"github.com/neilotoole/sq/cli/output/jsonw/internal"
+	jcolorenc "github.com/neilotoole/sq/cli/output/jsonw/internal/jcolorenc"
+	"github.com/neilotoole/sq/libsq/core/errz"
+)
+
+// writeJSON prints a JSON representation of v to out, using specs
+// from pr.
+func writeJSON(out io.Writer, pr *output.Printing, v any) error {
+	enc := jcolorenc.NewEncoder(out)
+	enc.SetColors(internal.NewColors(pr))
+	enc.SetEscapeHTML(false)
+	if pr.Pretty {
+		enc.SetIndent("", pr.Indent)
+	}
+
+	err := enc.Encode(v)
+	if err != nil {
+		return errz.Err(err)
+	}
+
+	return nil
+}

cli/output/jsonw/recordwriter.go

@@ -1,4 +1,3 @@
-// Package jsonw implements output writers for JSON.
 package jsonw
 
 import (
@@ -8,29 +7,10 @@ import (
 
 	"github.com/neilotoole/sq/cli/output"
 	"github.com/neilotoole/sq/cli/output/jsonw/internal"
-	jcolorenc "github.com/neilotoole/sq/cli/output/jsonw/internal/jcolorenc"
 	"github.com/neilotoole/sq/libsq/core/errz"
 	"github.com/neilotoole/sq/libsq/core/sqlz"
 )
 
-// writeJSON prints a JSON representation of v to out, using specs
-// from pr.
-func writeJSON(out io.Writer, pr *output.Printing, v any) error {
-	enc := jcolorenc.NewEncoder(out)
-	enc.SetColors(internal.NewColors(pr))
-	enc.SetEscapeHTML(false)
-	if pr.Pretty {
-		enc.SetIndent("", pr.Indent)
-	}
-
-	err := enc.Encode(v)
-	if err != nil {
-		return errz.Err(err)
-	}
-
-	return nil
-}
-
 // NewStdRecordWriter returns a record writer that outputs each
 // record as a JSON object that is an element of JSON array. This is
 // to say, standard JSON. For example:

cli/output/jsonw/sourcewriter.go

@@ -99,7 +99,7 @@ func (w *sourceWriter) Group(group *source.Group) error {
 		return nil
 	}
 
-	group.RedactLocations()
+	source.RedactGroup(group)
 	return writeJSON(w.out, w.pr, group)
 }
 
@@ -109,12 +109,12 @@ func (w *sourceWriter) SetActiveGroup(group *source.Group) error {
 		return nil
 	}
 
-	group.RedactLocations()
+	source.RedactGroup(group)
 	return writeJSON(w.out, w.pr, group)
 }
 
 // Groups implements output.SourceWriter.
 func (w *sourceWriter) Groups(tree *source.Group) error {
-	tree.RedactLocations()
+	source.RedactGroup(tree)
 	return writeJSON(w.out, w.pr, tree)
 }

cli/output/outputx/outputx.go

@@ -0,0 +1,33 @@
+// Package outputx contains extensions to pkg output, and helpers
+// for implementing output writers.
+package outputx
+
+import (
+	"reflect"
+
+	"github.com/neilotoole/sq/libsq/core/options"
+)
+
+// VerboseOpt is a verbose realization of an options.Opt value.
+type VerboseOpt struct {
+	Key          string `json:"key"`
+	Type         string `json:"type"`
+	IsSet        bool   `json:"is_set"`
+	DefaultValue any    `json:"default_value"`
+	Value        any    `json:"value"`
+	Comment      string `json:"comment"`
+}
+
+// NewVerboseOpt returns a VerboseOpt built from opt and o.
+func NewVerboseOpt(opt options.Opt, o options.Options) VerboseOpt {
+	v := VerboseOpt{
+		Key:          opt.Key(),
+		DefaultValue: opt.GetAny(nil),
+		IsSet:        o.IsSet(opt),
+		Comment:      opt.Comment(),
+		Value:        opt.GetAny(o),
+		Type:         reflect.TypeOf(opt.GetAny(nil)).String(),
+	}
+
+	return v
+}

cli/output/tablew/configwriter.go

@@ -38,6 +38,18 @@ func (w *configWriter) Location(path, origin string) error {
 	return nil
 }
 
+// Opt implements output.ConfigWriter.
+func (w *configWriter) Opt(o options.Options, opt options.Opt) error {
+	if o == nil || opt == nil {
+		return nil
+	}
+
+	o2 := options.Options{opt.Key(): o[opt.Key()]}
+	reg2 := &options.Registry{}
+	reg2.Add(opt)
+	return w.Options(reg2, o2)
+}
+
 // Options implements output.ConfigWriter.
 func (w *configWriter) Options(reg *options.Registry, o options.Options) error {
 	if o == nil {
@@ -134,7 +146,7 @@ func (w *configWriter) doPrintOptions(reg *options.Registry, o options.Options,
 }
 
 // SetOption implements output.ConfigWriter.
-func (w *configWriter) SetOption(_ *options.Registry, o options.Options, opt options.Opt) error {
+func (w *configWriter) SetOption(o options.Options, opt options.Opt) error {
 	if !w.tbl.pr.Verbose {
 		// No output unless verbose
 		return nil
@@ -151,6 +163,21 @@ func (w *configWriter) SetOption(_ *options.Registry, o options.Options, opt opt
 	return nil
 }
 
+// UnsetOption implements output.ConfigWriter.
+func (w *configWriter) UnsetOption(opt options.Opt) error {
+	if !w.tbl.pr.Verbose {
+		// No output unless verbose
+		return nil
+	}
+
+	reg := &options.Registry{}
+	reg.Add(opt)
+	o := options.Options{}
+
+	w.doPrintOptions(reg, o, true)
+	return nil
+}
+
 func getOptColor(pr *output.Printing, opt options.Opt) *color.Color {
 	if opt == nil {
 		return pr.Null

cli/output/writers.go

@@ -111,9 +111,15 @@ type ConfigWriter interface {
 	// of "flag", "env", "default".
 	Location(loc, origin string) error
 
+	// Opt prints a single options.Opt.
+	Opt(o options.Options, opt options.Opt) error
+
 	// Options prints config options.
 	Options(reg *options.Registry, o options.Options) error
 
 	// SetOption is called when an option is set.
-	SetOption(reg *options.Registry, o options.Options, opt options.Opt) error
+	SetOption(o options.Options, opt options.Opt) error
+
+	// UnsetOption is called when an option is unset.
+	UnsetOption(opt options.Opt) error
 }

cli/output/yamlw/configw.go

@@ -1,59 +0,0 @@
-package yamlw
-
-import (
-	"io"
-
-	"github.com/neilotoole/sq/libsq/core/options"
-
-	"github.com/goccy/go-yaml/printer"
-
-	"github.com/neilotoole/sq/cli/output"
-)
-
-var _ output.ConfigWriter = (*configWriter)(nil)
-
-// configWriter implements output.ConfigWriter.
-type configWriter struct {
-	p   printer.Printer
-	out io.Writer
-	pr  *output.Printing
-}
-
-// NewConfigWriter returns a new output.ConfigWriter.
-func NewConfigWriter(out io.Writer, pr *output.Printing) output.ConfigWriter {
-	return &configWriter{out: out, pr: pr, p: newPrinter(pr)}
-}
-
-// Location implements output.ConfigWriter.
-func (w *configWriter) Location(loc, origin string) error {
-	type cfgInfo struct {
-		Location string `json:"location"`
-		Origin   string `json:"origin,omitempty"`
-	}
-
-	c := cfgInfo{
-		Location: loc,
-		Origin:   origin,
-	}
-
-	return writeYAML(w.p, w.out, c)
-}
-
-// Options implements output.ConfigWriter.
-func (w *configWriter) Options(_ *options.Registry, o options.Options) error {
-	if len(o) == 0 {
-		return nil
-	}
-
-	return writeYAML(w.p, w.out, o)
-}
-
-// SetOption implements output.ConfigWriter.
-func (w *configWriter) SetOption(_ *options.Registry, o options.Options, opt options.Opt) error {
-	if !w.pr.Verbose {
-		return nil
-	}
-
-	o = options.Effective(o, opt)
-	return w.Options(nil, o)
-}

cli/output/yamlw/configwriter.go

@@ -0,0 +1,98 @@
+package yamlw
+
+import (
+	"io"
+
+	"github.com/neilotoole/sq/cli/output/outputx"
+
+	"github.com/neilotoole/sq/libsq/core/options"
+
+	"github.com/goccy/go-yaml/printer"
+
+	"github.com/neilotoole/sq/cli/output"
+)
+
+var _ output.ConfigWriter = (*configWriter)(nil)
+
+// configWriter implements output.ConfigWriter.
+type configWriter struct {
+	p   printer.Printer
+	out io.Writer
+	pr  *output.Printing
+}
+
+// NewConfigWriter returns a new output.ConfigWriter.
+func NewConfigWriter(out io.Writer, pr *output.Printing) output.ConfigWriter {
+	return &configWriter{out: out, pr: pr, p: newPrinter(pr)}
+}
+
+// Location implements output.ConfigWriter.
+func (w *configWriter) Location(loc, origin string) error {
+	type cfgInfo struct {
+		Location string `json:"location"`
+		Origin   string `json:"origin,omitempty"`
+	}
+
+	c := cfgInfo{
+		Location: loc,
+		Origin:   origin,
+	}
+
+	return writeYAML(w.out, w.p, c)
+}
+
+// Opt implements output.ConfigWriter.
+func (w *configWriter) Opt(o options.Options, opt options.Opt) error {
+	if o == nil || opt == nil {
+		return nil
+	}
+
+	o2 := options.Options{opt.Key(): o[opt.Key()]}
+
+	if !w.pr.Verbose {
+		return writeYAML(w.out, w.p, o2)
+	}
+
+	vo := outputx.NewVerboseOpt(opt, o2)
+	return writeYAML(w.out, w.p, vo)
+}
+
+// Options implements output.ConfigWriter.
+func (w *configWriter) Options(reg *options.Registry, o options.Options) error {
+	if len(o) == 0 {
+		return nil
+	}
+
+	if !w.pr.Verbose {
+		return writeYAML(w.out, w.p, o)
+	}
+
+	opts := reg.Opts()
+	m := map[string]outputx.VerboseOpt{}
+	for _, opt := range opts {
+		m[opt.Key()] = outputx.NewVerboseOpt(opt, o)
+	}
+
+	return writeYAML(w.out, w.p, m)
+}
+
+// SetOption implements output.ConfigWriter.
+func (w *configWriter) SetOption(o options.Options, opt options.Opt) error {
+	if !w.pr.Verbose {
+		return nil
+	}
+
+	vo := outputx.NewVerboseOpt(opt, o)
+	return writeYAML(w.out, w.p, vo)
+}
+
+// UnsetOption implements output.ConfigWriter.
+func (w *configWriter) UnsetOption(opt options.Opt) error {
+	if !w.pr.Verbose {
+		return nil
+	}
+
+	o := options.Options{opt.Key(): opt.GetAny(nil)}
+	vo := outputx.NewVerboseOpt(opt, o)
+	return writeYAML(w.out, w.p, vo)
+}

cli/output/yamlw/metadatawriter.go

@@ -25,12 +25,12 @@ func NewMetadataWriter(out io.Writer, pr *output.Printing) output.MetadataWriter
 
 // DriverMetadata implements output.MetadataWriter.
 func (w *mdWriter) DriverMetadata(md []driver.Metadata) error {
-	return writeYAML(w.yp, w.out, md)
+	return writeYAML(w.out, w.yp, md)
 }
 
 // TableMetadata implements output.MetadataWriter.
 func (w *mdWriter) TableMetadata(md *source.TableMetadata) error {
-	return writeYAML(w.yp, w.out, md)
+	return writeYAML(w.out, w.yp, md)
 }
 
 // SourceMetadata implements output.MetadataWriter.
@@ -38,5 +38,5 @@ func (w *mdWriter) SourceMetadata(md *source.Metadata) error {
 	md2 := *md // Shallow copy is fine
 	md2.Location = source.RedactLocation(md2.Location)
 
-	return writeYAML(w.yp, w.out, &md2)
+	return writeYAML(w.out, w.yp, &md2)
 }

cli/output/yamlw/sourcewriter.go

@@ -0,0 +1,123 @@
+package yamlw
+
+import (
+	"io"
+
+	"github.com/goccy/go-yaml/printer"
+
+	"golang.org/x/exp/slices"
+
+	"github.com/neilotoole/sq/cli/output"
+	"github.com/neilotoole/sq/libsq/source"
+)
+
+var _ output.SourceWriter = (*sourceWriter)(nil)
+
+type sourceWriter struct {
+	p   printer.Printer
+	out io.Writer
+	pr  *output.Printing
+}
+
+// NewSourceWriter returns a source writer that outputs source
+// details in text table format.
+func NewSourceWriter(out io.Writer, pr *output.Printing) output.SourceWriter {
+	return &sourceWriter{out: out, pr: pr, p: newPrinter(pr)}
+}
+
+// Collection implements output.SourceWriter.
+func (w *sourceWriter) Collection(coll *source.Collection) error {
+	if coll == nil {
+		return nil
+	}
+
+	// This is a bit hacky. Basically we want to YAML-print coll.Data().
+	// But, we want to do it just for the active group.
+	// So, our hack is that we clone the coll, and remove any
+	// sources that are not in the active group.
+	//
+	// This whole function, including what it outputs, should be revisited.
+	coll = coll.Clone()
+	group := coll.ActiveGroup()
+
+	// We store the active src handle
+	activeHandle := coll.ActiveHandle()
+
+	handles, err := coll.HandlesInGroup(group)
+	if err != nil {
+		return err
+	}
+
+	srcs := coll.Sources()
+	for _, src := range srcs {
+		if !slices.Contains(handles, src.Handle) {
+			if err = coll.Remove(src.Handle); err != nil {
+				// Should never happen
+				return err
+			}
+		}
+	}
+
+	srcs = coll.Sources()
+	for i := range srcs {
+		srcs[i].Location = srcs[i].RedactedLocation()
+	}
+
+	// HACK: we set the activeHandle back, even though that
+	// active source may have been removed (because it is not in
+	// the active group). This whole thing is a mess.
+	_, _ = coll.SetActive(activeHandle, true)
+
+	return writeYAML(w.out, w.p, coll.Data())
+}
+
+// Source implements output.SourceWriter.
+func (w *sourceWriter) Source(_ *source.Collection, src *source.Source) error {
+	if src == nil {
+		return nil
+	}
+
+	src = src.Clone()
+	src.Location = src.RedactedLocation()
+	return writeYAML(w.out, w.p, src)
+}
+
+// Removed implements output.SourceWriter.
+func (w *sourceWriter) Removed(srcs ...*source.Source) error {
+	if !w.pr.Verbose || len(srcs) == 0 {
+		return nil
+	}
+
+	srcs2 := make([]*source.Source, len(srcs))
+	for i := range srcs {
+		srcs2[i] = srcs[i].Clone()
+		srcs2[i].Location = srcs2[i].RedactedLocation()
+	}
+	return writeYAML(w.out, w.p, srcs2)
+}
+
+// Group implements output.SourceWriter.
+func (w *sourceWriter) Group(group *source.Group) error {
+	if group == nil {
+		return nil
+	}
+
+	source.RedactGroup(group)
+	return writeYAML(w.out, w.p, group)
+}
+
+// SetActiveGroup implements output.SourceWriter.
+func (w *sourceWriter) SetActiveGroup(group *source.Group) error {
+	if group == nil {
+		return nil
+	}
+
+	source.RedactGroup(group)
+	return writeYAML(w.out, w.p, group)
+}
+
+// Groups implements output.SourceWriter.
+func (w *sourceWriter) Groups(tree *source.Group) error {
+	source.RedactGroup(tree)
+	return writeYAML(w.out, w.p, tree)
+}

cli/output/yamlw/yamlw.go

@@ -16,7 +16,7 @@ import (
 
 // writeYAML prints a YAML representation of v to out, using specs
 // from pr.
-func writeYAML(p printer.Printer, out io.Writer, v any) error {
+func writeYAML(out io.Writer, p printer.Printer, v any) error {
 	b, err := goccy.Marshal(v)
 	if err != nil {
 		return errz.Err(err)

cli/runcontext.go

@@ -147,7 +147,7 @@ func newDefaultRunContext(ctx context.Context,
 	rc.Config, rc.ConfigStore, configErr = yamlstore.Load(ctx,
 		args, rc.OptionsRegistry, upgrades)
 
-	log, logHandler, logCloser, logErr := defaultLogging(ctx)
+	log, logHandler, logCloser, logErr := defaultLogging(ctx, args, rc.Config)
 	rc.clnup = cleanup.New().AddE(logCloser)
 	if logErr != nil {
 		stderrLog, h := stderrLogger()
@@ -160,10 +160,13 @@ func newDefaultRunContext(ctx context.Context,
 			return rc, log, err
 		}
 	}
-	if log != nil {
-		log = log.With(lga.Pid, os.Getpid())
+
+	if log == nil {
+		log = lg.Discard()
 	}
 
+	log = log.With(lga.Pid, os.Getpid())
+
 	if rc.Config == nil {
 		rc.Config = config.New()
 	}

cli/writers.go

@@ -165,6 +165,7 @@ func newWriters(cmd *cobra.Command, opts options.Options, out, errOut io.Writer,
 	case format.YAML:
 		w.configw = yamlw.NewConfigWriter(out2, pr)
 		w.metaw = yamlw.NewMetadataWriter(out2, pr)
+		w.srcw = yamlw.NewSourceWriter(out2, pr)
 	}
 
 	return w, out2, errOut2
@@ -238,11 +239,10 @@ func getPrinting(cmd *cobra.Command, opts options.Options, out, errOut io.Writer
 	return pr, out2, errOut2
 }
 
-func getFormat(cmd *cobra.Command, defaults options.Options) format.Format {
+func getFormat(cmd *cobra.Command, o options.Options) format.Format {
 	var fm format.Format
 
 	switch {
-	// cascade through the format flags in low-to-high order of precedence.
 	case cmdFlagChanged(cmd, flag.TSV):
 		fm = format.TSV
 	case cmdFlagChanged(cmd, flag.CSV):
@@ -269,7 +269,7 @@ func getFormat(cmd *cobra.Command, defaults options.Options) format.Format {
 		fm = format.YAML
 	default:
 		// no format flag, use the config value
-		fm = OptOutputFormat.Get(defaults)
+		fm = OptOutputFormat.Get(o)
 	}
 	return fm
 }

libsq/core/lg/lga/lga.go

@@ -14,8 +14,10 @@ const (
 	Driver    = "driver"
 	DefaultTo = "default_to"
 	Elapsed   = "elapsed"
+	Env       = "env"
 	Err       = "error"
 	From      = "from"
+	Flag      = "flag"
 	Handle    = "handle"
 	Index     = "index"
 	Key       = "key"

libsq/core/lg/slogbuf/slogbuf.go

@@ -63,6 +63,11 @@ func (b *Buffer) Flush(ctx context.Context, dest slog.Handler) error {
 
 	for i := range b.entries {
 		d, h, rec := dest, b.entries[i].handler, b.entries[i].record
+
+		if !d.Enabled(ctx, rec.Level) {
+			continue
+		}
+
 		d = d.WithAttrs(h.attrs)
 		for _, g := range h.groups {
 			d = d.WithGroup(g)

libsq/core/lg/userlogdir/userlogdir.go

@@ -0,0 +1,60 @@
+// Package userlogdir has a single function, UserLogDir, that returns
+// an OS-specific path for storing user logs.
+package userlogdir
+
+import (
+	"errors"
+	"os"
+	"runtime"
+)
+
+// UserLogDir returns the default root directory to use for user-specific
+// log data. Users should create their own application-specific subdirectory
+// within this one and use that.
+//
+// On Unix systems, it returns $XDG_CACHE_HOME as specified by
+// https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html if
+// non-empty, else $HOME/.cache.
+// On Darwin, it returns $HOME/Library/Logs.
+// On Windows, it returns %LocalAppData%.
+// On Plan 9, it returns $home/lib/cache.
+//
+// If the location cannot be determined (for example, $HOME is not defined),
+// then it will return an error.
+func UserLogDir() (string, error) {
+	var dir string
+
+	switch runtime.GOOS {
+	case "windows":
+		dir = os.Getenv("LocalAppData")
+		if dir == "" {
+			return "", errors.New("%LocalAppData% is not defined")
+		}
+
+	case "darwin", "ios":
+		dir = os.Getenv("HOME")
+		if dir == "" {
+			return "", errors.New("$HOME is not defined")
+		}
+		dir += "/Library/Logs"
+
+	case "plan9":
+		dir = os.Getenv("home")
+		if dir == "" {
+			return "", errors.New("$home is not defined")
+		}
+		dir += "/lib/cache"
+
+	default: // Unix
+		dir = os.Getenv("XDG_CACHE_HOME")
+		if dir == "" {
+			dir = os.Getenv("HOME")
+			if dir == "" {
+				return "", errors.New("neither $XDG_CACHE_HOME nor $HOME are defined")
+			}
+			dir += "/.cache"
+		}
+	}
+
+	return dir, nil
+}

libsq/source/collection.go

@@ -949,9 +949,11 @@ func (g *Group) AllSources() []*Source {
 	return srcs
 }
 
-// RedactLocations modifies g, cloning each descendant Source, and setting
+// RedactGroup modifies g, cloning each descendant Source, and setting
 // the Source.Location field of each contained source to its redacted value.
-func (g *Group) RedactLocations() {
+//
+// TODO: consider moving this to a function instead of a method.
+func RedactGroup(g *Group) {
 	if g == nil {
 		return
 	}
@@ -962,7 +964,7 @@ func (g *Group) RedactLocations() {
 	}
 
 	for i := range g.Groups {
-		g.Groups[i].RedactLocations()
+		RedactGroup(g.Groups[i])
 	}
 }