codebase refactor

.github/workflows/go.yml

@@ -1,28 +1,97 @@
 name: Go
 on: [push]
+
+
 jobs:
-
   build:
-    name: Build
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ macos-latest, ubuntu-latest, windows-latest]
+
+    runs-on: ${{ matrix.os }}
+
     steps:
+      - name: Set up Go 1.14
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.14
+        id: go
 
-    - name: Set up Go 1.13
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.13
-      id: go
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
 
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      - name: Get dependencies
+        run: |
+          go get -v -t -d ./...
 
-    - name: Get dependencies
-      run: |
-        go get -v -t -d ./...
-        if [ -f Gopkg.toml ]; then
-            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-            dep ensure
-        fi
+      - name: Build
+        run: go build -v .
 
-    - name: Build
-      run: go build -v .
+
+
+#
+#
+#  build-macos:
+#    name: Build macOS
+#    runs-on: macos-latest
+#
+#    steps:
+#      - name: Set up Go 1.14
+#        uses: actions/setup-go@v1
+#        with:
+#          go-version: 1.14
+#        id: go
+#
+#      - name: Check out code into the Go module directory
+#        uses: actions/checkout@v2
+#
+#      - name: Get dependencies
+#        run: |
+#          go get -v -t -d ./...
+#
+#      - name: Build
+#        run: go build -v .
+#
+#
+#  build-linux:
+#    name: Build Linux
+#    runs-on: ubuntu-latest
+#
+#    steps:
+#    - name: Set up Go 1.14
+#      uses: actions/setup-go@v1
+#      with:
+#        go-version: 1.14
+#      id: go
+#
+#    - name: Check out code into the Go module directory
+#      uses: actions/checkout@v2
+#
+#    - name: Get dependencies
+#      run: |
+#        go get -v -t -d ./...
+#
+#    - name: Build
+#      run: go build -v .
+#
+#  build-windows:
+#    name: Build Windows
+#    runs-on: windows-latest
+#
+#    steps:
+#    - name: Set up Go 1.14
+#      uses: actions/setup-go@v1
+#      with:
+#        go-version: 1.14
+#      id: go
+#
+#    - name: Check out code into the Go module directory
+#      uses: actions/checkout@v2
+#
+#    - name: Get dependencies
+#      run: |
+#        go get -v -t -d ./...
+#
+#    - name: Build
+#      run: go build -v .
+#

.gitignore

@@ -23,18 +23,21 @@ _testmain.go
 *.test
 *.prof
 
+.DS_Store
+
 /.idea
-/misc
-/build
-/bin
-/grammar/*.go
-/grammar/*.tokens
-/test/grun/java/*
-/sq.yml
-/sq.log
-/xlsx/~$test.xlsx
-"sq"
+/grammar/build
+*/~test$*
 /dist
+/projectFilesBackup
+/*.iml
+/.swp
+/sq
+/demo
+/scratch
 
+# Some apps create temp files when editing, e.g. Excel with drivers/xlsx/testdata/~$test_header.xlsx
+**/testdata/~*
 
-
+gorelease-snapshot.sh
+magefile_local.go

.golangci.yml

@@ -0,0 +1,134 @@
+linters-settings:
+  depguard:
+    list-type: blacklist
+    packages:
+      # logging is allowed only by logutils.Log, logrus
+      # is allowed to use only in logutils package
+#      - github.com/sirupsen/logrus
+    packages-with-error-message:
+#      - github.com/sirupsen/logrus: "logging is allowed only by logutils.Log"
+  dupl:
+    threshold: 100
+  funlen:
+    lines: 200
+    statements: 100
+  goconst:
+    min-len: 2
+    min-occurrences: 3
+  gocritic:
+    enabled-tags:
+      - diagnostic
+      - experimental
+      - opinionated
+      - performance
+      - style
+    disabled-checks:
+      - dupImport # https://github.com/go-critic/go-critic/issues/845
+      - ifElseChain
+      - octalLiteral
+      - whyNoLint
+      - wrapperFunc
+  gocognit:
+    # minimal code complexity to report, 30 by default (but we recommend 10-20)
+    min-complexity: 36
+  gocyclo:
+    min-complexity: 36
+  goimports:
+#    local-prefixes: github.com/golangci/golangci-lint
+  golint:
+    min-confidence: 0
+  gomnd:
+    settings:
+      mnd:
+        # don't include the "operation" and "assign"
+        checks: argument,case,condition,return
+  govet:
+    check-shadowing: true
+    settings:
+      printf:
+        funcs:
+          - (github.com/neilotoole/lg.Log).Debugf
+          - (github.com/neilotoole/lg.Log).Warnf
+          - (github.com/neilotoole/lg.Log).Errorf
+  lll:
+    line-length: 180
+  maligned:
+    suggest-new: true
+  misspell:
+    locale: US
+
+linters:
+  # please, do not use `enable-all`: it's deprecated and will be removed soon.
+  # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
+  disable-all: true
+  enable:
+    - bodyclose
+    - deadcode
+    - depguard
+    - dogsled
+    - dupl
+    - errcheck
+    - funlen
+    - gochecknoinits
+    - goconst
+    - gocritic
+    - gocyclo
+    - gofmt
+    - goimports
+    - golint
+    #- gomnd
+    - goprintffuncname
+    - gosec
+    - gosimple
+    - govet
+    - ineffassign
+    - interfacer
+    - lll
+#    - misspell
+    - nakedret
+    - rowserrcheck
+    - scopelint
+    - staticcheck
+    - structcheck
+    - stylecheck
+    - typecheck
+    - unconvert
+    - unparam
+    - unused
+    - varcheck
+    - whitespace
+#    - gochecknoglobals
+    - gocognit
+#    - godox
+    - maligned
+    - prealloc
+
+issues:
+  # Excluding configuration per-path, per-linter, per-text and per-source
+  exclude-rules:
+    - path: _test\.go
+      linters:
+        - gomnd
+        - lll
+        - maligned
+        - dupl
+        - funlen
+
+    - linters:
+      - gosec
+      text: "G[202]"
+
+
+run:
+  skip-dirs:
+    - testdata
+#    - internal/cache
+#    - internal/renameio
+#    - internal/robustio
+
+# golangci.com configuration
+# https://github.com/golangci/golangci/wiki/Configuration
+service:
+  golangci-lint-version: 1.27.x # use the fixed version to not introduce new linters unexpectedly
+  prepare:
+    - echo "here I can run custom commands, but no preparation needed for this repo"

.goreleaser.yml

@@ -0,0 +1,314 @@
+project_name: sq
+env:
+  - GO111MODULE=on
+  - CGO_ENABLED=1
+before:
+  hooks:
+    - go version  # prints the Go version used for build (can be seen when --debug flag is passed to goreleaser)
+    - go mod download
+
+builds:
+  - id: build_macos
+    ldflags: -s -w -X github.com/neilotoole/sq/cli/buildinfo.Version={{.Version}} -X github.com/neilotoole/sq/cli/buildinfo.Timestamp={{.Date}} -X github.com/neilotoole/sq/cli/buildinfo.Commit={{ .ShortCommit }}
+    binary: sq
+    env:
+      - CC=o64-clang
+      - CXX=o64-clang++
+    main: ./main.go
+    goos:
+      - darwin
+    goarch:
+      - amd64
+
+  - id: build_linux
+    binary: sq
+    main: ./main.go
+    goos:
+      - linux
+    goarch:
+      - amd64
+    # Note the additional ldflags (-linkmode etc), and the "-tags=netgo" in
+    # flags below. This is to build a static binary.
+    ldflags: -linkmode external -extldflags -static -s -w -X github.com/neilotoole/sq/cli/buildinfo.Version={{.Version}} -X github.com/neilotoole/sq/cli/buildinfo.Timestamp={{.Date}} -X github.com/neilotoole/sq/cli/buildinfo.Commit={{ .ShortCommit }}
+    flags:
+      - -tags=netgo
+      - -v
+
+
+  - id: build_windows
+    ldflags: -s -w -X github.com/neilotoole/sq/cli/buildinfo.Version={{.Version}} -X github.com/neilotoole/sq/cli/buildinfo.Timestamp={{.Date}} -X github.com/neilotoole/sq/cli/buildinfo.Commit={{ .ShortCommit }}
+    binary: sq
+    env:
+      - CC=x86_64-w64-mingw32-gcc
+      - CXX=x86_64-w64-mingw32-g++
+    main: ./main.go
+    goos:
+      - windows
+    goarch:
+      - amd64
+
+#
+#archives:
+#  - id: sq_macos_archive
+#    builds:
+#    - build_macos
+#    name_template: "{{.ProjectName}}-{{.Version}}-{{.Os}}"
+#    format: tar.gz
+#    replacements:
+#      darwin: macos
+#    files:
+#      - README.md
+#      - LICENSE
+#
+#  - id: sq_linux_archive
+#    builds:
+#    - build_linux
+#    name_template: "{{.ProjectName}}-{{.Version}}-{{.Os}}-{{.Arch}}"
+#    format: tar.gz
+#    replacements:
+#    files:
+#      - README.md
+#      - LICENSE
+#
+#  - id: sq_windows_archive
+#    builds:
+#    - build_windows
+#    name_template: "{{.ProjectName}}-{{.Version}}-{{.Os}}-{{.Arch}}"
+#    format: zip
+#    replacements:
+#    files:
+#      - README.md
+#      - LICENSE
+
+archives:
+  -
+    builds: ['build_macos', 'build_linux', 'build_windows']
+    name_template: "{{.ProjectName}}-{{.Version}}-{{.Os}}-{{.Arch}}"
+    format: tar.gz
+    files:
+      - README.md
+      - LICENSE
+    replacements:
+#      amd64: 64-bit
+#      386: 32-bit
+      darwin: macOS
+#      linux: Tux
+    format_overrides:
+      - goos: windows
+        format: zip
+
+
+checksum:
+  name_template: "{{.ProjectName}}-{{.Version}}-checksums.txt"
+
+snapshot:
+  name_template: "{{ .Version }}-snapshot"
+
+changelog:
+  skip: true
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'
+      - '^dev:'
+      - 'README'
+      - Merge pull request
+      - Merge branch
+
+
+release:
+  github:
+    owner: neilotoole
+    name: sq-preview
+  draft: true
+  prerelease: auto
+#
+#release:
+#  # Repo in which the release will be created.
+#  # We are using neilotoole/sq-preview for now.
+#  github:
+#    owner: neilotoole
+#    name: sq-preview
+#
+#  # IDs of the archives to use. Defaults to all.
+#  ids:
+#    - sq_macos_archive
+#    - sq_linux_archive
+#    - sq_windows_archive
+#
+#  # If set to true, will not auto-publish the release. Default is false.
+#  draft: false
+#
+#  # If set to auto, will mark the release as not ready for production
+#  # in case there is an indicator for this in the tag e.g. v1.0.0-rc1
+#  # If set to true, will mark the release as not ready for production.
+#  # Default is false.
+#  prerelease: auto
+#
+#  # You can change the name of the GitHub release.
+#  # Default is `{{.Tag}}`
+#  #  name_template: "{{.ProjectName}}-v{{.Version}} {{.Env.USER}}"
+#
+#  # You can disable this pipe in order to not upload any artifacts to
+#  # GitHub. Defaults to false.
+#  disable: false
+#
+#  # You can add extra pre-existing files to the release.
+#  # The filename on the release will be the last part of the path (base). If
+#  # another file with the same name exists, the latest one found will be used.
+#  # Defaults to empty.
+#  extra_files:
+#  #    - glob: ./path/to/file.txt
+#  #    - glob: ./glob/**/to/**/file/**/*
+#  #    - glob: ./glob/foo/to/bar/file/foobar/override_from_previous
+
+brews:
+  -
+    name: sq
+    homepage: "https://sq.io"
+    description: "sq is sed for structured data"
+    caveats: "This is a preview release of sq. Use with caution."
+#    ids:
+#      - sq_macos_archive
+
+    github:
+      owner: neilotoole
+      name: homebrew-sq
+
+    url_template: "https://github.com/neilotoole/sq-preview/releases/download/{{ .Tag }}/{{ .ArtifactName }}"
+
+    commit_author:
+      name: neilotoole
+      email: neilotoole@apache.org
+
+    folder: Formula
+
+    test: |
+      system "#{bin}/sq version"
+
+    install: |
+      bin.install "sq"
+    # Setting this will prevent goreleaser to actually try to commit the updated
+    # formula - instead, the formula file will be stored on the dist folder only,
+    # leaving the responsibility of publishing it to the user.
+    # If set to auto, the release will not be uploaded to the homebrew tap
+    # in case there is an indicator for prerelease in the tag e.g. v1.0.0-rc1
+    # Default is false.
+    skip_upload: false
+
+
+scoop:
+  # scoop is a package installer for Windows, like brew for macOS.
+  # For background, see https://github.com/lukesampson/scoop/wiki/Buckets
+  url_template: "https://github.com/neilotoole/sq-preview/releases/download/{{ .Tag }}/{{ .ArtifactName }}"
+  bucket:
+    owner: neilotoole
+    name: sq-preview
+  commit_author:
+    name: neilotoole
+    email: neilotoole@apache.org
+  homepage: "https://sq.io"
+  description: "sq is sed for structured data"
+  license: MIT
+
+
+nfpms:
+  -
+    builds: ['build_linux']
+    file_name_template: "{{.ProjectName}}-{{.Version}}-{{.Os}}-{{.Arch}}"
+    homepage:  https://sq.io
+    description: sq is sed for structured data
+    maintainer: Neil O'Toole <neilotoole@apache.org>
+    license: MIT
+    vendor: Neil O'Toole
+    formats:
+      - deb
+      - rpm
+
+
+snapcrafts:
+# For this to work, snapcraft needs to be installed.
+# On macOS, "brew install snapcraft", then "snapcraft login".
+
+  -
+    # ID of the snapcraft config, must be unique.
+    # Defaults to "default".
+    id: neilotoole-sq
+
+    builds:
+    - build_linux
+    # The name of the snap. This is optional.
+    # Default is project name.
+    name: neilotoole-sq
+
+
+#    name_template: '`{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}{{ if .Mips }}_{{ .Mips }}{{ end }}`'
+#    name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
+    name_template: "neilotoole-sq-{{ .Version }}-{{ .Os }}-{{ .Arch }}"
+#    name_template: "neilotoole-sq"
+    summary: "sq is sed for structured data"
+    description: |
+      sq is sed for structured data. See https://sq.io
+    grade: devel
+    confinement: devmode
+
+
+    # Whether to publish the snap to the snapcraft store.
+    # Remember you need to `snapcraft login` first.
+    # Defaults to false.
+    publish: true
+    license: MIT
+
+    # A snap of type base to be used as the execution environment for this snap.
+    # Valid values are:
+    # * bare - Empty base snap;
+    # * core - Ubuntu Core 16;
+    # * core18 - Ubuntu Core 18.
+    # Default is empty.
+    base: core18
+
+    # Each binary built by GoReleaser is an app inside the snap. In this section
+    # you can declare extra details for those binaries. It is optional.
+    apps:
+
+      # The name of the app must be the same name as the binary built or the snapcraft name.
+#      neilotoole-sq:
+      sq:
+        # Declare "home" and "network" plugs to grant access to
+        # the user home dir, and the network
+        plugs: ["home", "network"]
+
+        # If your app requires extra permissions to work outside of its default
+        # confined space, declare them here.
+        # You can read the documentation about the available plugs and the
+        # things they allow:
+        # https://snapcraft.io/docs/reference/interfaces.
+#        plugs: ["home", "network", "personal-files"]
+
+        # If you want your app to be autostarted and to always run in the
+        # background, you can make it a simple daemon.
+#        daemon: simple
+
+        # If you any to pass args to your binary, you can add them with the
+        # args option.
+#        args: --foo
+
+        # Bash completion snippet. More information about completion here:
+        # https://docs.snapcraft.io/tab-completion-for-snaps.
+#        completer: drumroll-completion.bash
+
+
+#name: foo
+#...
+#plugs:
+#  config-foo:
+#    interface: personal-files
+#    read:
+#      - $HOME/.config/foo
+#
+#apps:
+#  foo:
+#    plugs:
+#      - config-foo
+#    ...

.lnav.json

@@ -0,0 +1,65 @@
+{
+  "sq_log": {
+    "title": "sq",
+    "url": "https://sq,io",
+    "description": "Log format for sq",
+    "json": true,
+    "hide-extra": false,
+    "file-pattern": "sq.log",
+    "multiline": true,
+    "line-format": [
+      {
+        "field": "__timestamp__",
+        "timestamp-format": "%H:%M:%S.%L"
+      },
+      "\t",
+      {
+        "field": "level",
+        "text-transform": "uppercase"
+      },
+      "\t",
+      {
+        "field": "caller",
+        "max-width": 72,
+        "min-width": 72,
+        "overflow": "dot-dot"
+      },
+      "    ",
+      {
+        "field": "msg"
+      }
+    ],
+    "level-field": "level",
+    "level": {
+      "error": "error",
+      "debug": "debug",
+      "warning": "warn"
+    },
+    "highlights": {
+      "caller": {
+        "pattern": "caller",
+        "underline": true
+      }
+    },
+    "timestamp-field": "time",
+    "body-field": "msg",
+    "value": {
+      "time": {
+        "kind": "string",
+        "identifier": true
+      },
+      "level": {
+        "kind": "string",
+        "identifier": true
+      },
+      "caller": {
+        "kind": "string",
+        "identifier": true
+      },
+      "msg": {
+        "kind": "quoted",
+        "identifier": false
+      }
+    }
+  }
+}

Dockerfile

@@ -1,15 +0,0 @@
-FROM golang:1.7.1
-
-RUN apt-get update && apt-get install -y --no-install-recommends jq tar zip
-RUN rm -rf /var/lib/apt/lists/*
-
-ENV GOPATH /go
-ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
-RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
-
-ENV REPOPATH $GOPATH/src/github.com/neilotoole/sq
-RUN mkdir -p "$REPOPATH"
-ADD . "$REPOPATH"
-WORKDIR $REPOPATH
-
-RUN make install-go-tools

LICENSE

@@ -1,21 +1,3 @@
-The MIT License (MIT)
+Copyright (c) 2014-2020 Neil O'Toole <neilotoole@apache.org>
 
-Copyright (c) 2014-2016 Neil O'Toole <neilotoole@apache.org>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+sq is proprietary software created by Neil O'Toole.

Makefile

@@ -1,110 +0,0 @@
-# Makefile for sq - simple queryer for structured data
-# http://github.com/neilotoole/sq
-
-BINARY := sq
-BUILD_VERSION=0.41.7
-BUILD_TIMESTAMP := $(shell date +'%FT%T%z')
-
-# SOURCES is the .go files for the project, excluding test files
-SOURCES := $(shell find .  -name '*.go' ! -name '*_test.go' ! -path "./test/*" ! -path "./tools/*"  ! -path "./build/*" ! -path "./vendor/*")
-# SOURCES_NO_GENERATED is SOURCES excluding the generated parser files
-SOURCES_NO_GENERATED := $(shell find .  -name '*.go' ! -name '*_test.go' ! -path "./libsq/slq/*" ! -path "./test/*" ! -path "./tools/*"  ! -path "./build/*" ! -path "./vendor/*")
-
-# LDFLAGS are compiler flags, in this case to strip the binary of unneeded stuff
-LDFLAGS="-s -w"
-
-
-default: install
-
-
-install: $(SOURCES) imports build-assets
-	@go install
-
-
-build-assets:
-	@mkdir -p ./build/assets
-	@echo $(BUILD_VERSION) > ./build/assets/build_version.txt
-	@echo $(BUILD_TIMESTAMP) > ./build/assets/build_timestamp.txt
-	@cd ./build/assets && go-bindata -pkg assets -o ../../cmd/assets/assets.go .
-
-
-test: $(SOURCES) imports
-	govendor test -timeout 10s ./libsq/...
-	govendor test -timeout 10s ./cmd/...
-
-
-testv: $(SOURCES) imports
-	govendor test -v -timeout 10s ./libsq/...
-	govendor test -v -timeout 10s ./cmd/...
-
-
-clean:
-	rm -f ./sq # Delete the sq binary in the project root
-	rm -vf $(shell which sq)
-	rm -rf ./bin/*
-	rm -rf ./build/*
-	rm -rf ./dist/*
-
-
-imports:
-	@goimports -w  -srcdir ./vendor/github.com/neilotoole/sq/libsq/ ./libsq/
-	@goimports -w  -srcdir ./vendor/github.com/neilotoole/sq/cmd/ ./cmd/
-
-
-lint:
-
-	@golint ./cmd/...
-	@golint ./libsq			# Because we want to exclude the generated parser files from linting, we invoke go lint repeatedly (could be doing this wrong)
-	@golint ./libsq/ast/...
-	@golint ./libsq/drvr/...
-	@golint ./libsq/engine
-	@golint ./libsq/shutdown/...
-	@golint ./libsq/util/...
-
-
-vet:
-	@go vet ./libsq/...
-	@go vet ./cmd/...
-
-
-check: $(SOURCES) imports lint vet
-
-
-install-go-tools:
-	go get github.com/kardianos/govendor
-	go get github.com/golang/lint/golint/...
-	go get github.com/jteeuwen/go-bindata/...
-	go get golang.org/x/tools/cmd/goimports/...
-	go get github.com/karalabe/xgo/...
-
-
-list-src:
-	@echo $(SOURCES)
-
-
-dist: clean test build-assets
-	xgo -go=1.7.1 -dest=./dist -ldflags=$(LDFLAGS) -targets=darwin/amd64,linux/amd64,windows/amd64 .
-
-	mkdir -p ./dist/darwin64
-	mv ./dist/sq-darwin-10.6-amd64 ./dist/darwin64/sq
-	tar -C ./dist/darwin64 -cvzf ./dist/darwin64/sq-$(BUILD_VERSION)-darwin64.tar.gz sq
-
-	mkdir -p ./dist/linux64
-	mv ./dist/sq-linux-amd64 ./dist/linux64/sq
-	tar -C ./dist/linux64 -cvzf ./dist/linux64/sq-$(BUILD_VERSION)-linux64.tar.gz sq
-
-	mkdir -p ./dist/win64
-	mv ./dist/sq-windows-4.0-amd64.exe ./dist/win64/sq.exe
-	zip -jr ./dist/win64/sq-$(BUILD_VERSION)-win64.zip ./dist/win64/sq.exe
-
-
-smoke:
-	@./test/smoke/smoke.sh
-
-generate-parser:
-	@cd ./tools && ./gen-antlr.sh
-
-start-test-containers:
-	cd ./test/mysql && ./start.sh
-	cd ./test/postgres && ./start.sh
-

README.md

@@ -1,96 +1,39 @@
-# sq: simple queryer for structured data
+# sq: swiss army knife for data
 
-`sq` is a command-line tool that provides uniform access to structured data sources.
-This includes traditional SQL-style databases, or document formats such as JSON, XML, Excel etc.
+`sq` is a swiss army knife for data. `sq` provides uniform access to
+structured data sources like traditional SQL-style databases,
+or document formats such as CSV or Excel. `sq` can perform cross-source joins,  execute database-native SQL, and output to a multitude of formats including JSON, Excel, CSV, HTML markdown and XML, or output directly to a SQL database. `sq` can inspect sources to see metadata about the source structure (tables, columns, size) and has commands for frequent database operations such as copying or dropping tables.
+
+## Installation
 
 
-```
-> sq '.user | .uid, .username, .email'
-```
-```json
-[
-  {
-    "uid": 1,
-    "username": "neilotoole",
-    "email": "neilotoole@apache.org"
-  },
-  {
-    "uid": 2,
-    "username": "ksoze",
-    "email": "kaiser@soze.org"
-  },
-  {
-    "uid": 3,
-    "username": "kubla",
-    "email": "kubla@khan.mn"
-  }
-]
+### From source
+
+From the `sq` project dir:
+
+```shell script
+> go install
 ```
 
-> `sq` defines its own query language, seen above, formally known as `SLQ`.
-
-
-For usage information or to download the binaries, see the `sq` [manual](https://github.com/neilotoole/sq-manual/wiki).
-
-
-## Development
-
-These steps are for Mac OS X (tested on El Capitan `10.11.16`). The examples assume username  `ksoze`.
-
-
-### Prerequisites
-- [brew](http://brew.sh/)
-- [Xcode](https://itunes.apple.com/us/app/xcode/id497799835?mt=12) dev tools.
-- [jq](https://stedolan.github.io/jq/) `brew install jq 1.5`
-- [Go](https://golang.org/doc/install) `brew install go 1.7.1`
-- [Docker](https://docs.docker.com/docker-for-mac/)
-- [Java](http://www.oracle.com/technetwork/java/javase/downloads/index.html) is required if you're working on the *SLQ* grammar.
-
-
-
-### Fork
-Fork this [repo](https://github.com/neilotoole/sq), e.g. to  `https://github.com/ksoze/sq`.
-
-Clone the forked repo and set the `upstream` remote:
+The simple go install does not populate the binary with build info that
+is output via the `sq version` command. To do so, use [mage](https://magefile.org/).
 
+```shell script
+> brew install mage
+> mage install
 ```
-mkdir -p $GOPATH/src/github.com/neilotoole
-cd $GOPATH/src/github.com/neilotoole
-git clone https://github.com/ksoze/sq.git
-cd ./sq
-git remote add upstream https://github.com/neilotoole/sq.git
-# verify that the remote was set
-git remote -v
-```
-	
-### Make
-From  `$GOPATH/src/github.com/neilotoole/sq`, run `./test.sh`. This will run `make test`
-inside a docker container.
 
-For developing locally, this sequence should get you started:
+### Other installation options
 
-```
-make install-go-tools
-make start-test-containers
-make test
-make install
-make smoke
-make dist
-```
-	
-Note that running these steps may take some time (in particular due the use of
-Cgo and cross-compiling distributables). Try `sq ls`. Note that by default `sq` uses `~/.sq/sq.yml` as
-its config store, and outputs debug logs to `~/.sq/sq.log`.
+For homebrew, scoop, rpm etc, see the [wiki](https://github.com/neilotoole/sq-preview/wiki).
 
 
-Assuming the test containers are running (`make start-test-containers`), this workflow is suggested:
-
-- Make your changes
-- Run `make test && make install && make smoke`
 
 
-## Contributing
 
-When your changes are ready and tested, run `make dist` and a final `make smoke`.
-Push the changes to your own fork, and then open a PR against the upstream repo. The PR should include a link
-to the GitHub issue(s) that it addresses, and it must include the output of `make smoke`.
+## Acknowledgements
+
+- The [_Sakila_](https://dev.mysql.com/doc/sakila/en/) example databases were lifted from [jOOQ](https://github.com/jooq/jooq), which
+  in turn owe their heritage to earlier work on Sakila.
+- The `sq` query language was inspired by [jq](https://stedolan.github.io/jq/).
+

cli/buildinfo/buildinfo.go

@@ -0,0 +1,13 @@
+// Package buildinfo hosts build info variables populated via ldflags.
+package buildinfo
+
+var (
+	// Version is the build version.
+	Version string
+
+	// Commit is the commit hash.
+	Commit string
+
+	// Timestamp is the timestamp of when the cli was built.
+	Timestamp string
+)

cli/cli.go

@@ -0,0 +1,835 @@
+// Package cli implements sq's CLI. The spf13/cobra library
+// is used, with some notable modifications. Although cobra
+// provides excellent functionality, it has some issues.
+// Most prominently, its documentation suggests reliance
+// upon package-level constructs for initializing the
+// command tree (bad for testing). Also, it doesn't provide support
+// for context.Context: see https://github.com/spf13/cobra/pull/893
+// which has been lingering for a while at the time of writing.
+//
+// Thus, this cmd package deviates from cobra's suggested
+// usage pattern by eliminating all pkg-level constructs
+// (which makes testing easier), and also replaces cobra's
+// Command.RunE func signature with a signature that accepts
+// as its first argument the RunContext type.
+//
+// RunContext is similar to context.Context (and contains
+// an instance of that), but also encapsulates injectable
+// resources such as config and logging.
+//
+// The entry point to this pkg is the Execute function.
+package cli
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/mattn/go-colorable"
+	"github.com/mitchellh/go-homedir"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	"github.com/neilotoole/lg"
+	"github.com/neilotoole/lg/zaplg"
+
+	"github.com/neilotoole/sq/cli/config"
+	"github.com/neilotoole/sq/cli/output"
+	"github.com/neilotoole/sq/cli/output/csvw"
+	"github.com/neilotoole/sq/cli/output/htmlw"
+	"github.com/neilotoole/sq/cli/output/jsonw"
+	"github.com/neilotoole/sq/cli/output/markdownw"
+	"github.com/neilotoole/sq/cli/output/raww"
+	"github.com/neilotoole/sq/cli/output/tablew"
+	"github.com/neilotoole/sq/cli/output/xlsxw"
+	"github.com/neilotoole/sq/cli/output/xmlw"
+	"github.com/neilotoole/sq/drivers/csv"
+	"github.com/neilotoole/sq/drivers/mysql"
+	"github.com/neilotoole/sq/drivers/postgres"
+	"github.com/neilotoole/sq/drivers/sqlite3"
+	"github.com/neilotoole/sq/drivers/sqlserver"
+	"github.com/neilotoole/sq/drivers/userdriver"
+	"github.com/neilotoole/sq/drivers/userdriver/xmlud"
+	"github.com/neilotoole/sq/drivers/xlsx"
+	"github.com/neilotoole/sq/libsq/cleanup"
+	"github.com/neilotoole/sq/libsq/driver"
+	"github.com/neilotoole/sq/libsq/errz"
+	"github.com/neilotoole/sq/libsq/source"
+)
+
+// errNoMsg is a sentinel error indicating that a command
+// has failed, but that no error message should be printed.
+// This is useful in the case where any error information may
+// already have been printed as part of the command output.
+var errNoMsg = errors.New("")
+
+// Execute builds a RunContext using ctx and default
+// settings, and invokes ExecuteWith.
+func Execute(ctx context.Context, stdin *os.File, stdout, stderr io.Writer, args []string) error {
+	rc, err := newDefaultRunContext(ctx, stdin, stdout, stderr)
+	if err != nil {
+		printError(rc, err)
+		return err
+	}
+
+	defer rc.Close() // ok to call rc.Close on nil rc
+
+	return ExecuteWith(rc, args)
+}
+
+// ExecuteWith invokes the cobra CLI framework, ultimately
+// resulting in a command being executed. The caller must
+// invoke rc.Close.
+func ExecuteWith(rc *RunContext, args []string) error {
+	rc.Log.Debugf("EXECUTE: %s", strings.Join(args, " "))
+	rc.Log.Debugf("Using config: %s", rc.ConfigStore.Location())
+
+	rootCmd := newCommandTree(rc)
+
+	// The following is a workaround for the fact that cobra doesn't currently
+	// support executing the root command with arbitrary args. That is to say,
+	// if you execute:
+	//
+	//   sq arg1 arg2
+	//
+	// then cobra will look for a command named "arg1", and when it
+	// doesn't find such a command, it returns an "unknown command"
+	// error.
+	cmd, _, err := rootCmd.Find(args[1:])
+	if err != nil {
+		// This err will be the "unknown command" error.
+		// cobra still returns cmd though. It should be
+		// the root cmd.
+		if cmd == nil || cmd.Name() != rootCmd.Name() {
+			// should never happen
+			panic(fmt.Sprintf("bad cobra cmd state: %v", cmd))
+		}
+
+		// If we have args [sq, arg1, arg2] the we redirect
+		// to the "query" command by modifying args to
+		// look like: [query, arg1, arg2] -- noting that SetArgs
+		// doesn't want the first args element.
+		queryCmdArgs := append([]string{"slq"}, args[1:]...)
+		rootCmd.SetArgs(queryCmdArgs)
+	} else {
+		if cmd.Name() == rootCmd.Name() {
+			// Not sure why we have two paths to this, but it appears
+			// that we've found the root cmd again, so again
+			// we redirect to "query" cmd.
+
+			a := append([]string{"slq"}, args[1:]...)
+			rootCmd.SetArgs(a)
+		} else {
+			// It's just a normal command like "sq ls" or such.
+
+			// Explicitly set the args on rootCmd as this makes
+			// cobra happy when this func is executed via tests.
+			// Haven't explored the reason why.
+			rootCmd.SetArgs(args[1:])
+		}
+	}
+
+	// Execute the rootCmd; cobra will find the appropriate
+	// sub-command, and ultimately execute that command.
+	err = rootCmd.Execute()
+	if err != nil {
+		printError(rc, err)
+	}
+
+	return err
+}
+
+// newCommandTree builds sq's command tree, returning
+// the root cobra command.
+func newCommandTree(rc *RunContext) (rootCmd *cobra.Command) {
+	rootCmd = newRootCmd()
+
+	rootCmd.SetOut(rc.Out)
+	rootCmd.SetErr(rc.ErrOut)
+
+	// The --help flag must be explicitly added to rootCmd,
+	// or else cobra tries to do its own (unwanted) thing.
+	// The behavior of cobra in this regard seems to have
+	// changed? This particular incantation currently does the trick.
+	rootCmd.Flags().Bool(flagHelp, false, "Show sq help")
+	helpCmd := addCmd(rc, rootCmd, newHelpCmd)
+	rootCmd.SetHelpCommand(helpCmd)
+
+	addCmd(rc, rootCmd, newSLQCmd)
+	addCmd(rc, rootCmd, newSQLCmd)
+
+	addCmd(rc, rootCmd, newSrcCommand)
+	addCmd(rc, rootCmd, newSrcAddCmd)
+	addCmd(rc, rootCmd, newSrcListCmd)
+	addCmd(rc, rootCmd, newSrcRemoveCmd)
+	addCmd(rc, rootCmd, newScratchCmd)
+
+	addCmd(rc, rootCmd, newInspectCmd)
+	addCmd(rc, rootCmd, newPingCmd)
+
+	addCmd(rc, rootCmd, newVersionCmd)
+	addCmd(rc, rootCmd, newDriversCmd)
+
+	notifyCmd := addCmd(rc, rootCmd, newNotifyCmd)
+	addCmd(rc, notifyCmd, newNotifyListCmd)
+	addCmd(rc, notifyCmd, newNotifyRemoveCmd)
+	notifyAddCmd := addCmd(rc, notifyCmd, newNotifyAddCmd)
+	addCmd(rc, notifyAddCmd, newNotifyAddSlackCmd)
+	addCmd(rc, notifyAddCmd, newNotifyAddHipChatCmd)
+
+	tblCmd := addCmd(rc, rootCmd, newTblCmd)
+	addCmd(rc, tblCmd, newTblCopyCmd)
+	addCmd(rc, tblCmd, newTblTruncateCmd)
+	addCmd(rc, tblCmd, newTblDropCmd)
+
+	addCmd(rc, rootCmd, newInstallBashCompletionCmd)
+	addCmd(rc, rootCmd, newGenerateZshCompletionCmd)
+
+	return rootCmd
+}
+
+// runFunc is an expansion of cobra's RunE func that
+// adds a RunContext as the first param.
+type runFunc func(rc *RunContext, cmd *cobra.Command, args []string) error
+
+// addCmd adds the command returned by cmdFn to parentCmd.
+func addCmd(rc *RunContext, parentCmd *cobra.Command, cmdFn func() (*cobra.Command, runFunc)) *cobra.Command {
+	cmd, fn := cmdFn()
+
+	if cmd.Name() != "help" {
+		// Don't add the --help flag to the help command.
+		cmd.Flags().Bool(flagHelp, false, "help for "+cmd.Name())
+	}
+
+	cmd.PreRunE = func(cmd *cobra.Command, args []string) error {
+		rc.Cmd = cmd
+		rc.Args = args
+		err := rc.preRunE()
+		return err
+	}
+
+	cmd.RunE = func(cmd *cobra.Command, args []string) error {
+		rc.Log.Debugf("sq %s [%s]", cmd.Name(), strings.Join(args, ","))
+		if cmd.Flags().Changed(flagVersion) {
+			// Bit of a hack: flag --version on any command
+			// results in execVersion being invoked
+			return execVersion(rc, cmd, args)
+		}
+
+		return fn(rc, cmd, args)
+	}
+
+	// We handle the errors ourselves (rather than let cobra do it)
+	cmd.SilenceErrors = true
+	cmd.SilenceUsage = true
+
+	parentCmd.AddCommand(cmd)
+
+	return cmd
+}
+
+// RunContext is a container for injectable resources passed
+// to all execX funcs. The Close method should be invoked when
+// the RunContext is no longer needed.
+type RunContext struct {
+	// Context is the run's context.Context.
+	Context context.Context
+
+	// Stdin typically is os.Stdin, but can be changed for testing.
+	Stdin *os.File
+
+	// Out is the output destination.
+	// If nil, default to stdout.
+	Out io.Writer
+
+	// ErrOut is the error output destination.
+	// If nil, default to stderr.
+	ErrOut io.Writer
+
+	// Cmd is the command instance provided by cobra for
+	// the currently executing command. This field will
+	// be set before the command's runFunc is invoked.
+	Cmd *cobra.Command
+
+	// Args is the arg slice supplied by cobra for
+	// the currently executing command. This field will
+	// be set before the command's runFunc is invoked.
+	Args []string
+
+	// Config is the run's config.
+	Config *config.Config
+
+	// ConfigStore is run's config store.
+	ConfigStore config.Store
+
+	// Log is the run's logger.
+	Log lg.Log
+
+	wrtr   *writers
+	reg    *driver.Registry
+	files  *source.Files
+	dbases *driver.Databases
+	clnup  *cleanup.Cleanup
+}
+
+// newDefaultRunContext returns a RunContext configured
+// with standard values for logging, config, etc. This
+// effectively is the bootstrap mechanism for sq.
+//
+// Note: This func always returns a RunContext, even if
+// an error occurs during bootstrap of the RunContext (for
+// example if there's a config error). We do this to provide
+// enough framework so that such an error can be logged or
+// printed per the normal mechanisms if at all possible.
+func newDefaultRunContext(ctx context.Context, stdin *os.File, stdout, stderr io.Writer) (*RunContext, error) {
+	rc := &RunContext{
+		Context: ctx,
+		Stdin:   stdin,
+		Out:     stdout,
+		ErrOut:  stderr,
+	}
+
+	log, clnup, loggingErr := defaultLogging()
+	rc.Log = log
+	rc.clnup = clnup
+
+	cfg, cfgStore, configErr := defaultConfig()
+	rc.ConfigStore = cfgStore
+	rc.Config = cfg
+
+	switch {
+	case rc.Log == nil:
+		rc.Log = lg.Discard()
+	case rc.clnup == nil:
+		rc.clnup = cleanup.New()
+	case rc.Config == nil:
+		rc.Config = config.New()
+	}
+
+	if configErr != nil {
+		// configErr is more important, return that first
+		return rc, configErr
+	}
+
+	if loggingErr != nil {
+		return rc, loggingErr
+	}
+
+	return rc, nil
+}
+
+// preRunE is invoked by cobra prior to the command RunE being
+// invoked. It sets up the registry, databases, writer and related
+// fundamental components.
+func (rc *RunContext) preRunE() error {
+	rc.clnup = cleanup.New()
+	log, cfg := rc.Log, rc.Config
+
+	if cmdFlagChanged(rc.Cmd, flagOutput) {
+		fpath, _ := rc.Cmd.Flags().GetString(flagOutput)
+		fpath, err := filepath.Abs(fpath)
+		if err != nil {
+			return errz.Wrapf(err, "failed to get absolute path for --%s", flagOutput)
+		}
+
+		f, err := os.Create(fpath)
+		if err != nil {
+			return errz.Wrapf(err, "failed to open file specified by flag --%s", flagOutput)
+		}
+
+		rc.clnup.AddC(f) // Make sure the file gets closed eventually
+		rc.Out = f
+	}
+
+	rc.wrtr = newWriters(rc.Log, rc.Cmd, rc.Config.Options, rc.Out, rc.ErrOut)
+
+	var scratchSrcFunc driver.ScratchSrcFunc
+
+	// scratchSrc could be nil, and that's ok
+	scratchSrc := cfg.Sources.Scratch()
+	if scratchSrc == nil {
+		scratchSrcFunc = sqlite3.NewScratchSource
+	} else {
+		scratchSrcFunc = func(log lg.Log, name string) (src *source.Source, clnup func() error, err error) {
+			return scratchSrc, nil, nil
+		}
+	}
+
+	rc.reg = driver.NewRegistry(log)
+	rc.dbases = driver.NewDatabases(log, rc.reg, scratchSrcFunc)
+	rc.clnup.AddC(rc.dbases)
+
+	var err error
+	rc.files, err = source.NewFiles(log)
+	if err != nil {
+		return err
+	}
+	rc.files.AddTypeDetectors(source.DetectMagicNumber)
+
+	rc.reg.AddProvider(sqlite3.Type, &sqlite3.Provider{Log: log})
+	rc.reg.AddProvider(postgres.Type, &postgres.Provider{Log: log})
+	rc.reg.AddProvider(sqlserver.Type, &sqlserver.Provider{Log: log})
+	rc.reg.AddProvider(mysql.Type, &mysql.Provider{Log: log})
+	csvp := &csv.Provider{Log: log, Scratcher: rc.dbases, Files: rc.files}
+	rc.reg.AddProvider(csv.TypeCSV, csvp)
+	rc.reg.AddProvider(csv.TypeTSV, csvp)
+	rc.files.AddTypeDetectors(csv.DetectCSV, csv.DetectTSV)
+
+	rc.reg.AddProvider(xlsx.Type, &xlsx.Provider{Log: log, Scratcher: rc.dbases, Files: rc.files})
+	rc.files.AddTypeDetectors(xlsx.DetectXLSX)
+	// One day we may have more supported user driver genres.
+	userDriverImporters := map[string]userdriver.ImportFunc{
+		xmlud.Genre: xmlud.Import,
+	}
+
+	for i, userDriverDef := range cfg.Ext.UserDrivers {
+		userDriverDef := userDriverDef
+
+		errs := userdriver.ValidateDriverDef(userDriverDef)
+		if len(errs) > 0 {
+			err := errz.Combine(errs...)
+			err = errz.Wrapf(err, "failed validation of user driver definition [%d] (%q) from config",
+				i, userDriverDef.Name)
+			return err
+		}
+
+		importFn, ok := userDriverImporters[userDriverDef.Genre]
+		if !ok {
+			return errz.Errorf("unsupported genre %q for user driver %q specified via config",
+				userDriverDef.Genre, userDriverDef.Name)
+		}
+
+		// For each user driver definition, we register a
+		// distinct userdriver.Provider instance.
+		udp := &userdriver.Provider{
+			Log:       log,
+			DriverDef: userDriverDef,
+			ImportFn:  importFn,
+			Scratcher: rc.dbases,
+			Files:     rc.files,
+		}
+
+		rc.reg.AddProvider(source.Type(userDriverDef.Name), udp)
+		rc.files.AddTypeDetectors(udp.TypeDetectors()...)
+	}
+
+	return nil
+}
+
+// Close should be invoked to dispose of any open resources
+// held by rc. If an error occurs during close and rc.Log
+// is not nil, that error is logged at WARN level before
+// being returned.
+func (rc *RunContext) Close() error {
+	if rc == nil {
+		return nil
+	}
+
+	err := rc.clnup.Run()
+	if err != nil && rc.Log != nil {
+		rc.Log.Warnf("failed to close RunContext: %v", err)
+	}
+
+	return err
+}
+
+// writers returns this run context's writers instance.
+func (rc *RunContext) writers() *writers {
+	return rc.wrtr
+}
+
+// registry returns rc's Registry instance.
+func (rc *RunContext) registry() *driver.Registry {
+	return rc.reg
+}
+
+// registry returns rc's Databases instance.
+func (rc *RunContext) databases() *driver.Databases {
+	return rc.dbases
+}
+
+// writers is a container for the various output writer types.
+type writers struct {
+	fmt     *output.Formatting
+	recordw output.RecordWriter
+	metaw   output.MetadataWriter
+	srcw    output.SourceWriter
+	notifyw output.NotificationWriter
+	errw    output.ErrorWriter
+	pingw   output.PingWriter
+}
+
+func newWriters(log lg.Log, cmd *cobra.Command, opts config.Options, out, errOut io.Writer) *writers {
+	fm, out, errOut := getWriterFormatting(cmd, out, errOut)
+
+	// we need to determine --header here because the writer/format
+	// constructor functions, e.g. table.NewRecordWriter, require it.
+	hasHeader := false
+	switch {
+	case cmdFlagChanged(cmd, flagHeader):
+		hasHeader = true
+	case cmdFlagChanged(cmd, flagNoHeader):
+		hasHeader = false
+	default:
+		// get the default --header value from config
+		hasHeader = opts.Header
+	}
+
+	verbose := false
+	if cmdFlagChanged(cmd, flagVerbose) {
+		verbose, _ = cmd.Flags().GetBool(flagVerbose)
+	}
+
+	// Package tablew has writer impls for all of the writer interfaces,
+	// so we use its writers as the baseline. Later we check the format
+	// flags and set the various writer fields depending upon which
+	// writers the format implements.
+	w := &writers{
+		fmt:     fm,
+		recordw: tablew.NewRecordWriter(out, fm, hasHeader),
+		metaw:   tablew.NewMetadataWriter(out, fm),
+		srcw:    tablew.NewSourceWriter(out, fm, hasHeader, verbose),
+		pingw:   tablew.NewPingWriter(out, fm),
+		notifyw: tablew.NewNotifyWriter(out, fm, hasHeader),
+		errw:    tablew.NewErrorWriter(errOut, fm),
+	}
+
+	// Invoke getFormat to see if the format was specified
+	// via config or flag.
+	format := getFormat(cmd, opts)
+
+	switch format {
+	default:
+		// No format specified, use JSON
+		w.recordw = jsonw.NewStdRecordWriter(out, fm)
+		w.metaw = jsonw.NewMetadataWriter(out, fm)
+		w.errw = jsonw.NewErrorWriter(log, errOut, fm)
+
+	case config.FormatTable:
+	// Table is the base format, already set above, no need to do anything.
+
+	case config.FormatTSV:
+		w.recordw = csvw.NewRecordWriter(out, hasHeader, csvw.Tab)
+		w.pingw = csvw.NewPingWriter(out, csvw.Tab)
+
+	case config.FormatCSV:
+		w.recordw = csvw.NewRecordWriter(out, hasHeader, csvw.Comma)
+		w.pingw = csvw.NewPingWriter(out, csvw.Comma)
+
+	case config.FormatXML:
+		w.recordw = xmlw.NewRecordWriter(out, fm)
+
+	case config.FormatXLSX:
+		w.recordw = xlsxw.NewRecordWriter(out, hasHeader)
+
+	case config.FormatRaw:
+		w.recordw = raww.NewRecordWriter(out)
+
+	case config.FormatHTML:
+		w.recordw = htmlw.NewRecordWriter(out)
+
+	case config.FormatMarkdown:
+		w.recordw = markdownw.NewRecordWriter(out)
+
+	case config.FormatJSONA:
+		w.recordw = jsonw.NewArrayRecordWriter(out, fm)
+
+	case config.FormatJSONL:
+		w.recordw = jsonw.NewObjectRecordWriter(out, fm)
+	}
+
+	return w
+}
+
+// getWriterFormatting returns a Formatting instance and
+// colorable or non-colorable writers. It is permissible
+// for the cmd arg to be nil.
+func getWriterFormatting(cmd *cobra.Command, out, errOut io.Writer) (fm *output.Formatting, out2, errOut2 io.Writer) {
+	fm = output.NewFormatting()
+
+	if cmdFlagChanged(cmd, flagPretty) {
+		fm.Pretty, _ = cmd.Flags().GetBool(flagPretty)
+	}
+
+	// TODO: Should get this default value from config
+	colorize := true
+
+	if cmdFlagChanged(cmd, flagOutput) {
+		// We're outputting to a file, thus no color.
+		colorize = false
+	} else if cmdFlagChanged(cmd, flagMonochrome) {
+		if mono, _ := cmd.Flags().GetBool(flagMonochrome); mono {
+			colorize = false
+		}
+	}
+
+	if !colorize {
+		color.NoColor = true // TODO: shouldn't rely on package-level var
+		fm.EnableColor(false)
+		out2 = out
+		errOut2 = errOut
+		return fm, out2, errOut2
+	}
+
+	// We do want to colorize
+	if !isColorTerminal(out) {
+		// But out can't be colorized.
+		color.NoColor = true
+		fm.EnableColor(false)
+		out2, errOut2 = out, errOut
+		return fm, out2, errOut2
+	}
+
+	// out can be colorized.
+	color.NoColor = false
+	fm.EnableColor(true)
+	out2 = colorable.NewColorable(out.(*os.File))
+
+	// Check if we can colorize errOut
+	if isColorTerminal(errOut) {
+		errOut2 = colorable.NewColorable(errOut.(*os.File))
+	} else {
+		// errOut2 can't be colorized, but since we're colorizing
+		// out, we'll apply the non-colorable filter to errOut.
+		errOut2 = colorable.NewNonColorable(errOut)
+	}
+
+	return fm, out2, errOut2
+}
+
+func getFormat(cmd *cobra.Command, opts config.Options) config.Format {
+	var format config.Format
+
+	switch {
+	// cascade through the format flags in low-to-high order of precedence.
+	case cmdFlagChanged(cmd, flagTSV):
+		format = config.FormatTSV
+	case cmdFlagChanged(cmd, flagCSV):
+		format = config.FormatCSV
+	case cmdFlagChanged(cmd, flagXLSX):
+		format = config.FormatXLSX
+	case cmdFlagChanged(cmd, flagXML):
+		format = config.FormatXML
+	case cmdFlagChanged(cmd, flagRaw):
+		format = config.FormatRaw
+	case cmdFlagChanged(cmd, flagHTML):
+		format = config.FormatHTML
+	case cmdFlagChanged(cmd, flagMarkdown):
+		format = config.FormatMarkdown
+	case cmdFlagChanged(cmd, flagTable):
+		format = config.FormatTable
+	case cmdFlagChanged(cmd, flagJSONL):
+		format = config.FormatJSONL
+	case cmdFlagChanged(cmd, flagJSONA):
+		format = config.FormatJSONA
+	case cmdFlagChanged(cmd, flagJSON):
+		format = config.FormatJSON
+	default:
+		// no format flag, use the config value
+		format = opts.Format
+	}
+	return format
+}
+
+// defaultLogging returns a log (and its associated closer) if
+// logging has been enabled via envars.
+func defaultLogging() (lg.Log, *cleanup.Cleanup, error) {
+	truncate, _ := strconv.ParseBool(os.Getenv(envarLogTruncate))
+
+	logFilePath, ok := os.LookupEnv(envarLogPath)
+	if !ok || logFilePath == "" || strings.TrimSpace(logFilePath) == "" {
+		return lg.Discard(), nil, nil
+	}
+
+	// Let's try to create the dir holding the logfile... if it already exists,
+	// then os.MkdirAll will just no-op
+	parent := filepath.Dir(logFilePath)
+	err := os.MkdirAll(parent, os.ModePerm)
+	if err != nil {
+		return lg.Discard(), nil, errz.Wrapf(err, "failed to create parent dir of log file %s", logFilePath)
+	}
+
+	flag := os.O_APPEND
+	if truncate {
+		flag = os.O_TRUNC
+	}
+
+	logFile, err := os.OpenFile(logFilePath, os.O_RDWR|os.O_CREATE|flag, 0666)
+	if err != nil {
+		return lg.Discard(), nil, errz.Wrapf(err, "unable to open log file %q", logFilePath)
+	}
+	clnup := cleanup.New().AddE(logFile.Close)
+
+	log := zaplg.NewWith(logFile, "json", true, true, true, 0)
+	clnup.AddE(log.Sync)
+
+	return log, clnup, nil
+}
+
+// defaultConfig loads sq config from the default location
+// (~/.config/sq/sq.yml) or the location specified in envars.
+func defaultConfig() (*config.Config, config.Store, error) {
+	cfgDir, ok := os.LookupEnv(envarConfigDir)
+	if !ok {
+		// envar not set, let's use the default
+		home, err := homedir.Dir()
+		if err != nil {
+			// TODO: we should be able to run without the homedir... revisit this
+			return nil, nil, errz.Wrap(err, "unable to get user home dir for config purposes")
+		}
+
+		cfgDir = filepath.Join(home, ".config", "sq")
+	}
+
+	cfgPath := filepath.Join(cfgDir, "sq.yml")
+	extDir := filepath.Join(cfgDir, "ext")
+	cfgStore := &config.YAMLFileStore{Path: cfgPath, ExtPaths: []string{extDir}}
+
+	if !cfgStore.FileExists() {
+		cfg := config.New()
+		return cfg, cfgStore, nil
+	}
+
+	// file does exist, let's try to load it
+	cfg, err := cfgStore.Load()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return cfg, cfgStore, nil
+}
+
+// printError is the centralized function for printing
+// and logging errors. This func has a lot of (possibly needless)
+// redundancy; ultimately err will print if non-nil (even if
+// rc or any of its fields are nil).
+func printError(rc *RunContext, err error) {
+	log := lg.Discard()
+	if rc != nil && rc.Log != nil {
+		log = rc.Log
+	}
+
+	if err == nil {
+		log.Warnf("printError called with nil error")
+		return
+	}
+
+	if err == errNoMsg {
+		// errNoMsg is a sentinel err that sq doesn't want to print
+		return
+	}
+
+	switch errz.Cause(err) {
+	default:
+	case context.Canceled:
+		err = errz.New("stopped")
+	case context.DeadlineExceeded:
+		err = errz.New("timeout")
+	}
+
+	var cmd *cobra.Command
+	if rc != nil {
+		cmd = rc.Cmd
+
+		cmdName := "unknown"
+		if cmd != nil {
+			cmdName = fmt.Sprintf("[cmd:%s] ", cmd.Name())
+		}
+		log.Errorf("%s [%T] %+v", cmdName, err, err)
+
+		wrtrs := rc.writers()
+		if wrtrs != nil && wrtrs.errw != nil {
+			// If we have an errorWriter, we print to it
+			// and return.
+			wrtrs.errw.Error(err)
+			return
+		}
+
+		// Else we don't have an errorWriter, so we fall through
+	}
+
+	// If we get this far, something went badly wrong in bootstrap
+	// (probably the config is corrupt).
+	// At this point, we could just print err to os.Stderr and be done.
+	// However, our philosophy is to always provide the ability
+	// to output errors in json if possible. So, even though cobra
+	// may not have initialized and our own config may be borked, we
+	// will still try to determine if the user wants the error
+	// in json, specified via flags (by directly using the pflag
+	// package) or via sq config's default output format.
+
+	// getWriterFormatting works even if cmd is nil
+	fm, _, errOut := getWriterFormatting(cmd, os.Stdout, os.Stderr)
+
+	if bootstrapIsFormatJSON(rc) {
+		// The user wants JSON, either via defaults or flags.
+		jw := jsonw.NewErrorWriter(log, errOut, fm)
+		jw.Error(err)
+		return
+	}
+
+	// The user didn't want JSON, so we just print to stderr.
+	if isColorTerminal(os.Stderr) {
+		fm.Error.Fprintln(os.Stderr, "sq: "+err.Error())
+	} else {
+		fmt.Fprintln(os.Stderr, "sq: "+err.Error())
+	}
+}
+
+// cmdFlagChanged returns true if cmd is non-nil and
+// has the named flag and that flag been changed.
+func cmdFlagChanged(cmd *cobra.Command, name string) bool {
+	if cmd == nil {
+		return false
+	}
+
+	flag := cmd.Flag(name)
+	if flag == nil {
+		return false
+	}
+
+	return flag.Changed
+}
+
+// bootstrapIsFormatJSON is a last-gasp attempt to check if the user
+// supplied --json=true on the command line, to determine if a
+// bootstrap error (hopefully rare) should be output in JSON.
+func bootstrapIsFormatJSON(rc *RunContext) bool {
+	// If no RunContext, assume false
+	if rc == nil {
+		return false
+	}
+
+	defaultFormat := config.FormatTable
+	if rc.Config != nil {
+		defaultFormat = rc.Config.Options.Format
+	}
+
+	// If args were provided, create a new flag set and check
+	// for the --json flag.
+	if len(rc.Args) > 0 {
+		flags := pflag.NewFlagSet("bootstrap", pflag.ContinueOnError)
+
+		jsonFlag := flags.BoolP(flagJSON, flagJSONShort, false, flagJSONUsage)
+		err := flags.Parse(rc.Args)
+		if err != nil {
+			return false
+		}
+
+		// No --json flag, return true if the config file default is JSON
+		if jsonFlag == nil {
+			return defaultFormat == config.FormatJSON
+		}
+
+		return *jsonFlag
+	}
+
+	// No args, return true if the config file default is JSON
+	return defaultFormat == config.FormatJSON
+}

cli/cli_test.go

@@ -0,0 +1,162 @@
+package cli_test
+
+import (
+	"bytes"
+	"fmt"
+	"image/gif"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/neilotoole/lg/testlg"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/neilotoole/sq/cli"
+	"github.com/neilotoole/sq/libsq/sqlmodel"
+	"github.com/neilotoole/sq/libsq/sqlz"
+	"github.com/neilotoole/sq/libsq/stringz"
+	"github.com/neilotoole/sq/testh"
+	"github.com/neilotoole/sq/testh/fixt"
+	"github.com/neilotoole/sq/testh/proj"
+	"github.com/neilotoole/sq/testh/sakila"
+)
+
+func TestSmoke(t *testing.T) {
+	t.Parallel()
+	// Execute a bunch of smoke test cases.
+
+	sqargs := func(a ...string) []string {
+		return append([]string{"sq"}, a...)
+	}
+
+	testCases := []struct {
+		a []string
+		// errBecause, if non-empty, indicates an error is expected.
+		errBecause string
+	}{
+		{a: sqargs("ls")},
+		{a: sqargs("ls", "-v")},
+		{a: sqargs("ls", "--help")},
+		{a: sqargs("inspect"), errBecause: "no active data source"},
+		{a: sqargs("inspect", "--help")},
+		{a: sqargs("version")},
+		{a: sqargs("--version")},
+		{a: sqargs("help")},
+		{a: sqargs("--help")},
+		{a: sqargs("ping", "all")},
+		{a: sqargs("ping", "--help")},
+		{a: sqargs("ping"), errBecause: "no active data source"},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(strings.Join(tc.a, "_"), func(t *testing.T) {
+			t.Parallel()
+
+			rc, out, errOut := newTestRunCtx(testlg.New(t))
+			err := cli.ExecuteWith(rc, tc.a)
+
+			// We log sq's output before doing assert, because it reads
+			// better in testing's output that way.
+			if out.Len() > 0 {
+				t.Log(strings.TrimSuffix(out.String(), "\n"))
+			}
+			if errOut.Len() > 0 {
+				t.Log(strings.TrimSuffix(errOut.String(), "\n"))
+			}
+
+			if tc.errBecause != "" {
+				assert.Error(t, err, tc.errBecause)
+			} else {
+				assert.NoError(t, err, tc.errBecause)
+			}
+		})
+	}
+}
+
+func TestCreateTblTestBytes(t *testing.T) {
+	th, src, _, _ := testh.NewWith(t, sakila.Pg9)
+	th.NoDiff(src)
+
+	tblDef := sqlmodel.NewTableDef(
+		stringz.UniqTableName("test_bytes"),
+		[]string{"col_name", "col_bytes"},
+		[]sqlz.Kind{sqlz.KindText, sqlz.KindBytes},
+	)
+
+	fBytes := proj.ReadFile(fixt.GopherPath)
+	data := []interface{}{fixt.GopherFilename, fBytes}
+
+	require.Equal(t, int64(1), th.CreateTable(true, src, tblDef, data))
+}
+
+// TestOutputRaw verifies that the raw output format works.
+// We're particularly concerned that bytes output is correct.
+func TestOutputRaw(t *testing.T) {
+	for _, handle := range sakila.SQLAll {
+		handle := handle
+
+		t.Run(handle, func(t *testing.T) {
+			t.Parallel()
+
+			// Sanity check
+			wantBytes := proj.ReadFile(fixt.GopherPath)
+			require.Equal(t, fixt.GopherSize, len(wantBytes))
+			_, err := gif.Decode(bytes.NewReader(wantBytes))
+			require.NoError(t, err)
+
+			tblDef := sqlmodel.NewTableDef(
+				stringz.UniqTableName("test_bytes"),
+				[]string{"col_name", "col_bytes"},
+				[]sqlz.Kind{sqlz.KindText, sqlz.KindBytes},
+			)
+
+			th, src, _, _ := testh.NewWith(t, handle)
+
+			// Create the table and insert data
+			insertRow := []interface{}{fixt.GopherFilename, wantBytes}
+			require.Equal(t, int64(1), th.CreateTable(true, src, tblDef, insertRow))
+
+			// 1. Query and check that libsq is returning bytes correctly.
+			query := fmt.Sprintf("SELECT col_bytes FROM %s WHERE col_name = '%s'",
+				tblDef.Name, fixt.GopherFilename)
+			sink, err := th.QuerySQL(src, query)
+			require.NoError(t, err)
+
+			require.Equal(t, 1, len(sink.Recs))
+			require.Equal(t, sqlz.KindBytes, sink.RecMeta[0].Kind())
+			dbBytes := *(sink.Recs[0][0].(*[]byte))
+			require.Equal(t, fixt.GopherSize, len(dbBytes))
+			require.Equal(t, wantBytes, dbBytes)
+
+			// 1. Now that we've verified libsq, we'll test cli. First
+			// using using --output=/path/to/file
+			tmpDir, err := ioutil.TempDir("", "")
+			require.NoError(t, err)
+			outputPath := filepath.Join(tmpDir, "gopher.gif")
+			t.Cleanup(func() {
+				os.RemoveAll(outputPath)
+			})
+
+			ru := newRun(t).add(*src).hush()
+			err = ru.exec("sql", "--raw", "--output="+outputPath, query)
+			require.NoError(t, err)
+
+			outputBytes, err := ioutil.ReadFile(outputPath)
+			require.NoError(t, err)
+			require.Equal(t, fixt.GopherSize, len(outputBytes))
+			_, err = gif.Decode(bytes.NewReader(outputBytes))
+			require.NoError(t, err)
+
+			// 2. Now test that stdout also gets the same data
+			ru = newRun(t).add(*src)
+			err = ru.exec("sql", "--raw", query)
+			require.NoError(t, err)
+			require.Equal(t, wantBytes, ru.out.Bytes())
+		})
+	}
+}

cli/cmd_add.go

@@ -0,0 +1,192 @@
+package cli
+
+import (
+	"strings"
+
+	"github.com/spf13/cobra"
+
+	"github.com/neilotoole/sq/drivers/sqlite3"
+	"github.com/neilotoole/sq/libsq/errz"
+	"github.com/neilotoole/sq/libsq/options"
+	"github.com/neilotoole/sq/libsq/source"
+	"github.com/neilotoole/sq/libsq/stringz"
+)
+
+func newSrcAddCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use: "add [--driver=TYPE] [--handle=@HANDLE] LOCATION",
+		Example: `  # add a Postgres source; will have generated handle @sakila_pg
+  sq add 'postgres://user:pass@localhost/sakila?sslmode=disable'
+  
+  # same as above, but explicitly setting flags
+  sq add --handle=@sakila_pg --driver=postgres 'postgres://user:pass@localhost/sakila?sslmode=disable'
+
+  # same as above, but with short flags
+  sq add -h @sakila_pg --d postgres 'postgres://user:pass@localhost/sakila?sslmode=disable'
+
+  # add a SQL Server source; will have generated handle @sakila_mssql
+  sq add 'sqlserver://user:pass@localhost?database=sakila' 
+  
+  # add a sqlite db
+  sq add ./testdata/sqlite1.db
+
+  # add an Excel spreadsheet, with options
+  sq add ./testdata/test1.xlsx --opts=header=true
+  
+  # add a CSV source, with options
+  sq add ./testdata/person.csv --opts='header=true'
+
+  # add a CSV source from a server (will be downloaded)
+  sq add https://sq.io/testdata/actor.csv
+`,
+		Short: "Add data source",
+		Long: `Add data source specified by LOCATION and optionally identified by @HANDLE.
+The format of LOCATION varies, but is generally a DB connection string, a
+file path, or a URL.
+
+  DRIVER://USER:PASS@HOST:PORT/DBNAME
+  /path/to/local/file.ext
+  https://sq.io/data/test1.xlsx
+
+If flag --handle is omitted, sq will generate a handle based
+on LOCATION and the source driver type.
+
+If flag --driver is omitted, sq will attempt to determine the
+type from LOCATION via file suffix, content type, etc.. If the result
+is ambiguous, specify the driver tye type via flag --driver.
+
+Flag --opts sets source specific options. Generally opts are relevant
+to document source types (such as a CSV file). The most common
+use is to specify that the document has a header row:
+
+  sq add actor.csv --opts=header=true
+
+Available source driver types can be listed via "sq drivers".
+
+At a minimum, the following drivers are bundled:
+
+  sqlite3      SQLite3
+  postgres     Postgres
+  sqlserver    Microsoft SQL Server
+  xlsx         Microsoft Excel XLSX
+  mysql        MySQL
+  csv          Comma-Separated Values
+  tsv          Tab-Separated Values
+`,
+	}
+
+	cmd.Flags().StringP(flagDriver, flagDriverShort, "", flagDriverUsage)
+	cmd.Flags().StringP(flagSrcOptions, "", "", flagSrcOptionsUsage)
+	cmd.Flags().StringP(flagHandle, flagHandleShort, "", flagHandleUsage)
+	return cmd, execSrcAdd
+}
+
+func execSrcAdd(rc *RunContext, cmd *cobra.Command, args []string) error {
+	if len(args) != 1 {
+		return errz.Errorf(msgInvalidArgs)
+	}
+
+	cfg := rc.Config
+
+	loc := source.AbsLocation(strings.TrimSpace(args[0]))
+
+	var err error
+	var typ source.Type
+	if cmd.Flags().Changed(flagDriver) {
+		val, _ := cmd.Flags().GetString(flagDriver)
+		typ = source.Type(strings.TrimSpace(val))
+		if !rc.reg.HasProviderFor(typ) {
+			return errz.Errorf("unsupported source driver type %q", val)
+		}
+	} else {
+		typ, err = rc.files.Type(rc.Context, loc)
+		if err != nil {
+			return errz.Errorf("unable to determine source driver type: use --driver flag")
+		}
+	}
+
+	var handle string
+	if cmd.Flags().Changed(flagHandle) {
+		handle, _ = cmd.Flags().GetString(flagHandle)
+	} else {
+		handle, err = source.SuggestHandle(typ, loc, cfg.Sources.Exists)
+		if err != nil {
+			return errz.Wrap(err, "unable to suggest a handle: use --handle flag")
+		}
+	}
+
+	if stringz.InSlice(source.ReservedHandles(), handle) {
+		return errz.Errorf("handle reserved for system use: %s", handle)
+	}
+
+	err = source.VerifyLegalHandle(handle)
+	if err != nil {
+		return err
+	}
+
+	if cfg.Sources.Exists(handle) {
+		return errz.Errorf("source handle already exists: %s", handle)
+	}
+
+	var opts options.Options
+	if cmd.Flags().Changed(flagSrcOptions) {
+		val, _ := cmd.Flags().GetString(flagSrcOptions)
+		val = strings.TrimSpace(val)
+		if val != "" {
+			opts, err = options.ParseOptions(val)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// Special handling for SQLite, because it's a file-based SQL DB
+	// unlike the other SQL DBs sq supports so far.
+	// Both of these forms are allowed:
+	//
+	//  sq add sqlite3:///path/to/sakila.db
+	//  sq add /path/to/sakila.db
+	//
+	// The second form is particularly nice for bash completion etc.
+	if typ == sqlite3.Type {
+		if !strings.HasPrefix(loc, "sqlite3:") {
+			loc = "sqlite3:" + loc
+		}
+	}
+
+	src, err := newSource(rc.Log, rc.registry(), typ, handle, loc, opts)
+	if err != nil {
+		return err
+	}
+
+	err = cfg.Sources.Add(src)
+	if err != nil {
+		return err
+	}
+
+	if cfg.Sources.Active() == nil {
+		// If no current active data source, use this one.
+		_, err = cfg.Sources.SetActive(src.Handle)
+		if err != nil {
+			return err
+		}
+	}
+
+	drvr, err := rc.registry().DriverFor(src.Type)
+	if err != nil {
+		return err
+	}
+
+	// TODO: should we really be pinging this src right now?
+	err = drvr.Ping(rc.Context, src)
+	if err != nil {
+		return errz.Wrapf(err, "failed to ping %s [%s]", src.Handle, src.RedactedLocation())
+	}
+
+	err = rc.ConfigStore.Save(rc.Config)
+	if err != nil {
+		return err
+	}
+
+	return rc.writers().srcw.Source(src)
+}

cli/cmd_add_test.go

@@ -0,0 +1,85 @@
+package cli_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/neilotoole/sq/drivers/csv"
+	"github.com/neilotoole/sq/drivers/mysql"
+	"github.com/neilotoole/sq/drivers/postgres"
+	"github.com/neilotoole/sq/drivers/sqlite3"
+	"github.com/neilotoole/sq/drivers/sqlserver"
+	"github.com/neilotoole/sq/libsq/source"
+	"github.com/neilotoole/sq/testh"
+	"github.com/neilotoole/sq/testh/proj"
+	"github.com/neilotoole/sq/testh/sakila"
+)
+
+func TestCmdAdd(t *testing.T) {
+	th := testh.New(t)
+
+	testCases := []struct {
+		loc        string // first arg to "add" cmd
+		driver     string // --driver flag
+		handle     string // --handle flag
+		wantHandle string
+		wantType   source.Type
+		wantErr    bool
+	}{
+		{loc: "", wantErr: true},
+		{loc: "   ", wantErr: true},
+		{loc: "/", wantErr: true},
+		{loc: "../../", wantErr: true},
+		{loc: "does/not/exist", wantErr: true},
+		{loc: "_", wantErr: true},
+		{loc: ".", wantErr: true},
+		{loc: "/", wantErr: true},
+		{loc: "../does/not/exist.csv", wantErr: true},
+		{loc: proj.Rel(sakila.PathCSVActor), handle: "@h1", wantHandle: "@h1", wantType: csv.TypeCSV}, // relative path
+		{loc: proj.Abs(sakila.PathCSVActor), handle: "@h1", wantHandle: "@h1", wantType: csv.TypeCSV}, // absolute path
+		{loc: proj.Abs(sakila.PathCSVActor), wantHandle: "@actor_csv", wantType: csv.TypeCSV},
+		{loc: proj.Abs(sakila.PathCSVActor), driver: "csv", wantHandle: "@actor_csv", wantType: csv.TypeCSV},
+		{loc: proj.Abs(sakila.PathCSVActor), driver: "xlsx", wantErr: true},
+		// sqlite can be added both with and without the scheme "sqlite://"
+		{loc: "sqlite3://" + proj.Abs(sakila.PathSL3), wantHandle: "@sakila_sqlite", wantType: sqlite3.Type}, // with scheme
+		{loc: proj.Abs(sakila.PathSL3), wantHandle: "@sakila_sqlite", wantType: sqlite3.Type},                // without scheme, abs path
+		{loc: proj.Rel(sakila.PathSL3), wantHandle: "@sakila_sqlite", wantType: sqlite3.Type},                // without scheme, relative path
+		{loc: th.Source(sakila.Pg).Location, wantHandle: "@sakila_pg", wantType: postgres.Type},
+		{loc: th.Source(sakila.MS).Location, wantHandle: "@sakila_mssql", wantType: sqlserver.Type},
+		{loc: th.Source(sakila.My).Location, wantHandle: "@sakila_my", wantType: mysql.Type},
+		{loc: proj.Abs(sakila.PathCSVActor), handle: source.StdinHandle, wantErr: true},   // reserved handle
+		{loc: proj.Abs(sakila.PathCSVActor), handle: source.ActiveHandle, wantErr: true},  // reserved handle
+		{loc: proj.Abs(sakila.PathCSVActor), handle: source.ScratchHandle, wantErr: true}, // reserved handle
+		{loc: proj.Abs(sakila.PathCSVActor), handle: source.JoinHandle, wantErr: true},    // reserved handle
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(testh.TName(tc.wantHandle, tc.loc, tc.driver), func(t *testing.T) {
+			args := []string{"add", tc.loc}
+			if tc.handle != "" {
+				args = append(args, "--handle="+tc.handle)
+			}
+			if tc.driver != "" {
+				args = append(args, "--driver="+tc.driver)
+			}
+
+			ru := newRun(t)
+			err := ru.exec(args...)
+			if tc.wantErr {
+				require.Error(t, err)
+				return
+			}
+
+			require.NoError(t, err)
+
+			// Verify that the src was actually added
+			gotSrc, err := ru.rc.Config.Sources.Get(tc.wantHandle)
+			require.NoError(t, err)
+			require.Equal(t, tc.wantHandle, gotSrc.Handle)
+			require.Equal(t, tc.wantType, gotSrc.Type)
+		})
+	}
+}

cli/cmd_completion.go

@@ -0,0 +1,109 @@
+package cli
+
+import (
+	"os"
+	"runtime"
+
+	"github.com/spf13/cobra"
+)
+
+const bashCompletionFunc = `
+
+__sq_list_sources()
+{
+    local sq_output out
+    if sq_output=$(sq ls 2>/dev/null); then
+        out=($(echo "${sq_output}" | awk 'NR > 1 {print $1}'))
+        COMPREPLY=( $( compgen -W "${out[*]}" -- "$cur" ) )
+    fi
+}
+
+__sq_get_resource()
+{
+    if [[ ${#nouns[@]} -eq 0 ]]; then
+        return 1
+    fi
+    __sq_list_sources ${nouns[${#nouns[@]} -1]}
+    if [[ $? -eq 0 ]]; then
+        return 0
+    fi
+}
+
+__custom_func() {
+    case ${last_command} in
+        sq_ls | sq_src | sq_rm | sq_inspect )
+            __sq_list_sources
+            return
+            ;;
+        *)
+            ;;
+    esac
+}
+`
+
+func newInstallBashCompletionCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:    "install-bash-completion",
+		Short:  "Install bash completion script on Unix-ish systems.",
+		Hidden: true,
+	}
+
+	return cmd, execInstallBashCompletion
+}
+
+func execInstallBashCompletion(rc *RunContext, cmd *cobra.Command, args []string) error {
+	log := rc.Log
+	var path string
+
+	switch runtime.GOOS {
+	case "windows":
+		log.Warnf("skipping install bash completion on windows")
+		return nil
+	case "darwin":
+		path = "/usr/local/etc/bash_completion.d/sq"
+	default:
+		// it's unixish
+		path = " /etc/bash_completion.d/sq"
+	}
+
+	// TODO: only write if necessary (check for version/timestamp/checksum)
+	err := cmd.Root().GenBashCompletionFile(path)
+	if err != nil {
+		log.Warnf("failed to write bash completion to %q: %v", path, err)
+		return err
+	}
+
+	return nil
+}
+
+func newGenerateZshCompletionCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:    "gen-zsh-completion",
+		Short:  "Generate zsh completion script on Unix-ish systems.",
+		Hidden: true,
+	}
+	return cmd, execGenerateZshCompletion
+}
+
+func execGenerateZshCompletion(rc *RunContext, cmd *cobra.Command, args []string) error {
+	log := rc.Log
+	var path string
+
+	switch runtime.GOOS {
+	case "windows":
+		log.Warnf("skipping install zsh completion on windows")
+		return nil
+	case "darwin":
+		path = "/usr/local/etc/bash_completion.d/sq"
+	default:
+		// it's unixish
+		path = " /etc/bash_completion.d/sq"
+	}
+
+	err := cmd.Root().GenZshCompletion(os.Stdout)
+	if err != nil {
+		log.Warnf("failed to write zsh completion to %q: %v", path, err)
+		return err
+	}
+	return nil
+}

cli/cmd_drivers.go

@@ -0,0 +1,31 @@
+package cli
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/neilotoole/sq/libsq/errz"
+)
+
+func newDriversCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:   "drivers",
+		Short: "List available drivers",
+	}
+
+	cmd.Flags().BoolP(flagJSON, flagJSONShort, false, flagJSONUsage)
+	cmd.Flags().BoolP(flagTable, flagTableShort, false, flagTableUsage)
+	cmd.Flags().BoolP(flagHeader, flagHeaderShort, false, flagHeaderUsage)
+	cmd.Flags().BoolP(flagNoHeader, flagNoHeaderShort, false, flagNoHeaderUsage)
+	cmd.Flags().BoolP(flagMonochrome, flagMonochromeShort, false, flagMonochromeUsage)
+
+	return cmd, execDrivers
+}
+
+func execDrivers(rc *RunContext, cmd *cobra.Command, args []string) error {
+	if len(args) > 0 {
+		return errz.Errorf("invalid arguments: zero arguments expected")
+	}
+
+	drvrs := rc.registry().DriversMetadata()
+	return rc.writers().metaw.DriverMetadata(drvrs)
+}

cli/cmd_help.go

@@ -0,0 +1,17 @@
+package cli
+
+import "github.com/spf13/cobra"
+
+func newHelpCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:    "help",
+		Short:  "Show sq help",
+		Hidden: true,
+	}
+
+	return cmd, execHelp
+}
+
+func execHelp(rc *RunContext, cmd *cobra.Command, args []string) error {
+	return cmd.Root().Help()
+}

cli/cmd_inspect.go

@@ -0,0 +1,137 @@
+package cli
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/neilotoole/sq/libsq/errz"
+	"github.com/neilotoole/sq/libsq/source"
+)
+
+func newInspectCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use: "inspect [@HANDLE|@HANDLE.TABLE|.TABLE]",
+		Example: `  # inspect active data source
+  sq inspect
+  
+  # inspect @pg1 data source
+  sq inspect @pg1
+  
+  # inspect 'tbluser' in @pg1 data source
+  sq inspect @pg1.tbluser
+  
+  # inspect 'tbluser' in active data source
+  sq inspect .tbluser
+  
+  # inspect piped data
+  cat data.xlsx | sq inspect`,
+		Short: "Inspect data source schema and stats",
+		Long: `Inspect a data source, including table schemata, columns, etc.
+If @HANDLE is not provided, use the active data source.`,
+	}
+
+	cmd.Flags().BoolP(flagJSON, flagJSONShort, false, flagJSONUsage)
+	cmd.Flags().BoolP(flagTable, flagTableShort, false, flagTableUsage)
+	cmd.Flags().Bool(flagInspectFull, false, flagInspectFullUsage)
+
+	return cmd, execInspect
+}
+
+func execInspect(rc *RunContext, cmd *cobra.Command, args []string) error {
+	if len(args) > 1 {
+		return errz.Errorf("too many arguments")
+	}
+
+	srcs := rc.Config.Sources
+
+	var src *source.Source
+	var table string
+	var err error
+
+	if len(args) == 0 {
+		// No args supplied.
+
+		// There are two paths from here:
+		// - There's input on stdin, which we'll inspect, or
+		// - We're inspecting the active src
+
+		// check if there's input on stdin
+		src, err = checkStdinSource(rc)
+		if err != nil {
+			return err
+		}
+
+		if src != nil {
+			// We have a valid source on stdin.
+
+			// Add the source to the set.
+			err = srcs.Add(src)
+			if err != nil {
+				return err
+			}
+
+			// Set the stdin pipe data source as the active source,
+			// as it's commonly the only data source the user is acting upon.
+			src, err = srcs.SetActive(src.Handle)
+			if err != nil {
+				return err
+			}
+		} else {
+			// No source on stdin. Let's see if there's an active source.
+			src = srcs.Active()
+			if src == nil {
+				return errz.Errorf("no data source specified and no active data source")
+			}
+		}
+	} else {
+		// We received an argument, which can be one of these forms:
+		//   @my1			-- inspect the named source
+		//   @my1.tbluser	-- inspect a table of the named source
+		//   .tbluser		-- inspect a table from the active source
+		var handle string
+		handle, table, err = source.ParseTableHandle(args[0])
+		if err != nil {
+			return errz.Wrap(err, "invalid input")
+		}
+
+		if handle == "" {
+			src = srcs.Active()
+			if src == nil {
+				return errz.Errorf("no data source specified and no active data source")
+			}
+		} else {
+			src, err = srcs.Get(handle)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	dbase, err := rc.databases().Open(rc.Context, src)
+	if err != nil {
+		return errz.Wrapf(err, "failed to inspect %s", src.Handle)
+	}
+	defer rc.Log.WarnIfCloseError(dbase)
+
+	if table != "" {
+		var tblMeta *source.TableMetadata
+		tblMeta, err = dbase.TableMetadata(rc.Context, table)
+		if err != nil {
+			return err
+		}
+
+		return rc.writers().metaw.TableMetadata(tblMeta)
+	}
+
+	meta, err := dbase.SourceMetadata(rc.Context)
+	if err != nil {
+		return errz.Wrapf(err, "failed to read %s source metadata", src.Handle)
+	}
+
+	// This is a bit hacky, but it works... if not "--full", then just zap
+	// the DBVars, as we usually don't want to see those
+	if !cmd.Flags().Changed(flagInspectFull) {
+		meta.DBVars = nil
+	}
+
+	return rc.writers().metaw.SourceMetadata(meta)
+}

cli/cmd_inspect_test.go

@@ -0,0 +1,98 @@
+package cli_test
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/neilotoole/sq/drivers/csv"
+	"github.com/neilotoole/sq/drivers/sqlite3"
+	"github.com/neilotoole/sq/drivers/xlsx"
+	"github.com/neilotoole/sq/libsq/source"
+	"github.com/neilotoole/sq/testh"
+	"github.com/neilotoole/sq/testh/proj"
+	"github.com/neilotoole/sq/testh/sakila"
+)
+
+func TestCmdInspect(t *testing.T) {
+	th := testh.New(t)
+	src := th.Source(sakila.SL3)
+	ru := newRun(t)
+
+	err := ru.exec("inspect")
+	require.Error(t, err, "should fail because no active src")
+
+	ru = newRun(t)
+	ru.add(*src) // now have an active src
+
+	err = ru.exec("inspect", "--json")
+	require.NoError(t, err, "should pass because there is an active src")
+
+	md := &source.Metadata{}
+	require.NoError(t, json.Unmarshal(ru.out.Bytes(), md))
+	require.Equal(t, sqlite3.Type, md.SourceType)
+	require.Equal(t, sakila.SL3, md.Handle)
+	require.Equal(t, src.Location, md.Location)
+	require.Equal(t, sakila.AllTbls, md.TableNames())
+
+	// Try one more source for good measure
+	ru = newRun(t)
+	src = th.Source(sakila.CSVActor)
+	ru.add(*src)
+
+	err = ru.exec("inspect", "--json", src.Handle)
+	require.NoError(t, err)
+
+	md = &source.Metadata{}
+	require.NoError(t, json.Unmarshal(ru.out.Bytes(), md))
+	require.Equal(t, csv.TypeCSV, md.SourceType)
+	require.Equal(t, sakila.CSVActor, md.Handle)
+	require.Equal(t, src.Location, md.Location)
+	require.Equal(t, []string{source.MonotableName}, md.TableNames())
+}
+
+func TestCmdInspect_Stdin(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		fpath    string
+		wantErr  bool
+		wantType source.Type
+		wantTbls []string
+	}{
+		{fpath: proj.Abs(sakila.PathCSVActor), wantType: csv.TypeCSV, wantTbls: []string{source.MonotableName}},
+		{fpath: proj.Abs(sakila.PathTSVActor), wantType: csv.TypeTSV, wantTbls: []string{source.MonotableName}},
+		{fpath: proj.Abs(sakila.PathXLSX), wantType: xlsx.Type, wantTbls: sakila.AllTbls},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(testh.TName(tc.fpath), func(t *testing.T) {
+			testh.SkipShort(t, tc.wantType == xlsx.Type)
+			t.Parallel()
+
+			f, err := os.Open(tc.fpath)
+			require.NoError(t, err)
+			ru := newRun(t)
+			ru.rc.Stdin = f
+
+			err = ru.exec("inspect", "--json")
+			if tc.wantErr {
+				require.Error(t, err)
+				return
+			}
+
+			require.NoError(t, err, "should read from stdin")
+
+			md := &source.Metadata{}
+			require.NoError(t, json.Unmarshal(ru.out.Bytes(), md))
+			require.Equal(t, tc.wantType, md.SourceType)
+			require.Equal(t, source.StdinHandle, md.Handle)
+			require.Equal(t, source.StdinHandle, md.Location)
+			require.Equal(t, tc.wantTbls, md.TableNames())
+		})
+	}
+}

cli/cmd_list.go

@@ -0,0 +1,27 @@
+package cli
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/neilotoole/sq/libsq/errz"
+)
+
+func newSrcListCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:   "ls",
+		Short: "List data sources",
+	}
+
+	cmd.Flags().BoolP(flagVerbose, flagVerboseShort, false, flagVerboseUsage)
+	cmd.Flags().BoolP(flagHeader, flagHeaderShort, false, flagHeaderUsage)
+	cmd.Flags().BoolP(flagNoHeader, flagNoHeaderShort, false, flagNoHeaderUsage)
+	return cmd, execSrcList
+}
+
+func execSrcList(rc *RunContext, cmd *cobra.Command, args []string) error {
+	if len(args) != 0 {
+		return errz.Errorf(msgInvalidArgs)
+	}
+
+	return rc.writers().srcw.SourceSet(rc.Config.Sources)
+}

cli/cmd_notify.go

@@ -0,0 +1,201 @@
+package cli
+
+import (
+	"fmt"
+
+	"strings"
+
+	"github.com/spf13/cobra"
+
+	"github.com/neilotoole/sq/libsq/errz"
+	"github.com/neilotoole/sq/libsq/notify"
+)
+
+func newNotifyCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:    "notify",
+		Hidden: true, // Not advertising this feature right now
+		Short:  "Manage notification destinations",
+		Example: `sq notify ls
+sq notify add slack devops [...]
+sq notify rm devops
+sq notify add --help`,
+	}
+
+	return cmd, func(rc *RunContext, cmd *cobra.Command, args []string) error {
+		return cmd.Help()
+	}
+}
+
+func newNotifyListCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:     "ls",
+		Aliases: []string{"list"},
+		Short:   "List notification destinations",
+	}
+
+	return cmd, execNotifyList
+}
+
+func execNotifyList(rc *RunContext, cmd *cobra.Command, args []string) error {
+	return rc.writers().notifyw.NotifyDestinations(rc.Config.Notification.Destinations)
+}
+
+func newNotifyRemoveCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:     "rm",
+		Aliases: []string{"remove"},
+		Short:   "Remove notification destination",
+	}
+
+	return cmd, execNotifyRemove
+}
+
+func execNotifyRemove(rc *RunContext, cmd *cobra.Command, args []string) error {
+	if len(args) != 1 {
+		return errz.Errorf("this command takes exactly one argument")
+	}
+
+	cfg := rc.Config
+
+	if len(cfg.Notification.Destinations) == 0 {
+		return errz.Errorf("the notification destination %q does not exist", args[0])
+	}
+
+	var dests []notify.Destination
+	for _, dest := range cfg.Notification.Destinations {
+		if dest.Label == args[0] {
+			continue
+		}
+		dests = append(dests, dest)
+	}
+
+	if len(dests) == len(cfg.Notification.Destinations) {
+		return errz.Errorf("the notification destination %q does not exist", args[0])
+	}
+
+	cfg.Notification.Destinations = dests
+	err := rc.ConfigStore.Save(rc.Config)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func newNotifyAddCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:   "add",
+		Short: "Add notification destination",
+		Example: `sq notify add slack #devops xoxp-892529...911b8a
+sq notify add slack --help
+sq notify add hipchat myteam ABAD098ASDF...99AB
+sq notify add hipchat --help
+`,
+	}
+
+	return cmd, nil
+}
+
+func newNotifyAddSlackCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:   "slack CHANNEL [HANDLE] TOKEN",
+		Short: "Add Slack channel",
+		Long: `Add Slack channel. The CHANNEL param should not include the leading '#'.
+	The HANDLE param is optional; if not provided, a handle
+will be generated. To generate the auth token using your browser, login to
+https://TEAM.slack.com and then visit https://api.slack.com/custom-integrations/legacy-tokens
+and use the "Legacy token generator" to get the token value.`,
+		Example: `sq notify add slack devops xoxp-892529...911b8a`,
+	}
+
+	return cmd, execNotifyAddSlack
+}
+
+func execNotifyAddSlack(rc *RunContext, cmd *cobra.Command, args []string) error {
+	if len(args) != 2 && len(args) != 3 {
+		return errz.Errorf(`this command takes either 2 or 3 arguments: see "sq notify add slack --help"`)
+	}
+
+	cfg := rc.Config
+
+	labelAvailableFn := func(label string) bool {
+		for _, dest := range cfg.Notification.Destinations {
+			if dest.Label == label {
+				return false
+			}
+		}
+		return true
+	}
+
+	provider, err := notify.ProviderFor("slack")
+	if err != nil {
+		return err
+	}
+
+	target := args[0]
+	var label string
+	var token string
+
+	if len(args) == 2 {
+		token = args[1]
+	} else {
+		label = args[1]
+		token = args[2]
+
+		if !labelAvailableFn(label) {
+			return errz.Errorf("a notifier with the label %q already exists", label)
+		}
+
+		err = notify.ValidHandle(label)
+		if err != nil {
+			return err
+		}
+	}
+
+	dest, err := provider.Destination(notify.DestType("slack"), target, label, token, labelAvailableFn)
+	if err != nil {
+		return err
+	}
+
+	cfg.Notification.Destinations = append(cfg.Notification.Destinations, *dest)
+
+	err = rc.ConfigStore.Save(rc.Config)
+	if err != nil {
+		return err
+	}
+
+	return rc.writers().notifyw.NotifyDestinations([]notify.Destination{*dest})
+}
+
+func newNotifyAddHipChatCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:     "hipchat ROOM TOKEN",
+		Short:   "Add HipChat room",
+		Example: `sq notify add hipchat devops --label="hip_devops" BOuyOe...VRBksq6`,
+	}
+
+	cmd.Flags().String(flagNotifierLabel, "", flagNotifierLabelUsage)
+
+	return cmd, execNotifyAddHipChat
+}
+
+func execNotifyAddHipChat(rc *RunContext, cmd *cobra.Command, args []string) error {
+	fmt.Println("Add HipChat room")
+	fmt.Println(strings.Join(args, " | "))
+
+	var label string
+	var err error
+	if cmd.Flags().Changed(flagNotifierLabel) {
+		label, err = cmd.Flags().GetString(flagNotifierLabel)
+		if err != nil {
+			return errz.Err(err)
+		}
+	}
+
+	if label != "" {
+		fmt.Printf("Label: %s", label)
+	}
+
+	return nil
+}

cli/cmd_ping.go

@@ -0,0 +1,194 @@
+package cli
+
+import (
+	"context"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"github.com/neilotoole/lg"
+
+	"github.com/neilotoole/sq/cli/output"
+	"github.com/neilotoole/sq/libsq/driver"
+	"github.com/neilotoole/sq/libsq/errz"
+	"github.com/neilotoole/sq/libsq/source"
+)
+
+func newPingCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use: "ping [@HANDLE|all]",
+		Example: `  # ping active data source
+  sq ping
+
+  # ping all data sources
+  sq ping all
+
+  # ping @my1 with 2s timeout
+  sq ping @my1 --timeout=2s
+		
+  # ping @my1 and @pg1
+  sq ping @my1 @pg1
+
+  # output in TSV format
+  sq ping --tsv @my1`,
+
+		Short: "Check data source connection health",
+		Long: `Ping data sources to check connection health. If no arguments provided, the
+active data source is pinged. The exit code is 1 if ping fails for any of the sources.`,
+	}
+
+	cmd.Flags().BoolP(flagTable, flagTableShort, false, flagTableUsage)
+	cmd.Flags().BoolP(flagCSV, flagCSVShort, false, flagCSVUsage)
+	cmd.Flags().BoolP(flagTSV, flagTSVShort, false, flagTSVUsage)
+	cmd.Flags().Duration(flagTimeout, time.Second*10, flagTimeoutPingUsage)
+
+	return cmd, execPing
+}
+
+func execPing(rc *RunContext, cmd *cobra.Command, args []string) error {
+	cfg := rc.Config
+	var srcs []*source.Source
+	var gotAll bool
+
+	// args can be:
+	// [empty] : ping active source
+	// all     : ping all sources
+	// @handle1 @handleN: ping multiple sources
+	if len(args) == 0 {
+		src := cfg.Sources.Active()
+		if src == nil {
+			return errz.New(msgNoActiveSrc)
+		}
+		srcs = []*source.Source{src}
+	} else {
+		for i, arg := range args {
+			if arg == "all" {
+				if gotAll || i != 0 {
+					// If "all" is an arg, it must the the only one
+					return errz.New("arg 'all' must be supplied without other args")
+				}
+
+				gotAll = true
+				srcs = cfg.Sources.Items()
+				continue
+			}
+
+			if gotAll {
+				// This can happen if arg "all" is mixed in with
+				// handle args, e.g. [@handle1 all @handle2]
+				return errz.New("arg 'all' must be supplied without other args")
+			}
+
+			err := source.VerifyLegalHandle(arg)
+			if err != nil {
+				return err
+			}
+
+			src, err := cfg.Sources.Get(arg)
+			if err != nil {
+				return err
+			}
+
+			srcs = append(srcs, src)
+		}
+	}
+
+	timeout := cfg.Options.Timeout
+	if cmdFlagChanged(cmd, flagTimeout) {
+		timeout, _ = cmd.Flags().GetDuration(flagTimeout)
+	}
+
+	rc.Log.Debugf("Using timeout value: %s", timeout)
+
+	return pingSources(rc.Context, rc.Log, rc.registry(), srcs, rc.writers().pingw, timeout)
+}
+
+// pingSources pings each of the sources in srcs, and prints results
+// to w. If any error occurs pinging any of srcs, that error is printed
+// inline as part of the ping results, and an errNoMsg is returned.
+func pingSources(ctx context.Context, log lg.Log, dp driver.Provider, srcs []*source.Source, w output.PingWriter, timeout time.Duration) error {
+	w.Open(srcs)
+	defer log.WarnIfFuncError(w.Close)
+
+	resultCh := make(chan pingResult, len(srcs))
+
+	// pingErrExists is set to true if there was an error for
+	// any of the pings. This later determines if an error
+	// is returned from this func.
+	var pingErrExists bool
+
+	for _, src := range srcs {
+		go pingSource(ctx, dp, src, timeout, resultCh)
+	}
+
+	// This func doesn't check for context.Canceled itself; instead
+	// it checks if any of the goroutines return that value on
+	// resultCh.
+	for i := 0; i < len(srcs); i++ {
+		result := <-resultCh
+
+		switch {
+		case result.err == context.Canceled:
+			// If any one of the goroutines have received context.Canceled,
+			// then we'll bubble that up and ignore the remaining goroutines.
+			return context.Canceled
+
+		case result.err == context.DeadlineExceeded:
+			// If timeout occurred, set the duration to timeout.
+			result.duration = timeout
+			pingErrExists = true
+
+		case result.err != nil:
+			pingErrExists = true
+		}
+
+		w.Result(result.src, result.duration, result.err)
+	}
+
+	// If there's at least one error, we return the
+	// sentinel errNoMsg so that sq can os.Exit(1) without printing
+	// an additional error message (as the error message will already have
+	// been printed by PingWriter).
+	if pingErrExists {
+		return errNoMsg
+	}
+
+	return nil
+}
+
+// pingSource pings an individual driver.Source. It always returns a
+// result on resultCh, even when ctx is done.
+func pingSource(ctx context.Context, dp driver.Provider, src *source.Source, timeout time.Duration, resultCh chan<- pingResult) {
+	drvr, err := dp.DriverFor(src.Type)
+	if err != nil {
+		resultCh <- pingResult{src: src, err: err}
+		return
+	}
+
+	if timeout > 0 {
+		var cancelFn context.CancelFunc
+		ctx, cancelFn = context.WithTimeout(ctx, timeout)
+		defer cancelFn()
+	}
+
+	doneCh := make(chan pingResult)
+	start := time.Now()
+
+	go func() {
+		err = drvr.Ping(ctx, src)
+		doneCh <- pingResult{src: src, duration: time.Since(start), err: err}
+	}()
+
+	select {
+	case <-ctx.Done():
+		resultCh <- pingResult{src: src, err: ctx.Err()}
+	case result := <-doneCh:
+		resultCh <- result
+	}
+}
+
+type pingResult struct {
+	src      *source.Source
+	duration time.Duration
+	err      error
+}

cli/cmd_ping_test.go

@@ -0,0 +1,73 @@
+package cli_test
+
+import (
+	"encoding/csv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/neilotoole/sq/libsq/source"
+	"github.com/neilotoole/sq/testh"
+	"github.com/neilotoole/sq/testh/sakila"
+)
+
+func TestCmdPing(t *testing.T) {
+	t.Parallel()
+
+	err := newRun(t).exec("ping")
+	require.Error(t, err, "no active data source")
+
+	err = newRun(t).exec("ping", "invalid_handle")
+	require.Error(t, err)
+
+	err = newRun(t).exec("ping", "@not_a_handle")
+	require.Error(t, err)
+
+	var ru *run
+
+	th := testh.New(t)
+	src1, src2 := th.Source(sakila.CSVActor), th.Source(sakila.CSVActorNoHeader)
+
+	ru = newRun(t).add(*src1)
+	err = ru.exec("ping", "--csv", src1.Handle)
+	require.NoError(t, err)
+	checkPingOutputCSV(t, ru, *src1)
+
+	ru = newRun(t).add(*src2)
+	err = ru.exec("ping", "--csv", src2.Handle)
+	require.NoError(t, err)
+	checkPingOutputCSV(t, ru, *src2)
+
+	ru = newRun(t).add(*src1, *src2)
+	err = ru.exec("ping", "--csv", src1.Handle, src2.Handle)
+	require.NoError(t, err)
+	checkPingOutputCSV(t, ru, *src1, *src2)
+}
+
+// checkPintOutputCSV reads CSV records from h.out, and verifies
+// that there's an appropriate record for each of srcs.
+func checkPingOutputCSV(t *testing.T, h *run, srcs ...source.Source) {
+	recs, err := csv.NewReader(h.out).ReadAll()
+	require.NoError(t, err)
+	require.Equal(t, len(srcs), len(recs))
+
+	if len(srcs) > 0 {
+		require.Equal(t, 3, len(recs[0]), "each ping record should have 3 fields, but got %d fields", len(recs[0]))
+	}
+
+	handles := make(map[string]bool)
+	for _, src := range srcs {
+		handles[src.Handle] = true
+	}
+
+	for i := 0; i < len(recs); i++ {
+		recHandle := recs[i][0]
+		require.True(t, handles[recHandle], "should have handle %q in map", recHandle)
+
+		_, err := time.ParseDuration(recs[i][1])
+		require.NoError(t, err, "should be a valid duration value")
+
+		require.Empty(t, recs[i][2], "error field should be empty")
+	}
+}

cli/cmd_remove.go

@@ -0,0 +1,48 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/neilotoole/sq/libsq/errz"
+)
+
+func newSrcRemoveCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:     "rm @HANDLE",
+		Example: `  sq rm @my1`,
+		Aliases: []string{"remove"},
+		Short:   "Remove data source",
+	}
+
+	return cmd, execSrcRemove
+}
+
+func execSrcRemove(rc *RunContext, cmd *cobra.Command, args []string) error {
+	if len(args) != 1 {
+		return errz.Errorf(msgInvalidArgs)
+	}
+
+	cfg := rc.Config
+	src, err := cfg.Sources.Get(args[0])
+	if err != nil {
+		return err
+	}
+
+	err = cfg.Sources.Remove(src.Handle)
+	if err != nil {
+		return err
+	}
+
+	err = rc.ConfigStore.Save(cfg)
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintf(rc.Out, "Removed data source ")
+	_, _ = rc.wrtr.fmt.Hilite.Fprintf(rc.Out, "%s", src.Handle)
+	fmt.Fprintln(rc.Out)
+
+	return nil
+}

cli/cmd_remove_test.go

@@ -0,0 +1,34 @@
+package cli_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/neilotoole/sq/testh"
+	"github.com/neilotoole/sq/testh/sakila"
+)
+
+func TestCmdRemove(t *testing.T) {
+	th := testh.New(t)
+
+	// 1. Should fail if bad handle
+	ru := newRun(t)
+	err := ru.exec("rm", "@not_a_source")
+	require.Error(t, err)
+
+	// 2. Check normal operation
+	src := th.Source(sakila.SL3)
+	ru = newRun(t).add(*src)
+
+	// The src we just added should be the active src
+	activeSrc := ru.rc.Config.Sources.Active()
+	require.NotNil(t, activeSrc)
+	require.Equal(t, src.Handle, activeSrc.Handle)
+
+	err = ru.exec("rm", src.Handle)
+	require.NoError(t, err)
+
+	activeSrc = ru.rc.Config.Sources.Active()
+	require.Nil(t, activeSrc, "should be no active src anymore")
+}

cli/cmd_root.go

@@ -0,0 +1,89 @@
+package cli
+
+import (
+	"github.com/spf13/cobra"
+
+	// Import the providers package to initialize provider implementations
+	_ "github.com/neilotoole/sq/drivers"
+)
+
+func newRootCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   `sq QUERY`,
+		Short: "sq",
+		Long: `sq is a swiss army knife for data.
+
+Use sq to uniformly query Postgres, SQLite, SQLServer, MySQL, CSV, TSV,
+Excel (and more), and output in text, JSON, CSV, Excel, etc.
+
+You can query using sq's own jq-like syntax, or in native SQL.
+
+More at https://sq.io
+`,
+		Example: `  # pipe an Excel file and output the first 10 rows from sheet1
+  cat data.xlsx | sq '.sheet1 | .[0:10]'
+
+  # add Postgres source identified by handle @sakila_pg
+  sq add --handle=@sakila_pg 'postgres://user:pass@localhost:5432/sakila?sslmode=disable'
+
+  # add SQL Server source; will have generated handle @sakila_mssql
+  sq add 'sqlserver://user:pass@localhost?database=sakila'
+
+  # list available data sources
+  sq ls
+
+  # ping all data sources
+  sq ping all
+
+  # set active data source
+  sq src @sakila_pg
+
+  # get specified cols from table address in active data source
+  sq '.address |  .address_id, .city, .country'
+
+  # get metadata (schema, stats etc) for data source
+  sq inspect @sakila_pg
+
+  # get metadata for a table
+  sq inspect @pg1.person
+
+  # output in JSON
+  sq -j '.person | .uid, .username, .email'
+
+  # output in table format (with header)
+  sq -th '.person | .uid, .username, .email'
+
+  # output in table format (no header)
+  sq -t '.person | .uid, .username, .email'
+
+  # output to a HTML file
+  sq --html '@sakila_sl3.actor' -o actor.html
+
+  # join across data sources
+  sq '@my1.person, @pg1.address | join(.uid) | .username, .email, .city'
+
+  # insert query results into a table in another data source
+  sq --insert=@pg1.person '@my1.person | .username, .email'
+
+  # execute a database-native SQL query, specifying the source
+  sq sql --src=@pg1 'SELECT uid, username, email FROM person LIMIT 2'
+
+  # copy a table (in the same source)
+  sq tbl copy @sakila_sl3.actor .actor2
+
+  # truncate tables
+  sq tbl truncate @sakila_sl3.actor2
+
+  # drop table
+  sq tbl drop @sakila_sl3.actor2
+`,
+		BashCompletionFunction: bashCompletionFunc,
+	}
+
+	// the query cmd does the real work when the root cmd is invoked
+	addQueryCmdFlags(cmd)
+	cmd.Flags().Bool(flagVersion, false, flagVersionUsage)
+	cmd.PersistentFlags().BoolP(flagMonochrome, flagMonochromeShort, false, flagMonochromeUsage)
+
+	return cmd
+}

cli/cmd_scratch.go

@@ -0,0 +1,83 @@
+package cli
+
+import (
+	"github.com/neilotoole/sq/drivers/sqlite3"
+	"github.com/neilotoole/sq/libsq/errz"
+	"github.com/neilotoole/sq/libsq/source"
+
+	"github.com/spf13/cobra"
+)
+
+// TODO: dump all this "internal" stuff: make the options as follows: @HANDLE, file, memory
+
+func newScratchCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use: "scratch [@HANDLE|internal|internal:file|internal:mem|@scratch]",
+		Example: `   # get scratch data source
+   sq scratch
+   # set @my1 as scratch data source
+   sq scratch @my1
+   # use the default embedded db
+   sq scratch internal
+   # explicitly specify use of embedded file db
+   sq scratch internal:file
+   # explicitly specify use of embedded memory db
+   sq scratch internal:mem
+   # restore default scratch db (equivalent to "internal")
+   sq scratch @scratch`,
+		Short: "Get or set scratch data source",
+		Long: `Get or set scratch data source. The scratch db is used internally by sq for multiple purposes such as
+importing non-SQL data, or cross-database joins. If no argument provided, get the current scratch data
+source. Otherwise, set @HANDLE or an internal db as the scratch data source. The reserved handle "@scratch" resets the
+`,
+	}
+
+	return cmd, execScratch
+}
+
+func execScratch(rc *RunContext, cmd *cobra.Command, args []string) error {
+	if len(args) > 1 {
+		return errz.Errorf(msgInvalidArgs)
+	}
+
+	cfg := rc.Config
+
+	var src *source.Source
+	var err error
+	defaultScratch := &source.Source{
+		Handle:   source.ScratchHandle,
+		Location: "internal:file",
+		Type:     sqlite3.Type,
+	}
+
+	if len(args) == 0 {
+		// Print the scratch src
+		src = cfg.Sources.Scratch()
+		if src == nil {
+			src = defaultScratch
+		}
+
+		return rc.writers().srcw.Source(src)
+	}
+
+	// Set the scratch src
+
+	switch args[0] {
+	case "internal", "internal:file", "internal:mem":
+		// TODO: currently only supports file sqlite3 db, fairly trivial to do mem as well
+		_, _ = cfg.Sources.SetScratch("")
+		src = defaultScratch
+	default:
+		src, err = cfg.Sources.SetScratch(args[0])
+		if err != nil {
+			return err
+		}
+	}
+
+	err = rc.ConfigStore.Save(rc.Config)
+	if err != nil {
+		return err
+	}
+
+	return rc.writers().srcw.Source(src)
+}

cli/cmd_slq.go

@@ -0,0 +1,307 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/spf13/cobra"
+
+	"github.com/neilotoole/sq/cli/output"
+	"github.com/neilotoole/sq/libsq"
+	"github.com/neilotoole/sq/libsq/errz"
+	"github.com/neilotoole/sq/libsq/source"
+	"github.com/neilotoole/sq/libsq/stringz"
+)
+
+func newSLQCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:    "slq",
+		Short:  "",
+		Hidden: true,
+	}
+
+	addQueryCmdFlags(cmd)
+	cmd.Flags().Bool(flagVersion, false, flagVersionUsage)
+	cmd.SetUsageFunc(func(cmd *cobra.Command) error {
+		return cmd.Root().Help()
+	})
+
+	return cmd, execSLQ
+}
+
+func execSLQ(rc *RunContext, cmd *cobra.Command, args []string) error {
+	srcs := rc.Config.Sources
+
+	// check if there's input on stdin
+	src, err := checkStdinSource(rc)
+	if err != nil {
+		return err
+	}
+
+	if src != nil {
+		// We have a valid source on stdin.
+
+		// Add the source to the set.
+		err = srcs.Add(src)
+		if err != nil {
+			return err
+		}
+
+		// Set the stdin pipe data source as the active source,
+		// as it's commonly the only data source the user is acting upon.
+		_, err = srcs.SetActive(src.Handle)
+		if err != nil {
+			return err
+		}
+	} else {
+		// No source on stdin, so we're using the source set.
+		src = srcs.Active()
+		if src == nil {
+			// TODO: Should sq be modified to support executing queries
+			// 	even when there's no active data source. Probably.
+			return errz.New(msgNoActiveSrc)
+		}
+	}
+
+	if !cmdFlagChanged(cmd, flagInsert) {
+		// The user didn't specify the --insert=@src.tbl flag,
+		// so we just want to print the records.
+		return execSLQPrint(rc)
+	}
+
+	// Instead of printing the records, they will be
+	// written to another database
+	insertTo, _ := cmd.Flags().GetString(flagInsert)
+	if insertTo == "" {
+		return errz.Errorf("invalid --%s value: empty", flagInsert)
+	}
+
+	destHandle, destTbl, err := source.ParseTableHandle(insertTo)
+	if err != nil {
+		return errz.Wrapf(err, "invalid --%s value", flagInsert)
+	}
+
+	destSrc, err := srcs.Get(destHandle)
+	if err != nil {
+		return err
+	}
+
+	return execSLQInsert(rc, destSrc, destTbl)
+}
+
+// execSQLInsert executes the SQL and inserts resulting records
+// into destTbl in destSrc.
+func execSLQInsert(rc *RunContext, destSrc *source.Source, destTbl string) error {
+	args, srcs, dbases := rc.Args, rc.Config.Sources, rc.databases()
+	slq, err := preprocessUserSLQ(rc, args)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancelFn := context.WithCancel(rc.Context)
+	defer cancelFn()
+
+	destDB, err := dbases.Open(ctx, destSrc)
+	if err != nil {
+		return err
+	}
+
+	// Note: We don't need to worry about closing fromConn and
+	// destConn because they are closed by databases.Close, which
+	// is invoked by rc.Close, and rc is closed further up the
+	// stack.
+
+	inserter := libsq.NewDBWriter(rc.Log, destDB, destTbl, libsq.DefaultRecordChSize)
+	err = libsq.ExecuteSLQ(ctx, rc.Log, rc.dbases, rc.dbases, srcs, slq, inserter)
+	if err != nil {
+		return errz.Wrapf(err, "insert %s.%s failed", destSrc.Handle, destTbl)
+	}
+
+	affected, err := inserter.Wait() // Wait for the writer to finish processing
+	if err != nil {
+		return errz.Wrapf(err, "insert %s.%s failed", destSrc.Handle, destTbl)
+	}
+
+	fmt.Fprintf(rc.Out, stringz.Plu("Inserted %d row(s) into %s.%s\n", int(affected)), affected, destSrc.Handle, destTbl)
+	return nil
+}
+
+func execSLQPrint(rc *RunContext) error {
+	slq, err := preprocessUserSLQ(rc, rc.Args)
+	if err != nil {
+		return err
+	}
+
+	recw := output.NewRecordWriterAdapter(rc.writers().recordw)
+	err = libsq.ExecuteSLQ(rc.Context, rc.Log, rc.dbases, rc.dbases, rc.Config.Sources, slq, recw)
+	if err != nil {
+		return err
+	}
+
+	_, err = recw.Wait()
+	if err != nil {
+		return err
+	}
+
+	return err
+}
+
+// preprocessUserSLQ does a bit of validation and munging on the
+// SLQ input (provided in args), returning the SLQ query. This
+// function is something of a hangover from the early days of
+// seek and may need to be rethought.
+//
+// 1. If there's piped input but no query args, the first table
+// from the pipe source becomes the query. Invoked like this:
+//
+//  $ cat something.csv | sq
+//
+// The query effectively becomes:
+//
+//  $ cat something.csv | sq @stdin.data
+//
+// For non-monotable sources, the first table is used:
+//
+//  $ cat something.xlsx | sq @stdin.sheet1
+//
+// 2. If the query doesn't contain a source selector segment
+// starting with @HANDLE, the active src handle is prepended
+// to the query. This allows a query where the first selector
+// segment is the table name.
+//
+//  $ sq '.person'  -->  sq '@active.person'
+func preprocessUserSLQ(rc *RunContext, args []string) (string, error) {
+	log, reg, dbases, srcs := rc.Log, rc.registry(), rc.databases(), rc.Config.Sources
+	activeSrc := srcs.Active()
+
+	if len(args) == 0 {
+		// Special handling for the case where no args are supplied
+		// but sq is receiving pipe input. Let's say the user does this:
+		//
+		//  $ cat something.csv | sq  # query becomes ".stdin.data"
+		if activeSrc == nil {
+			// Piped input would result in an active @stdin src. We don't
+			// have that; we don't have any active src.
+			return "", errz.New(msgEmptyQueryString)
+		}
+
+		if activeSrc.Handle != source.StdinHandle {
+			// It's not piped input.
+			return "", errz.New(msgEmptyQueryString)
+		}
+
+		// We know for sure that we've got pipe input
+		drvr, err := reg.DriverFor(activeSrc.Type)
+		if err != nil {
+			return "", err
+		}
+
+		tblName := source.MonotableName
+
+		if !drvr.DriverMetadata().Monotable {
+			// This isn't a monotable src, so we can't
+			// just select @stdin.data. Instead we'll select
+			// the first table name, as found in the source meta.
+			dbase, err := dbases.Open(rc.Context, activeSrc)
+			if err != nil {
+				return "", err
+			}
+			defer log.WarnIfCloseError(dbase)
+
+			srcMeta, err := dbase.SourceMetadata(rc.Context)
+			if err != nil {
+				return "", err
+			}
+
+			if len(srcMeta.Tables) == 0 {
+				return "", errz.New(msgSrcNoData)
+			}
+
+			tblName = srcMeta.Tables[0].Name
+			if tblName == "" {
+				return "", errz.New(msgSrcEmptyTableName)
+			}
+
+			log.Debug("Using first table name from document source metadata as table selector: ", tblName)
+		}
+
+		selector := source.StdinHandle + "." + tblName
+		log.Debug("Added selector to argument-less piped query: ", selector)
+
+		return selector, nil
+	}
+
+	// We have at least one query arg
+	for i, arg := range args {
+		args[i] = strings.TrimSpace(arg)
+	}
+
+	start := strings.TrimSpace(args[0])
+	parts := strings.Split(start, " ")
+
+	if parts[0][0] == '@' {
+		// The query starts with a handle, e.g. sq '@my | .person'.
+		// Let's perform some basic checks on it.
+
+		// We split on . because both @my1.person and @my1 need to be checked.
+		dsParts := strings.Split(parts[0], ".")
+
+		handle := dsParts[0]
+		if len(handle) < 2 {
+			// handle name is too short
+			return "", errz.Errorf("invalid data source: %q", handle)
+		}
+
+		// Check that the handle actual exists
+		_, err := srcs.Get(handle)
+		if err != nil {
+			return "", err
+		}
+
+		// All is good, return the query.
+		query := strings.Join(args, " ")
+		return query, nil
+	}
+
+	// The query doesn't start with a handle selector; let's prepend
+	// a handle selector segment.
+	if activeSrc == nil {
+		return "", errz.New("no data source provided, and no active data source")
+	}
+
+	query := strings.Join(args, " ")
+	query = fmt.Sprintf("%s | %s", activeSrc.Handle, query)
+
+	log.Debug("The query didn't start with @handle, so the active src was prepended: ", query)
+
+	return query, nil
+}
+
+// addQueryCmdFlags sets flags for the slq/sql commands.
+func addQueryCmdFlags(cmd *cobra.Command) {
+	cmd.Flags().StringP(flagOutput, flagOutputShort, "", flagOutputUsage)
+
+	cmd.Flags().BoolP(flagJSON, flagJSONShort, false, flagJSONUsage)
+	cmd.Flags().BoolP(flagJSONA, flagJSONAShort, false, flagJSONAUsage)
+	cmd.Flags().BoolP(flagJSONL, flagJSONLShort, false, flagJSONLUsage)
+	cmd.Flags().BoolP(flagTable, flagTableShort, false, flagTableUsage)
+	cmd.Flags().BoolP(flagXML, flagXMLShort, false, flagXMLUsage)
+	cmd.Flags().BoolP(flagXLSX, flagXLSXShort, false, flagXLSXUsage)
+	cmd.Flags().BoolP(flagCSV, flagCSVShort, false, flagCSVUsage)
+	cmd.Flags().BoolP(flagTSV, flagTSVShort, false, flagTSVUsage)
+	cmd.Flags().BoolP(flagRaw, flagRawShort, false, flagRawUsage)
+	cmd.Flags().Bool(flagHTML, false, flagHTMLUsage)
+	cmd.Flags().Bool(flagMarkdown, false, flagMarkdownUsage)
+
+	cmd.Flags().BoolP(flagHeader, flagHeaderShort, false, flagHeaderUsage)
+	cmd.Flags().BoolP(flagNoHeader, flagNoHeaderShort, false, flagNoHeaderUsage)
+	cmd.Flags().BoolP(flagPretty, "", true, flagPrettyUsage)
+
+	cmd.Flags().StringP(flagInsert, "", "", flagInsertUsage)
+	cmd.Flags().StringP(flagActiveSrc, "", "", flagActiveSrcUsage)
+
+	// The driver flag can be used if data is piped to sq over stdin
+	cmd.Flags().StringP(flagDriver, "", "", flagQueryDriverUsage)
+	cmd.Flags().StringP(flagSrcOptions, "", "", flagQuerySrcOptionsUsage)
+}

cli/cmd_slq_test.go

@@ -0,0 +1,174 @@
+package cli_test
+
+import (
+	"encoding/csv"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/neilotoole/sq/libsq/source"
+	"github.com/neilotoole/sq/libsq/stringz"
+	"github.com/neilotoole/sq/testh"
+	"github.com/neilotoole/sq/testh/sakila"
+)
+
+func TestCmdSLQ_Insert_LONG(t *testing.T) {
+	testh.SkipShort(t, true)
+	testCmdSLQ_Insert(t, sakila.All, sakila.SQLAll)
+}
+
+func TestCmdSLQ_Insert(t *testing.T) {
+	testCmdSLQ_Insert(t, sakila.SQLLatest, sakila.SQLLatest)
+}
+
+// testCmdSLQ_Insert tests "sq slq QUERY --insert=dest.tbl".
+func testCmdSLQ_Insert(t *testing.T, origins, dests []string) {
+	for _, origin := range origins {
+		origin := origin
+
+		t.Run("origin_"+origin, func(t *testing.T) {
+			testh.SkipShort(t, origin == sakila.XLSX)
+
+			for _, dest := range dests {
+				dest := dest
+
+				t.Run("dest_"+dest, func(t *testing.T) {
+					t.Parallel()
+
+					th := testh.New(t)
+					originSrc, destSrc := th.Source(origin), th.Source(dest)
+					srcTbl := sakila.TblActor
+					if th.IsMonotable(originSrc) {
+						srcTbl = source.MonotableName
+					}
+
+					// To avoid dirtying the destination table, we make a copy
+					// of it (without data).
+					actualDestTbl := th.CopyTable(false, destSrc, sakila.TblActor, "", false)
+					t.Cleanup(func() { th.DropTable(destSrc, actualDestTbl) })
+
+					ru := newRun(t).add(*originSrc)
+					if destSrc.Handle != originSrc.Handle {
+						ru.add(*destSrc)
+					}
+
+					insertTo := fmt.Sprintf("%s.%s", destSrc.Handle, actualDestTbl)
+					cols := stringz.PrefixSlice(sakila.TblActorCols, ".")
+					query := fmt.Sprintf("%s.%s | %s", originSrc.Handle, srcTbl, strings.Join(cols, ", "))
+
+					err := ru.exec("slq", "--insert="+insertTo, query)
+					require.NoError(t, err)
+
+					sink, err := th.QuerySQL(destSrc, "select * from "+actualDestTbl)
+					require.NoError(t, err)
+					require.Equal(t, sakila.TblActorCount, len(sink.Recs))
+				})
+			}
+		})
+	}
+}
+
+func TestCmdSLQ_CSV(t *testing.T) {
+	t.Parallel()
+
+	src := testh.New(t).Source(sakila.CSVActor)
+	ru := newRun(t).add(*src)
+	err := ru.exec("slq", "--no-header", "--csv", fmt.Sprintf("%s.data", src.Handle))
+	require.NoError(t, err)
+
+	recs := ru.mustReadCSV()
+	require.Equal(t, sakila.TblActorCount, len(recs))
+}
+
+// TestCmdSLQ_OutputFlag verifies that flag --output=<file> works.
+func TestCmdSLQ_OutputFlag(t *testing.T) {
+	t.Parallel()
+
+	src := testh.New(t).Source(sakila.SL3)
+	ru := newRun(t).add(*src)
+	outputFile, err := ioutil.TempFile("", t.Name())
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		assert.NoError(t, outputFile.Close())
+		assert.NoError(t, os.Remove(outputFile.Name()))
+	})
+
+	err = ru.exec("slq",
+		"--no-header", "--csv", fmt.Sprintf("%s.%s", src.Handle, sakila.TblActor),
+		"--output", outputFile.Name())
+	require.NoError(t, err)
+
+	recs, err := csv.NewReader(outputFile).ReadAll()
+	require.NoError(t, err)
+	require.Equal(t, sakila.TblActorCount, len(recs))
+}
+
+func TestCmdSLQ_Join(t *testing.T) {
+	const queryTpl = `%s.customer, %s.address | join(.address_id) | .customer_id == %d | .[0] | .customer_id, .email, .city_id`
+	handles := sakila.SQLAll
+
+	// Attempt to join every SQL test source against every SQL test source.
+	for _, h1 := range handles {
+		h1 := h1
+
+		t.Run("origin_"+h1, func(t *testing.T) {
+			for _, h2 := range handles {
+				h2 := h2
+
+				t.Run("dest_"+h2, func(t *testing.T) {
+					t.Parallel()
+
+					th := testh.New(t)
+					src1, src2 := th.Source(h1), th.Source(h2)
+
+					ru := newRun(t).add(*src1)
+					if src2.Handle != src1.Handle {
+						ru.add(*src2)
+					}
+
+					query := fmt.Sprintf(queryTpl, src1.Handle, src2.Handle, sakila.MillerCustID)
+
+					err := ru.exec("slq", "--no-header", "--csv", query)
+					require.NoError(t, err)
+
+					recs := ru.mustReadCSV()
+					require.Equal(t, 1, len(recs), "should only be one matching record")
+					require.Equal(t, 3, len(recs[0]), "should have three fields")
+					require.Equal(t, strconv.Itoa(sakila.MillerCustID), recs[0][0])
+					require.Equal(t, sakila.MillerEmail, recs[0][1])
+					require.Equal(t, strconv.Itoa(sakila.MillerCityID), recs[0][2])
+				})
+			}
+		})
+	}
+}
+
+// TestCmdSLQ_ActiveSrcHandle verifies that source.ActiveHandle is
+// interpreted as the active src in a SLQ query.
+func TestCmdSLQ_ActiveSrcHandle(t *testing.T) {
+	src := testh.New(t).Source(sakila.SL3)
+
+	// 1. Verify that the query works as expected using the actual src handle
+	ru := newRun(t).add(*src).hush()
+
+	require.Equal(t, src.Handle, ru.rc.Config.Sources.Active().Handle)
+	err := ru.exec("slq", "--no-header", "--csv", "@sakila_sl3.actor")
+	require.NoError(t, err)
+	recs := ru.mustReadCSV()
+	require.Equal(t, sakila.TblActorCount, len(recs))
+
+	// 2. Verify that it works using source.ActiveHandle as the src handle
+	ru = newRun(t).add(*src).hush()
+	require.Equal(t, src.Handle, ru.rc.Config.Sources.Active().Handle)
+	err = ru.exec("slq", "--no-header", "--csv", source.ActiveHandle+".actor")
+	require.NoError(t, err)
+	recs = ru.mustReadCSV()
+	require.Equal(t, sakila.TblActorCount, len(recs))
+}

cli/cmd_sql.go

@@ -0,0 +1,158 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/neilotoole/sq/libsq"
+	"github.com/neilotoole/sq/libsq/errz"
+	"github.com/neilotoole/sq/libsq/source"
+
+	"github.com/spf13/cobra"
+
+	"github.com/neilotoole/sq/cli/output"
+	"github.com/neilotoole/sq/libsq/stringz"
+)
+
+func newSQLCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:   "sql QUERY|STMT",
+		Short: "Execute DB-native SQL query or statement",
+		Long: `Execute a SQL query or statement using the source's SQL dialect
+against the active source. Use the --src=@HANDLE to specify
+an alternative source.
+
+If flag --query is set, sq will run the input as a query
+(SELECT) and return the query rows. If the --exec flag is set,
+sq will execute the input and return the result. If neither
+flag is set, sq determines the appropriate mode.`,
+		Example: `  # Select from active source
+  sq sql 'SELECT * FROM actor'
+
+  # Select from a specified source
+  sq sql --src=@sakila_pg12 'SELECT * FROM actor'
+
+  # Drop table @sakila_pg12.actor
+  sq sql --exec --src=@sakila_pg12 'DROP TABLE actor'
+
+  # Select from active source and write results to @pg1.actor
+  sq sql 'SELECT * FROM actor' --insert=@sakila_pg12.actor`,
+	}
+
+	addQueryCmdFlags(cmd)
+
+	// User explicitly wants to execute the SQL using sql.DB.Query
+	cmd.Flags().Bool(flagSQLQuery, false, flagSQLQueryUsage)
+	// User explicitly wants to execute the SQL using sql.DB.Exec
+	cmd.Flags().Bool(flagSQLExec, false, flagSQLExecUsage)
+
+	return cmd, execSQL
+}
+
+func execSQL(rc *RunContext, cmd *cobra.Command, args []string) error {
+	switch len(args) {
+	default:
+		// FIXME: we should allow multiple args and concat them
+		return errz.New("a single query string is required")
+	case 0:
+		return errz.New("empty SQL query string")
+	case 1:
+		if strings.TrimSpace(args[0]) == "" {
+			return errz.New("empty SQL query string")
+		}
+	}
+
+	err := determineSources(rc)
+	if err != nil {
+		return err
+	}
+
+	srcs := rc.Config.Sources
+	// activeSrc is guaranteed to be non-nil after
+	// determineSources successfully returns.
+	activeSrc := srcs.Active()
+
+	if !cmdFlagChanged(cmd, flagInsert) {
+		// The user didn't specify the --insert=@src.tbl flag,
+		// so we just want to print the records.
+		return execSQLPrint(rc, activeSrc)
+	}
+
+	// Instead of printing the records, they will be
+	// written to another database
+	insertTo, _ := cmd.Flags().GetString(flagInsert)
+	if insertTo == "" {
+		return errz.Errorf("invalid --%s value: empty", flagInsert)
+	}
+
+	destHandle, destTbl, err := source.ParseTableHandle(insertTo)
+	if err != nil {
+		return errz.Wrapf(err, "invalid --%s value", flagInsert)
+	}
+
+	destSrc, err := srcs.Get(destHandle)
+	if err != nil {
+		return err
+	}
+
+	return execSQLInsert(rc, activeSrc, destSrc, destTbl)
+}
+
+// execSQLPrint executes the SQL and prints resulting records
+// to the configured writer.
+func execSQLPrint(rc *RunContext, fromSrc *source.Source) error {
+	args := rc.Args
+	dbase, err := rc.databases().Open(rc.Context, fromSrc)
+	if err != nil {
+		return err
+	}
+
+	recw := output.NewRecordWriterAdapter(rc.writers().recordw)
+	err = libsq.QuerySQL(rc.Context, rc.Log, dbase, recw, args[0])
+	if err != nil {
+		return err
+	}
+	_, err = recw.Wait() // Wait for the writer to finish processing
+	return err
+}
+
+// execSQLInsert executes the SQL and inserts resulting records
+// into destTbl in destSrc.
+func execSQLInsert(rc *RunContext, fromSrc, destSrc *source.Source, destTbl string) error {
+	args := rc.Args
+	dbases := rc.databases()
+	ctx, cancelFn := context.WithCancel(rc.Context)
+	defer cancelFn()
+
+	fromDB, err := dbases.Open(ctx, fromSrc)
+	if err != nil {
+		return err
+	}
+
+	destDB, err := dbases.Open(ctx, destSrc)
+	if err != nil {
+		return err
+	}
+
+	// Note: We don't need to worry about closing fromDB and
+	// destDB because they are closed by dbases.Close, which
+	// is invoked by rc.Close, and rc is closed further up the
+	// stack.
+
+	inserter := libsq.NewDBWriter(rc.Log, destDB, destTbl, libsq.DefaultRecordChSize)
+	err = libsq.QuerySQL(ctx, rc.Log, fromDB, inserter, args[0])
+	if err != nil {
+		return errz.Wrapf(err, "insert %s.%s failed", destSrc.Handle, destTbl)
+	}
+
+	affected, err := inserter.Wait() // Wait for the writer to finish processing
+	if err != nil {
+		return errz.Wrapf(err, "insert %s.%s failed", destSrc.Handle, destTbl)
+	}
+
+	rc.Log.Debugf("Rows affected: %d", affected)
+
+	fmt.Fprintf(rc.Out, stringz.Plu("Inserted %d row(s) into %s.%s\n", int(affected)), affected, destSrc.Handle, destTbl)
+	return nil
+}

cli/cmd_sql_test.go

@@ -0,0 +1,156 @@
+package cli_test
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/neilotoole/sq/drivers/userdriver"
+	"github.com/neilotoole/sq/libsq/source"
+	"github.com/neilotoole/sq/testh"
+	"github.com/neilotoole/sq/testh/proj"
+	"github.com/neilotoole/sq/testh/sakila"
+	"github.com/neilotoole/sq/testh/testsrc"
+)
+
+func TestCmdSQL_Insert_LONG(t *testing.T) {
+	testh.SkipShort(t, true)
+	testCmdSQL_Insert(t, sakila.All, sakila.All)
+}
+func TestCmdSQL_Insert(t *testing.T) {
+	testCmdSQL_Insert(t, sakila.SQLLatest, sakila.SQLLatest)
+}
+
+// testCmdSQL_Insert tests "sq sql QUERY --insert=dest.tbl".
+func testCmdSQL_Insert(t *testing.T, origins, dests []string) {
+	for _, origin := range origins {
+		origin := origin
+
+		t.Run("origin_"+origin, func(t *testing.T) {
+			testh.SkipShort(t, origin == sakila.XLSX)
+
+			for _, dest := range dests {
+				dest := dest
+
+				t.Run("dest_"+dest, func(t *testing.T) {
+					t.Parallel()
+
+					th := testh.New(t)
+					originSrc, destSrc := th.Source(origin), th.Source(dest)
+					originTbl := sakila.TblActor
+
+					if th.IsMonotable(originSrc) {
+						originTbl = source.MonotableName
+					}
+
+					// To avoid dirtying the destination table, we make a copy
+					// of it (without data).
+					actualDestTbl := th.CopyTable(false, destSrc, sakila.TblActor, "", false)
+					t.Cleanup(func() { th.DropTable(destSrc, actualDestTbl) })
+
+					ru := newRun(t).add(*originSrc)
+					if destSrc.Handle != originSrc.Handle {
+						ru.add(*destSrc)
+					}
+
+					insertTo := fmt.Sprintf("%s.%s", destSrc.Handle, actualDestTbl)
+					query := fmt.Sprintf("SELECT %s FROM %s", strings.Join(sakila.TblActorCols, ", "), originTbl)
+
+					err := ru.exec("sql", "--insert="+insertTo, query)
+					require.NoError(t, err)
+
+					sink, err := th.QuerySQL(destSrc, "select * from "+actualDestTbl)
+					require.NoError(t, err)
+					require.Equal(t, sakila.TblActorCount, len(sink.Recs))
+				})
+			}
+		})
+	}
+}
+
+func TestCmdSQL_SelectFromUserDriver(t *testing.T) {
+	testCases := map[string][]struct {
+		tblName  string
+		wantRows int
+		wantCols int
+	}{
+		testsrc.PplUD: {
+			{tblName: "person", wantRows: 3, wantCols: 7},
+			{tblName: "skill", wantRows: 6, wantCols: 3},
+		},
+		testsrc.RSSNYTLocalUD: {
+			{tblName: "category", wantRows: 251, wantCols: 4},
+			{tblName: "channel", wantRows: 1, wantCols: 7},
+			{tblName: "item", wantRows: 45, wantCols: 9},
+		},
+	}
+
+	for handle, wantTbls := range testCases {
+		for _, wantTbl := range wantTbls {
+			handle, wantTbl := handle, wantTbl
+			t.Run(handle+"__"+wantTbl.tblName, func(t *testing.T) {
+				t.Parallel()
+
+				th := testh.New(t)
+				src := th.Source(handle)
+
+				ru := newRun(t).add(*src)
+
+				udDefs := testh.DriverDefsFrom(t, testsrc.PathDriverDefPpl, testsrc.PathDriverDefRSS)
+				for _, udDef := range udDefs {
+					require.Empty(t, userdriver.ValidateDriverDef(udDef))
+				}
+				ru.rc.Config.Ext.UserDrivers = append(ru.rc.Config.Ext.UserDrivers, udDefs...)
+
+				err := ru.exec("sql", "--csv", "--no-header", "SELECT * FROM "+wantTbl.tblName)
+				require.NoError(t, err)
+				recs := ru.mustReadCSV()
+				require.Equal(t, wantTbl.wantRows, len(recs), "expected %d rows in tbl %q but got %s", wantTbl.wantRows, wantTbl, len(recs))
+				require.Equal(t, wantTbl.wantCols, len(recs[0]), "expected %d cols in tbl %q but got %s", wantTbl.wantCols, wantTbl, len(recs[0]))
+			})
+		}
+	}
+}
+
+// TestCmdSQL_StdinQuery verifies that cmd sql can read from stdin.
+func TestCmdSQL_StdinQuery(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		fpath     string
+		tbl       string
+		wantCount int
+		wantErr   bool
+	}{
+		{fpath: proj.Abs(sakila.PathCSVActor), tbl: source.MonotableName, wantCount: sakila.TblActorCount + 1}, // +1 is for the header row
+		{fpath: proj.Abs(sakila.PathXLSXSubset), tbl: sakila.TblActor, wantCount: sakila.TblActorCount + 1},
+		{fpath: proj.Abs("README.md"), wantErr: true},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(testh.TName(tc.fpath), func(t *testing.T) {
+			t.Parallel()
+
+			f, err := os.Open(tc.fpath)
+			require.NoError(t, err)
+
+			ru := newRun(t).hush()
+			ru.rc.Stdin = f
+
+			err = ru.exec("sql", "SELECT * FROM "+tc.tbl)
+			if tc.wantErr {
+				require.Error(t, err)
+				return
+			}
+
+			require.NoError(t, err)
+			results := ru.mustReadCSV()
+			require.Equal(t, tc.wantCount, len(results))
+		})
+	}
+}

cli/cmd_src.go

@@ -0,0 +1,59 @@
+package cli
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/neilotoole/sq/libsq/errz"
+)
+
+func newSrcCommand() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use: "src [@HANDLE]",
+		Example: `  # get active data source
+   sq src
+   # set @my1 as active data source
+   sq src @my1`,
+		// RunE:    execSrc,
+		Short:   "Get or set active data source",
+		Aliases: []string{"using"},
+		Long: `Get or set active data source. If no argument provided, get the active data
+source. Otherwise, set @HANDLE as the active data source.`,
+	}
+
+	cmd.Flags().BoolP(flagJSON, flagJSONShort, false, flagJSONUsage)
+	cmd.Flags().BoolP(flagTable, flagTableShort, false, flagTableUsage)
+	cmd.Flags().BoolP(flagHeader, flagHeaderShort, false, flagHeaderUsage)
+	cmd.Flags().BoolP(flagNoHeader, flagNoHeaderShort, false, flagNoHeaderUsage)
+
+	return cmd, execSrc
+}
+
+func execSrc(rc *RunContext, cmd *cobra.Command, args []string) error {
+	if len(args) > 1 {
+		return errz.Errorf(msgInvalidArgs)
+	}
+
+	cfg := rc.Config
+
+	if len(args) == 0 {
+		// Get the active data source
+		src := cfg.Sources.Active()
+		if src == nil {
+			return nil
+		}
+
+		return rc.writers().srcw.Source(src)
+	}
+
+	src, err := cfg.Sources.SetActive(args[0])
+	if err != nil {
+		return err
+	}
+
+	err = rc.ConfigStore.Save(cfg)
+	if err != nil {
+		return err
+	}
+
+	return rc.writers().srcw.Source(src)
+}

cli/cmd_tbl.go

@@ -0,0 +1,298 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/neilotoole/sq/libsq/driver"
+	"github.com/neilotoole/sq/libsq/errz"
+	"github.com/neilotoole/sq/libsq/source"
+	"github.com/neilotoole/sq/libsq/stringz"
+)
+
+func newTblCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:   "tbl",
+		Short: "Common actions on tables (copy, truncate, drop)",
+
+		Example: `  # Copy table actor to new table actor2
+  sq tbl copy @sakila_sl3.actor actor2
+
+  # Truncate table actor2
+  sq tbl truncate @sakila_sl3.actor2
+
+  # Drop table actor2
+  sq tbl drop @sakila_sl3.actor2`,
+	}
+
+	return cmd, func(rc *RunContext, cmd *cobra.Command, args []string) error {
+		return cmd.Help()
+	}
+}
+
+func newTblCopyCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:   "copy @HANDLE.TABLE NEWTABLE",
+		Short: "Make a copy of a table",
+		Long:  `Make a copy of a table in the same database. The table data is also copied by default.`,
+		Example: `  # Copy table "actor" in @sakila_sl3" to new table "actor2"
+  sq tbl copy @sakila_sl3.actor .actor2
+
+  # Copy table "actor" in active src to table "actor2"
+  sq tbl copy .actor .actor2
+
+  # Copy table "actor" in active src to generated table name (e.g. "@sakila_sl3.actor_copy__1ae03e9b")
+  sq tbl copy .actor
+
+  # Copy table structure, but don't copy table data
+  sq tbl copy --data=false .actor
+`,
+	}
+
+	cmd.Flags().BoolP(flagJSON, flagJSONShort, false, flagJSONUsage)
+	cmd.Flags().Bool(flagTblData, true, flagTblDataUsage)
+
+	return cmd, execTblCopy
+}
+
+func execTblCopy(rc *RunContext, cmd *cobra.Command, args []string) error {
+	if len(args) == 0 || len(args) > 2 {
+		return errz.New("one or two table args required")
+	}
+
+	tblHandles, err := parseTableHandleArgs(rc.reg, rc.Config.Sources, args)
+	if err != nil {
+		return err
+	}
+
+	if tblHandles[0].tbl == "" {
+		return errz.Errorf("arg %q does not specify a table name")
+	}
+
+	switch len(tblHandles) {
+	case 1:
+		// Make a copy of the first tbl handle
+		tblHandles = append(tblHandles, tblHandles[0])
+		// But we can't copy the table to itself, so we create a new name
+		tblHandles[1].tbl = stringz.UniqTableName(tblHandles[0].tbl + "_copy")
+	case 2:
+		if tblHandles[1].tbl == "" {
+			tblHandles[1].tbl = stringz.UniqTableName(tblHandles[0].tbl + "_copy")
+		}
+	default:
+		return errz.New("one or two table args required")
+	}
+
+	if tblHandles[0].src.Handle != tblHandles[1].src.Handle {
+		return errz.Errorf("tbl copy only works on the same source, but got %s.%s --> %s.%s",
+			tblHandles[0].handle, tblHandles[0].tbl,
+			tblHandles[1].handle, tblHandles[1].tbl)
+	}
+
+	if tblHandles[0].tbl == tblHandles[1].tbl {
+		return errz.Errorf("cannot copy table %s.%s to itself", tblHandles[0].handle, tblHandles[0].tbl)
+	}
+
+	sqlDrvr, ok := tblHandles[0].drvr.(driver.SQLDriver)
+	if !ok {
+		return errz.Errorf("source type %q (%s) doesn't support dropping tables", tblHandles[0].src.Type, tblHandles[0].src.Handle)
+	}
+
+	copyData := true // copy data by default
+	if cmdFlagChanged(cmd, flagTblData) {
+		copyData, err = cmd.Flags().GetBool(flagTblData)
+		if err != nil {
+			return errz.Err(err)
+		}
+	}
+
+	var dbase driver.Database
+	dbase, err = rc.databases().Open(rc.Context, tblHandles[0].src)
+	if err != nil {
+		return err
+	}
+
+	copied, err := sqlDrvr.CopyTable(rc.Context, dbase.DB(), tblHandles[0].tbl, tblHandles[1].tbl, copyData)
+	if err != nil {
+		return errz.Wrapf(err, "failed tbl copy %s.%s --> %s.%s",
+			tblHandles[0].handle, tblHandles[0].tbl,
+			tblHandles[1].handle, tblHandles[1].tbl)
+	}
+
+	msg := fmt.Sprintf("Copied table: %s.%s --> %s.%s",
+		tblHandles[0].handle, tblHandles[0].tbl,
+		tblHandles[1].handle, tblHandles[1].tbl)
+
+	if copyData {
+		switch copied {
+		case 1:
+			msg += " (1 row copied)"
+		default:
+			msg += fmt.Sprintf(" (%d rows copied)", copied)
+		}
+	}
+
+	fmt.Fprintln(rc.Out, msg)
+	return nil
+}
+
+func newTblTruncateCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:   "truncate @HANDLE.TABLE",
+		Short: "Truncate one or more tables",
+		Example: `  # truncate table "actor"" in source @sakila_sl3
+  sq tbl truncate @sakila_sl3.actor
+
+  # truncate table "payment"" in the active src
+  sq tbl truncate .payment
+
+  # truncate multiple tables
+  sq tbl truncate .payment @sakila_sl3.actor
+`,
+	}
+
+	cmd.Flags().BoolP(flagJSON, flagJSONShort, false, flagJSONUsage)
+	cmd.Flags().BoolP(flagTable, flagTableShort, false, flagTableUsage)
+
+	return cmd, execTblTruncate
+}
+
+func execTblTruncate(rc *RunContext, cmd *cobra.Command, args []string) (err error) {
+	var tblHandles []tblHandle
+	tblHandles, err = parseTableHandleArgs(rc.reg, rc.Config.Sources, args)
+	if err != nil {
+		return err
+	}
+
+	for _, tblH := range tblHandles {
+		var affected int64
+		affected, err = tblH.drvr.Truncate(rc.Context, tblH.src, tblH.tbl, true)
+		if err != nil {
+			return err
+		}
+
+		msg := fmt.Sprintf("Truncated %d row(s) from %s.%s", affected, tblH.src.Handle, tblH.tbl)
+		msg = stringz.Plu(msg, int(affected))
+		fmt.Fprintln(rc.Out, msg)
+	}
+
+	return nil
+}
+
+func newTblDropCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:   "drop @HANDLE.TABLE",
+		Short: "Drop one or more tables",
+		Example: `# drop table "actor" in src @sakila_sl3
+  sq tbl drop @sakila_sl3.actor
+
+  # drop table "payment"" in the active src
+  sq tbl drop .payment
+
+  # drop multiple tables
+  sq drop .payment @sakila_sl3.actor
+`,
+	}
+
+	return cmd, execTblDrop
+}
+
+func execTblDrop(rc *RunContext, cmd *cobra.Command, args []string) (err error) {
+	var tblHandles []tblHandle
+	tblHandles, err = parseTableHandleArgs(rc.reg, rc.Config.Sources, args)
+	if err != nil {
+		return err
+	}
+
+	for _, tblH := range tblHandles {
+		sqlDrvr, ok := tblH.drvr.(driver.SQLDriver)
+		if !ok {
+			return errz.Errorf("source type %q (%s) doesn't support dropping tables", tblH.src.Type, tblH.src.Handle)
+		}
+
+		var dbase driver.Database
+		dbase, err = rc.databases().Open(rc.Context, tblH.src)
+		if err != nil {
+			return err
+		}
+		err = sqlDrvr.DropTable(rc.Context, dbase.DB(), tblH.tbl, false)
+		if err != nil {
+			return err
+		}
+
+		fmt.Fprintf(rc.Out, "Dropped table %s.%s\n", tblH.src.Handle, tblH.tbl)
+	}
+
+	return nil
+}
+
+// parseTableHandleArgs parses args of the form:
+//
+//  @HANDLE1.TABLE1 .TABLE2 .TABLE3 @HANDLE2.TABLE4 .TABLEN
+//
+// It returns a slice of tblHandle, one for each arg. If an arg
+// does not have a HANDLE, the active src is assumed: it's an error
+// if no active src. It is also an error if len(args) is zero.
+func parseTableHandleArgs(dp driver.Provider, srcs *source.Set, args []string) ([]tblHandle, error) {
+	if len(args) == 0 {
+		return nil, errz.New(msgInvalidArgs)
+	}
+
+	var tblHandles []tblHandle
+	activeSrc := srcs.Active()
+
+	// We iterate over the args several times, because we want
+	// to present error checks consistently.
+	for _, arg := range args {
+		handle, tbl, err := source.ParseTableHandle(arg)
+		if err != nil {
+			return nil, err
+		}
+
+		tblHandles = append(tblHandles, tblHandle{
+			handle: handle,
+			tbl:    tbl,
+		})
+	}
+
+	for i := range tblHandles {
+		if tblHandles[i].tbl == "" {
+			return nil, errz.Errorf("arg[%d] %q doesn't specify a table", i, args[i])
+		}
+
+		if tblHandles[i].handle == "" {
+			// It's a table name without a handle, so we use the active src
+			if activeSrc == nil {
+				return nil, errz.Errorf("arg[%d] %q doesn't specify a handle and there's no active source",
+					i, args[i])
+			}
+
+			tblHandles[i].handle = activeSrc.Handle
+		}
+
+		src, err := srcs.Get(tblHandles[i].handle)
+		if err != nil {
+			return nil, err
+		}
+
+		drvr, err := dp.DriverFor(src.Type)
+		if err != nil {
+			return nil, err
+		}
+
+		tblHandles[i].src = src
+		tblHandles[i].drvr = drvr
+	}
+
+	return tblHandles, nil
+}
+
+// tblHandle represents a @HANDLE.TABLE, with the handle's associated
+// src and driver.
+type tblHandle struct {
+	handle string
+	tbl    string
+	src    *source.Source
+	drvr   driver.Driver
+}

cli/cmd_tbl_test.go

@@ -0,0 +1,96 @@
+package cli_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/neilotoole/sq/libsq/stringz"
+	"github.com/neilotoole/sq/testh"
+	"github.com/neilotoole/sq/testh/sakila"
+)
+
+func TestCmdTblCopy(t *testing.T) {
+	t.Parallel()
+
+	for _, handle := range sakila.SQLAll {
+		handle := handle
+
+		t.Run(handle, func(t *testing.T) {
+			t.Parallel()
+
+			th := testh.New(t)
+			src := th.Source(handle)
+			srcTblHandle := src.Handle + "." + sakila.TblActor
+			destTblName := stringz.UniqTableName(sakila.TblActor)
+
+			ru := newRun(t).add(*src)
+			err := ru.exec("tbl", "copy", "--data=false", srcTblHandle, src.Handle+"."+destTblName)
+			require.NoError(t, err)
+			t.Cleanup(func() { th.DropTable(src, destTblName) })
+			require.Equal(t, int64(0), th.RowCount(src, destTblName), "should not have copied any rows because --data=false")
+
+			destTblName = stringz.UniqTableName(sakila.TblActor)
+			err = ru.exec("tbl", "copy", "--data=true", srcTblHandle, src.Handle+"."+destTblName)
+			require.NoError(t, err)
+			t.Cleanup(func() { th.DropTable(src, destTblName) })
+			require.Equal(t, int64(sakila.TblActorCount), th.RowCount(src, destTblName), "should have copied rows because --data=true")
+		})
+	}
+}
+
+func TestCmdTblDrop(t *testing.T) {
+	t.Parallel()
+
+	for _, handle := range sakila.SQLAll {
+		handle := handle
+
+		t.Run(handle, func(t *testing.T) {
+			t.Parallel()
+
+			th := testh.New(t)
+			src := th.Source(handle)
+			destTblName := th.CopyTable(false, src, sakila.TblActor, "", true)
+			t.Cleanup(func() { th.DropTable(src, destTblName) })
+
+			tblMeta, err := th.Open(src).TableMetadata(th.Context, destTblName)
+			require.NoError(t, err) // verify that the table exists
+			require.Equal(t, destTblName, tblMeta.Name)
+			require.Equal(t, int64(sakila.TblActorCount), tblMeta.RowCount)
+
+			err = newRun(t).add(*src).exec("tbl", "drop", src.Handle+"."+destTblName)
+			require.NoError(t, err)
+			tblMeta, err = th.Open(src).TableMetadata(th.Context, destTblName)
+			require.Error(t, err, "should get an error because the table was dropped")
+			require.Nil(t, tblMeta)
+		})
+	}
+}
+
+func TestCmdTblTruncate(t *testing.T) {
+	t.Parallel()
+
+	for _, handle := range sakila.SQLAll {
+		handle := handle
+
+		t.Run(handle, func(t *testing.T) {
+			t.Parallel()
+
+			th := testh.New(t)
+			src := th.Source(handle)
+			destTblName := th.CopyTable(false, src, sakila.TblActor, "", true)
+			t.Cleanup(func() { th.DropTable(src, destTblName) })
+
+			tblMeta, err := th.Open(src).TableMetadata(th.Context, destTblName)
+			require.NoError(t, err) // verify that the table exists
+			require.Equal(t, destTblName, tblMeta.Name)
+			require.Equal(t, int64(sakila.TblActorCount), tblMeta.RowCount)
+
+			err = newRun(t).add(*src).exec("tbl", "truncate", src.Handle+"."+destTblName)
+			require.NoError(t, err)
+			tblMeta, err = th.Open(src).TableMetadata(th.Context, destTblName)
+			require.NoError(t, err)
+			require.Equal(t, int64(0), tblMeta.RowCount)
+		})
+	}
+}

cli/cmd_version.go

@@ -0,0 +1,43 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/neilotoole/sq/cli/buildinfo"
+)
+
+func newVersionCmd() (*cobra.Command, runFunc) {
+	cmd := &cobra.Command{
+		Use:   "version",
+		Short: "Print sq version",
+	}
+
+	return cmd, execVersion
+}
+
+func execVersion(rc *RunContext, cmd *cobra.Command, args []string) error {
+	version := buildinfo.Version
+
+	// If buildinfo.Version is not set (building without ldflags),
+	// then we set a dummy version.
+	if version == "" {
+		version = "0.0.0.dev"
+	}
+
+	rc.wrtr.fmt.Hilite.Fprintf(rc.Out, "sq %s", version)
+
+	if len(buildinfo.Commit) > 0 {
+		fmt.Fprintf(rc.Out, "                ")
+		rc.wrtr.fmt.Faint.Fprintf(rc.Out, "#"+buildinfo.Commit)
+	}
+
+	if len(buildinfo.Timestamp) > 0 {
+		fmt.Fprintf(rc.Out, "       ")
+		rc.wrtr.fmt.Faint.Fprintf(rc.Out, buildinfo.Timestamp)
+	}
+
+	fmt.Fprintln(rc.Out)
+	return nil
+}

cli/config/config.go

@@ -0,0 +1,109 @@
+// Package config holds CLI configuration.
+package config
+
+import (
+	"time"
+
+	"github.com/neilotoole/sq/drivers/userdriver"
+	"github.com/neilotoole/sq/libsq/errz"
+	"github.com/neilotoole/sq/libsq/notify"
+	"github.com/neilotoole/sq/libsq/source"
+	"github.com/neilotoole/sq/libsq/stringz"
+)
+
+// Config holds application config/session data.
+type Config struct {
+	Options      Options       `yaml:"options" json:"options"`
+	Sources      *source.Set   `yaml:"sources" json:"sources"`
+	Notification *Notification `yaml:"notification" json:"notification"`
+
+	// Ext holds sq config extensions, such as user driver config.
+	Ext Ext `yaml:"-" json:"-"`
+}
+
+func (c *Config) String() string {
+	return stringz.SprintJSON(c)
+}
+
+// Ext holds additional config (extensions) loaded from other
+// config files, e.g. ~/.config/sq/ext/*.sq.yml
+type Ext struct {
+	UserDrivers []*userdriver.DriverDef `yaml:"user_drivers" json:"user_drivers"`
+}
+
+// Options contains sq default values.
+type Options struct {
+	Timeout time.Duration `yaml:"timeout" json:"timeout"`
+	Format  Format        `yaml:"output_format" json:"output_format"`
+	Header  bool          `yaml:"output_header" json:"output_header"`
+}
+
+// Notification holds notification configuration.
+type Notification struct {
+	Enabled      []string             `yaml:"enabled" json:"enabled"`
+	Destinations []notify.Destination `yaml:"destinations" json:"destinations"`
+}
+
+// New returns a config instance with default options set.
+func New() *Config {
+	cfg := &Config{}
+
+	// By default, we want header to be true; this is
+	// ugly wrt applyDefaults, as the zero value of a bool
+	// is false, but we actually want it to be true for Header.
+	cfg.Options.Header = true
+
+	applyDefaults(cfg)
+	return cfg
+}
+
+// applyDefaults checks if required values are present, and if not, sets them.
+func applyDefaults(cfg *Config) {
+	if cfg.Sources == nil {
+		cfg.Sources = &source.Set{}
+	}
+
+	if cfg.Notification == nil {
+		cfg.Notification = &Notification{}
+	}
+
+	if cfg.Options.Format == "" {
+		cfg.Options.Format = FormatTable
+	}
+
+	if cfg.Options.Timeout == 0 {
+		cfg.Options.Timeout = 10 * time.Second
+	}
+}
+
+// Format is a sq output format such as json or xml.
+type Format string
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (f *Format) UnmarshalText(text []byte) error {
+	switch Format(text) {
+	default:
+		return errz.Errorf("unknown output format %q", string(text))
+	case FormatJSON, FormatJSONA, FormatJSONL, FormatTable, FormatGrid, FormatRaw,
+		FormatHTML, FormatMarkdown, FormatXLSX, FormatXML, FormatCSV, FormatTSV:
+	}
+
+	*f = Format(text)
+	return nil
+}
+
+// Constants
+const (
+	FormatJSON     Format = "json"
+	FormatJSONL    Format = "jsonl"
+	FormatJSONA    Format = "jsona"
+	FormatTable    Format = "table" // FIXME: rename to FormatText
+	FormatGrid     Format = "grid"
+	FormatRaw      Format = "raw"
+	FormatHTML     Format = "html"
+	FormatMarkdown Format = "markdown"
+	FormatXLSX     Format = "xlsx"
+	FormatXML      Format = "xml"
+	FormatCSV      Format = "csv"
+	FormatTSV      Format = "tsv"
+)

cli/config/config_test.go

@@ -0,0 +1,84 @@
+package config_test
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/neilotoole/sq/cli/config"
+	"github.com/neilotoole/sq/testh/proj"
+)
+
+func TestFileStore_Nil_Save(t *testing.T) {
+	t.Parallel()
+
+	var f *config.YAMLFileStore
+
+	// noinspection GoNilness
+	err := f.Save(config.New())
+	require.Error(t, err)
+}
+
+func TestFileStore_LoadSaveLoad(t *testing.T) {
+	t.Parallel()
+
+	// good.01.sq.yml has a bunch of fixtures in it
+	fs := &config.YAMLFileStore{Path: "testdata/good.01.sq.yml", HookLoad: hookExpand}
+	const expectGood01SrcCount = 34
+
+	cfg, err := fs.Load()
+	require.NoError(t, err)
+	require.NotNil(t, cfg)
+	require.NotNil(t, cfg.Sources)
+	require.Equal(t, expectGood01SrcCount, len(cfg.Sources.Items()))
+
+	f, err := ioutil.TempFile("", "*.sq.yml")
+	require.NoError(t, err)
+	t.Cleanup(func() { assert.NoError(t, f.Close()) })
+
+	fs.Path = f.Name()
+	t.Logf("writing to tmp file: %s", fs.Path)
+
+	err = fs.Save(cfg)
+	require.NoError(t, err)
+
+	cfg2, err := fs.Load()
+	require.NoError(t, err)
+	require.NotNil(t, cfg2)
+	require.Equal(t, expectGood01SrcCount, len(cfg2.Sources.Items()))
+	require.EqualValues(t, cfg, cfg2)
+}
+
+// hookExpand expands variables in data, e.g. ${SQ_ROOT}.
+var hookExpand = func(data []byte) ([]byte, error) {
+	return []byte(proj.Expand(string(data))), nil
+}
+
+func TestFileStore_Load(t *testing.T) {
+	t.Parallel()
+
+	good, err := filepath.Glob("testdata/good.*")
+	require.NoError(t, err)
+	bad, err := filepath.Glob("testdata/bad.*")
+	require.NoError(t, err)
+
+	t.Logf("%d good fixtures, %d bad fixtures", len(good), len(bad))
+
+	fs := &config.YAMLFileStore{HookLoad: hookExpand}
+
+	for _, match := range good {
+		fs.Path = match
+		_, err := fs.Load()
+		require.NoError(t, err, match)
+	}
+
+	for _, match := range bad {
+		fs.Path = match
+		_, err := fs.Load()
+		require.Error(t, err, match)
+		t.Logf("got err: %s", err)
+	}
+}

cli/config/store.go

@@ -0,0 +1,209 @@
+package config
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"strings"
+
+	"github.com/neilotoole/sq/libsq/errz"
+	"github.com/neilotoole/sq/libsq/source"
+
+	"gopkg.in/yaml.v2"
+)
+
+// Store saves and loads config.
+type Store interface {
+	// Save writes config to the store.
+	Save(cfg *Config) error
+
+	// Load reads config from the store.
+	Load() (*Config, error)
+
+	// Location returns the location of the store, typically
+	// a file path.
+	Location() string
+}
+
+// YAMLFileStore provides persistence of config via YAML file.
+type YAMLFileStore struct {
+	// Path is the location of the config file
+	Path string
+
+	// If HookLoad is non-nil, it is invoked by Load
+	// on Path's bytes before the YAML is unmarshaled.
+	// This allows expansion of variables etc.
+	HookLoad func(data []byte) ([]byte, error)
+
+	// ExtPaths holds locations of potential ext config, both dirs and files (with suffix ".sq.yml")
+	ExtPaths []string
+}
+
+func (fs *YAMLFileStore) String() string {
+	return fmt.Sprintf("config filestore: %v", fs.Path)
+}
+
+// Location implements Store.
+func (fs *YAMLFileStore) Location() string {
+	return fs.Path
+}
+
+// Load reads config from disk.
+func (fs *YAMLFileStore) Load() (*Config, error) {
+	bytes, err := ioutil.ReadFile(fs.Path)
+	if err != nil {
+		return nil, errz.Wrapf(err, "config: failed to load file %q", fs.Path)
+	}
+
+	loadHookFn := fs.HookLoad
+	if loadHookFn != nil {
+		bytes, err = loadHookFn(bytes)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	cfg := &Config{}
+	err = yaml.Unmarshal(bytes, cfg)
+	if err != nil {
+		return nil, errz.Wrapf(err, "config: %s: failed to unmarshal config YAML", fs.Path)
+	}
+
+	applyDefaults(cfg)
+
+	err = source.VerifySetIntegrity(cfg.Sources)
+	if err != nil {
+		return nil, errz.Wrapf(err, "config: %s", fs.Path)
+	}
+
+	err = fs.loadExt(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return cfg, nil
+}
+
+// loadExt loads extension config files into cfg.
+func (fs *YAMLFileStore) loadExt(cfg *Config) error {
+	const extSuffix = ".sq.yml"
+	var extCfgCandidates []string
+
+	for _, extPath := range fs.ExtPaths {
+		// TODO: This seems overly complicated: could just use glob
+		//  for any files in the same or child dir?
+		if fiExtPath, err := os.Stat(extPath); err == nil {
+			// path exists
+
+			if fiExtPath.IsDir() {
+				files, err := ioutil.ReadDir(extPath)
+				if err != nil {
+					// just continue; no means of logging this yet (logging may
+					// not have bootstrapped), and we shouldn't stop bootstrap
+					// because of bad sqext files.
+					continue
+				}
+
+				for _, file := range files {
+					if file.IsDir() {
+						// We don't currently descend through sub dirs
+						continue
+					}
+
+					if !strings.HasSuffix(file.Name(), extSuffix) {
+						continue
+					}
+
+					extCfgCandidates = append(extCfgCandidates, filepath.Join(extPath, file.Name()))
+				}
+
+				continue
+			}
+
+			// it's a file
+			if !strings.HasSuffix(fiExtPath.Name(), extSuffix) {
+				continue
+			}
+			extCfgCandidates = append(extCfgCandidates, filepath.Join(extPath, fiExtPath.Name()))
+		} else {
+			continue
+			// Again, won't stop bootstrap because of sqext failures
+			// return nil, errz.Wrapf(err, "failed to load ext config from %q", extPath)
+		}
+		// else file does not exist
+	}
+
+	for _, f := range extCfgCandidates {
+		bytes, err := ioutil.ReadFile(f)
+		if err != nil {
+			return errz.Wrapf(err, "error reading config ext file %q", f)
+		}
+		ext := &Ext{}
+
+		err = yaml.Unmarshal(bytes, ext)
+		if err != nil {
+			return errz.Wrapf(err, "error parsing config ext file %q", f)
+		}
+
+		cfg.Ext.UserDrivers = append(cfg.Ext.UserDrivers, ext.UserDrivers...)
+	}
+
+	return nil
+}
+
+// Save writes config to disk.
+func (fs *YAMLFileStore) Save(cfg *Config) error {
+	if fs == nil {
+		return errz.New("config file store is nil")
+	}
+
+	bytes, err := yaml.Marshal(cfg)
+	if err != nil {
+		return errz.Wrap(err, "failed to marshal config to YAML")
+	}
+
+	// It's possible that the parent dir of fs.Path doesn't exist.
+	dir := filepath.Dir(fs.Path)
+	err = os.MkdirAll(dir, os.ModePerm)
+	if err != nil {
+		return errz.Wrapf(err, "failed to make parent dir of sq config file: %s", dir)
+	}
+
+	err = ioutil.WriteFile(fs.Path, bytes, os.ModePerm)
+	if err != nil {
+		return errz.Wrap(err, "failed to save config file")
+	}
+
+	return nil
+}
+
+// FileExists returns true if the backing file can be accessed, false if it doesn't
+// exist or on any error.
+func (fs *YAMLFileStore) FileExists() bool {
+	_, err := os.Stat(fs.Path)
+	return err == nil
+}
+
+// DiscardStore implements Store but its Save method is no-op
+// and Load always returns a new empty Config. Useful for testing.
+type DiscardStore struct {
+}
+
+var _ Store = (*DiscardStore)(nil)
+
+// Load returns a new empty Config.
+func (DiscardStore) Load() (*Config, error) {
+	return New(), nil
+}
+
+// Save is no-op.
+func (DiscardStore) Save(*Config) error {
+	return nil
+}
+
+// Location returns /dev/null
+func (DiscardStore) Location() string {
+	return "/dev/null"
+}

cli/config/testdata/bad.01.sq.yml

@@ -0,0 +1,2 @@
+options:
+  timeout: not_a_duration

cli/config/testdata/bad.02.sq.yml

@@ -0,0 +1,2 @@
+options:
+  output_format: not_a_format

cli/config/testdata/bad.03.sq.yml

@@ -0,0 +1,2 @@
+options:
+  output_header: not_a_bool

cli/config/testdata/bad.04.sq.yml

@@ -0,0 +1,2 @@
+sources:
+  active: '@does_not_exist'

cli/config/testdata/bad.05.sq.yml

@@ -0,0 +1,5 @@
+sources:
+  items:
+    - handle: "illegal_handle"
+      type: sqlite3
+      location: sqlite3://${SQ_ROOT}/drivers/sqlite3/testdata/sqtest.db

cli/config/testdata/bad.06.sq.yml

@@ -0,0 +1,6 @@
+sources:
+  items:
+    - handle: "@sl1"
+      type: sqlite3
+      # location is empty: illegal
+      location:

cli/config/testdata/bad.07.sq.yml

@@ -0,0 +1,6 @@
+sources:
+  items:
+    - handle: "@sl1"
+      # type is empty: illegal
+      type:
+      location: sqlite3://${SQ_ROOT}/drivers/sqlite3/testdata/sqtest.db

cli/config/testdata/good.01.sq.yml

@@ -0,0 +1,166 @@
+options:
+  timeout: 10s
+  output_format: table
+  output_header: true
+sources:
+  active: '@sl1'
+  scratch: ""
+  items:
+    - handle: '@sl1'
+      type: sqlite3
+      location: sqlite3://${SQ_ROOT}/drivers/sqlite3/testdata/sqtest.db
+
+    - handle: '@xl1'
+      type: xlsx
+      location: ${SQ_ROOT}/drivers/xlsx/testdata/test_header.xlsx
+      options:
+        header:
+          - "true"
+
+    - handle: '@csv1'
+      type: csv
+      location: ${SQ_ROOT}/drivers/csv/testdata/person_comma_header.csv
+      options:
+        header:
+          - "true"
+
+    - handle: '@ms1'
+      type: sqlserver
+      location: sqlserver://sq:p_ssW0rd@localhost?database=sqtest
+
+    - handle: '@my1'
+      type: mysql
+      location: mysql://sq:p_ssW0rd@localhost:3306/sqtest
+
+    - handle: '@pg1'
+      type: postgres
+      location: postgres://sq:p_ssW0rd@localhost/sqtest?sslmode=disable
+
+    - handle: '@ms_sqtest'
+      type: sqlserver
+      location: sqlserver://sq:p_ssW0rd@localhost?database=sqtest
+
+    - handle: '@ms_sqtype'
+      type: sqlserver
+      location: sqlserver://sq:p_ssW0rd@localhost?database=sqtype
+
+    - handle: '@pg_sqtest'
+      type: postgres
+      location: postgres://sq:p_ssW0rd@localhost/sqtest?sslmode=disable
+
+    - handle: '@pg_sqtype'
+      type: postgres
+      location: postgres://sq:p_ssW0rd@localhost/sqtype?sslmode=disable
+
+    - handle: '@sl_sqtest'
+      type: sqlite3
+      location: sqlite3://${SQ_ROOT}/drivers/sqlite3/testdata/sqtest.db
+
+    - handle: '@sl_sqtype'
+      type: sqlite3
+      location: sqlite3://${SQ_ROOT}/drivers/sqlite3/testdata/sqtype.db
+
+    - handle: '@my_sqtest'
+      type: mysql
+      location: mysql://sq:p_ssW0rd@localhost:3306/sqtest
+
+    - handle: '@my_sqtype'
+      type: mysql
+      location: mysql://sq:p_ssW0rd@localhost:3306/sqtype
+
+    - handle: '@xl_header'
+      type: xlsx
+      location: ${SQ_ROOT}/drivers/xlsx/testdata/test_header.xlsx
+      options:
+        header:
+          - "true"
+
+    - handle: '@xl_noheader'
+      type: xlsx
+      location: ${SQ_ROOT}/drivers/xlsx/testdata/test_noheader.xlsx
+
+    - handle: '@xl_remote'
+      type: xlsx
+      location: http://neilotoole.io/sq/test/test1.xlsx
+
+    - handle: '@csv_person_comma_header'
+      type: csv
+      location: ${SQ_ROOT}/drivers/csv/testdata/person_comma_header.csv
+      options:
+        header:
+          - "true"
+
+    - handle: '@csv_person_comma_noheader'
+      type: csv
+      location: ${SQ_ROOT}/drivers/csv/testdata/person_comma_noheader.csv
+
+    - handle: '@tsv_person_header'
+      type: tsv
+      location: ${SQ_ROOT}/drivers/csv/testdata/person_header.tsv
+
+    - handle: '@tsv_person_noheader'
+      type: tsv
+      location: ${SQ_ROOT}/drivers/csv/testdata/person_noheader.tsv
+
+    - handle: '@tsv_person_noheader_cols'
+      type: tsv
+      location: ${SQ_ROOT}/drivers/csv/testdata/person_noheader.tsv
+      options:
+        cols:
+          - uid,username,email
+
+    - handle: '@rss_basic'
+      type: rss
+      location: ${SQ_ROOT}/libsq/driver/userdriver/testdata/basic.rss.xml
+
+    - handle: '@nytimes'
+      type: rss
+      location: http://www.nytimes.com/services/xml/rss/nyt/World.xml
+
+    - handle: '@myfriends'
+      type: ppl
+      location: ${SQ_ROOT}/libsq/driver/userdriver/testdata/people.xml
+
+    - handle: '@peeps'
+      type: ppl
+      location: ${SQ_ROOT}/libsq/driver/userdriver/testdata/people2.xml
+
+    - handle: '@ds_invalid_creds'
+      type: mysql
+      location: mysql://root:badpass@localhost:3306/sqtest
+
+    - handle: '@ds_invalid_port'
+      type: mysql
+      location: mysql://root:root@localhost:33661/sqtest
+
+    - handle: '@ds_invalid_host'
+      type: mysql
+      location: mysql://root:root@news.google.com:80/sqtest
+
+    - handle: '@ds_invalid_db'
+      type: mysql
+      location: mysql://sq:sq@localhost:3306/not_a_db
+
+    - handle: '@csvbig'
+      type: csv
+      location: ${SQ_ROOT}/drivers/csv/testdata/person_comma_header_big.csv
+      options:
+        header:
+          - "true"
+
+    - handle: '@sl_sakila'
+      type: sqlite3
+      location: sqlite3://${SQ_ROOT}/examples/sakila/sqlite-sakila-db/sakila.db
+
+    - handle: '@my_sakila'
+      type: mysql
+      location: mysql://root:sakila@localhost:33067/sakila
+
+    - handle: '@pg_sakila'
+      type: postgres
+      location: postgres://sq:p_ssW0rd@localhost:54321/sakila?sslmode=disable
+
+
+notification:
+  enabled: []
+  destinations: []

cli/config/testdata/good.02.sq.yml

@@ -0,0 +1,7 @@
+options:
+
+sources:
+  active: ""
+  scratch: ""
+  items:
+

cli/config/testdata/good.03.sq.yml

@@ -0,0 +1,7 @@
+
+
+sources:
+  active: ""
+  scratch: ""
+  items:
+

cli/config/testdata/good.04.sq.yml

@@ -0,0 +1,5 @@
+options:
+
+sources:
+
+

cli/consts.go

@@ -0,0 +1,122 @@
+package cli
+
+// cli flags
+const (
+	flagActiveSrc      = "src"
+	flagActiveSrcUsage = "Override the active source for this query"
+
+	flagCSV      = "csv"
+	flagCSVShort = "c"
+	flagCSVUsage = "CSV output"
+
+	flagDriver      = "driver"
+	flagDriverShort = "d"
+	flagDriverUsage = "Explicitly specify the data source driver to use"
+
+	flagHTML      = "html"
+	flagHTMLUsage = "HTML table output"
+
+	flagHeader      = "header"
+	flagHeaderShort = "h"
+	flagHeaderUsage = "Print header row in output"
+
+	flagHandle      = "handle"
+	flagHandleShort = "h"
+	flagHandleUsage = "Handle for the source"
+
+	flagHelp = "help"
+
+	flagInsert      = "insert"
+	flagInsertUsage = "Insert query results into @HANDLE.TABLE"
+
+	flagInspectFull      = "full"
+	flagInspectFullUsage = "Output full data source details (JSON only)"
+
+	flagJSON       = "json"
+	flagJSONUsage  = "JSON output"
+	flagJSONShort  = "j"
+	flagJSONA      = "jsona"
+	flagJSONAShort = "A"
+	flagJSONAUsage = "JSON: output each record's values as JSON array on its own line"
+	flagJSONL      = "jsonl"
+	flagJSONLShort = "l"
+	flagJSONLUsage = "JSON: output each record as a JSON object on its own line"
+
+	flagMarkdown      = "markdown"
+	flagMarkdownUsage = "Markdown table output"
+
+	flagMonochrome      = "monochrome"
+	flagMonochromeShort = "M"
+	flagMonochromeUsage = "Don't colorize output"
+
+	flagNoHeader      = "no-header"
+	flagNoHeaderShort = "H"
+	flagNoHeaderUsage = "Don't print header row in output"
+
+	flagNotifierLabel      = "label"
+	flagNotifierLabelUsage = "Optional label for the notification destination"
+
+	flagOutput      = "output"
+	flagOutputShort = "o"
+	flagOutputUsage = "Write output to <file> instead of stdout"
+
+	flagPretty      = "pretty"
+	flagPrettyUsage = "Pretty-print output for certain formats such as JSON or XML"
+
+	flagQueryDriverUsage     = "Explicitly specify the data source driver to use when piping input"
+	flagQuerySrcOptionsUsage = "Driver-dependent data source options when piping input"
+
+	flagRaw      = "raw"
+	flagRawShort = "r"
+	flagRawUsage = "Output each record field in raw format without any encoding or delimiter"
+
+	flagSQLExec      = "exec"
+	flagSQLExecUsage = "Execute the SQL as a statement (as opposed to query)"
+
+	flagSQLQuery      = "query"
+	flagSQLQueryUsage = "Execute the SQL as a query (as opposed to statement)"
+
+	flagSrcOptions      = "opts"
+	flagSrcOptionsUsage = "Driver-dependent data source options"
+
+	flagTSV      = "tsv"
+	flagTSVShort = "T"
+	flagTSVUsage = "TSV output"
+
+	flagTable      = "table"
+	flagTableShort = "t"
+	flagTableUsage = "Table output"
+
+	flagTblData      = "data"
+	flagTblDataUsage = "Copy table data (defualt true)"
+
+	flagTimeout          = "timeout"
+	flagTimeoutPingUsage = "Max time to wait for ping"
+
+	flagVerbose      = "verbose"
+	flagVerboseShort = "v"
+	flagVerboseUsage = "Print verbose data, if applicable"
+
+	flagVersion      = "version"
+	flagVersionUsage = "Print sq version"
+
+	flagXLSX      = "xlsx"
+	flagXLSXShort = "x"
+	flagXLSXUsage = "Excel XLSX output"
+
+	flagXML      = "xml"
+	flagXMLShort = "X"
+	flagXMLUsage = "XML output"
+)
+
+const (
+	msgInvalidArgs       = "invalid args"
+	msgNoActiveSrc       = "no active data source"
+	msgEmptyQueryString  = "query string is empty"
+	msgSrcNoData         = "source has no data"
+	msgSrcEmptyTableName = "source has empty table name"
+
+	envarLogPath     = "SQ_LOGFILE"
+	envarLogTruncate = "SQ_LOGFILE_TRUNCATE"
+	envarConfigDir   = "SQ_CONFIGDIR"
+)

cli/ioutilz.go

@@ -0,0 +1,45 @@
+package cli
+
+import (
+	"io"
+	"os"
+
+	"github.com/mattn/go-isatty"
+	"golang.org/x/crypto/ssh/terminal"
+)
+
+// isTerminal returns true if w is a terminal.
+func isTerminal(w io.Writer) bool {
+	switch v := w.(type) {
+	case *os.File:
+		return terminal.IsTerminal(int(v.Fd()))
+	default:
+		return false
+	}
+}
+
+// isColorTerminal returns true if w is a colorable terminal.
+func isColorTerminal(w io.Writer) bool {
+	if w == nil {
+		return false
+	}
+
+	if !isTerminal(w) {
+		return false
+	}
+
+	if os.Getenv("TERM") == "dumb" {
+		return false
+	}
+
+	f, ok := w.(*os.File)
+	if !ok {
+		return false
+	}
+
+	if isatty.IsCygwinTerminal(f.Fd()) {
+		return false
+	}
+
+	return true
+}

cli/output/adapter.go

@@ -0,0 +1,178 @@
+package output
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"go.uber.org/atomic"
+
+	"github.com/neilotoole/sq/libsq/errz"
+	"github.com/neilotoole/sq/libsq/sqlz"
+)
+
+// RecordWriterAdapter implements libsq.RecordWriter and
+// wraps an output.RecordWriter instance, providing a
+// bridge between the asynchronous libsq.RecordWriter and
+// synchronous output.RecordWriter interfaces.
+//
+// Note that a writer implementation such as the JSON or
+// CSV writer could directly implement libsq.RecordWriter.
+// But that interface is non-trivial to implement, hence
+// this bridge type.
+//
+// The FlushAfterN and FlushAfterDuration fields control
+// flushing of the writer.
+type RecordWriterAdapter struct {
+	rw       RecordWriter
+	wg       *sync.WaitGroup
+	recCh    chan sqlz.Record
+	errCh    chan error
+	errs     []error
+	written  *atomic.Int64
+	cancelFn context.CancelFunc
+
+	// FlushAfterN indicates that the writer's Flush method
+	// should be invoked after N invocations of WriteRecords.
+	// A value of 0 will flush every time a record is written.
+	// Set to -1 to disable.
+	FlushAfterN int64
+
+	// FlushAfterDuration controls whether the writer's Flush method
+	// is invoked periodically. A duration <= 0 disables periodic flushing.
+	FlushAfterDuration time.Duration
+}
+
+// RecordWriterAdapterChSize is the size of the record chan (effectively
+// the buffer) used by RecordWriterAdapter.
+// Possibly this value should be user-configurable.
+const RecordWriterAdapterChSize = 1000
+
+// NewRecordWriterAdapter returns a new RecordWriterAdapter.
+func NewRecordWriterAdapter(rw RecordWriter) *RecordWriterAdapter {
+	recCh := make(chan sqlz.Record, RecordWriterAdapterChSize)
+	return &RecordWriterAdapter{rw: rw, recCh: recCh, wg: &sync.WaitGroup{}, written: atomic.NewInt64(0)}
+}
+
+// Open implements libsq.RecordWriter.
+func (w *RecordWriterAdapter) Open(ctx context.Context, cancelFn context.CancelFunc, recMeta sqlz.RecordMeta) (chan<- sqlz.Record, <-chan error, error) {
+	w.cancelFn = cancelFn
+
+	err := w.rw.Open(recMeta)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// errCh has size 2 because that's the maximum number of
+	// errs that could be sent. Typically only one err is sent,
+	// but in the case of ctx.Done, we send ctx.Err, followed
+	// by any error returned by r.rw.Close.
+	w.errCh = make(chan error, 2)
+	w.wg.Add(1)
+
+	go func() {
+		defer func() {
+			w.wg.Done()
+			close(w.errCh)
+		}()
+
+		var lastFlushN, recN int64
+		var flushTimer *time.Timer
+		var flushCh <-chan time.Time
+
+		if w.FlushAfterDuration > 0 {
+			flushTimer = time.NewTimer(w.FlushAfterDuration)
+			flushCh = flushTimer.C
+			defer flushTimer.Stop()
+		}
+
+		for {
+			select {
+			case <-ctx.Done():
+				w.addErrs(ctx.Err(), w.rw.Close())
+				return
+
+			case <-flushCh:
+				// The flushTimer has expired, time to flush.
+				err = w.rw.Flush()
+				if err != nil {
+					w.addErrs(err)
+					return
+				}
+				lastFlushN = recN
+				flushTimer.Reset(w.FlushAfterDuration)
+				continue
+
+			case rec := <-w.recCh:
+				if rec == nil { // no more results on recCh, it has been closed
+					err = w.rw.Close()
+					if err != nil {
+						w.addErrs()
+					}
+					return
+				}
+
+				// rec is not nil, therefore we write it out.
+
+				// We could accumulate a bunch of recs into a slice here,
+				// but we'll worry about that if benchmarking shows it'll matter.
+				writeErr := w.rw.WriteRecords([]sqlz.Record{rec})
+				if writeErr != nil {
+					w.addErrs(writeErr)
+					return
+				}
+
+				recN = w.written.Inc()
+
+				// Check if we should flush
+				if w.FlushAfterN >= 0 && (recN-lastFlushN >= w.FlushAfterN) {
+					err = w.rw.Flush()
+					if err != nil {
+						w.addErrs(err)
+						return
+					}
+					lastFlushN = recN
+
+					if flushTimer != nil {
+						// Reset the timer, but we need to stop and drain it first.
+						// See the timer.Reset docs.
+						if !flushTimer.Stop() {
+							<-flushTimer.C
+						}
+
+						flushTimer.Reset(w.FlushAfterDuration)
+					}
+				}
+
+				// If we got this far, we successfully wrote rec to rw.
+				// Therefore continue to wait/select for the next
+				// element on recCh (or for recCh to close)
+				// or for ctx.Done indicating timeout or cancel etc.
+				continue
+			}
+		}
+	}()
+
+	return w.recCh, w.errCh, nil
+}
+
+// Wait implements libsq.RecordWriter.
+func (w *RecordWriterAdapter) Wait() (written int64, err error) {
+	w.wg.Wait()
+	if w.cancelFn != nil {
+		w.cancelFn()
+	}
+
+	return w.written.Load(), errz.Combine(w.errs...)
+}
+
+// addErrs handles any non-nil err in errs by appending it to w.errs
+// and sending it on w.errCh.
+func (w *RecordWriterAdapter) addErrs(errs ...error) {
+	for _, err := range errs {
+		if err != nil {
+			w.errs = append(w.errs, err)
+			w.errCh <- err
+		}
+	}
+}

cli/output/adapter_test.go

@@ -0,0 +1,167 @@
+package output_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/neilotoole/sq/cli/output"
+	"github.com/neilotoole/sq/libsq"
+	"github.com/neilotoole/sq/libsq/sqlz"
+	"github.com/neilotoole/sq/testh"
+	"github.com/neilotoole/sq/testh/sakila"
+	"github.com/neilotoole/sq/testh/testsrc"
+)
+
+var _ libsq.RecordWriter = (*output.RecordWriterAdapter)(nil)
+
+func TestRecordWriterAdapter(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		handle       string
+		sqlQuery     string
+		wantRowCount int
+		wantColCount int
+	}{
+		{
+			handle:       sakila.SL3,
+			sqlQuery:     "SELECT * FROM actor",
+			wantRowCount: sakila.TblActorCount,
+			wantColCount: len(sakila.TblActorCols),
+		},
+		{
+			handle:       testsrc.CSVPersonBig,
+			sqlQuery:     "SELECT * FROM data",
+			wantRowCount: 10000,
+			wantColCount: len(sakila.TblActorCols),
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(tc.handle, func(t *testing.T) {
+			t.Parallel()
+
+			th := testh.New(t)
+			src := th.Source(tc.handle)
+			dbase := th.Open(src)
+
+			sink := &testh.RecordSink{}
+			recw := output.NewRecordWriterAdapter(sink)
+			err := libsq.QuerySQL(th.Context, th.Log, dbase, recw, tc.sqlQuery)
+			require.NoError(t, err)
+			written, err := recw.Wait()
+			require.NoError(t, err)
+			require.Equal(t, int64(tc.wantRowCount), written)
+
+			require.True(t, len(sink.Closed) == 1)
+			require.Equal(t, tc.wantRowCount, len(sink.Recs))
+			require.Equal(t, tc.wantColCount, len(sink.Recs[0]))
+		})
+	}
+}
+
+func TestRecordWriterAdapter_FlushAfterN(t *testing.T) {
+	const writeRecCount = 200
+
+	testCases := map[int]int{
+		-1:                0,
+		0:                 writeRecCount,
+		1:                 writeRecCount,
+		2:                 writeRecCount / 2,
+		10:                writeRecCount / 10,
+		100:               writeRecCount / 100,
+		writeRecCount + 1: 0,
+	}
+	// Get some recMeta to feed to RecordWriter.Open.
+	// In this case, the field is "actor_id", which is an int.
+	recMeta := testh.NewRecordMeta([]string{"col_int"}, []sqlz.Kind{sqlz.KindInt})
+
+	for flushAfterN, wantFlushed := range testCases {
+		flushAfterN, wantFlushed := flushAfterN, wantFlushed
+
+		t.Run(fmt.Sprintf("flustAfter_%d__wantFlushed_%d", flushAfterN, wantFlushed), func(t *testing.T) {
+			t.Parallel()
+
+			sink := &testh.RecordSink{}
+			recw := output.NewRecordWriterAdapter(sink)
+
+			recw.FlushAfterN = int64(flushAfterN)
+			recw.FlushAfterDuration = -1 // not testing duration
+			recCh, _, err := recw.Open(context.Background(), nil, recMeta)
+			require.NoError(t, err)
+
+			// Write some records
+			for i := 0; i < writeRecCount; i++ {
+				recCh <- []interface{}{1}
+			}
+			close(recCh)
+
+			written, err := recw.Wait()
+			require.NoError(t, err)
+			require.Equal(t, int64(writeRecCount), written)
+			require.Equal(t, writeRecCount, len(sink.Recs))
+
+			require.Equal(t, wantFlushed, len(sink.Flushed))
+		})
+	}
+}
+
+func TestRecordWriterAdapter_FlushAfterDuration(t *testing.T) {
+	// Don't run this as t.Parallel because it's timing sensitive.
+	const (
+		sleepTime     = time.Millisecond * 10
+		writeRecCount = 10
+	)
+
+	testCases := []struct {
+		flushAfter  time.Duration
+		wantFlushed int
+		assertFn    testh.AssertCompareFunc
+	}{
+		{flushAfter: -1, wantFlushed: 0, assertFn: require.Equal},
+		{flushAfter: 0, wantFlushed: 0, assertFn: require.Equal},
+		{flushAfter: 1, wantFlushed: 10, assertFn: require.GreaterOrEqual},
+		{flushAfter: time.Millisecond * 20, wantFlushed: 2, assertFn: require.GreaterOrEqual},
+		{flushAfter: time.Second, wantFlushed: 0, assertFn: require.Equal},
+	}
+
+	// Get some recMeta to feed to RecordWriter.Open.
+	// In this case, the field is "actor_id", which is an int.
+	recMeta := testh.NewRecordMeta([]string{"col_int"}, []sqlz.Kind{sqlz.KindInt})
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(fmt.Sprintf("flushAfter_%s__wantFlushed_%d", tc.flushAfter, tc.wantFlushed), func(t *testing.T) {
+			t.Parallel()
+
+			sink := &testh.RecordSink{}
+			recw := output.NewRecordWriterAdapter(sink)
+
+			recw.FlushAfterN = -1 // not testing FlushAfterN
+			recw.FlushAfterDuration = tc.flushAfter
+
+			recCh, _, err := recw.Open(context.Background(), nil, recMeta)
+			require.NoError(t, err)
+
+			// Write some records
+			for i := 0; i < writeRecCount; i++ {
+				recCh <- []interface{}{1}
+				time.Sleep(sleepTime)
+			}
+			close(recCh)
+
+			written, err := recw.Wait()
+			require.NoError(t, err)
+			require.Equal(t, int64(writeRecCount), written)
+			require.Equal(t, writeRecCount, len(sink.Recs))
+
+			tc.assertFn(t, len(sink.Flushed), tc.wantFlushed)
+		})
+	}
+}

cli/output/csvw/csvw_test.go

@@ -0,0 +1,35 @@
+package csvw_test
+
+import (
+	"bytes"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/neilotoole/sq/cli/output/csvw"
+	"github.com/neilotoole/sq/libsq/sqlz"
+	"github.com/neilotoole/sq/testh"
+)
+
+func TestDateTimeHandling(t *testing.T) {
+	var (
+		colNames = []string{"col_datetime", "col_date", "col_time"}
+		kinds    = []sqlz.Kind{sqlz.KindDatetime, sqlz.KindDate, sqlz.KindTime}
+		when     = time.Unix(0, 0).UTC()
+	)
+	const want = "1970-01-01T00:00:00Z\t1970-01-01\t00:00:00\n"
+
+	recMeta := testh.NewRecordMeta(colNames, kinds)
+	buf := &bytes.Buffer{}
+
+	w := csvw.NewRecordWriter(buf, false, csvw.Tab)
+	require.NoError(t, w.Open(recMeta))
+
+	rec := sqlz.Record{&when, &when, &when}
+	require.NoError(t, w.WriteRecords([]sqlz.Record{rec}))
+	require.NoError(t, w.Close())
+
+	require.Equal(t, want, buf.String())
+	println(buf.String())
+}

cli/output/csvw/csvwriter.go

@@ -0,0 +1,108 @@
+// Package csvw implements writers for CSV.
+package csvw
+
+import (
+	"encoding/csv"
+	"fmt"
+	"io"
+	"strconv"
+	"time"
+
+	"github.com/neilotoole/sq/libsq/errz"
+	"github.com/neilotoole/sq/libsq/sqlz"
+	"github.com/neilotoole/sq/libsq/stringz"
+)
+
+const (
+	// Tab is the tab rune.
+	Tab = '\t'
+
+	// Comma is the comma rune.
+	Comma = ','
+)
+
+// RecordWriter implements output.RecordWriter.
+type RecordWriter struct {
+	recMeta     sqlz.RecordMeta
+	csvW        *csv.Writer
+	needsHeader bool
+}
+
+// NewRecordWriter returns a writer instance.
+func NewRecordWriter(out io.Writer, header bool, sep rune) *RecordWriter {
+	csvW := csv.NewWriter(out)
+	csvW.Comma = sep
+	return &RecordWriter{needsHeader: header, csvW: csvW}
+}
+
+// Open implements output.RecordWriter.
+func (w *RecordWriter) Open(recMeta sqlz.RecordMeta) error {
+	w.recMeta = recMeta
+	return nil
+}
+
+// Flush implements output.RecordWriter.
+func (w *RecordWriter) Flush() error {
+	w.csvW.Flush()
+	return nil
+}
+
+// Close implements output.RecordWriter.
+func (w *RecordWriter) Close() error {
+	w.csvW.Flush()
+	return w.csvW.Error()
+}
+
+// WriteRecords implements output.RecordWriter.
+func (w *RecordWriter) WriteRecords(recs []sqlz.Record) error {
+	if w.needsHeader {
+		headerRow := w.recMeta.Names()
+
+		err := w.csvW.Write(headerRow)
+		if err != nil {
+			return errz.Wrap(err, "failed to write header record")
+		}
+		w.needsHeader = false
+	}
+
+	for _, rec := range recs {
+		fields := make([]string, len(rec))
+
+		for i, val := range rec {
+			switch val := val.(type) {
+			default:
+				// should never happen
+				fields[i] = fmt.Sprintf("%v", val)
+			case nil:
+				// nil is rendered as empty string, which this cell already is
+			case *int64:
+				fields[i] = strconv.FormatInt(*val, 10)
+			case *string:
+				fields[i] = *val
+			case *bool:
+				fields[i] = strconv.FormatBool(*val)
+			case *float64:
+				fields[i] = fmt.Sprintf("%v", *val)
+			case *[]byte:
+				fields[i] = fmt.Sprintf("%v", string(*val))
+			case *time.Time:
+				switch w.recMeta[i].Kind() {
+				default:
+					fields[i] = val.Format(stringz.DatetimeFormat)
+				case sqlz.KindTime:
+					fields[i] = val.Format(stringz.TimeFormat)
+				case sqlz.KindDate:
+					fields[i] = val.Format(stringz.DateFormat)
+				}
+			}
+		}
+
+		err := w.csvW.Write(fields)
+		if err != nil {
+			return errz.Wrap(err, "failed to write records")
+		}
+	}
+
+	w.csvW.Flush()
+	return w.csvW.Error()
+}

cli/output/csvw/pingwriter.go

@@ -0,0 +1,53 @@
+package csvw
+
+import (
+	"context"
+	"encoding/csv"
+	"io"
+	"time"
+
+	"github.com/neilotoole/sq/libsq/errz"
+
+	"github.com/neilotoole/sq/cli/output"
+
+	"github.com/neilotoole/sq/libsq/source"
+)
+
+// NewPingWriter returns a new instance.
+func NewPingWriter(out io.Writer, sep rune) output.PingWriter {
+	csvw := csv.NewWriter(out)
+	csvw.Comma = sep
+	return &pingWriter{csvw: csvw}
+}
+
+// pingWriter implements out.pingWriter.
+type pingWriter struct {
+	csvw *csv.Writer
+}
+
+// Open implements output.PingWriter.
+func (p *pingWriter) Open(srcs []*source.Source) {
+}
+
+// Result implements output.PingWriter.
+func (p *pingWriter) Result(src *source.Source, d time.Duration, err error) {
+	rec := make([]string, 3)
+	rec[0] = src.Handle
+	rec[1] = d.Truncate(time.Millisecond).String()
+	if err != nil {
+		if err == context.DeadlineExceeded {
+			rec[2] = "timeout exceeded"
+		} else {
+			rec[2] = err.Error()
+		}
+	}
+
+	_ = p.csvw.Write(rec)
+	p.csvw.Flush()
+}
+
+// Close implements output.PingWriter.
+func (p *pingWriter) Close() error {
+	p.csvw.Flush()
+	return errz.Err(p.csvw.Error())
+}

cli/output/formatting.go

@@ -0,0 +1,133 @@
+package output
+
+import "github.com/fatih/color"
+
+// Formatting describes color and pretty-printing options.
+type Formatting struct {
+	monochrome bool
+
+	// Pretty indicates that output should be pretty-printed.
+	// Typically this means indentation, new lines, etc, but
+	// varies by output format.
+	Pretty bool
+
+	// Indent is the indent string to use when pretty-printing,
+	// typically two spaces.
+	Indent string
+
+	// Bool is the color for boolean values.
+	Bool *color.Color
+
+	// Bytes is the color for byte / binary values.
+	Bytes *color.Color
+
+	// Datetime is the color for time-related values.
+	Datetime *color.Color
+
+	// Null is the color for null.
+	Null *color.Color
+
+	// Number is the color for number values, including int,
+	// float, decimal etc.
+	Number *color.Color
+
+	// String is the color for string values.
+	String *color.Color
+
+	// Success is the color for success elements.
+	Success *color.Color
+
+	// Error is the color for error elements such as an error message.
+	Error *color.Color
+
+	// Handle is the color for source handles such as "@my_db"
+	Handle *color.Color
+
+	// Header is the color for header elements in a table.
+	Header *color.Color
+
+	// Hilite is the color for highlighted elements.
+	Hilite *color.Color
+
+	// Faint is the color for faint elements - the opposite of Hilite.
+	Faint *color.Color
+
+	// Bold is the color for bold elements.
+	Bold *color.Color
+
+	// Key is the color for keys such as a JSON field name.
+	Key *color.Color
+
+	// Punc is the color for punctuation such as colons, braces, etc.
+	// Frequently Punc will just be color.Bold.
+	Punc *color.Color
+}
+
+// NewFormatting returns a Formatting instance. Color and pretty-print
+// are enabled. The default indent is two spaces.
+func NewFormatting() *Formatting {
+	fm := &Formatting{
+		Pretty:     true,
+		monochrome: false,
+		Indent:     "  ",
+		Bool:       color.New(color.FgCyan),
+		Bold:       color.New(color.Bold),
+		Bytes:      color.New(color.FgGreen),
+		Datetime:   color.New(color.FgGreen),
+		Error:      color.New(color.FgRed, color.Bold),
+		Faint:      color.New(color.Faint),
+		Handle:     color.New(color.FgBlue),
+		Header:     color.New(color.FgBlue, color.Bold),
+		Hilite:     color.New(color.FgHiBlue),
+		Key:        color.New(color.FgBlue, color.Bold),
+		Null:       color.New(color.FgBlue, color.Faint),
+		Number:     color.New(),
+		String:     color.New(color.FgGreen),
+		Success:    color.New(color.FgGreen, color.Bold),
+		Punc:       color.New(color.Bold),
+	}
+
+	fm.EnableColor(true)
+	return fm
+}
+
+// IsMonochrome returns true if in monochrome (no color) mode.
+// Default is false (color enabled) for a new instance.
+func (f *Formatting) IsMonochrome() bool {
+	return f.monochrome
+}
+
+// EnableColor enables or disables all colors.
+func (f *Formatting) EnableColor(enable bool) {
+	if enable {
+		f.monochrome = false
+
+		f.Bool.EnableColor()
+		f.Bytes.EnableColor()
+		f.Datetime.EnableColor()
+		f.Error.EnableColor()
+		f.Faint.EnableColor()
+		f.Header.EnableColor()
+		f.Hilite.EnableColor()
+		f.Key.EnableColor()
+		f.Null.EnableColor()
+		f.Success.EnableColor()
+		f.Punc.EnableColor()
+		f.Bold.EnableColor()
+	} else {
+		f.monochrome = true
+
+		f.Bool.DisableColor()
+		f.Bytes.DisableColor()
+		f.Datetime.DisableColor()
+		f.Error.DisableColor()
+		f.Faint.DisableColor()
+		f.Header.DisableColor()
+		f.Hilite.DisableColor()
+		f.Key.DisableColor()
+		f.Null.DisableColor()
+		f.Success.DisableColor()
+		f.Punc.DisableColor()
+		f.Bold.DisableColor()
+	}
+}

cli/output/formatting_test.go

@@ -0,0 +1 @@
+package output_test

cli/output/htmlw/htmlw.go

@@ -0,0 +1,142 @@
+// Package htmlw implements a RecordWriter for HTML.
+package htmlw
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"html"
+	"io"
+	"strconv"
+	"time"
+
+	"github.com/neilotoole/sq/cli/output"
+
+	"github.com/neilotoole/sq/libsq/errz"
+	"github.com/neilotoole/sq/libsq/sqlz"
+	"github.com/neilotoole/sq/libsq/stringz"
+)
+
+// RecordWriter implements output.RecordWriter.
+type recordWriter struct {
+	recMeta sqlz.RecordMeta
+	out     io.Writer
+	buf     *bytes.Buffer
+}
+
+// NewRecordWriter an output.RecordWriter for HTML.
+func NewRecordWriter(out io.Writer) output.RecordWriter {
+	return &recordWriter{out: out}
+}
+
+// Open implements output.RecordWriter.
+func (w *recordWriter) Open(recMeta sqlz.RecordMeta) error {
+	w.recMeta = recMeta
+	w.buf = &bytes.Buffer{}
+
+	const header = `<!doctype html>
+<html>
+<head>
+<title>sq output</title>
+</head>
+<body>
+
+<table>
+  <colgroup>
+`
+
+	w.buf.WriteString(header)
+	for _, field := range recMeta {
+		w.buf.WriteString("    <col class=\"kind-")
+		w.buf.WriteString(field.Kind().String())
+		w.buf.WriteString("\" />\n")
+	}
+	w.buf.WriteString("  </colgroup>\n  <thead>\n    <tr>\n")
+	for _, field := range recMeta {
+		w.buf.WriteString("      <th scope=\"col\">")
+		w.buf.WriteString(field.Name())
+		w.buf.WriteString("</th>\n")
+	}
+	w.buf.WriteString("    </tr>\n  </thead>\n  <tbody>\n")
+	return nil
+}
+
+// Flush implements output.RecordWriter.
+func (w *recordWriter) Flush() error {
+	_, err := w.buf.WriteTo(w.out) // resets buf
+	return errz.Err(err)
+}
+
+// Close implements output.RecordWriter.
+func (w *recordWriter) Close() error {
+	err := w.Flush()
+	if err != nil {
+		return err
+	}
+
+	const footer = `  </tbody>
+</table>
+
+</body>
+</html>
+`
+
+	_, err = w.out.Write([]byte(footer))
+	return errz.Err(err)
+}
+
+func (w *recordWriter) writeRecord(rec sqlz.Record) error {
+	w.buf.WriteString("    <tr>\n")
+
+	var s string
+	for i, field := range rec {
+		w.buf.WriteString("      <td>")
+
+		switch val := field.(type) {
+		default:
+			s = html.EscapeString(fmt.Sprintf("%v", val))
+			// should never happen
+		case nil:
+			// nil is rendered as empty string, which this cell already is
+		case *int64:
+			s = strconv.FormatInt(*val, 10)
+		case *string:
+			s = html.EscapeString(*val)
+		case *bool:
+			s = strconv.FormatBool(*val)
+		case *float64:
+			s = stringz.FormatFloat(*val)
+		case *[]byte:
+			s = base64.StdEncoding.EncodeToString(*val)
+		case *time.Time:
+			switch w.recMeta[i].Kind() {
+			default:
+				s = val.Format(stringz.DatetimeFormat)
+			case sqlz.KindTime:
+				s = val.Format(stringz.TimeFormat)
+			case sqlz.KindDate:
+				s = val.Format(stringz.DateFormat)
+			}
+		}
+
+		w.buf.WriteString(s)
+		w.buf.WriteString("</td>\n")
+	}
+
+	w.buf.WriteString("    </tr>\n")
+
+	return nil
+}
+
+// WriteRecords implements output.RecordWriter.
+func (w *recordWriter) WriteRecords(recs []sqlz.Record) error {
+	var err error
+	for _, rec := range recs {
+		err = w.writeRecord(rec)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cli/output/htmlw/htmlw_test.go

@@ -0,0 +1,44 @@
+package htmlw_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/neilotoole/sq/cli/output/htmlw"
+	"github.com/neilotoole/sq/testh"
+	"github.com/neilotoole/sq/testh/sakila"
+)
+
+func TestRecordWriter(t *testing.T) {
+	testCases := []struct {
+		name     string
+		numRecs  int
+		fixtPath string
+	}{
+		{name: "actor_0", numRecs: 0, fixtPath: "testdata/actor_0_rows.html"},
+		{name: "actor_3", numRecs: 3, fixtPath: "testdata/actor_3_rows.html"},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			recMeta, recs := testh.RecordsFromTbl(t, sakila.SL3, sakila.TblActor)
+			recs = recs[0:tc.numRecs]
+
+			buf := &bytes.Buffer{}
+			w := htmlw.NewRecordWriter(buf)
+			require.NoError(t, w.Open(recMeta))
+
+			require.NoError(t, w.WriteRecords(recs))
+			require.NoError(t, w.Close())
+
+			want, err := ioutil.ReadFile(tc.fixtPath)
+			require.NoError(t, err)
+			require.Equal(t, string(want), buf.String())
+		})
+	}
+}

cli/output/htmlw/testdata/actor_0_rows.html

@@ -0,0 +1,28 @@
+<!doctype html>
+<html>
+<head>
+<title>sq output</title>
+</head>
+<body>
+
+<table>
+  <colgroup>
+    <col class="kind-int" />
+    <col class="kind-text" />
+    <col class="kind-text" />
+    <col class="kind-datetime" />
+  </colgroup>
+  <thead>
+    <tr>
+      <th scope="col">actor_id</th>
+      <th scope="col">first_name</th>
+      <th scope="col">last_name</th>
+      <th scope="col">last_update</th>
+    </tr>
+  </thead>
+  <tbody>
+  </tbody>
+</table>
+
+</body>
+</html>

cli/output/htmlw/testdata/actor_3_rows.html

@@ -0,0 +1,46 @@
+<!doctype html>
+<html>
+<head>
+<title>sq output</title>
+</head>
+<body>
+
+<table>
+  <colgroup>
+    <col class="kind-int" />
+    <col class="kind-text" />
+    <col class="kind-text" />
+    <col class="kind-datetime" />
+  </colgroup>
+  <thead>
+    <tr>
+      <th scope="col">actor_id</th>
+      <th scope="col">first_name</th>
+      <th scope="col">last_name</th>
+      <th scope="col">last_update</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td>1</td>
+      <td>PENELOPE</td>
+      <td>GUINESS</td>
+      <td>2020-06-11T02:50:54Z</td>
+    </tr>
+    <tr>
+      <td>2</td>
+      <td>NICK</td>
+      <td>WAHLBERG</td>
+      <td>2020-06-11T02:50:54Z</td>
+    </tr>
+    <tr>
+      <td>3</td>
+      <td>ED</td>
+      <td>CHASE</td>
+      <td>2020-06-11T02:50:54Z</td>
+    </tr>
+  </tbody>
+</table>
+
+</body>
+</html>

cli/output/jsonw/README.md

@@ -0,0 +1,57 @@
+# Package `jsonw`
+
+Package `jsonw` implements JSON output writers.
+
+Note that there are three implementations of `output.RecordWriter`.
+
+- `NewStdRecordWriter` returns a writer that outputs in "standard" JSON.
+- `NewArrayRecordWriter` outputs each record on its own line as an element of a JSON array.
+- `NewObjectRecordWriter` outputs each record as a JSON object on its own line.
+
+These `RecordWriter`s correspond to the `--json`, `--jsona`, and `--jsonl` flags (where `jsonl` means "JSON Lines"). There are also other writer implementations, such as an `output.ErrorWriter` and an `output.MetadataWriter`.
+
+
+#### Standard JSON `--json`:
+
+```json
+[
+  {
+    "actor_id": 1,
+    "first_name": "PENELOPE",
+    "last_name": "GUINESS",
+    "last_update": "2020-06-11T02:50:54Z"
+  },
+  {
+    "actor_id": 2,
+    "first_name": "NICK",
+    "last_name": "WAHLBERG",
+    "last_update": "2020-06-11T02:50:54Z"
+  }
+]
+```
+
+#### JSON Array `--jsona`:
+
+```json
+[1, "PENELOPE", "GUINESS", "2020-06-11T02:50:54Z"]
+[2, "NICK", "WAHLBERG", "2020-06-11T02:50:54Z"]
+```
+
+#### Object aka JSON Lines `--jsonl`: 
+
+```json
+{"actor_id": 1, "first_name": "PENELOPE", "last_name": "GUINESS", "last_update": "2020-06-11T02:50:54Z"}
+{"actor_id": 2, "first_name": "NICK", "last_name": "WAHLBERG", "last_update": "2020-06-11T02:50:54Z"}
+```
+
+## Notes
+
+At the time of development there was not a JSON encoder library available that provided the functionality that `sq` required. These requirements:
+
+- Optional colorization
+- Optional pretty-printing (indentation, spacing)
+- Preservation of the order of record fields (columns).
+
+For the `RecordWriter` implementations, given the known "flat" structure of a record, it was relatively straightforward to create custom writers for each type of JSON output.
+
+For general-purpose JSON output (such as metadata output), it was necessary to modify an existing JSON library to provide colorization (and also on-the-fly indentation). After benchmarking, the [segmentio.io encoder](https://github.com/segmentio/encoding) was selected as the base library. Rather than a separate forked project (which probably would not make sense to ever merge with its parent project), the modified encoder is found in `jsonw/internal/jcolorenc`. 

cli/output/jsonw/encode.go

@@ -0,0 +1,397 @@
+package jsonw
+
+import (
+	"encoding/base64"
+	"strconv"
+	"time"
+	"unicode/utf8"
+
+	"github.com/neilotoole/sq/cli/output"
+	"github.com/neilotoole/sq/cli/output/jsonw/internal"
+	"github.com/neilotoole/sq/libsq/errz"
+	"github.com/neilotoole/sq/libsq/sqlz"
+	"github.com/neilotoole/sq/libsq/stringz"
+)
+
+// monoEncoder provides methods for encoding JSON values
+// without colorization (that is, in monochrome).
+type monoEncoder struct {
+}
+
+func (e monoEncoder) encodeTime(b []byte, v interface{}) ([]byte, error) {
+	return e.doEncodeTime(b, v, stringz.TimeFormat)
+}
+
+func (e monoEncoder) encodeDatetime(b []byte, v interface{}) ([]byte, error) {
+	return e.doEncodeTime(b, v, stringz.DatetimeFormat)
+}
+
+func (e monoEncoder) encodeDate(b []byte, v interface{}) ([]byte, error) {
+	return e.doEncodeTime(b, v, stringz.DateFormat)
+}
+
+func (e monoEncoder) doEncodeTime(b []byte, v interface{}, layout string) ([]byte, error) {
+	switch v := v.(type) {
+	case nil:
+		return append(b, "null"...), nil
+	case *time.Time:
+		b = append(b, '"')
+		b = v.AppendFormat(b, layout)
+		b = append(b, '"')
+		return b, nil
+	case *string:
+		// If we've got a string, assume it's in the correct format
+		return encodeString(b, *v, false)
+	default:
+		return b, errz.Errorf("unsupported time type %T: %v", v, v)
+	}
+}
+
+func (e monoEncoder) encodeAny(b []byte, v interface{}) ([]byte, error) {
+	switch v := v.(type) {
+	default:
+		return b, errz.Errorf("unexpected record field type %T: %#v", v, v)
+
+	case nil:
+		return append(b, "null"...), nil
+
+	case *int64:
+		return strconv.AppendInt(b, *v, 10), nil
+
+	case *float64:
+		return append(b, stringz.FormatFloat(*v)...), nil
+
+	case *bool:
+		return strconv.AppendBool(b, *v), nil
+
+	case *[]byte:
+		var err error
+		b, err = encodeBytes(b, *v)
+		if err != nil {
+			return b, errz.Err(err)
+		}
+		return b, nil
+
+	case *string:
+		var err error
+		b, err = encodeString(b, *v, false)
+		if err != nil {
+			return b, errz.Err(err)
+		}
+		return b, nil
+
+	case *time.Time:
+		b = append(b, '"')
+		b = v.AppendFormat(b, stringz.DatetimeFormat)
+		b = append(b, '"')
+		return b, nil
+	}
+}
+
+// colorEncoder provides methods for encoding JSON values
+// with color.
+type colorEncoder struct {
+	clrs internal.Colors
+}
+
+func (e *colorEncoder) encodeTime(b []byte, v interface{}) ([]byte, error) {
+	return e.doEncodeTime(b, v, stringz.TimeFormat)
+}
+
+func (e *colorEncoder) encodeDatetime(b []byte, v interface{}) ([]byte, error) {
+	return e.doEncodeTime(b, v, stringz.DatetimeFormat)
+}
+
+func (e *colorEncoder) encodeDate(b []byte, v interface{}) ([]byte, error) {
+	return e.doEncodeTime(b, v, stringz.DateFormat)
+}
+
+func (e *colorEncoder) doEncodeTime(b []byte, v interface{}, layout string) ([]byte, error) {
+	start := len(b)
+
+	switch v := v.(type) {
+	case nil:
+		return e.clrs.AppendNull(b), nil
+	case *time.Time:
+		b = append(b, e.clrs.Time.Prefix...)
+		b = append(b, '"')
+		b = v.AppendFormat(b, layout)
+		b = append(b, '"')
+		b = append(b, e.clrs.Time.Suffix...)
+		return b, nil
+	case *string:
+		// If we've got a string, assume it's in the correct format
+		b = append(b, e.clrs.Time.Prefix...)
+		var err error
+		b, err = encodeString(b, *v, false)
+		if err != nil {
+			return b[0:start], err
+		}
+		b = append(b, e.clrs.Time.Suffix...)
+		return b, nil
+	default:
+		return b, errz.Errorf("unsupported time type %T: %v", v, v)
+	}
+}
+
+func (e *colorEncoder) encodeAny(b []byte, v interface{}) ([]byte, error) {
+	switch v := v.(type) {
+	default:
+		return b, errz.Errorf("unexpected record field type %T: %#v", v, v)
+
+	case nil:
+		return e.clrs.AppendNull(b), nil
+
+	case *int64:
+		b = append(b, e.clrs.Number.Prefix...)
+		b = strconv.AppendInt(b, *v, 10)
+		return append(b, e.clrs.Number.Suffix...), nil
+
+	case *float64:
+		b = append(b, e.clrs.Number.Prefix...)
+		b = append(b, stringz.FormatFloat(*v)...)
+		return append(b, e.clrs.Number.Suffix...), nil
+
+	case *bool:
+		b = append(b, e.clrs.Bool.Prefix...)
+		b = strconv.AppendBool(b, *v)
+		return append(b, e.clrs.Bool.Suffix...), nil
+
+	case *[]byte:
+		var err error
+		b = append(b, e.clrs.Bytes.Prefix...)
+		b, err = encodeBytes(b, *v)
+		if err != nil {
+			return b, errz.Err(err)
+		}
+		b = append(b, e.clrs.Bytes.Suffix...)
+		return b, nil
+
+	case *string:
+		b = append(b, e.clrs.String.Prefix...)
+		var err error
+		b, err = encodeString(b, *v, false)
+		if err != nil {
+			return b, errz.Err(err)
+		}
+		return append(b, e.clrs.String.Suffix...), nil
+
+	case *time.Time:
+		b = append(b, e.clrs.Time.Prefix...)
+		b = append(b, '"')
+		b = v.AppendFormat(b, stringz.DatetimeFormat)
+		b = append(b, '"')
+		return append(b, e.clrs.Time.Suffix...), nil
+	}
+}
+
+// punc holds the byte values of JSON punctuation chars
+// like left bracket "[", right brace "}" etc. When
+// colorizing, these values will include the terminal color codes.
+type punc struct {
+	comma    []byte
+	colon    []byte
+	lBrace   []byte
+	rBrace   []byte
+	lBracket []byte
+	rBracket []byte
+	// null is also included in punc just for convenience
+	null []byte
+}
+
+func newPunc(fm *output.Formatting) punc {
+	var p punc
+
+	if fm == nil || fm.IsMonochrome() || !fm.Pretty {
+		p.comma = append(p.comma, ',')
+		p.colon = append(p.colon, ':')
+		p.lBrace = append(p.lBrace, '{')
+		p.rBrace = append(p.rBrace, '}')
+		p.lBracket = append(p.lBracket, '[')
+		p.rBracket = append(p.rBracket, ']')
+		p.null = append(p.null, "null"...)
+		return p
+	}
+
+	clrs := internal.NewColors(fm)
+	p.comma = clrs.AppendPunc(p.comma, ',')
+	p.colon = clrs.AppendPunc(p.colon, ':')
+	p.lBrace = clrs.AppendPunc(p.lBrace, '{')
+	p.rBrace = clrs.AppendPunc(p.rBrace, '}')
+	p.lBracket = clrs.AppendPunc(p.lBracket, '[')
+	p.rBracket = clrs.AppendPunc(p.rBracket, ']')
+	p.null = clrs.AppendNull(p.null)
+	return p
+}
+
+func getFieldEncoders(recMeta sqlz.RecordMeta, fm *output.Formatting) []func(b []byte, v interface{}) ([]byte, error) {
+	encodeFns := make([]func(b []byte, v interface{}) ([]byte, error), len(recMeta))
+
+	if fm.IsMonochrome() {
+		enc := monoEncoder{}
+
+		for i := 0; i < len(recMeta); i++ {
+			switch recMeta[i].Kind() {
+			case sqlz.KindTime:
+				encodeFns[i] = enc.encodeTime
+			case sqlz.KindDate:
+				encodeFns[i] = enc.encodeDate
+			case sqlz.KindDatetime:
+				encodeFns[i] = enc.encodeDatetime
+			default:
+				encodeFns[i] = enc.encodeAny
+			}
+		}
+
+		return encodeFns
+	}
+
+	clrs := internal.NewColors(fm)
+
+	// Else, we want color encoders
+	enc := &colorEncoder{clrs: clrs}
+	for i := 0; i < len(recMeta); i++ {
+		switch recMeta[i].Kind() {
+		case sqlz.KindTime:
+			encodeFns[i] = enc.encodeTime
+		case sqlz.KindDate:
+			encodeFns[i] = enc.encodeDate
+		case sqlz.KindDatetime:
+			encodeFns[i] = enc.encodeDatetime
+		default:
+			encodeFns[i] = enc.encodeAny
+		}
+	}
+
+	return encodeFns
+}
+
+// encodeString encodes s, appending to b and returning
+// the resulting []byte.
+func encodeString(b []byte, s string, escapeHTML bool) ([]byte, error) { //nolint:unparam
+	// This function is copied from the segment.io JSON encoder.
+	const hex = "0123456789abcdef"
+
+	i := 0
+	j := 0
+
+	b = append(b, '"')
+
+	for j < len(s) {
+		c := s[j]
+
+		if c >= 0x20 && c <= 0x7f && c != '\\' && c != '"' && (!escapeHTML || (c != '<' && c != '>' && c != '&')) {
+			// fast path: most of the time, printable ascii characters are used
+			j++
+			continue
+		}
+
+		switch c {
+		case '\\', '"':
+			b = append(b, s[i:j]...)
+			b = append(b, '\\', c)
+			i = j + 1
+			j++
+			continue
+
+		case '\n':
+			b = append(b, s[i:j]...)
+			b = append(b, '\\', 'n')
+			i = j + 1
+			j++
+			continue
+
+		case '\r':
+			b = append(b, s[i:j]...)
+			b = append(b, '\\', 'r')
+			i = j + 1
+			j++
+			continue
+
+		case '\t':
+			b = append(b, s[i:j]...)
+			b = append(b, '\\', 't')
+			i = j + 1
+			j++
+			continue
+
+		case '<', '>', '&':
+			b = append(b, s[i:j]...)
+			b = append(b, `\u00`...)
+			b = append(b, hex[c>>4], hex[c&0xF])
+			i = j + 1
+			j++
+			continue
+		}
+
+		// This encodes bytes < 0x20 except for \t, \n and \r.
+		if c < 0x20 {
+			b = append(b, s[i:j]...)
+			b = append(b, `\u00`...)
+			b = append(b, hex[c>>4], hex[c&0xF])
+			i = j + 1
+			j++
+			continue
+		}
+
+		r, size := utf8.DecodeRuneInString(s[j:])
+
+		if r == utf8.RuneError && size == 1 {
+			b = append(b, s[i:j]...)
+			b = append(b, `\ufffd`...)
+			i = j + size
+			j += size
+			continue
+		}
+
+		switch r {
+		case '\u2028', '\u2029':
+			// U+2028 is LINE SEPARATOR.
+			// U+2029 is PARAGRAPH SEPARATOR.
+			// They are both technically valid characters in JSON strings,
+			// but don't work in JSONP, which has to be evaluated as JavaScript,
+			// and can lead to security holes there. It is valid JSON to
+			// escape them, so we do so unconditionally.
+			// See http://timelessrepo.com/json-isnt-a-javascript-subset for discussion.
+			b = append(b, s[i:j]...)
+			b = append(b, `\u202`...)
+			b = append(b, hex[r&0xF])
+			i = j + size
+			j += size
+			continue
+		}
+
+		j += size
+	}
+
+	b = append(b, s[i:]...)
+	b = append(b, '"')
+	return b, nil
+}
+
+// encodeBytes encodes v in base64 and appends to b, returning
+// the resulting slice.
+func encodeBytes(b, v []byte) ([]byte, error) { // nolint:unparam
+	// This function is copied from the segment.io JSON encoder.
+
+	if v == nil {
+		return append(b, "null"...), nil
+	}
+
+	n := base64.StdEncoding.EncodedLen(len(v)) + 2
+
+	if avail := cap(b) - len(b); avail < n {
+		newB := make([]byte, cap(b)+(n-avail))
+		copy(newB, b)
+		b = newB[:len(b)]
+	}
+
+	i := len(b)
+	j := len(b) + n
+
+	b = b[:j]
+	b[i] = '"'
+	base64.StdEncoding.Encode(b[i+1:j-1], v)
+	b[j-1] = '"'
+	return b, nil
+}

cli/output/jsonw/errorw.go

@@ -0,0 +1,43 @@
+package jsonw
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/neilotoole/lg"
+
+	"github.com/neilotoole/sq/cli/output"
+)
+
+// errorWriter implements output.ErrorWriter.
+type errorWriter struct {
+	log lg.Log
+	out io.Writer
+	fm  *output.Formatting
+}
+
+// NewErrorWriter returns an output.ErrorWriter that outputs in JSON.
+func NewErrorWriter(log lg.Log, out io.Writer, fm *output.Formatting) output.ErrorWriter {
+	return &errorWriter{log: log, out: out, fm: fm}
+}
+
+// Error implements output.ErrorWriter.
+func (w *errorWriter) Error(err error) {
+	const tplNoPretty = "{%s: %s}"
+	tplPretty := "{\n" + w.fm.Indent + "%s" + ": %s\n}"
+
+	b, err2 := encodeString(nil, err.Error(), false)
+	w.log.WarnIfError(err2)
+
+	key := w.fm.Key.Sprint(`"error"`)
+	val := w.fm.Error.Sprint(string(b)) // trim the newline
+
+	var s string
+	if w.fm.Pretty {
+		s = fmt.Sprintf(tplPretty, key, val)
+	} else {
+		s = fmt.Sprintf(tplNoPretty, key, val)
+	}
+
+	fmt.Fprintln(w.out, s)
+}

cli/output/jsonw/internal/benchmark_test.go

@@ -0,0 +1,125 @@
+package internal_test
+
+import (
+	stdj "encoding/json"
+	"io/ioutil"
+	"testing"
+
+	segmentj "github.com/segmentio/encoding/json"
+
+	jcolorenc "github.com/neilotoole/sq/cli/output/jsonw/internal/jcolorenc"
+
+	"github.com/neilotoole/sq/testh"
+	"github.com/neilotoole/sq/testh/sakila"
+)
+
+// The following benchmarks compare the encoding performance
+// of JSON encoders. These are:
+//
+// - stdj: the std lib json encoder
+// - segmentj: the encoder by segment.io
+// - jcolorenc: sq's fork of segmentj that supports color
+
+func BenchmarkStdj(b *testing.B) {
+	_, recs := testh.RecordsFromTbl(b, sakila.SL3, "payment")
+	b.ResetTimer()
+
+	for n := 0; n < b.N; n++ {
+		enc := stdj.NewEncoder(ioutil.Discard)
+		enc.SetEscapeHTML(false)
+
+		for i := range recs {
+			err := enc.Encode(recs[i])
+			if err != nil {
+				b.Error(err)
+			}
+		}
+	}
+}
+
+func BenchmarkStdj_Indent(b *testing.B) {
+	_, recs := testh.RecordsFromTbl(b, sakila.SL3, "payment")
+	b.ResetTimer()
+
+	for n := 0; n < b.N; n++ {
+		enc := stdj.NewEncoder(ioutil.Discard)
+		enc.SetEscapeHTML(false)
+		enc.SetIndent("", "  ")
+
+		for i := range recs {
+			err := enc.Encode(recs[i])
+			if err != nil {
+				b.Error(err)
+			}
+		}
+	}
+}
+
+func BenchmarkSegmentj(b *testing.B) {
+	_, recs := testh.RecordsFromTbl(b, sakila.SL3, "payment")
+	b.ResetTimer()
+
+	for n := 0; n < b.N; n++ {
+		enc := segmentj.NewEncoder(ioutil.Discard)
+		enc.SetEscapeHTML(false)
+
+		for i := range recs {
+			err := enc.Encode(recs[i])
+			if err != nil {
+				b.Error(err)
+			}
+		}
+	}
+}
+
+func BenchmarkSegmentj_Indent(b *testing.B) {
+	_, recs := testh.RecordsFromTbl(b, sakila.SL3, "payment")
+	b.ResetTimer()
+
+	for n := 0; n < b.N; n++ {
+		enc := segmentj.NewEncoder(ioutil.Discard)
+		enc.SetEscapeHTML(false)
+		enc.SetIndent("", "  ")
+
+		for i := range recs {
+			err := enc.Encode(recs[i])
+			if err != nil {
+				b.Error(err)
+			}
+		}
+	}
+}
+func BenchmarkJColorEnc(b *testing.B) {
+	_, recs := testh.RecordsFromTbl(b, sakila.SL3, "payment")
+	b.ResetTimer()
+
+	for n := 0; n < b.N; n++ {
+		enc := jcolorenc.NewEncoder(ioutil.Discard)
+		enc.SetEscapeHTML(false)
+
+		for i := range recs {
+			err := enc.Encode(recs[i])
+			if err != nil {
+				b.Error(err)
+			}
+		}
+	}
+}
+
+func BenchmarkJColorEnc_Indent(b *testing.B) {
+	_, recs := testh.RecordsFromTbl(b, sakila.SL3, "payment")
+	b.ResetTimer()
+
+	for n := 0; n < b.N; n++ {
+		enc := jcolorenc.NewEncoder(ioutil.Discard)
+		enc.SetEscapeHTML(false)
+		enc.SetIndent("", "  ")
+
+		for i := range recs {
+			err := enc.Encode(recs[i])
+			if err != nil {
+				b.Error(err)
+			}
+		}
+	}
+}

cli/output/jsonw/internal/internal.go

@@ -0,0 +1,122 @@
+package internal
+
+import (
+	"bytes"
+	"strconv"
+
+	fcolor "github.com/fatih/color"
+
+	"github.com/neilotoole/sq/cli/output"
+)
+
+// Colors encapsulates colorization of JSON output.
+type Colors struct {
+	Null   Color
+	Bool   Color
+	Number Color
+	String Color
+	Key    Color
+	Bytes  Color
+	Time   Color
+	Punc   Color
+}
+
+// AppendNull appends a colorized "null" to b.
+func (c Colors) AppendNull(b []byte) []byte {
+	b = append(b, c.Null.Prefix...)
+	b = append(b, "null"...)
+	return append(b, c.Null.Suffix...)
+}
+
+// AppendBool appends the colorized bool v to b.
+func (c Colors) AppendBool(b []byte, v bool) []byte {
+	b = append(b, c.Bool.Prefix...)
+
+	if v {
+		b = append(b, "true"...)
+	} else {
+		b = append(b, "false"...)
+	}
+
+	return append(b, c.Bool.Suffix...)
+}
+
+// AppendKey appends the colorized key v to b.
+func (c Colors) AppendKey(b []byte, v []byte) []byte {
+	b = append(b, c.Key.Prefix...)
+	b = append(b, v...)
+	return append(b, c.Key.Suffix...)
+}
+
+// AppendInt64 appends the colorized int64 v to b.
+func (c Colors) AppendInt64(b []byte, v int64) []byte {
+	b = append(b, c.Number.Prefix...)
+	b = strconv.AppendInt(b, v, 10)
+	return append(b, c.Number.Suffix...)
+}
+
+// AppendUint64 appends the colorized uint64 v to b.
+func (c Colors) AppendUint64(b []byte, v uint64) []byte {
+	b = append(b, c.Number.Prefix...)
+	b = strconv.AppendUint(b, v, 10)
+	return append(b, c.Number.Suffix...)
+}
+
+// AppendPunc appends the colorized punctuation mark v to b.
+func (c Colors) AppendPunc(b []byte, v byte) []byte {
+	b = append(b, c.Punc.Prefix...)
+	b = append(b, v)
+	return append(b, c.Punc.Suffix...)
+}
+
+// Color is used to render terminal colors. The Prefix
+// value is written, then the actual value, then the suffix.
+type Color struct {
+	// Prefix is the terminal color code prefix to print before the value (may be empty).
+	Prefix []byte
+
+	// Suffix is the terminal color code suffix to print after the value (may be empty).
+	Suffix []byte
+}
+
+// newColor creates a Color instance from a fatih/color instance.
+func newColor(c *fcolor.Color) Color {
+	// Dirty conversion function ahead: print
+	// a space using c, then grab the bytes printed
+	// before and after the space, and those are the
+	// bytes we need for the prefix and suffix.
+
+	if c == nil {
+		return Color{}
+	}
+
+	// Make a copy because the pkg-level color.NoColor could be false.
+	c2 := *c
+	c2.EnableColor()
+
+	b := []byte(c2.Sprint(" "))
+	i := bytes.IndexByte(b, ' ')
+	if i <= 0 {
+		return Color{}
+	}
+
+	return Color{Prefix: b[:i], Suffix: b[i+1:]}
+}
+
+// NewColors builds a Colors instance from a Formatting instance.
+func NewColors(fm *output.Formatting) Colors {
+	if fm == nil || fm.IsMonochrome() {
+		return Colors{}
+	}
+
+	return Colors{
+		Null:   newColor(fm.Null),
+		Bool:   newColor(fm.Bool),
+		Bytes:  newColor(fm.Bytes),
+		Number: newColor(fm.Number),
+		String: newColor(fm.String),
+		Time:   newColor(fm.Datetime),
+		Key:    newColor(fm.Key),
+		Punc:   newColor(fm.Punc),
+	}
+}

cli/output/jsonw/internal/internal_test.go

@@ -0,0 +1,497 @@
+package internal_test
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	stdjson "encoding/json"
+
+	"github.com/neilotoole/sq/cli/output"
+	"github.com/neilotoole/sq/cli/output/jsonw/internal"
+	jcolorenc "github.com/neilotoole/sq/cli/output/jsonw/internal/jcolorenc"
+)
+
+// Encoder encapsulates the methods of a JSON encoder.
+type Encoder interface {
+	Encode(v interface{}) error
+	SetEscapeHTML(on bool)
+	SetIndent(prefix, indent string)
+}
+
+var _ Encoder = (*jcolorenc.Encoder)(nil)
+
+func TestEncode(t *testing.T) {
+	testCases := []struct {
+		name    string
+		pretty  bool
+		color   bool
+		sortMap bool
+		v       interface{}
+		want    string
+	}{
+		{name: "nil", pretty: true, v: nil, want: "null\n"},
+		{name: "slice_empty", pretty: true, v: []int{}, want: "[]\n"},
+		{name: "slice_1_pretty", pretty: true, v: []interface{}{1}, want: "[\n  1\n]\n"},
+		{name: "slice_1_no_pretty", v: []interface{}{1}, want: "[1]\n"},
+		{name: "slice_2_pretty", pretty: true, v: []interface{}{1, true}, want: "[\n  1,\n  true\n]\n"},
+		{name: "slice_2_no_pretty", v: []interface{}{1, true}, want: "[1,true]\n"},
+		{name: "map_int_empty", pretty: true, v: map[string]int{}, want: "{}\n"},
+		{name: "map_interface_empty", pretty: true, v: map[string]interface{}{}, want: "{}\n"},
+		{name: "map_interface_empty_sorted", pretty: true, sortMap: true, v: map[string]interface{}{}, want: "{}\n"},
+		{name: "map_1_pretty", pretty: true, sortMap: true, v: map[string]interface{}{"one": 1}, want: "{\n  \"one\": 1\n}\n"},
+		{name: "map_1_no_pretty", sortMap: true, v: map[string]interface{}{"one": 1}, want: "{\"one\":1}\n"},
+		{name: "map_2_pretty", pretty: true, sortMap: true, v: map[string]interface{}{"one": 1, "two": 2}, want: "{\n  \"one\": 1,\n  \"two\": 2\n}\n"},
+		{name: "map_2_no_pretty", sortMap: true, v: map[string]interface{}{"one": 1, "two": 2}, want: "{\"one\":1,\"two\":2}\n"},
+		{name: "tinystruct", pretty: true, v: TinyStruct{FBool: true}, want: "{\n  \"f_bool\": true\n}\n"},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			fm := output.NewFormatting()
+			fm.Pretty = tc.pretty
+			fm.EnableColor(tc.color)
+
+			buf := &bytes.Buffer{}
+			enc := jcolorenc.NewEncoder(buf)
+			enc.SetEscapeHTML(false)
+			enc.SetSortMapKeys(tc.sortMap)
+			enc.SetColors(internal.NewColors(fm))
+
+			if fm.Pretty {
+				enc.SetIndent("", fm.Indent)
+			}
+
+			require.NoError(t, enc.Encode(tc.v))
+			require.True(t, stdjson.Valid(buf.Bytes()))
+			require.Equal(t, tc.want, buf.String())
+		})
+	}
+}
+
+func TestEncode_Slice(t *testing.T) {
+	testCases := []struct {
+		name   string
+		pretty bool
+		color  bool
+		v      []interface{}
+		want   string
+	}{
+		{name: "nil", pretty: true, v: nil, want: "null\n"},
+		{name: "empty", pretty: true, v: []interface{}{}, want: "[]\n"},
+		{name: "one", pretty: true, v: []interface{}{1}, want: "[\n  1\n]\n"},
+		{name: "two", pretty: true, v: []interface{}{1, true}, want: "[\n  1,\n  true\n]\n"},
+		{name: "three", pretty: true, v: []interface{}{1, true, "hello"}, want: "[\n  1,\n  true,\n  \"hello\"\n]\n"},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			fm := output.NewFormatting()
+			fm.Pretty = tc.pretty
+			fm.EnableColor(tc.color)
+
+			buf := &bytes.Buffer{}
+			enc := jcolorenc.NewEncoder(buf)
+			enc.SetEscapeHTML(false)
+			enc.SetColors(internal.NewColors(fm))
+			if fm.Pretty {
+				enc.SetIndent("", "  ")
+			}
+
+			require.NoError(t, enc.Encode(tc.v))
+			require.True(t, stdjson.Valid(buf.Bytes()))
+			require.Equal(t, tc.want, buf.String())
+		})
+	}
+}
+
+func TestEncode_SmallStruct(t *testing.T) {
+	v := SmallStruct{
+		FInt:   7,
+		FSlice: []interface{}{64, true},
+		FMap: map[string]interface{}{
+			"m_float64": 64.64,
+			"m_string":  "hello",
+		},
+		FTinyStruct: TinyStruct{FBool: true},
+		FString:     "hello",
+	}
+
+	testCases := []struct {
+		pretty bool
+		color  bool
+		want   string
+	}{
+		{pretty: false, color: false, want: "{\"f_int\":7,\"f_slice\":[64,true],\"f_map\":{\"m_float64\":64.64,\"m_string\":\"hello\"},\"f_tinystruct\":{\"f_bool\":true},\"f_string\":\"hello\"}\n"},
+		{pretty: true, color: false, want: "{\n  \"f_int\": 7,\n  \"f_slice\": [\n    64,\n    true\n  ],\n  \"f_map\": {\n    \"m_float64\": 64.64,\n    \"m_string\": \"hello\"\n  },\n  \"f_tinystruct\": {\n    \"f_bool\": true\n  },\n  \"f_string\": \"hello\"\n}\n"},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(fmt.Sprintf("pretty_%v__color_%v", tc.pretty, tc.color), func(t *testing.T) {
+			fm := output.NewFormatting()
+			fm.Pretty = tc.pretty
+			fm.EnableColor(tc.color)
+
+			buf := &bytes.Buffer{}
+			enc := jcolorenc.NewEncoder(buf)
+			enc.SetEscapeHTML(false)
+			enc.SetSortMapKeys(true)
+			enc.SetColors(internal.NewColors(fm))
+
+			if fm.Pretty {
+				enc.SetIndent("", "  ")
+			}
+
+			require.NoError(t, enc.Encode(v))
+			require.True(t, stdjson.Valid(buf.Bytes()))
+			require.Equal(t, tc.want, buf.String())
+		})
+	}
+}
+
+func TestEncode_Map_Nested(t *testing.T) {
+	v := map[string]interface{}{
+		"m_bool1": true,
+		"m_nest1": map[string]interface{}{
+			"m_nest1_bool": true,
+			"m_nest2": map[string]interface{}{
+				"m_nest2_bool": true,
+				"m_nest3": map[string]interface{}{
+					"m_nest3_bool": true,
+				},
+			},
+		},
+		"m_string1": "hello",
+	}
+
+	testCases := []struct {
+		pretty bool
+		color  bool
+		want   string
+	}{
+		{pretty: false, want: "{\"m_bool1\":true,\"m_nest1\":{\"m_nest1_bool\":true,\"m_nest2\":{\"m_nest2_bool\":true,\"m_nest3\":{\"m_nest3_bool\":true}}},\"m_string1\":\"hello\"}\n"},
+		{pretty: true, want: "{\n  \"m_bool1\": true,\n  \"m_nest1\": {\n    \"m_nest1_bool\": true,\n    \"m_nest2\": {\n      \"m_nest2_bool\": true,\n      \"m_nest3\": {\n        \"m_nest3_bool\": true\n      }\n    }\n  },\n  \"m_string1\": \"hello\"\n}\n"},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(fmt.Sprintf("pretty_%v__color_%v", tc.pretty, tc.color), func(t *testing.T) {
+			fm := output.NewFormatting()
+			fm.Pretty = tc.pretty
+			fm.EnableColor(tc.color)
+
+			buf := &bytes.Buffer{}
+			enc := jcolorenc.NewEncoder(buf)
+			enc.SetEscapeHTML(false)
+			enc.SetSortMapKeys(true)
+			enc.SetColors(internal.NewColors(fm))
+
+			if fm.Pretty {
+				enc.SetIndent("", "  ")
+			}
+
+			require.NoError(t, enc.Encode(v))
+			require.True(t, stdjson.Valid(buf.Bytes()))
+			require.Equal(t, tc.want, buf.String())
+		})
+	}
+}
+
+// TestEncode_Map_StringNotInterface tests maps with a string key
+// but the value type is not interface{}.
+// For example, map[string]bool. This test is necessary because the
+// encoder has a fast path for map[string]interface{}
+func TestEncode_Map_StringNotInterface(t *testing.T) {
+	testCases := []struct {
+		pretty  bool
+		color   bool
+		sortMap bool
+		v       map[string]bool
+		want    string
+	}{
+		{pretty: false, sortMap: true, v: map[string]bool{}, want: "{}\n"},
+		{pretty: false, sortMap: false, v: map[string]bool{}, want: "{}\n"},
+		{pretty: true, sortMap: true, v: map[string]bool{}, want: "{}\n"},
+		{pretty: true, sortMap: false, v: map[string]bool{}, want: "{}\n"},
+		{pretty: false, sortMap: true, v: map[string]bool{"one": true}, want: "{\"one\":true}\n"},
+		{pretty: false, sortMap: false, v: map[string]bool{"one": true}, want: "{\"one\":true}\n"},
+		{pretty: false, sortMap: true, v: map[string]bool{"one": true, "two": false}, want: "{\"one\":true,\"two\":false}\n"},
+		{pretty: true, sortMap: true, v: map[string]bool{"one": true, "two": false}, want: "{\n  \"one\": true,\n  \"two\": false\n}\n"},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(fmt.Sprintf("size_%d__pretty_%v__color_%v", len(tc.v), tc.pretty, tc.color), func(t *testing.T) {
+			fm := output.NewFormatting()
+			fm.Pretty = tc.pretty
+			fm.EnableColor(tc.color)
+
+			buf := &bytes.Buffer{}
+			enc := jcolorenc.NewEncoder(buf)
+			enc.SetEscapeHTML(false)
+			enc.SetSortMapKeys(tc.sortMap)
+			enc.SetColors(internal.NewColors(fm))
+			if fm.Pretty {
+				enc.SetIndent("", fm.Indent)
+			}
+
+			require.NoError(t, enc.Encode(tc.v))
+			require.True(t, stdjson.Valid(buf.Bytes()))
+			require.Equal(t, tc.want, buf.String())
+		})
+	}
+}
+
+func TestEncode_RawMessage(t *testing.T) {
+	type RawStruct struct {
+		FString string               `json:"f_string"`
+		FRaw    jcolorenc.RawMessage `json:"f_raw"`
+	}
+
+	raw := jcolorenc.RawMessage(`{"one":1,"two":2}`)
+
+	testCases := []struct {
+		name   string
+		pretty bool
+		color  bool
+		v      interface{}
+		want   string
+	}{
+		{name: "empty", pretty: false, v: jcolorenc.RawMessage(`{}`), want: "{}\n"},
+		{name: "no_pretty", pretty: false, v: raw, want: "{\"one\":1,\"two\":2}\n"},
+		{name: "pretty", pretty: true, v: raw, want: "{\n  \"one\": 1,\n  \"two\": 2\n}\n"},
+		{name: "pretty_struct", pretty: true, v: RawStruct{FString: "hello", FRaw: raw}, want: "{\n  \"f_string\": \"hello\",\n  \"f_raw\": {\n    \"one\": 1,\n    \"two\": 2\n  }\n}\n"},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			fm := output.NewFormatting()
+			fm.Pretty = tc.pretty
+			fm.EnableColor(tc.color)
+
+			buf := &bytes.Buffer{}
+			enc := jcolorenc.NewEncoder(buf)
+			enc.SetEscapeHTML(false)
+			enc.SetSortMapKeys(true)
+			enc.SetColors(internal.NewColors(fm))
+			if fm.Pretty {
+				enc.SetIndent("", fm.Indent)
+			}
+
+			err := enc.Encode(tc.v)
+			require.NoError(t, err)
+			require.True(t, stdjson.Valid(buf.Bytes()))
+			require.Equal(t, tc.want, buf.String())
+		})
+	}
+}
+
+// TestEncode_Map_StringNotInterface tests map[string]json.RawMessage.
+// This test is necessary because the encoder has a fast path
+// for map[string]interface{}
+func TestEncode_Map_StringRawMessage(t *testing.T) {
+	raw := jcolorenc.RawMessage(`{"one":1,"two":2}`)
+
+	testCases := []struct {
+		pretty  bool
+		color   bool
+		sortMap bool
+		v       map[string]jcolorenc.RawMessage
+		want    string
+	}{
+		{pretty: false, sortMap: true, v: map[string]jcolorenc.RawMessage{}, want: "{}\n"},
+		{pretty: false, sortMap: false, v: map[string]jcolorenc.RawMessage{}, want: "{}\n"},
+		{pretty: true, sortMap: true, v: map[string]jcolorenc.RawMessage{}, want: "{}\n"},
+		{pretty: true, sortMap: false, v: map[string]jcolorenc.RawMessage{}, want: "{}\n"},
+		{pretty: false, sortMap: true, v: map[string]jcolorenc.RawMessage{"msg1": raw, "msg2": raw}, want: "{\"msg1\":{\"one\":1,\"two\":2},\"msg2\":{\"one\":1,\"two\":2}}\n"},
+		{pretty: true, sortMap: true, v: map[string]jcolorenc.RawMessage{"msg1": raw, "msg2": raw}, want: "{\n  \"msg1\": {\n    \"one\": 1,\n    \"two\": 2\n  },\n  \"msg2\": {\n    \"one\": 1,\n    \"two\": 2\n  }\n}\n"},
+		{pretty: true, sortMap: false, v: map[string]jcolorenc.RawMessage{"msg1": raw}, want: "{\n  \"msg1\": {\n    \"one\": 1,\n    \"two\": 2\n  }\n}\n"},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		name := fmt.Sprintf("size_%d__pretty_%v__color_%v__sort_%v", len(tc.v), tc.pretty, tc.color, tc.sortMap)
+		t.Run(name, func(t *testing.T) {
+			fm := output.NewFormatting()
+			fm.Pretty = tc.pretty
+			fm.EnableColor(tc.color)
+
+			buf := &bytes.Buffer{}
+			enc := jcolorenc.NewEncoder(buf)
+			enc.SetEscapeHTML(false)
+			enc.SetSortMapKeys(tc.sortMap)
+			enc.SetColors(internal.NewColors(fm))
+			if fm.Pretty {
+				enc.SetIndent("", fm.Indent)
+			}
+
+			require.NoError(t, enc.Encode(tc.v))
+			require.True(t, stdjson.Valid(buf.Bytes()))
+			require.Equal(t, tc.want, buf.String())
+		})
+	}
+}
+
+func TestEncode_BigStruct(t *testing.T) {
+	v := newBigStruct()
+
+	testCases := []struct {
+		pretty bool
+		color  bool
+		want   string
+	}{
+		{pretty: false, want: "{\"f_int\":-7,\"f_int8\":-8,\"f_int16\":-16,\"f_int32\":-32,\"f_int64\":-64,\"f_uint\":7,\"f_uint8\":8,\"f_uint16\":16,\"f_uint32\":32,\"f_uint64\":64,\"f_float32\":32.32,\"f_float64\":64.64,\"f_bool\":true,\"f_bytes\":\"aGVsbG8=\",\"f_nil\":null,\"f_string\":\"hello\",\"f_map\":{\"m_bool\":true,\"m_int64\":64,\"m_nil\":null,\"m_smallstruct\":{\"f_int\":7,\"f_slice\":[64,true],\"f_map\":{\"m_float64\":64.64,\"m_string\":\"hello\"},\"f_tinystruct\":{\"f_bool\":true},\"f_string\":\"hello\"},\"m_string\":\"hello\"},\"f_smallstruct\":{\"f_int\":7,\"f_slice\":[64,true],\"f_map\":{\"m_float64\":64.64,\"m_string\":\"hello\"},\"f_tinystruct\":{\"f_bool\":true},\"f_string\":\"hello\"},\"f_interface\":\"hello\",\"f_interfaces\":[64,\"hello\",true]}\n"},
+		{pretty: true, want: "{\n  \"f_int\": -7,\n  \"f_int8\": -8,\n  \"f_int16\": -16,\n  \"f_int32\": -32,\n  \"f_int64\": -64,\n  \"f_uint\": 7,\n  \"f_uint8\": 8,\n  \"f_uint16\": 16,\n  \"f_uint32\": 32,\n  \"f_uint64\": 64,\n  \"f_float32\": 32.32,\n  \"f_float64\": 64.64,\n  \"f_bool\": true,\n  \"f_bytes\": \"aGVsbG8=\",\n  \"f_nil\": null,\n  \"f_string\": \"hello\",\n  \"f_map\": {\n    \"m_bool\": true,\n    \"m_int64\": 64,\n    \"m_nil\": null,\n    \"m_smallstruct\": {\n      \"f_int\": 7,\n      \"f_slice\": [\n        64,\n        true\n      ],\n      \"f_map\": {\n        \"m_float64\": 64.64,\n        \"m_string\": \"hello\"\n      },\n      \"f_tinystruct\": {\n        \"f_bool\": true\n      },\n      \"f_string\": \"hello\"\n    },\n    \"m_string\": \"hello\"\n  },\n  \"f_smallstruct\": {\n    \"f_int\": 7,\n    \"f_slice\": [\n      64,\n      true\n    ],\n    \"f_map\": {\n      \"m_float64\": 64.64,\n      \"m_string\": \"hello\"\n    },\n    \"f_tinystruct\": {\n      \"f_bool\": true\n    },\n    \"f_string\": \"hello\"\n  },\n  \"f_interface\": \"hello\",\n  \"f_interfaces\": [\n    64,\n    \"hello\",\n    true\n  ]\n}\n"},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(fmt.Sprintf("pretty_%v__color_%v", tc.pretty, tc.color), func(t *testing.T) {
+			fm := output.NewFormatting()
+			fm.Pretty = tc.pretty
+			fm.EnableColor(tc.color)
+
+			buf := &bytes.Buffer{}
+			enc := jcolorenc.NewEncoder(buf)
+			enc.SetEscapeHTML(false)
+			enc.SetSortMapKeys(true)
+			enc.SetColors(internal.NewColors(fm))
+
+			if fm.Pretty {
+				enc.SetIndent("", "  ")
+			}
+
+			require.NoError(t, enc.Encode(v))
+			require.True(t, stdjson.Valid(buf.Bytes()))
+			require.Equal(t, tc.want, buf.String())
+		})
+	}
+}
+
+// TestEncode_Map_Not_StringInterface tests map encoding where
+// the map is not map[string]interface{} (for which the encoder
+// has a fast path).
+//
+// NOTE: Currently the encoder is broken wrt colors enabled
+//  for non-string map keys. It's possible we don't actually need
+//  to address this for sq purposes.
+func TestEncode_Map_Not_StringInterface(t *testing.T) {
+	fm := output.NewFormatting()
+	fm.Pretty = true
+	fm.EnableColor(true)
+
+	buf := &bytes.Buffer{}
+	enc := jcolorenc.NewEncoder(buf)
+	enc.SetEscapeHTML(false)
+	enc.SetSortMapKeys(true)
+	enc.SetColors(internal.NewColors(fm))
+	if fm.Pretty {
+		enc.SetIndent("", "  ")
+	}
+
+	v := map[int32]string{
+		0: "zero",
+		1: "one",
+		2: "two",
+	}
+
+	require.NoError(t, enc.Encode(v))
+	require.False(t, stdjson.Valid(buf.Bytes()),
+		"expected to be invalid JSON because the encoder currently doesn't handle maps with non-string keys")
+}
+
+// BigStruct is a big test struct.
+type BigStruct struct {
+	FInt         int                    `json:"f_int"`
+	FInt8        int8                   `json:"f_int8"`
+	FInt16       int16                  `json:"f_int16"`
+	FInt32       int32                  `json:"f_int32"`
+	FInt64       int64                  `json:"f_int64"`
+	FUint        uint                   `json:"f_uint"`
+	FUint8       uint8                  `json:"f_uint8"`
+	FUint16      uint16                 `json:"f_uint16"`
+	FUint32      uint32                 `json:"f_uint32"`
+	FUint64      uint64                 `json:"f_uint64"`
+	FFloat32     float32                `json:"f_float32"`
+	FFloat64     float64                `json:"f_float64"`
+	FBool        bool                   `json:"f_bool"`
+	FBytes       []byte                 `json:"f_bytes"`
+	FNil         interface{}            `json:"f_nil"`
+	FString      string                 `json:"f_string"`
+	FMap         map[string]interface{} `json:"f_map"`
+	FSmallStruct SmallStruct            `json:"f_smallstruct"`
+	FInterface   interface{}            `json:"f_interface"`
+	FInterfaces  []interface{}          `json:"f_interfaces"`
+}
+
+// SmallStruct is a small test struct.
+type SmallStruct struct {
+	FInt        int                    `json:"f_int"`
+	FSlice      []interface{}          `json:"f_slice"`
+	FMap        map[string]interface{} `json:"f_map"`
+	FTinyStruct TinyStruct             `json:"f_tinystruct"`
+	FString     string                 `json:"f_string"`
+}
+
+// Tiny Struct is a tiny test struct.
+type TinyStruct struct {
+	FBool bool `json:"f_bool"`
+}
+
+func newBigStruct() BigStruct {
+	return BigStruct{
+		FInt:     -7,
+		FInt8:    -8,
+		FInt16:   -16,
+		FInt32:   -32,
+		FInt64:   -64,
+		FUint:    7,
+		FUint8:   8,
+		FUint16:  16,
+		FUint32:  32,
+		FUint64:  64,
+		FFloat32: 32.32,
+		FFloat64: 64.64,
+		FBool:    true,
+		FBytes:   []byte("hello"),
+		FNil:     nil,
+		FString:  "hello",
+		FMap: map[string]interface{}{
+			"m_int64":       int64(64),
+			"m_string":      "hello",
+			"m_bool":        true,
+			"m_nil":         nil,
+			"m_smallstruct": newSmallStruct(),
+		},
+		FSmallStruct: newSmallStruct(),
+		FInterface:   interface{}("hello"),
+		FInterfaces:  []interface{}{int64(64), "hello", true},
+	}
+}
+
+func newSmallStruct() SmallStruct {
+	return SmallStruct{
+		FInt:   7,
+		FSlice: []interface{}{64, true},
+		FMap: map[string]interface{}{
+			"m_float64": 64.64,
+			"m_string":  "hello",
+		},
+		FTinyStruct: TinyStruct{FBool: true},
+		FString:     "hello",
+	}
+}

cli/output/jsonw/internal/jcolorenc/README.md

@@ -0,0 +1,76 @@
+# encoding/json [![GoDoc](https://godoc.org/github.com/segmentio/encoding/json?status.svg)](https://godoc.org/github.com/segmentio/encoding/json)
+
+Go package offering a replacement implementation of the standard library's
+[`encoding/json`](https://golang.org/pkg/encoding/json/) package, with much
+better performance.
+
+## Usage
+
+The exported API of this package mirrors the standard library's
+[`encoding/json`](https://golang.org/pkg/encoding/json/) package, the only
+change needed to take advantage of the performance improvements is the import
+path of the `json` package, from:
+```go
+import (
+    "encoding/json"
+)
+```
+to
+```go
+import (
+    "github.com/segmentio/encoding/json"
+)
+```
+
+One way to gain higher encoding throughput is to disable HTML escaping.
+It allows the string encoding to use a much more efficient code path which
+does not require parsing UTF-8 runes most of the time.
+
+## Performance Improvements
+
+The internal implementation uses a fair amount of unsafe operations (untyped
+code, pointer arithmetic, etc...) to avoid using reflection as much as possible,
+which is often the reason why serialization code has a large CPU and memory
+footprint.
+
+The package aims for zero unnecessary dynamic memory allocations and hot code
+paths that are mostly free from calls into the reflect package.
+
+## Compatibility with encoding/json
+
+This package aims to be a drop-in replacement, therefore it is tested to behave
+exactly like the standard library's package. However, there are still a few
+missing features that have not been ported yet:
+
+- Streaming decoder, currently the `Decoder` implementation offered by the
+package does not support progressively reading values from a JSON array (unlike
+the standard library). In our experience this is a very rare use-case, if you
+need it you're better off sticking to the standard library, or spend a bit of
+time implementing it in here ;)
+
+Note that none of those features should result in performance degradations if
+they were implemented in the package, and we welcome contributions!
+
+## Trade-offs
+
+As one would expect, we had to make a couple of trade-offs to achieve greater
+performance than the standard library, but there were also features that we
+did not want to give away.
+
+Other open-source packages offering a reduced CPU and memory footprint usually
+do so by designing a different API, or require code generation (therefore adding
+complexity to the build process). These were not acceptable conditions for us,
+as we were not willing to trade off developer productivity for better runtime
+performance. To achieve this, we chose to exactly replicate the standard
+library interfaces and behavior, which meant the package implementation was the
+only area that we were able to work with. The internals of this package make
+heavy use of unsafe pointer arithmetics and other performance optimizations,
+and therefore are not as approachable as typical Go programs. Basically, we put
+a bigger burden on maintainers to achieve better runtime cost without
+sacrificing developer productivity.
+
+For these reasons, we also don't believe that this code should be ported upstream
+to the standard `encoding/json` package. The standard library has to remain
+readable and approachable to maximize stability and maintainability, and make
+projects like this one possible because a high quality reference implementation
+already exists.

cli/output/jsonw/internal/jcolorenc/encode.go

@@ -0,0 +1,989 @@
+package json
+
+import (
+	"bytes"
+	"encoding"
+	"encoding/base64"
+	"math"
+	"reflect"
+	"sort"
+	"strconv"
+	"sync"
+	"time"
+	"unicode/utf8"
+	"unsafe"
+)
+
+const hex = "0123456789abcdef"
+
+func (e encoder) encodeNull(b []byte, p unsafe.Pointer) ([]byte, error) {
+	return e.clrs.AppendNull(b), nil
+}
+
+func (e encoder) encodeBool(b []byte, p unsafe.Pointer) ([]byte, error) {
+	return e.clrs.AppendBool(b, *(*bool)(p)), nil
+}
+
+func (e encoder) encodeInt(b []byte, p unsafe.Pointer) ([]byte, error) {
+	return e.clrs.AppendInt64(b, int64(*(*int)(p))), nil
+}
+
+func (e encoder) encodeInt8(b []byte, p unsafe.Pointer) ([]byte, error) {
+	return e.clrs.AppendInt64(b, int64(*(*int8)(p))), nil
+}
+
+func (e encoder) encodeInt16(b []byte, p unsafe.Pointer) ([]byte, error) {
+	return e.clrs.AppendInt64(b, int64(*(*int16)(p))), nil
+}
+
+func (e encoder) encodeInt32(b []byte, p unsafe.Pointer) ([]byte, error) {
+	return e.clrs.AppendInt64(b, int64(*(*int32)(p))), nil
+}
+
+func (e encoder) encodeInt64(b []byte, p unsafe.Pointer) ([]byte, error) {
+	return e.clrs.AppendInt64(b, int64(*(*int64)(p))), nil
+}
+
+func (e encoder) encodeUint(b []byte, p unsafe.Pointer) ([]byte, error) {
+	return e.clrs.AppendUint64(b, uint64(*(*uint)(p))), nil
+}
+
+func (e encoder) encodeUintptr(b []byte, p unsafe.Pointer) ([]byte, error) {
+	return e.clrs.AppendUint64(b, uint64(*(*uintptr)(p))), nil
+}
+
+func (e encoder) encodeUint8(b []byte, p unsafe.Pointer) ([]byte, error) {
+	return e.clrs.AppendUint64(b, uint64(*(*uint8)(p))), nil
+}
+
+func (e encoder) encodeUint16(b []byte, p unsafe.Pointer) ([]byte, error) {
+	return e.clrs.AppendUint64(b, uint64(*(*uint16)(p))), nil
+}
+
+func (e encoder) encodeUint32(b []byte, p unsafe.Pointer) ([]byte, error) {
+	return e.clrs.AppendUint64(b, uint64(*(*uint32)(p))), nil
+}
+
+func (e encoder) encodeUint64(b []byte, p unsafe.Pointer) ([]byte, error) {
+	return e.clrs.AppendUint64(b, uint64(*(*uint64)(p))), nil
+}
+
+func (e encoder) encodeFloat32(b []byte, p unsafe.Pointer) ([]byte, error) {
+	b = append(b, e.clrs.Number.Prefix...)
+	var err error
+	b, err = e.encodeFloat(b, float64(*(*float32)(p)), 32)
+	b = append(b, e.clrs.Number.Suffix...)
+	return b, err
+}
+
+func (e encoder) encodeFloat64(b []byte, p unsafe.Pointer) ([]byte, error) {
+	b = append(b, e.clrs.Number.Prefix...)
+	var err error
+	b, err = e.encodeFloat(b, *(*float64)(p), 64)
+	b = append(b, e.clrs.Number.Suffix...)
+	return b, err
+}
+
+func (e encoder) encodeFloat(b []byte, f float64, bits int) ([]byte, error) {
+	switch {
+	case math.IsNaN(f):
+		return b, &UnsupportedValueError{Value: reflect.ValueOf(f), Str: "NaN"}
+	case math.IsInf(f, 0):
+		return b, &UnsupportedValueError{Value: reflect.ValueOf(f), Str: "inf"}
+	}
+
+	// Convert as if by ES6 number to string conversion.
+	// This matches most other JSON generators.
+	// See golang.org/issue/6384 and golang.org/issue/14135.
+	// Like fmt %g, but the exponent cutoffs are different
+	// and exponents themselves are not padded to two digits.
+	abs := math.Abs(f)
+	fmt := byte('f')
+	// Note: Must use float32 comparisons for underlying float32 value to get precise cutoffs right.
+	if abs != 0 {
+		if bits == 64 && (abs < 1e-6 || abs >= 1e21) || bits == 32 && (float32(abs) < 1e-6 || float32(abs) >= 1e21) {
+			fmt = 'e'
+		}
+	}
+
+	b = strconv.AppendFloat(b, f, fmt, -1, int(bits))
+
+	if fmt == 'e' {
+		// clean up e-09 to e-9
+		n := len(b)
+		if n >= 4 && b[n-4] == 'e' && b[n-3] == '-' && b[n-2] == '0' {
+			b[n-2] = b[n-1]
+			b = b[:n-1]
+		}
+	}
+
+	return b, nil
+}
+
+func (e encoder) encodeNumber(b []byte, p unsafe.Pointer) ([]byte, error) {
+	n := *(*Number)(p)
+	if n == "" {
+		n = "0"
+	}
+
+	_, _, err := parseNumber(stringToBytes(string(n)))
+	if err != nil {
+		return b, err
+	}
+
+	b = append(b, e.clrs.Number.Prefix...)
+	b = append(b, n...)
+	b = append(b, e.clrs.Number.Suffix...)
+	return b, nil
+}
+
+func (e encoder) encodeKey(b []byte, p unsafe.Pointer) ([]byte, error) {
+	b = append(b, e.clrs.Key.Prefix...)
+	var err error
+	b, err = e.doEncodeString(b, p)
+	b = append(b, e.clrs.Key.Suffix...)
+	return b, err
+}
+
+func (e encoder) encodeString(b []byte, p unsafe.Pointer) ([]byte, error) {
+	b = append(b, e.clrs.String.Prefix...)
+	var err error
+	b, err = e.doEncodeString(b, p)
+	b = append(b, e.clrs.String.Suffix...)
+	return b, err
+}
+
+func (e encoder) doEncodeString(b []byte, p unsafe.Pointer) ([]byte, error) {
+	s := *(*string)(p)
+	i := 0
+	j := 0
+	escapeHTML := (e.flags & EscapeHTML) != 0
+
+	b = append(b, '"')
+
+	for j < len(s) {
+		c := s[j]
+
+		if c >= 0x20 && c <= 0x7f && c != '\\' && c != '"' && (!escapeHTML || (c != '<' && c != '>' && c != '&')) {
+			// fast path: most of the time, printable ascii characters are used
+			j++
+			continue
+		}
+
+		switch c {
+		case '\\', '"':
+			b = append(b, s[i:j]...)
+			b = append(b, '\\', c)
+			i = j + 1
+			j = j + 1
+			continue
+
+		case '\n':
+			b = append(b, s[i:j]...)
+			b = append(b, '\\', 'n')
+			i = j + 1
+			j = j + 1
+			continue
+
+		case '\r':
+			b = append(b, s[i:j]...)
+			b = append(b, '\\', 'r')
+			i = j + 1
+			j = j + 1
+			continue
+
+		case '\t':
+			b = append(b, s[i:j]...)
+			b = append(b, '\\', 't')
+			i = j + 1
+			j = j + 1
+			continue
+
+		case '<', '>', '&':
+			b = append(b, s[i:j]...)
+			b = append(b, `\u00`...)
+			b = append(b, hex[c>>4], hex[c&0xF])
+			i = j + 1
+			j = j + 1
+			continue
+		}
+
+		// This encodes bytes < 0x20 except for \t, \n and \r.
+		if c < 0x20 {
+			b = append(b, s[i:j]...)
+			b = append(b, `\u00`...)
+			b = append(b, hex[c>>4], hex[c&0xF])
+			i = j + 1
+			j = j + 1
+			continue
+		}
+
+		r, size := utf8.DecodeRuneInString(s[j:])
+
+		if r == utf8.RuneError && size == 1 {
+			b = append(b, s[i:j]...)
+			b = append(b, `\ufffd`...)
+			i = j + size
+			j = j + size
+			continue
+		}
+
+		switch r {
+		case '\u2028', '\u2029':
+			// U+2028 is LINE SEPARATOR.
+			// U+2029 is PARAGRAPH SEPARATOR.
+			// They are both technically valid characters in JSON strings,
+			// but don't work in JSONP, which has to be evaluated as JavaScript,
+			// and can lead to security holes there. It is valid JSON to
+			// escape them, so we do so unconditionally.
+			// See http://timelessrepo.com/json-isnt-a-javascript-subset for discussion.
+			b = append(b, s[i:j]...)
+			b = append(b, `\u202`...)
+			b = append(b, hex[r&0xF])
+			i = j + size
+			j = j + size
+			continue
+		}
+
+		j += size
+	}
+
+	b = append(b, s[i:]...)
+	b = append(b, '"')
+	return b, nil
+}
+
+func (e encoder) encodeToString(b []byte, p unsafe.Pointer, encode encodeFunc) ([]byte, error) {
+	i := len(b)
+
+	b, err := encode(e, b, p)
+	if err != nil {
+		return b, err
+	}
+
+	j := len(b)
+	s := b[i:]
+
+	if b, err = e.doEncodeString(b, unsafe.Pointer(&s)); err != nil {
+		return b, err
+	}
+
+	n := copy(b[i:], b[j:])
+	return b[:i+n], nil
+}
+
+func (e encoder) encodeBytes(b []byte, p unsafe.Pointer) ([]byte, error) {
+	b = append(b, e.clrs.Bytes.Prefix...)
+	var err error
+	b, err = e.doEncodeBytes(b, p)
+	return append(b, e.clrs.Bytes.Suffix...), err
+}
+
+func (e encoder) doEncodeBytes(b []byte, p unsafe.Pointer) ([]byte, error) {
+	v := *(*[]byte)(p)
+	if v == nil {
+		return e.clrs.AppendNull(b), nil
+	}
+
+	n := base64.StdEncoding.EncodedLen(len(v)) + 2
+
+	if avail := cap(b) - len(b); avail < n {
+		newB := make([]byte, cap(b)+(n-avail))
+		copy(newB, b)
+		b = newB[:len(b)]
+	}
+
+	i := len(b)
+	j := len(b) + n
+
+	b = b[:j]
+	b[i] = '"'
+	base64.StdEncoding.Encode(b[i+1:j-1], v)
+	b[j-1] = '"'
+	return b, nil
+}
+
+func (e encoder) encodeDuration(b []byte, p unsafe.Pointer) ([]byte, error) {
+	b = append(b, e.clrs.Time.Prefix...)
+	b = append(b, '"')
+	b = appendDuration(b, *(*time.Duration)(p))
+	b = append(b, '"')
+	b = append(b, e.clrs.Time.Suffix...)
+	return b, nil
+}
+
+func (e encoder) encodeTime(b []byte, p unsafe.Pointer) ([]byte, error) {
+	t := *(*time.Time)(p)
+	b = append(b, e.clrs.Time.Prefix...)
+	b = append(b, '"')
+	b = t.AppendFormat(b, time.RFC3339Nano)
+	b = append(b, '"')
+	b = append(b, e.clrs.Time.Suffix...)
+	return b, nil
+}
+
+func (e encoder) encodeArray(b []byte, p unsafe.Pointer, n int, size uintptr, t reflect.Type, encode encodeFunc) ([]byte, error) {
+	var start = len(b)
+	var err error
+
+	b = append(b, '[')
+
+	if n > 0 {
+		e.indenter.Push()
+		for i := 0; i < n; i++ {
+			if i != 0 {
+				b = append(b, ',')
+			}
+
+			b = e.indenter.AppendByte(b, '\n')
+			b = e.indenter.AppendIndent(b)
+
+			if b, err = encode(e, b, unsafe.Pointer(uintptr(p)+(uintptr(i)*size))); err != nil {
+				return b[:start], err
+			}
+		}
+		e.indenter.Pop()
+		b = e.indenter.AppendByte(b, '\n')
+		b = e.indenter.AppendIndent(b)
+	}
+
+	b = append(b, ']')
+
+	return b, nil
+}
+
+func (e encoder) encodeSlice(b []byte, p unsafe.Pointer, size uintptr, t reflect.Type, encode encodeFunc) ([]byte, error) {
+	s := (*slice)(p)
+
+	if s.data == nil && s.len == 0 && s.cap == 0 {
+		return e.clrs.AppendNull(b), nil
+	}
+
+	return e.encodeArray(b, s.data, s.len, size, t, encode)
+}
+
+func (e encoder) encodeMap(b []byte, p unsafe.Pointer, t reflect.Type, encodeKey, encodeValue encodeFunc, sortKeys sortFunc) ([]byte, error) {
+	m := reflect.NewAt(t, p).Elem()
+	if m.IsNil() {
+		return e.clrs.AppendNull(b), nil
+	}
+
+	keys := m.MapKeys()
+	if sortKeys != nil && (e.flags&SortMapKeys) != 0 {
+		sortKeys(keys)
+	}
+
+	var start = len(b)
+	var err error
+	b = append(b, '{')
+
+	if len(keys) != 0 {
+		b = e.indenter.AppendByte(b, '\n')
+
+		e.indenter.Push()
+		for i, k := range keys {
+			v := m.MapIndex(k)
+
+			if i != 0 {
+				b = append(b, ',')
+				b = e.indenter.AppendByte(b, '\n')
+			}
+
+			b = e.indenter.AppendIndent(b)
+			if b, err = encodeKey(e, b, (*iface)(unsafe.Pointer(&k)).ptr); err != nil {
+				return b[:start], err
+			}
+
+			b = append(b, ':')
+			b = e.indenter.AppendByte(b, ' ')
+
+			if b, err = encodeValue(e, b, (*iface)(unsafe.Pointer(&v)).ptr); err != nil {
+				return b[:start], err
+			}
+		}
+		b = e.indenter.AppendByte(b, '\n')
+		e.indenter.Pop()
+		b = e.indenter.AppendIndent(b)
+	}
+
+	b = append(b, '}')
+	return b, nil
+}
+
+type element struct {
+	key string
+	val interface{}
+	raw RawMessage
+}
+
+type mapslice struct {
+	elements []element
+}
+
+func (m *mapslice) Len() int           { return len(m.elements) }
+func (m *mapslice) Less(i, j int) bool { return m.elements[i].key < m.elements[j].key }
+func (m *mapslice) Swap(i, j int)      { m.elements[i], m.elements[j] = m.elements[j], m.elements[i] }
+
+var mapslicePool = sync.Pool{
+	New: func() interface{} { return new(mapslice) },
+}
+
+func (e encoder) encodeMapStringInterface(b []byte, p unsafe.Pointer) ([]byte, error) {
+	m := *(*map[string]interface{})(p)
+	if m == nil {
+		return e.clrs.AppendNull(b), nil
+	}
+
+	if (e.flags & SortMapKeys) == 0 {
+		// Optimized code path when the program does not need the map keys to be
+		// sorted.
+		b = append(b, '{')
+
+		if len(m) != 0 {
+			b = e.indenter.AppendByte(b, '\n')
+
+			var err error
+			var i = 0
+
+			e.indenter.Push()
+			for k, v := range m {
+				if i != 0 {
+					b = append(b, ',')
+					b = e.indenter.AppendByte(b, '\n')
+				}
+
+				b = e.indenter.AppendIndent(b)
+
+				b, err = e.encodeKey(b, unsafe.Pointer(&k))
+				if err != nil {
+					return b, err
+				}
+
+				b = append(b, ':')
+				b = e.indenter.AppendByte(b, ' ')
+
+				b, err = Append(b, v, e.flags, e.clrs, e.indenter)
+				if err != nil {
+					return b, err
+				}
+
+				i++
+			}
+			b = e.indenter.AppendByte(b, '\n')
+			e.indenter.Pop()
+			b = e.indenter.AppendIndent(b)
+		}
+
+		b = append(b, '}')
+		return b, nil
+	}
+
+	s := mapslicePool.Get().(*mapslice)
+	if cap(s.elements) < len(m) {
+		s.elements = make([]element, 0, align(10, uintptr(len(m))))
+	}
+	for key, val := range m {
+		s.elements = append(s.elements, element{key: key, val: val})
+	}
+	sort.Sort(s)
+
+	var start = len(b)
+	var err error
+	b = append(b, '{')
+
+	if len(s.elements) > 0 {
+		b = e.indenter.AppendByte(b, '\n')
+
+		e.indenter.Push()
+		for i, elem := range s.elements {
+			if i != 0 {
+				b = append(b, ',')
+				b = e.indenter.AppendByte(b, '\n')
+			}
+
+			b = e.indenter.AppendIndent(b)
+
+			b, _ = e.encodeKey(b, unsafe.Pointer(&elem.key))
+			b = append(b, ':')
+			b = e.indenter.AppendByte(b, ' ')
+
+			b, err = Append(b, elem.val, e.flags, e.clrs, e.indenter)
+			if err != nil {
+				break
+			}
+		}
+		b = e.indenter.AppendByte(b, '\n')
+		e.indenter.Pop()
+		b = e.indenter.AppendIndent(b)
+	}
+
+	for i := range s.elements {
+		s.elements[i] = element{}
+	}
+
+	s.elements = s.elements[:0]
+	mapslicePool.Put(s)
+
+	if err != nil {
+		return b[:start], err
+	}
+
+	b = append(b, '}')
+	return b, nil
+}
+
+func (e encoder) encodeMapStringRawMessage(b []byte, p unsafe.Pointer) ([]byte, error) {
+	m := *(*map[string]RawMessage)(p)
+	if m == nil {
+		return e.clrs.AppendNull(b), nil
+	}
+
+	if (e.flags & SortMapKeys) == 0 {
+		// Optimized code path when the program does not need the map keys to be
+		// sorted.
+		b = append(b, '{')
+
+		if len(m) != 0 {
+			b = e.indenter.AppendByte(b, '\n')
+
+			var err error
+			var i = 0
+
+			e.indenter.Push()
+			for k, v := range m {
+				if i != 0 {
+					b = append(b, ',')
+					b = e.indenter.AppendByte(b, '\n')
+				}
+
+				b = e.indenter.AppendIndent(b)
+
+				b, _ = e.encodeKey(b, unsafe.Pointer(&k))
+
+				b = append(b, ':')
+				b = e.indenter.AppendByte(b, ' ')
+
+				b, err = e.encodeRawMessage(b, unsafe.Pointer(&v))
+				if err != nil {
+					break
+				}
+
+				i++
+			}
+			b = e.indenter.AppendByte(b, '\n')
+			e.indenter.Pop()
+			b = e.indenter.AppendIndent(b)
+		}
+
+		b = append(b, '}')
+		return b, nil
+	}
+
+	s := mapslicePool.Get().(*mapslice)
+	if cap(s.elements) < len(m) {
+		s.elements = make([]element, 0, align(10, uintptr(len(m))))
+	}
+	for key, raw := range m {
+		s.elements = append(s.elements, element{key: key, raw: raw})
+	}
+	sort.Sort(s)
+
+	var start = len(b)
+	var err error
+	b = append(b, '{')
+
+	if len(s.elements) > 0 {
+		b = e.indenter.AppendByte(b, '\n')
+
+		e.indenter.Push()
+
+		for i, elem := range s.elements {
+			if i != 0 {
+				b = append(b, ',')
+				b = e.indenter.AppendByte(b, '\n')
+			}
+
+			b = e.indenter.AppendIndent(b)
+
+			b, _ = e.encodeKey(b, unsafe.Pointer(&elem.key))
+			b = append(b, ':')
+			b = e.indenter.AppendByte(b, ' ')
+
+			b, err = e.encodeRawMessage(b, unsafe.Pointer(&elem.raw))
+			if err != nil {
+				break
+			}
+		}
+		b = e.indenter.AppendByte(b, '\n')
+		e.indenter.Pop()
+		b = e.indenter.AppendIndent(b)
+	}
+
+	for i := range s.elements {
+		s.elements[i] = element{}
+	}
+
+	s.elements = s.elements[:0]
+	mapslicePool.Put(s)
+
+	if err != nil {
+		return b[:start], err
+	}
+
+	b = append(b, '}')
+	return b, nil
+}
+
+func (e encoder) encodeStruct(b []byte, p unsafe.Pointer, st *structType) ([]byte, error) {
+	var start = len(b)
+	var err error
+	var k string
+	var n int
+
+	b = append(b, '{')
+
+	if len(st.fields) > 0 {
+		b = e.indenter.AppendByte(b, '\n')
+	}
+
+	e.indenter.Push()
+
+	for i := range st.fields {
+		f := &st.fields[i]
+		v := unsafe.Pointer(uintptr(p) + f.offset)
+
+		if f.omitempty && f.empty(v) {
+			continue
+		}
+
+		if n != 0 {
+			b = append(b, ',')
+			b = e.indenter.AppendByte(b, '\n')
+		}
+
+		if (e.flags & EscapeHTML) != 0 {
+			k = f.html
+		} else {
+			k = f.json
+		}
+
+		lengthBeforeKey := len(b)
+		b = e.indenter.AppendIndent(b)
+
+		b = append(b, e.clrs.Key.Prefix...)
+		b = append(b, k...)
+		b = append(b, e.clrs.Key.Suffix...)
+		b = append(b, ':')
+
+		b = e.indenter.AppendByte(b, ' ')
+
+		if b, err = f.codec.encode(e, b, v); err != nil {
+			if err == (rollback{}) {
+				b = b[:lengthBeforeKey]
+				continue
+			}
+			return b[:start], err
+		}
+
+		n++
+	}
+
+	if n > 0 {
+		b = e.indenter.AppendByte(b, '\n')
+	}
+
+	e.indenter.Pop()
+	b = e.indenter.AppendIndent(b)
+
+	b = append(b, '}')
+	return b, nil
+}
+
+type rollback struct{}
+
+func (rollback) Error() string { return "rollback" }
+
+func (e encoder) encodeEmbeddedStructPointer(b []byte, p unsafe.Pointer, t reflect.Type, unexported bool, offset uintptr, encode encodeFunc) ([]byte, error) {
+	p = *(*unsafe.Pointer)(p)
+	if p == nil {
+		return b, rollback{}
+	}
+	return encode(e, b, unsafe.Pointer(uintptr(p)+offset))
+}
+
+func (e encoder) encodePointer(b []byte, p unsafe.Pointer, t reflect.Type, encode encodeFunc) ([]byte, error) {
+	if p = *(*unsafe.Pointer)(p); p != nil {
+		return encode(e, b, p)
+	}
+	return e.encodeNull(b, nil)
+}
+
+func (e encoder) encodeInterface(b []byte, p unsafe.Pointer) ([]byte, error) {
+	return Append(b, *(*interface{})(p), e.flags, e.clrs, e.indenter)
+}
+
+func (e encoder) encodeMaybeEmptyInterface(b []byte, p unsafe.Pointer, t reflect.Type) ([]byte, error) {
+	return Append(b, reflect.NewAt(t, p).Elem().Interface(), e.flags, e.clrs, e.indenter)
+}
+
+func (e encoder) encodeUnsupportedTypeError(b []byte, p unsafe.Pointer, t reflect.Type) ([]byte, error) {
+	return b, &UnsupportedTypeError{Type: t}
+}
+
+func (e encoder) encodeRawMessage(b []byte, p unsafe.Pointer) ([]byte, error) {
+	v := *(*RawMessage)(p)
+
+	if v == nil {
+
+		return e.clrs.AppendNull(b), nil
+	}
+
+	var s []byte
+
+	if (e.flags & TrustRawMessage) != 0 {
+		s = v
+	} else {
+		var err error
+		s, _, err = parseValue(v)
+		if err != nil {
+			return b, &UnsupportedValueError{Value: reflect.ValueOf(v), Str: err.Error()}
+		}
+	}
+
+	if e.indenter == nil {
+		if (e.flags & EscapeHTML) != 0 {
+			return appendCompactEscapeHTML(b, s), nil
+		}
+
+		return append(b, s...), nil
+	}
+
+	// In order to get the tests inherited from the original segmentio
+	// encoder to work, we need to support indentation. However, due to
+	// the complexity of parsing and then colorizing, we're not going to
+	// go to the effort of adding color support for JSONMarshaler right
+	// now. Possibly revisit this in future if needed.
+
+	// This below is sloppy, but seems to work.
+	if (e.flags & EscapeHTML) != 0 {
+		s = appendCompactEscapeHTML(nil, s)
+	}
+
+	// The "prefix" arg to Indent is the current indentation.
+	pre := e.indenter.AppendIndent(nil)
+
+	buf := &bytes.Buffer{}
+	// And now we just make use of the existing Indent function.
+	err := Indent(buf, s, string(pre), e.indenter.Indent)
+	if err != nil {
+		return b, err
+	}
+
+	s = buf.Bytes()
+
+	return append(b, s...), nil
+}
+
+func (e encoder) encodeJSONMarshaler(b []byte, p unsafe.Pointer, t reflect.Type, pointer bool) ([]byte, error) {
+	v := reflect.NewAt(t, p)
+
+	if !pointer {
+		v = v.Elem()
+	}
+
+	switch v.Kind() {
+	case reflect.Ptr, reflect.Interface:
+		if v.IsNil() {
+			return e.clrs.AppendNull(b), nil
+		}
+	}
+
+	j, err := v.Interface().(Marshaler).MarshalJSON()
+	if err != nil {
+		return b, err
+	}
+
+	s, _, err := parseValue(j)
+	if err != nil {
+		return b, &MarshalerError{Type: t, Err: err}
+	}
+
+	if e.indenter == nil {
+		if (e.flags & EscapeHTML) != 0 {
+			return appendCompactEscapeHTML(b, s), nil
+		}
+
+		return append(b, s...), nil
+	}
+
+	// In order to get the tests inherited from the original segmentio
+	// encoder to work, we need to support indentation. However, due to
+	// the complexity of parsing and then colorizing, we're not going to
+	// go to the effort of supporting color for JSONMarshaler.
+	// Possibly revisit this in future if needed.
+
+	// This below is sloppy, but seems to work.
+	if (e.flags & EscapeHTML) != 0 {
+		s = appendCompactEscapeHTML(nil, s)
+	}
+
+	// The "prefix" arg to Indent is the current indentation.
+	pre := e.indenter.AppendIndent(nil)
+
+	buf := &bytes.Buffer{}
+	// And now we just make use of the existing Indent function.
+	err = Indent(buf, s, string(pre), e.indenter.Indent)
+	if err != nil {
+		return b, err
+	}
+
+	s = buf.Bytes()
+
+	return append(b, s...), nil
+
+}
+
+func (e encoder) encodeTextMarshaler(b []byte, p unsafe.Pointer, t reflect.Type, pointer bool) ([]byte, error) {
+	v := reflect.NewAt(t, p)
+
+	if !pointer {
+		v = v.Elem()
+	}
+
+	switch v.Kind() {
+	case reflect.Ptr, reflect.Interface:
+		if v.IsNil() {
+			return append(b, `null`...), nil
+		}
+	}
+
+	s, err := v.Interface().(encoding.TextMarshaler).MarshalText()
+	if err != nil {
+		return b, err
+	}
+
+	return e.doEncodeString(b, unsafe.Pointer(&s))
+}
+
+func appendCompactEscapeHTML(dst []byte, src []byte) []byte {
+	start := 0
+	escape := false
+	inString := false
+
+	for i, c := range src {
+		if !inString {
+			switch c {
+			case '"': // enter string
+				inString = true
+			case ' ', '\n', '\r', '\t': // skip space
+				if start < i {
+					dst = append(dst, src[start:i]...)
+				}
+				start = i + 1
+			}
+			continue
+		}
+
+		if escape {
+			escape = false
+			continue
+		}
+
+		if c == '\\' {
+			escape = true
+			continue
+		}
+
+		if c == '"' {
+			inString = false
+			continue
+		}
+
+		if c == '<' || c == '>' || c == '&' {
+			if start < i {
+				dst = append(dst, src[start:i]...)
+			}
+			dst = append(dst, `\u00`...)
+			dst = append(dst, hex[c>>4], hex[c&0xF])
+			start = i + 1
+			continue
+		}
+
+		// Convert U+2028 and U+2029 (E2 80 A8 and E2 80 A9).
+		if c == 0xE2 && i+2 < len(src) && src[i+1] == 0x80 && src[i+2]&^1 == 0xA8 {
+			if start < i {
+				dst = append(dst, src[start:i]...)
+			}
+			dst = append(dst, `\u202`...)
+			dst = append(dst, hex[src[i+2]&0xF])
+			start = i + 3
+			continue
+		}
+	}
+
+	if start < len(src) {
+		dst = append(dst, src[start:]...)
+	}
+
+	return dst
+}
+
+// Indenter is used to indent JSON. The Push and Pop methods
+// change indentation level. The AppendIndent method appends the
+// computed indentation. The AppendByte method appends a byte. All
+// methods are safe to use with a nil receiver.
+type Indenter struct {
+	disabled bool
+	Prefix   string
+	Indent   string
+	depth    int
+}
+
+// NewIndenter returns a new Indenter instance. If prefix and
+// indent are both empty, the indenter is effectively disabled,
+// and the AppendIndent and AppendByte methods are no-op.
+func NewIndenter(prefix, indent string) *Indenter {
+	return &Indenter{
+		disabled: prefix == "" && indent == "",
+		Prefix:   prefix,
+		Indent:   indent,
+	}
+}
+
+// Push increases the indentation level.
+func (in *Indenter) Push() {
+	if in != nil {
+		in.depth++
+	}
+}
+
+// Pop decreases the indentation level.
+func (in *Indenter) Pop() {
+	if in != nil {
+		in.depth--
+	}
+}
+
+// AppendByte appends a to b if the indenter is non-nil and enabled.
+// Otherwise b is returned unmodified.
+func (in *Indenter) AppendByte(b []byte, a byte) []byte {
+	if in == nil || in.disabled {
+		return b
+	}
+
+	return append(b, a)
+}
+
+// AppendIndent writes indentation to b, returning the resulting slice.
+// If the indenter is nil or disabled b is returned unchanged.
+func (in *Indenter) AppendIndent(b []byte) []byte {
+	if in == nil || in.disabled {
+		return b
+	}
+
+	b = append(b, in.Prefix...)
+	for i := 0; i < in.depth; i++ {
+		b = append(b, in.Indent...)
+	}
+	return b
+}

cli/output/jsonw/internal/jcolorenc/golang_bench_test.go

@@ -0,0 +1,374 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Large data benchmark.
+// The JSON data is a summary of agl's changes in the
+// go, webkit, and chromium open source projects.
+// We benchmark converting between the JSON form
+// and in-memory data structures.
+
+package json
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"reflect"
+	"runtime"
+	"strings"
+	"sync"
+	"testing"
+)
+
+type codeResponse struct {
+	Tree     *codeNode `json:"tree"`
+	Username string    `json:"username"`
+}
+
+type codeNode struct {
+	Name     string      `json:"name"`
+	Kids     []*codeNode `json:"kids"`
+	CLWeight float64     `json:"cl_weight"`
+	Touches  int         `json:"touches"`
+	MinT     int64       `json:"min_t"`
+	MaxT     int64       `json:"max_t"`
+	MeanT    int64       `json:"mean_t"`
+}
+
+var codeJSON []byte
+var codeStruct codeResponse
+
+func codeInit() {
+	f, err := os.Open("testdata/code.json.gz")
+	if err != nil {
+		panic(err)
+	}
+	defer f.Close()
+	gz, err := gzip.NewReader(f)
+	if err != nil {
+		panic(err)
+	}
+	data, err := ioutil.ReadAll(gz)
+	if err != nil {
+		panic(err)
+	}
+
+	codeJSON = data
+
+	if err := Unmarshal(codeJSON, &codeStruct); err != nil {
+		panic("unmarshal code.json: " + err.Error())
+	}
+
+	if data, err = Marshal(&codeStruct); err != nil {
+		panic("marshal code.json: " + err.Error())
+	}
+
+	if !bytes.Equal(data, codeJSON) {
+		println("different lengths", len(data), len(codeJSON))
+		for i := 0; i < len(data) && i < len(codeJSON); i++ {
+			if data[i] != codeJSON[i] {
+				println("re-marshal: changed at byte", i)
+				println("orig: ", string(codeJSON[i-10:i+10]))
+				println("new: ", string(data[i-10:i+10]))
+				break
+			}
+		}
+		panic("re-marshal code.json: different result")
+	}
+}
+
+func BenchmarkCodeEncoder(b *testing.B) {
+	b.ReportAllocs()
+	if codeJSON == nil {
+		b.StopTimer()
+		codeInit()
+		b.StartTimer()
+	}
+	b.RunParallel(func(pb *testing.PB) {
+		enc := NewEncoder(ioutil.Discard)
+		for pb.Next() {
+			if err := enc.Encode(&codeStruct); err != nil {
+				b.Fatal("Encode:", err)
+			}
+		}
+	})
+	b.SetBytes(int64(len(codeJSON)))
+}
+
+func BenchmarkCodeMarshal(b *testing.B) {
+	b.ReportAllocs()
+	if codeJSON == nil {
+		b.StopTimer()
+		codeInit()
+		b.StartTimer()
+	}
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			if _, err := Marshal(&codeStruct); err != nil {
+				b.Fatal("Marshal:", err)
+			}
+		}
+	})
+	b.SetBytes(int64(len(codeJSON)))
+}
+
+func benchMarshalBytes(n int) func(*testing.B) {
+	sample := []byte("hello world")
+	// Use a struct pointer, to avoid an allocation when passing it as an
+	// interface parameter to Marshal.
+	v := &struct {
+		Bytes []byte
+	}{
+		bytes.Repeat(sample, (n/len(sample))+1)[:n],
+	}
+	return func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			if _, err := Marshal(v); err != nil {
+				b.Fatal("Marshal:", err)
+			}
+		}
+	}
+}
+
+func BenchmarkMarshalBytes(b *testing.B) {
+	b.ReportAllocs()
+	// 32 fits within encodeState.scratch.
+	b.Run("32", benchMarshalBytes(32))
+	// 256 doesn't fit in encodeState.scratch, but is small enough to
+	// allocate and avoid the slower base64.NewEncoder.
+	b.Run("256", benchMarshalBytes(256))
+	// 4096 is large enough that we want to avoid allocating for it.
+	b.Run("4096", benchMarshalBytes(4096))
+}
+
+func BenchmarkCodeDecoder(b *testing.B) {
+	b.ReportAllocs()
+	if codeJSON == nil {
+		b.StopTimer()
+		codeInit()
+		b.StartTimer()
+	}
+	b.RunParallel(func(pb *testing.PB) {
+		var buf bytes.Buffer
+		dec := NewDecoder(&buf)
+		var r codeResponse
+		for pb.Next() {
+			buf.Write(codeJSON)
+			// hide EOF
+			buf.WriteByte('\n')
+			buf.WriteByte('\n')
+			buf.WriteByte('\n')
+			if err := dec.Decode(&r); err != nil {
+				b.Fatal("Decode:", err)
+			}
+		}
+	})
+	b.SetBytes(int64(len(codeJSON)))
+}
+
+func BenchmarkUnicodeDecoder(b *testing.B) {
+	b.ReportAllocs()
+	j := []byte(`"\uD83D\uDE01"`)
+	b.SetBytes(int64(len(j)))
+	r := bytes.NewReader(j)
+	dec := NewDecoder(r)
+	var out string
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := dec.Decode(&out); err != nil {
+			b.Fatal("Decode:", err)
+		}
+		r.Seek(0, 0)
+	}
+}
+
+func BenchmarkDecoderStream(b *testing.B) {
+	b.ReportAllocs()
+	b.StopTimer()
+	var buf bytes.Buffer
+	dec := NewDecoder(&buf)
+	buf.WriteString(`"` + strings.Repeat("x", 1000000) + `"` + "\n\n\n")
+	var x interface{}
+	if err := dec.Decode(&x); err != nil {
+		b.Fatal("Decode:", err)
+	}
+	ones := strings.Repeat(" 1\n", 300000) + "\n\n\n"
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		if i%300000 == 0 {
+			buf.WriteString(ones)
+		}
+		x = nil
+		if err := dec.Decode(&x); err != nil || x != 1.0 {
+			b.Fatalf("Decode: %v after %d", err, i)
+		}
+	}
+}
+
+func BenchmarkCodeUnmarshal(b *testing.B) {
+	b.ReportAllocs()
+	if codeJSON == nil {
+		b.StopTimer()
+		codeInit()
+		b.StartTimer()
+	}
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			var r codeResponse
+			if err := Unmarshal(codeJSON, &r); err != nil {
+				b.Fatal("Unmarshal:", err)
+			}
+		}
+	})
+	b.SetBytes(int64(len(codeJSON)))
+}
+
+func BenchmarkCodeUnmarshalReuse(b *testing.B) {
+	b.ReportAllocs()
+	if codeJSON == nil {
+		b.StopTimer()
+		codeInit()
+		b.StartTimer()
+	}
+	b.RunParallel(func(pb *testing.PB) {
+		var r codeResponse
+		for pb.Next() {
+			if err := Unmarshal(codeJSON, &r); err != nil {
+				b.Fatal("Unmarshal:", err)
+			}
+		}
+	})
+	b.SetBytes(int64(len(codeJSON)))
+}
+
+func BenchmarkUnmarshalString(b *testing.B) {
+	b.ReportAllocs()
+	data := []byte(`"hello, world"`)
+	b.RunParallel(func(pb *testing.PB) {
+		var s string
+		for pb.Next() {
+			if err := Unmarshal(data, &s); err != nil {
+				b.Fatal("Unmarshal:", err)
+			}
+		}
+	})
+}
+
+func BenchmarkUnmarshalFloat64(b *testing.B) {
+	b.ReportAllocs()
+	data := []byte(`3.14`)
+	b.RunParallel(func(pb *testing.PB) {
+		var f float64
+		for pb.Next() {
+			if err := Unmarshal(data, &f); err != nil {
+				b.Fatal("Unmarshal:", err)
+			}
+		}
+	})
+}
+
+func BenchmarkUnmarshalInt64(b *testing.B) {
+	b.ReportAllocs()
+	data := []byte(`3`)
+	b.RunParallel(func(pb *testing.PB) {
+		var x int64
+		for pb.Next() {
+			if err := Unmarshal(data, &x); err != nil {
+				b.Fatal("Unmarshal:", err)
+			}
+		}
+	})
+}
+
+func BenchmarkIssue10335(b *testing.B) {
+	b.ReportAllocs()
+	j := []byte(`{"a":{ }}`)
+	b.RunParallel(func(pb *testing.PB) {
+		var s struct{}
+		for pb.Next() {
+			if err := Unmarshal(j, &s); err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+}
+
+func BenchmarkUnmapped(b *testing.B) {
+	b.ReportAllocs()
+	j := []byte(`{"s": "hello", "y": 2, "o": {"x": 0}, "a": [1, 99, {"x": 1}]}`)
+	b.RunParallel(func(pb *testing.PB) {
+		var s struct{}
+		for pb.Next() {
+			if err := Unmarshal(j, &s); err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+}
+
+func BenchmarkTypeFieldsCache(b *testing.B) {
+	b.ReportAllocs()
+	var maxTypes int = 1e6
+	if testenv.Builder() != "" {
+		maxTypes = 1e3 // restrict cache sizes on builders
+	}
+
+	// Dynamically generate many new types.
+	types := make([]reflect.Type, maxTypes)
+	fs := []reflect.StructField{{
+		Type:  reflect.TypeOf(""),
+		Index: []int{0},
+	}}
+	for i := range types {
+		fs[0].Name = fmt.Sprintf("TypeFieldsCache%d", i)
+		types[i] = reflect.StructOf(fs)
+	}
+
+	// clearClear clears the cache. Other JSON operations, must not be running.
+	clearCache := func() {
+		fieldCache = sync.Map{}
+	}
+
+	// MissTypes tests the performance of repeated cache misses.
+	// This measures the time to rebuild a cache of size nt.
+	for nt := 1; nt <= maxTypes; nt *= 10 {
+		ts := types[:nt]
+		b.Run(fmt.Sprintf("MissTypes%d", nt), func(b *testing.B) {
+			nc := runtime.GOMAXPROCS(0)
+			for i := 0; i < b.N; i++ {
+				clearCache()
+				var wg sync.WaitGroup
+				for j := 0; j < nc; j++ {
+					wg.Add(1)
+					go func(j int) {
+						for _, t := range ts[(j*len(ts))/nc : ((j+1)*len(ts))/nc] {
+							cachedTypeFields(t)
+						}
+						wg.Done()
+					}(j)
+				}
+				wg.Wait()
+			}
+		})
+	}
+
+	// HitTypes tests the performance of repeated cache hits.
+	// This measures the average time of each cache lookup.
+	for nt := 1; nt <= maxTypes; nt *= 10 {
+		// Pre-warm a cache of size nt.
+		clearCache()
+		for _, t := range types[:nt] {
+			cachedTypeFields(t)
+		}
+		b.Run(fmt.Sprintf("HitTypes%d", nt), func(b *testing.B) {
+			b.RunParallel(func(pb *testing.PB) {
+				for pb.Next() {
+					cachedTypeFields(types[0])
+				}
+			})
+		})
+	}
+}

cli/output/jsonw/internal/jcolorenc/golang_example_marshaling_test.go

@@ -0,0 +1,73 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"strings"
+)
+
+type Animal int
+
+const (
+	Unknown Animal = iota
+	Gopher
+	Zebra
+)
+
+func (a *Animal) UnmarshalJSON(b []byte) error {
+	var s string
+	if err := json.Unmarshal(b, &s); err != nil {
+		return err
+	}
+	switch strings.ToLower(s) {
+	default:
+		*a = Unknown
+	case "gopher":
+		*a = Gopher
+	case "zebra":
+		*a = Zebra
+	}
+
+	return nil
+}
+
+func (a Animal) MarshalJSON() ([]byte, error) {
+	var s string
+	switch a {
+	default:
+		s = "unknown"
+	case Gopher:
+		s = "gopher"
+	case Zebra:
+		s = "zebra"
+	}
+
+	return json.Marshal(s)
+}
+
+func Example_customMarshalJSON() {
+	blob := `["gopher","armadillo","zebra","unknown","gopher","bee","gopher","zebra"]`
+	var zoo []Animal
+	if err := json.Unmarshal([]byte(blob), &zoo); err != nil {
+		log.Fatal(err)
+	}
+
+	census := make(map[Animal]int)
+	for _, animal := range zoo {
+		census[animal] += 1
+	}
+
+	fmt.Printf("Zoo Census:\n* Gophers: %d\n* Zebras:  %d\n* Unknown: %d\n",
+		census[Gopher], census[Zebra], census[Unknown])
+
+	// Output:
+	// Zoo Census:
+	// * Gophers: 3
+	// * Zebras:  2
+	// * Unknown: 3
+}

cli/output/jsonw/internal/jcolorenc/golang_example_test.go

@@ -0,0 +1,310 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"strings"
+)
+
+func ExampleMarshal() {
+	type ColorGroup struct {
+		ID     int
+		Name   string
+		Colors []string
+	}
+	group := ColorGroup{
+		ID:     1,
+		Name:   "Reds",
+		Colors: []string{"Crimson", "Red", "Ruby", "Maroon"},
+	}
+	b, err := json.Marshal(group)
+	if err != nil {
+		fmt.Println("error:", err)
+	}
+	os.Stdout.Write(b)
+	// Output:
+	// {"ID":1,"Name":"Reds","Colors":["Crimson","Red","Ruby","Maroon"]}
+}
+
+func ExampleUnmarshal() {
+	var jsonBlob = []byte(`[
+	{"Name": "Platypus", "Order": "Monotremata"},
+	{"Name": "Quoll",    "Order": "Dasyuromorphia"}
+]`)
+	type Animal struct {
+		Name  string
+		Order string
+	}
+	var animals []Animal
+	err := json.Unmarshal(jsonBlob, &animals)
+	if err != nil {
+		fmt.Println("error:", err)
+	}
+	fmt.Printf("%+v", animals)
+	// Output:
+	// [{Name:Platypus Order:Monotremata} {Name:Quoll Order:Dasyuromorphia}]
+}
+
+// This example uses a Decoder to decode a stream of distinct JSON values.
+func ExampleDecoder() {
+	const jsonStream = `
+	{"Name": "Ed", "Text": "Knock knock."}
+	{"Name": "Sam", "Text": "Who's there?"}
+	{"Name": "Ed", "Text": "Go fmt."}
+	{"Name": "Sam", "Text": "Go fmt who?"}
+	{"Name": "Ed", "Text": "Go fmt yourself!"}
+`
+	type Message struct {
+		Name, Text string
+	}
+	dec := json.NewDecoder(strings.NewReader(jsonStream))
+	for {
+		var m Message
+		if err := dec.Decode(&m); err == io.EOF {
+			break
+		} else if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Printf("%s: %s\n", m.Name, m.Text)
+	}
+	// Output:
+	// Ed: Knock knock.
+	// Sam: Who's there?
+	// Ed: Go fmt.
+	// Sam: Go fmt who?
+	// Ed: Go fmt yourself!
+}
+
+// This example uses a Decoder to decode a stream of distinct JSON values.
+func ExampleDecoder_Token() {
+	const jsonStream = `
+	{"Message": "Hello", "Array": [1, 2, 3], "Null": null, "Number": 1.234}
+`
+	dec := json.NewDecoder(strings.NewReader(jsonStream))
+	for {
+		t, err := dec.Token()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Printf("%T: %v", t, t)
+		if dec.More() {
+			fmt.Printf(" (more)")
+		}
+		fmt.Printf("\n")
+	}
+	// Output:
+	// json.Delim: { (more)
+	// string: Message (more)
+	// string: Hello (more)
+	// string: Array (more)
+	// json.Delim: [ (more)
+	// float64: 1 (more)
+	// float64: 2 (more)
+	// float64: 3
+	// json.Delim: ] (more)
+	// string: Null (more)
+	// <nil>: <nil> (more)
+	// string: Number (more)
+	// float64: 1.234
+	// json.Delim: }
+}
+
+// This example uses a Decoder to decode a streaming array of JSON objects.
+func ExampleDecoder_Decode_stream() {
+	const jsonStream = `
+	[
+		{"Name": "Ed", "Text": "Knock knock."},
+		{"Name": "Sam", "Text": "Who's there?"},
+		{"Name": "Ed", "Text": "Go fmt."},
+		{"Name": "Sam", "Text": "Go fmt who?"},
+		{"Name": "Ed", "Text": "Go fmt yourself!"}
+	]
+`
+	type Message struct {
+		Name, Text string
+	}
+	dec := json.NewDecoder(strings.NewReader(jsonStream))
+
+	// read open bracket
+	t, err := dec.Token()
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("%T: %v\n", t, t)
+
+	// while the array contains values
+	for dec.More() {
+		var m Message
+		// decode an array value (Message)
+		err := dec.Decode(&m)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		fmt.Printf("%v: %v\n", m.Name, m.Text)
+	}
+
+	// read closing bracket
+	t, err = dec.Token()
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("%T: %v\n", t, t)
+
+	// Output:
+	// json.Delim: [
+	// Ed: Knock knock.
+	// Sam: Who's there?
+	// Ed: Go fmt.
+	// Sam: Go fmt who?
+	// Ed: Go fmt yourself!
+	// json.Delim: ]
+}
+
+// This example uses RawMessage to delay parsing part of a JSON message.
+func ExampleRawMessage_unmarshal() {
+	type Color struct {
+		Space string
+		Point json.RawMessage // delay parsing until we know the color space
+	}
+	type RGB struct {
+		R uint8
+		G uint8
+		B uint8
+	}
+	type YCbCr struct {
+		Y  uint8
+		Cb int8
+		Cr int8
+	}
+
+	var j = []byte(`[
+	{"Space": "YCbCr", "Point": {"Y": 255, "Cb": 0, "Cr": -10}},
+	{"Space": "RGB",   "Point": {"R": 98, "G": 218, "B": 255}}
+]`)
+	var colors []Color
+	err := json.Unmarshal(j, &colors)
+	if err != nil {
+		log.Fatalln("error:", err)
+	}
+
+	for _, c := range colors {
+		var dst interface{}
+		switch c.Space {
+		case "RGB":
+			dst = new(RGB)
+		case "YCbCr":
+			dst = new(YCbCr)
+		}
+		err := json.Unmarshal(c.Point, dst)
+		if err != nil {
+			log.Fatalln("error:", err)
+		}
+		fmt.Println(c.Space, dst)
+	}
+	// Output:
+	// YCbCr &{255 0 -10}
+	// RGB &{98 218 255}
+}
+
+// This example uses RawMessage to use a precomputed JSON during marshal.
+func ExampleRawMessage_marshal() {
+	h := json.RawMessage(`{"precomputed": true}`)
+
+	c := struct {
+		Header *json.RawMessage `json:"header"`
+		Body   string           `json:"body"`
+	}{Header: &h, Body: "Hello Gophers!"}
+
+	b, err := json.MarshalIndent(&c, "", "\t")
+	if err != nil {
+		fmt.Println("error:", err)
+	}
+	os.Stdout.Write(b)
+
+	// Output:
+	// {
+	// 	"header": {
+	// 		"precomputed": true
+	// 	},
+	// 	"body": "Hello Gophers!"
+	// }
+}
+
+func ExampleIndent() {
+	type Road struct {
+		Name   string
+		Number int
+	}
+	roads := []Road{
+		{"Diamond Fork", 29},
+		{"Sheep Creek", 51},
+	}
+
+	b, err := json.Marshal(roads)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var out bytes.Buffer
+	json.Indent(&out, b, "=", "\t")
+	out.WriteTo(os.Stdout)
+	// Output:
+	// [
+	// =	{
+	// =		"Name": "Diamond Fork",
+	// =		"Number": 29
+	// =	},
+	// =	{
+	// =		"Name": "Sheep Creek",
+	// =		"Number": 51
+	// =	}
+	// =]
+}
+
+func ExampleMarshalIndent() {
+	data := map[string]int{
+		"a": 1,
+		"b": 2,
+	}
+
+	json, err := json.MarshalIndent(data, "<prefix>", "<indent>")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(string(json))
+	// Output:
+	// {
+	// <prefix><indent>"a": 1,
+	// <prefix><indent>"b": 2
+	// <prefix>}
+}
+
+func ExampleValid() {
+	goodJSON := `{"example": 1}`
+	badJSON := `{"example":2:]}}`
+
+	fmt.Println(json.Valid([]byte(goodJSON)), json.Valid([]byte(badJSON)))
+	// Output:
+	// true false
+}
+
+func ExampleHTMLEscape() {
+	var out bytes.Buffer
+	json.HTMLEscape(&out, []byte(`{"Name":"<b>HTML content</b>"}`))
+	out.WriteTo(os.Stdout)
+	// Output:
+	//{"Name":"\u003cb\u003eHTML content\u003c/b\u003e"}
+}

cli/output/jsonw/internal/jcolorenc/golang_number_test.go

@@ -0,0 +1,129 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"regexp"
+	"testing"
+)
+
+func TestNumberIsValid(t *testing.T) {
+	// From: https://stackoverflow.com/a/13340826
+	var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`)
+
+	validTests := []string{
+		"0",
+		"-0",
+		"1",
+		"-1",
+		"0.1",
+		"-0.1",
+		"1234",
+		"-1234",
+		"12.34",
+		"-12.34",
+		"12E0",
+		"12E1",
+		"12e34",
+		"12E-0",
+		"12e+1",
+		"12e-34",
+		"-12E0",
+		"-12E1",
+		"-12e34",
+		"-12E-0",
+		"-12e+1",
+		"-12e-34",
+		"1.2E0",
+		"1.2E1",
+		"1.2e34",
+		"1.2E-0",
+		"1.2e+1",
+		"1.2e-34",
+		"-1.2E0",
+		"-1.2E1",
+		"-1.2e34",
+		"-1.2E-0",
+		"-1.2e+1",
+		"-1.2e-34",
+		"0E0",
+		"0E1",
+		"0e34",
+		"0E-0",
+		"0e+1",
+		"0e-34",
+		"-0E0",
+		"-0E1",
+		"-0e34",
+		"-0E-0",
+		"-0e+1",
+		"-0e-34",
+	}
+
+	for _, test := range validTests {
+		if !isValidNumber(test) {
+			t.Errorf("%s should be valid", test)
+		}
+
+		var f float64
+		if err := Unmarshal([]byte(test), &f); err != nil {
+			t.Errorf("%s should be valid but Unmarshal failed: %v", test, err)
+		}
+
+		if !jsonNumberRegexp.MatchString(test) {
+			t.Errorf("%s should be valid but regexp does not match", test)
+		}
+	}
+
+	invalidTests := []string{
+		"",
+		"invalid",
+		"1.0.1",
+		"1..1",
+		"-1-2",
+		"012a42",
+		"01.2",
+		"012",
+		"12E12.12",
+		"1e2e3",
+		"1e+-2",
+		"1e--23",
+		"1e",
+		"e1",
+		"1e+",
+		"1ea",
+		"1a",
+		"1.a",
+		"1.",
+		"01",
+		"1.e1",
+	}
+
+	for _, test := range invalidTests {
+		var f float64
+		if err := Unmarshal([]byte(test), &f); err == nil {
+			t.Errorf("%s should be invalid but unmarshal wrote %v", test, f)
+		}
+
+		if jsonNumberRegexp.MatchString(test) {
+			t.Errorf("%s should be invalid but matches regexp", test)
+		}
+	}
+}
+
+func BenchmarkNumberIsValid(b *testing.B) {
+	s := "-61657.61667E+61673"
+	for i := 0; i < b.N; i++ {
+		isValidNumber(s)
+	}
+}
+
+func BenchmarkNumberIsValidRegexp(b *testing.B) {
+	var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`)
+	s := "-61657.61667E+61673"
+	for i := 0; i < b.N; i++ {
+		jsonNumberRegexp.MatchString(s)
+	}
+}

cli/output/jsonw/internal/jcolorenc/golang_scanner_test.go

@@ -1,4 +1,4 @@
-// Copyright 2010 The Go Authors.  All rights reserved.
+// Copyright 2010 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
@@ -8,10 +8,29 @@ import (
 	"bytes"
 	"math"
 	"math/rand"
-	"reflect"
 	"testing"
 )
 
+var validTests = []struct {
+	data string
+	ok   bool
+}{
+	{`foo`, false},
+	{`}{`, false},
+	{`{]`, false},
+	{`{}`, true},
+	{`{"foo":"bar"}`, true},
+	{`{"foo":"bar","bar":{"baz":["qux"]}}`, true},
+}
+
+func TestValid(t *testing.T) {
+	for _, tt := range validTests {
+		if ok := Valid([]byte(tt.data)); ok != tt.ok {
+			t.Errorf("Valid(%#q) = %v, want %v", tt.data, ok, tt.ok)
+		}
+	}
+}
+
 // Tests of simple examples.
 
 type example struct {
@@ -28,6 +47,7 @@ var examples = []example{
 	{`[1,2,3]`, "[\n\t1,\n\t2,\n\t3\n]"},
 	{`{"x":1}`, "{\n\t\"x\": 1\n}"},
 	{ex1, ex1i},
+	{"{\"\":\"<>&\u2028\u2029\"}", "{\n\t\"\": \"<>&\u2028\u2029\"\n}"}, // See golang.org/issue/34070
 }
 
 var ex1 = `[true,false,null,"x",1,1.5,0,-5e+2]`
@@ -69,8 +89,8 @@ func TestCompactSeparators(t *testing.T) {
 	tests := []struct {
 		in, compact string
 	}{
-		{"{\"\u2028\": 1}", `{"\u2028":1}`},
-		{"{\"\u2029\" :2}", `{"\u2029":2}`},
+		{"{\"\u2028\": 1}", "{\"\u2028\":1}"},
+		{"{\"\u2029\" :2}", "{\"\u2029\":2}"},
 	}
 	for _, tt := range tests {
 		var buf bytes.Buffer
@@ -119,6 +139,7 @@ func TestCompactBig(t *testing.T) {
 }
 
 func TestIndentBig(t *testing.T) {
+	t.Parallel()
 	initBig()
 	var buf bytes.Buffer
 	if err := Indent(&buf, jsonBig, "", "\t"); err != nil {
@@ -162,59 +183,19 @@ type indentErrorTest struct {
 }
 
 var indentErrorTests = []indentErrorTest{
-	{`{"X": "foo", "Y"}`, &SyntaxError{"invalid character '}' after object key", 17}},
-	{`{"X": "foo" "Y": "bar"}`, &SyntaxError{"invalid character '\"' after object key:value pair", 13}},
+	{`{"X": "foo", "Y"}`, &testSyntaxError{"invalid character '}' after object key", 17}},
+	{`{"X": "foo" "Y": "bar"}`, &testSyntaxError{"invalid character '\"' after object key:value pair", 13}},
 }
 
 func TestIndentErrors(t *testing.T) {
 	for i, tt := range indentErrorTests {
 		slice := make([]uint8, 0)
 		buf := bytes.NewBuffer(slice)
-		if err := Indent(buf, []uint8(tt.in), "", ""); err != nil {
-			if !reflect.DeepEqual(err, tt.err) {
-				t.Errorf("#%d: Indent: %#v", i, err)
-				continue
-			}
-		}
+		err := Indent(buf, []uint8(tt.in), "", "")
+		assertErrorPresence(t, tt.err, err, i)
 	}
 }
 
-func TestNextValueBig(t *testing.T) {
-	initBig()
-	var scan Scanner
-	item, rest, err := NextValue(jsonBig, &scan)
-	if err != nil {
-		t.Fatalf("NextValue: %s", err)
-	}
-	if len(item) != len(jsonBig) || &item[0] != &jsonBig[0] {
-		t.Errorf("invalid item: %d %d", len(item), len(jsonBig))
-	}
-	if len(rest) != 0 {
-		t.Errorf("invalid rest: %d", len(rest))
-	}
-
-	item, rest, err = NextValue(append(jsonBig, "HELLO WORLD"...), &scan)
-	if err != nil {
-		t.Fatalf("NextValue extra: %s", err)
-	}
-	if len(item) != len(jsonBig) {
-		t.Errorf("invalid item: %d %d", len(item), len(jsonBig))
-	}
-	if string(rest) != "HELLO WORLD" {
-		t.Errorf("invalid rest: %d", len(rest))
-	}
-}
-
-var benchScan Scanner
-
-func BenchmarkSkipValue(b *testing.B) {
-	initBig()
-	for i := 0; i < b.N; i++ {
-		NextValue(jsonBig, &benchScan)
-	}
-	b.SetBytes(int64(len(jsonBig)))
-}
-
 func diff(t *testing.T, a, b []byte) {
 	for i := 0; ; i++ {
 		if i >= len(a) || i >= len(b) || a[i] != b[i] {

cli/output/jsonw/internal/jcolorenc/golang_shim_test.go

@@ -0,0 +1,72 @@
+// This file is a shim for dependencies of golang_*_test.go files that are normally provided by the standard library.
+// It helps importing those files with minimal changes.
+package json
+
+import (
+	"bytes"
+	"reflect"
+	"sync"
+	"testing"
+)
+
+// Field cache used in golang_bench_test.go
+var fieldCache = sync.Map{}
+
+func cachedTypeFields(reflect.Type) {}
+
+// Fake test env for golang_bench_test.go
+type testenvShim struct {
+}
+
+func (ts testenvShim) Builder() string {
+	return ""
+}
+
+var testenv testenvShim
+
+// Fake scanner for golang_decode_test.go
+type scanner struct {
+}
+
+func checkValid(in []byte, scan *scanner) error {
+	return nil
+}
+
+// Actual isSpace implementation
+func isSpace(c byte) bool {
+	return c == ' ' || c == '\t' || c == '\r' || c == '\n'
+}
+
+// Fake encoder for golang_encode_test.go
+type encodeState struct {
+	Buffer bytes.Buffer
+}
+
+func (es *encodeState) string(s string, escapeHTML bool) {
+}
+
+func (es *encodeState) stringBytes(b []byte, escapeHTML bool) {
+}
+
+// Fake number test
+func isValidNumber(n string) bool {
+	return true
+}
+
+func assertErrorPresence(t *testing.T, expected error, actual error, prefixes ...interface{}) {
+	if expected != nil && actual == nil {
+		errorWithPrefixes(t, prefixes, "expected error, but did not get an error")
+	} else if expected == nil && actual != nil {
+		errorWithPrefixes(t, prefixes, "did not expect error but got %v", actual)
+	}
+}
+
+func errorWithPrefixes(t *testing.T, prefixes []interface{}, format string, elements ...interface{}) {
+	fullFormat := format
+	allElements := append(prefixes, elements...)
+
+	for range prefixes {
+		fullFormat = "%v: " + fullFormat
+	}
+	t.Errorf(fullFormat, allElements...)
+}

cli/output/jsonw/internal/jcolorenc/golang_tagkey_test.go

@@ -1,4 +1,4 @@
-// Copyright 2011 The Go Authors.  All rights reserved.
+// Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
@@ -37,11 +37,15 @@ type miscPlaneTag struct {
 }
 
 type percentSlashTag struct {
-	V string `json:"text/html%"` // http://golang.org/issue/2718
+	V string `json:"text/html%"` // https://golang.org/issue/2718
 }
 
 type punctuationTag struct {
-	V string `json:"!#$%&()*+-./:<=>?@[]^_{|}~"` // http://golang.org/issue/3546
+	V string `json:"!#$%&()*+-./:<=>?@[]^_{|}~"` // https://golang.org/issue/3546
+}
+
+type dashTag struct {
+	V string `json:"-,"`
 }
 
 type emptyTag struct {
@@ -80,6 +84,7 @@ var structTagObjectKeyTests = []struct {
 	{basicLatin6xTag{"6x"}, "6x", "abcdefghijklmno"},
 	{basicLatin7xTag{"7x"}, "7x", "pqrstuvwxyz"},
 	{miscPlaneTag{"いろはにほへと"}, "いろはにほへと", "色は匂へど"},
+	{dashTag{"foo"}, "foo", "-"},
 	{emptyTag{"Pour Moi"}, "Pour Moi", "W"},
 	{misnamedTag{"Animal Kingdom"}, "Animal Kingdom", "X"},
 	{badFormatTag{"Orfevre"}, "Orfevre", "Y"},

cli/output/jsonw/internal/jcolorenc/json.go

@@ -0,0 +1,460 @@
+package json
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"reflect"
+	"runtime"
+	"sync"
+	"unsafe"
+
+	"github.com/neilotoole/sq/cli/output/jsonw/internal"
+)
+
+// Delim is documented at https://golang.org/pkg/encoding/json/#Delim
+type Delim = json.Delim
+
+// InvalidUTF8Error is documented at https://golang.org/pkg/encoding/json/#InvalidUTF8Error
+type InvalidUTF8Error = json.InvalidUTF8Error
+
+// InvalidUnmarshalError is documented at https://golang.org/pkg/encoding/json/#InvalidUnmarshalError
+type InvalidUnmarshalError = json.InvalidUnmarshalError
+
+// Marshaler is documented at https://golang.org/pkg/encoding/json/#Marshaler
+type Marshaler = json.Marshaler
+
+// MarshalerError is documented at https://golang.org/pkg/encoding/json/#MarshalerError
+type MarshalerError = json.MarshalerError
+
+// Number is documented at https://golang.org/pkg/encoding/json/#Number
+type Number = json.Number
+
+// RawMessage is documented at https://golang.org/pkg/encoding/json/#RawMessage
+type RawMessage = json.RawMessage
+
+// A SyntaxError is a description of a JSON syntax error.
+type SyntaxError = json.SyntaxError
+
+// Token is documented at https://golang.org/pkg/encoding/json/#Token
+type Token = json.Token
+
+// UnmarshalFieldError is documented at https://golang.org/pkg/encoding/json/#UnmarshalFieldError
+type UnmarshalFieldError = json.UnmarshalFieldError
+
+// UnmarshalTypeError is documented at https://golang.org/pkg/encoding/json/#UnmarshalTypeError
+type UnmarshalTypeError = json.UnmarshalTypeError
+
+// Unmarshaler is documented at https://golang.org/pkg/encoding/json/#Unmarshaler
+type Unmarshaler = json.Unmarshaler
+
+// UnsupportedTypeError is documented at https://golang.org/pkg/encoding/json/#UnsupportedTypeError
+type UnsupportedTypeError = json.UnsupportedTypeError
+
+// UnsupportedValueError is documented at https://golang.org/pkg/encoding/json/#UnsupportedValueError
+type UnsupportedValueError = json.UnsupportedValueError
+
+// AppendFlags is a type used to represent configuration options that can be
+// applied when formatting json output.
+type AppendFlags int
+
+const (
+	// EscapeHTML is a formatting flag used to to escape HTML in json strings.
+	EscapeHTML AppendFlags = 1 << iota
+
+	// SortMapKeys is formatting flag used to enable sorting of map keys when
+	// encoding JSON (this matches the behavior of the standard encoding/json
+	// package).
+	SortMapKeys
+
+	// TrustRawMessage is a performance optimization flag to skip value
+	// checking of raw messages. It should only be used if the values are
+	// known to be valid json (e.g., they were created by json.Unmarshal).
+	TrustRawMessage
+)
+
+// ParseFlags is a type used to represent configuration options that can be
+// applied when parsing json input.
+type ParseFlags int
+
+const (
+	// DisallowUnknownFields is a parsing flag used to prevent decoding of
+	// objects to Go struct values when a field of the input does not match
+	// with any of the struct fields.
+	DisallowUnknownFields ParseFlags = 1 << iota
+
+	// UseNumber is a parsing flag used to load numeric values as Number
+	// instead of float64.
+	UseNumber
+
+	// DontCopyString is a parsing flag used to provide zero-copy support when
+	// loading string values from a json payload. It is not always possible to
+	// avoid dynamic memory allocations, for example when a string is escaped in
+	// the json data a new buffer has to be allocated, but when the `wire` value
+	// can be used as content of a Go value the decoder will simply point into
+	// the input buffer.
+	DontCopyString
+
+	// DontCopyNumber is a parsing flag used to provide zero-copy support when
+	// loading Number values (see DontCopyString and DontCopyRawMessage).
+	DontCopyNumber
+
+	// DontCopyRawMessage is a parsing flag used to provide zero-copy support
+	// when loading RawMessage values from a json payload. When used, the
+	// RawMessage values will not be allocated into new memory buffers and
+	// will instead point directly to the area of the input buffer where the
+	// value was found.
+	DontCopyRawMessage
+
+	// DontMatchCaseInsensitiveStructFields is a parsing flag used to prevent
+	// matching fields in a case-insensitive way. This can prevent degrading
+	// performance on case conversions, and can also act as a stricter decoding
+	// mode.
+	DontMatchCaseInsensitiveStructFields
+
+	// ZeroCopy is a parsing flag that combines all the copy optimizations
+	// available in the package.
+	//
+	// The zero-copy optimizations are better used in request-handler style
+	// code where none of the values are retained after the handler returns.
+	ZeroCopy = DontCopyString | DontCopyNumber | DontCopyRawMessage
+)
+
+// Append acts like Marshal but appends the json representation to b instead of
+// always reallocating a new slice.
+func Append(b []byte, x interface{}, flags AppendFlags, clrs internal.Colors, indenter *Indenter) ([]byte, error) {
+	if x == nil {
+		// Special case for nil values because it makes the rest of the code
+		// simpler to assume that it won't be seeing nil pointers.
+		return clrs.AppendNull(b), nil
+	}
+
+	t := reflect.TypeOf(x)
+	p := (*iface)(unsafe.Pointer(&x)).ptr
+
+	cache := cacheLoad()
+	c, found := cache[typeid(t)]
+
+	if !found {
+		c = constructCachedCodec(t, cache)
+	}
+
+	b, err := c.encode(encoder{flags: flags, clrs: clrs, indenter: indenter}, b, p)
+	runtime.KeepAlive(x)
+	return b, err
+}
+
+// Compact is documented at https://golang.org/pkg/encoding/json/#Compact
+func Compact(dst *bytes.Buffer, src []byte) error {
+	return json.Compact(dst, src)
+}
+
+// HTMLEscape is documented at https://golang.org/pkg/encoding/json/#HTMLEscape
+func HTMLEscape(dst *bytes.Buffer, src []byte) {
+	json.HTMLEscape(dst, src)
+}
+
+// Indent is documented at https://golang.org/pkg/encoding/json/#Indent
+func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error {
+	return json.Indent(dst, src, prefix, indent)
+}
+
+// Marshal is documented at https://golang.org/pkg/encoding/json/#Marshal
+func Marshal(x interface{}) ([]byte, error) {
+	var err error
+	var buf = encoderBufferPool.Get().(*encoderBuffer)
+
+	if buf.data, err = Append(buf.data[:0], x, EscapeHTML|SortMapKeys, internal.Colors{}, nil); err != nil {
+		return nil, err
+	}
+
+	b := make([]byte, len(buf.data))
+	copy(b, buf.data)
+	encoderBufferPool.Put(buf)
+	return b, nil
+}
+
+// MarshalIndent is documented at https://golang.org/pkg/encoding/json/#MarshalIndent
+func MarshalIndent(x interface{}, prefix, indent string) ([]byte, error) {
+	b, err := Marshal(x)
+
+	if err == nil {
+		tmp := &bytes.Buffer{}
+		tmp.Grow(2 * len(b))
+
+		Indent(tmp, b, prefix, indent)
+		b = tmp.Bytes()
+	}
+
+	return b, err
+}
+
+// Unmarshal is documented at https://golang.org/pkg/encoding/json/#Unmarshal
+func Unmarshal(b []byte, x interface{}) error {
+	r, err := Parse(b, x, 0)
+	if len(r) != 0 {
+		if _, ok := err.(*SyntaxError); !ok {
+			// The encoding/json package prioritizes reporting errors caused by
+			// unexpected trailing bytes over other issues; here we emulate this
+			// behavior by overriding the error.
+			err = syntaxError(r, "invalid character '%c' after top-level value", r[0])
+		}
+	}
+	return err
+}
+
+// Parse behaves like Unmarshal but the caller can pass a set of flags to
+// configure the parsing behavior.
+func Parse(b []byte, x interface{}, flags ParseFlags) ([]byte, error) {
+	t := reflect.TypeOf(x)
+	p := (*iface)(unsafe.Pointer(&x)).ptr
+
+	if t == nil || p == nil || t.Kind() != reflect.Ptr {
+		_, r, err := parseValue(skipSpaces(b))
+		r = skipSpaces(r)
+		if err != nil {
+			return r, err
+		}
+		return r, &InvalidUnmarshalError{Type: t}
+	}
+	t = t.Elem()
+
+	cache := cacheLoad()
+	c, found := cache[typeid(t)]
+
+	if !found {
+		c = constructCachedCodec(t, cache)
+	}
+
+	r, err := c.decode(decoder{flags: flags}, skipSpaces(b), p)
+	return skipSpaces(r), err
+}
+
+// Valid is documented at https://golang.org/pkg/encoding/json/#Valid
+func Valid(data []byte) bool {
+	_, data, err := parseValue(skipSpaces(data))
+	if err != nil {
+		return false
+	}
+	return len(skipSpaces(data)) == 0
+}
+
+// Decoder is documented at https://golang.org/pkg/encoding/json/#Decoder
+type Decoder struct {
+	reader      io.Reader
+	buffer      []byte
+	remain      []byte
+	inputOffset int64
+	err         error
+	flags       ParseFlags
+}
+
+// NewDecoder is documented at https://golang.org/pkg/encoding/json/#NewDecoder
+func NewDecoder(r io.Reader) *Decoder { return &Decoder{reader: r} }
+
+// Buffered is documented at https://golang.org/pkg/encoding/json/#Decoder.Buffered
+func (dec *Decoder) Buffered() io.Reader {
+	return bytes.NewReader(dec.remain)
+}
+
+// Decode is documented at https://golang.org/pkg/encoding/json/#Decoder.Decode
+func (dec *Decoder) Decode(v interface{}) error {
+	raw, err := dec.readValue()
+	if err != nil {
+		return err
+	}
+	_, err = Parse(raw, v, dec.flags)
+	return err
+}
+
+const (
+	minBufferSize = 32768
+	minReadSize   = 4096
+)
+
+// readValue reads one JSON value from the buffer and returns its raw bytes. It
+// is optimized for the "one JSON value per line" case.
+func (dec *Decoder) readValue() (v []byte, err error) {
+	var n int
+	var r []byte
+
+	for {
+		if len(dec.remain) != 0 {
+			v, r, err = parseValue(dec.remain)
+			if err == nil {
+				dec.remain, n = skipSpacesN(r)
+				dec.inputOffset += int64(len(v) + n)
+				return
+			}
+			if len(r) != 0 {
+				// Parsing of the next JSON value stopped at a position other
+				// than the end of the input buffer, which indicaates that a
+				// syntax error was encountered.
+				return
+			}
+		}
+
+		if err = dec.err; err != nil {
+			if len(dec.remain) != 0 && err == io.EOF {
+				err = io.ErrUnexpectedEOF
+			}
+			return
+		}
+
+		if dec.buffer == nil {
+			dec.buffer = make([]byte, 0, minBufferSize)
+		} else {
+			dec.buffer = dec.buffer[:copy(dec.buffer[:cap(dec.buffer)], dec.remain)]
+			dec.remain = nil
+		}
+
+		if (cap(dec.buffer) - len(dec.buffer)) < minReadSize {
+			buf := make([]byte, len(dec.buffer), 2*cap(dec.buffer))
+			copy(buf, dec.buffer)
+			dec.buffer = buf
+		}
+
+		n, err = io.ReadFull(dec.reader, dec.buffer[len(dec.buffer):cap(dec.buffer)])
+		if n > 0 {
+			dec.buffer = dec.buffer[:len(dec.buffer)+n]
+			if err != nil {
+				err = nil
+			}
+		} else if err == io.ErrUnexpectedEOF {
+			err = io.EOF
+		}
+		dec.remain, n = skipSpacesN(dec.buffer)
+		dec.inputOffset += int64(n)
+		dec.err = err
+	}
+}
+
+// DisallowUnknownFields is documented at https://golang.org/pkg/encoding/json/#Decoder.DisallowUnknownFields
+func (dec *Decoder) DisallowUnknownFields() { dec.flags |= DisallowUnknownFields }
+
+// UseNumber is documented at https://golang.org/pkg/encoding/json/#Decoder.UseNumber
+func (dec *Decoder) UseNumber() { dec.flags |= UseNumber }
+
+// DontCopyString is an extension to the standard encoding/json package
+// which instructs the decoder to not copy strings loaded from the json
+// payloads when possible.
+func (dec *Decoder) DontCopyString() { dec.flags |= DontCopyString }
+
+// DontCopyNumber is an extension to the standard encoding/json package
+// which instructs the decoder to not copy numbers loaded from the json
+// payloads.
+func (dec *Decoder) DontCopyNumber() { dec.flags |= DontCopyNumber }
+
+// DontCopyRawMessage is an extension to the standard encoding/json package
+// which instructs the decoder to not allocate RawMessage values in separate
+// memory buffers (see the documentation of the DontcopyRawMessage flag for
+// more detais).
+func (dec *Decoder) DontCopyRawMessage() { dec.flags |= DontCopyRawMessage }
+
+// DontMatchCaseInsensitiveStructFields is an extension to the standard
+// encoding/json package which instructs the decoder to not match object fields
+// against struct fields in a case-insensitive way, the field names have to
+// match exactly to be decoded into the struct field values.
+func (dec *Decoder) DontMatchCaseInsensitiveStructFields() {
+	dec.flags |= DontMatchCaseInsensitiveStructFields
+}
+
+// ZeroCopy is an extension to the standard encoding/json package which enables
+// all the copy optimizations of the decoder.
+func (dec *Decoder) ZeroCopy() { dec.flags |= ZeroCopy }
+
+// InputOffset returns the input stream byte offset of the current decoder position.
+// The offset gives the location of the end of the most recently returned token
+// and the beginning of the next token.
+func (dec *Decoder) InputOffset() int64 {
+	return dec.inputOffset
+}
+
+// Encoder is documented at https://golang.org/pkg/encoding/json/#Encoder
+type Encoder struct {
+	writer io.Writer
+	// prefix   string
+	// indent   string
+	buffer   *bytes.Buffer
+	err      error
+	flags    AppendFlags
+	clrs     internal.Colors
+	indenter *Indenter
+}
+
+// NewEncoder is documented at https://golang.org/pkg/encoding/json/#NewEncoder
+func NewEncoder(w io.Writer) *Encoder { return &Encoder{writer: w, flags: EscapeHTML | SortMapKeys} }
+
+// SetColors sets the colors for the encoder to use.
+func (enc *Encoder) SetColors(c internal.Colors) {
+	enc.clrs = c
+}
+
+// Encode is documented at https://golang.org/pkg/encoding/json/#Encoder.Encode
+func (enc *Encoder) Encode(v interface{}) error {
+	if enc.err != nil {
+		return enc.err
+	}
+
+	var err error
+	var buf = encoderBufferPool.Get().(*encoderBuffer)
+
+	// Note: unlike the original segmentio encoder, indentation is
+	// performed via the Append function.
+	buf.data, err = Append(buf.data[:0], v, enc.flags, enc.clrs, enc.indenter)
+	if err != nil {
+		encoderBufferPool.Put(buf)
+		return err
+	}
+
+	buf.data = append(buf.data, '\n')
+	b := buf.data
+
+	if _, err := enc.writer.Write(b); err != nil {
+		enc.err = err
+	}
+
+	encoderBufferPool.Put(buf)
+	return err
+}
+
+// SetEscapeHTML is documented at https://golang.org/pkg/encoding/json/#Encoder.SetEscapeHTML
+func (enc *Encoder) SetEscapeHTML(on bool) {
+	if on {
+		enc.flags |= EscapeHTML
+	} else {
+		enc.flags &= ^EscapeHTML
+	}
+}
+
+// SetIndent is documented at https://golang.org/pkg/encoding/json/#Encoder.SetIndent
+func (enc *Encoder) SetIndent(prefix, indent string) {
+	enc.indenter = NewIndenter(prefix, indent)
+}
+
+// SetSortMapKeys is an extension to the standard encoding/json package which
+// allows the program to toggle sorting of map keys on and off.
+func (enc *Encoder) SetSortMapKeys(on bool) {
+	if on {
+		enc.flags |= SortMapKeys
+	} else {
+		enc.flags &= ^SortMapKeys
+	}
+}
+
+// SetTrustRawMessage skips value checking when encoding a raw json message. It should only
+// be used if the values are known to be valid json, e.g. because they were originally created
+// by json.Unmarshal.
+func (enc *Encoder) SetTrustRawMessage(on bool) {
+	if on {
+		enc.flags |= TrustRawMessage
+	} else {
+		enc.flags &= ^TrustRawMessage
+	}
+}
+
+var encoderBufferPool = sync.Pool{
+	New: func() interface{} { return &encoderBuffer{data: make([]byte, 0, 4096)} },
+}
+
+type encoderBuffer struct{ data []byte }

cli/output/jsonw/internal/jcolorenc/parse.go

@@ -0,0 +1,737 @@
+package json
+
+import (
+	"bytes"
+	"math"
+	"reflect"
+	"unicode"
+	"unicode/utf16"
+	"unicode/utf8"
+
+	"github.com/segmentio/encoding/ascii"
+)
+
+// All spaces characters defined in the json specification.
+const (
+	sp = ' '
+	ht = '\t'
+	nl = '\n'
+	cr = '\r'
+)
+
+const (
+	escape = '\\'
+	quote  = '"'
+)
+
+func skipSpaces(b []byte) []byte {
+	b, _ = skipSpacesN(b)
+	return b
+}
+
+func skipSpacesN(b []byte) ([]byte, int) {
+	for i := range b {
+		switch b[i] {
+		case sp, ht, nl, cr:
+		default:
+			return b[i:], i
+		}
+	}
+	return nil, 0
+}
+
+// parseInt parses a decimanl representation of an int64 from b.
+//
+// The function is equivalent to calling strconv.ParseInt(string(b), 10, 64) but
+// it prevents Go from making a memory allocation for converting a byte slice to
+// a string (escape analysis fails due to the error returned by strconv.ParseInt).
+//
+// Because it only works with base 10 the function is also significantly faster
+// than strconv.ParseInt.
+func parseInt(b []byte, t reflect.Type) (int64, []byte, error) {
+	var value int64
+	var count int
+
+	if len(b) == 0 {
+		return 0, b, syntaxError(b, "cannot decode integer from an empty input")
+	}
+
+	if b[0] == '-' {
+		const max = math.MinInt64
+		const lim = max / 10
+
+		if len(b) == 1 {
+			return 0, b, syntaxError(b, "cannot decode integer from '-'")
+		}
+
+		if len(b) > 2 && b[1] == '0' && '0' <= b[2] && b[2] <= '9' {
+			return 0, b, syntaxError(b, "invalid leading character '0' in integer")
+		}
+
+		for _, d := range b[1:] {
+			if !(d >= '0' && d <= '9') {
+				if count == 0 {
+					b, err := inputError(b, t)
+					return 0, b, err
+				}
+				break
+			}
+
+			if value < lim {
+				return 0, b, unmarshalOverflow(b, t)
+			}
+
+			value *= 10
+			x := int64(d - '0')
+
+			if value < (max + x) {
+				return 0, b, unmarshalOverflow(b, t)
+			}
+
+			value -= x
+			count++
+		}
+
+		count++
+	} else {
+		const max = math.MaxInt64
+		const lim = max / 10
+
+		if len(b) > 1 && b[0] == '0' && '0' <= b[1] && b[1] <= '9' {
+			return 0, b, syntaxError(b, "invalid leading character '0' in integer")
+		}
+
+		for _, d := range b {
+			if !(d >= '0' && d <= '9') {
+				if count == 0 {
+					b, err := inputError(b, t)
+					return 0, b, err
+				}
+				break
+			}
+			x := int64(d - '0')
+
+			if value > lim {
+				return 0, b, unmarshalOverflow(b, t)
+			}
+
+			if value *= 10; value > (max - x) {
+				return 0, b, unmarshalOverflow(b, t)
+			}
+
+			value += x
+			count++
+		}
+	}
+
+	if count < len(b) {
+		switch b[count] {
+		case '.', 'e', 'E': // was this actually a float?
+			v, r, err := parseNumber(b)
+			if err != nil {
+				v, r = b[:count+1], b[count+1:]
+			}
+			return 0, r, unmarshalTypeError(v, t)
+		}
+	}
+
+	return value, b[count:], nil
+}
+
+// parseUint is like parseInt but for unsigned integers.
+func parseUint(b []byte, t reflect.Type) (uint64, []byte, error) {
+	const max = math.MaxUint64
+	const lim = max / 10
+
+	var value uint64
+	var count int
+
+	if len(b) == 0 {
+		return 0, b, syntaxError(b, "cannot decode integer value from an empty input")
+	}
+
+	if len(b) > 1 && b[0] == '0' && '0' <= b[1] && b[1] <= '9' {
+		return 0, b, syntaxError(b, "invalid leading character '0' in integer")
+	}
+
+	for _, d := range b {
+		if !(d >= '0' && d <= '9') {
+			if count == 0 {
+				b, err := inputError(b, t)
+				return 0, b, err
+			}
+			break
+		}
+		x := uint64(d - '0')
+
+		if value > lim {
+			return 0, b, unmarshalOverflow(b, t)
+		}
+
+		if value *= 10; value > (max - x) {
+			return 0, b, unmarshalOverflow(b, t)
+		}
+
+		value += x
+		count++
+	}
+
+	if count < len(b) {
+		switch b[count] {
+		case '.', 'e', 'E': // was this actually a float?
+			v, r, err := parseNumber(b)
+			if err != nil {
+				v, r = b[:count+1], b[count+1:]
+			}
+			return 0, r, unmarshalTypeError(v, t)
+		}
+	}
+
+	return value, b[count:], nil
+}
+
+// parseUintHex parses a hexadecimanl representation of a uint64 from b.
+//
+// The function is equivalent to calling strconv.ParseUint(string(b), 16, 64) but
+// it prevents Go from making a memory allocation for converting a byte slice to
+// a string (escape analysis fails due to the error returned by strconv.ParseUint).
+//
+// Because it only works with base 16 the function is also significantly faster
+// than strconv.ParseUint.
+func parseUintHex(b []byte) (uint64, []byte, error) {
+	const max = math.MaxUint64
+	const lim = max / 0x10
+
+	var value uint64
+	var count int
+
+	if len(b) == 0 {
+		return 0, b, syntaxError(b, "cannot decode hexadecimal value from an empty input")
+	}
+
+parseLoop:
+	for i, d := range b {
+		var x uint64
+
+		switch {
+		case d >= '0' && d <= '9':
+			x = uint64(d - '0')
+
+		case d >= 'A' && d <= 'F':
+			x = uint64(d-'A') + 0xA
+
+		case d >= 'a' && d <= 'f':
+			x = uint64(d-'a') + 0xA
+
+		default:
+			if i == 0 {
+				return 0, b, syntaxError(b, "expected hexadecimal digit but found '%c'", d)
+			}
+			break parseLoop
+		}
+
+		if value > lim {
+			return 0, b, syntaxError(b, "hexadecimal value out of range")
+		}
+
+		if value *= 0x10; value > (max - x) {
+			return 0, b, syntaxError(b, "hexadecimal value out of range")
+		}
+
+		value += x
+		count++
+	}
+
+	return value, b[count:], nil
+}
+
+func parseNull(b []byte) ([]byte, []byte, error) {
+	if hasNullPrefix(b) {
+		return b[:4], b[4:], nil
+	}
+	if len(b) < 4 {
+		return nil, b[len(b):], unexpectedEOF(b)
+	}
+	return nil, b, syntaxError(b, "expected 'null' but found invalid token")
+}
+
+func parseTrue(b []byte) ([]byte, []byte, error) {
+	if hasTruePrefix(b) {
+		return b[:4], b[4:], nil
+	}
+	if len(b) < 4 {
+		return nil, b[len(b):], unexpectedEOF(b)
+	}
+	return nil, b, syntaxError(b, "expected 'true' but found invalid token")
+}
+
+func parseFalse(b []byte) ([]byte, []byte, error) {
+	if hasFalsePrefix(b) {
+		return b[:5], b[5:], nil
+	}
+	if len(b) < 5 {
+		return nil, b[len(b):], unexpectedEOF(b)
+	}
+	return nil, b, syntaxError(b, "expected 'false' but found invalid token")
+}
+
+func parseNumber(b []byte) (v, r []byte, err error) {
+	if len(b) == 0 {
+		r, err = b, unexpectedEOF(b)
+		return
+	}
+
+	i := 0
+	// sign
+	if b[i] == '-' {
+		i++
+	}
+
+	if i == len(b) {
+		r, err = b[i:], syntaxError(b, "missing number value after sign")
+		return
+	}
+
+	if b[i] < '0' || b[i] > '9' {
+		r, err = b[i:], syntaxError(b, "expected digit but got '%c'", b[i])
+		return
+	}
+
+	// integer part
+	if b[i] == '0' {
+		i++
+		if i == len(b) || (b[i] != '.' && b[i] != 'e' && b[i] != 'E') {
+			v, r = b[:i], b[i:]
+			return
+		}
+		if '0' <= b[i] && b[i] <= '9' {
+			r, err = b[i:], syntaxError(b, "cannot decode number with leading '0' character")
+			return
+		}
+	}
+
+	for i < len(b) && '0' <= b[i] && b[i] <= '9' {
+		i++
+	}
+
+	// decimal part
+	if i < len(b) && b[i] == '.' {
+		i++
+		decimalStart := i
+
+		for i < len(b) {
+			if c := b[i]; !('0' <= c && c <= '9') {
+				if i == decimalStart {
+					r, err = b[i:], syntaxError(b, "expected digit but found '%c'", c)
+					return
+				}
+				break
+			}
+			i++
+		}
+
+		if i == decimalStart {
+			r, err = b[i:], syntaxError(b, "expected decimal part after '.'")
+			return
+		}
+	}
+
+	// exponent part
+	if i < len(b) && (b[i] == 'e' || b[i] == 'E') {
+		i++
+
+		if i < len(b) {
+			if c := b[i]; c == '+' || c == '-' {
+				i++
+			}
+		}
+
+		if i == len(b) {
+			r, err = b[i:], syntaxError(b, "missing exponent in number")
+			return
+		}
+
+		exponentStart := i
+
+		for i < len(b) {
+			if c := b[i]; !('0' <= c && c <= '9') {
+				if i == exponentStart {
+					err = syntaxError(b, "expected digit but found '%c'", c)
+					return
+				}
+				break
+			}
+			i++
+		}
+	}
+
+	v, r = b[:i], b[i:]
+	return
+}
+
+func parseUnicode(b []byte) (rune, int, error) {
+	if len(b) < 4 {
+		return 0, 0, syntaxError(b, "unicode code point must have at least 4 characters")
+	}
+
+	u, r, err := parseUintHex(b[:4])
+	if err != nil {
+		return 0, 0, syntaxError(b, "parsing unicode code point: %s", err)
+	}
+
+	if len(r) != 0 {
+		return 0, 0, syntaxError(b, "invalid unicode code point")
+	}
+
+	return rune(u), 4, nil
+}
+
+func parseStringFast(b []byte) ([]byte, []byte, bool, error) {
+	if len(b) < 2 {
+		return nil, b[len(b):], false, unexpectedEOF(b)
+	}
+	if b[0] != '"' {
+		return nil, b, false, syntaxError(b, "expected '\"' at the beginning of a string value")
+	}
+
+	n := bytes.IndexByte(b[1:], '"') + 2
+	if n <= 1 {
+		return nil, b[len(b):], false, syntaxError(b, "missing '\"' at the end of a string value")
+	}
+	if bytes.IndexByte(b[1:n], '\\') < 0 && ascii.ValidPrint(b[1:n]) {
+		return b[:n], b[n:], false, nil
+	}
+
+	for i := 1; i < len(b); i++ {
+		switch b[i] {
+		case '\\':
+			if i++; i < len(b) {
+				switch b[i] {
+				case '"', '\\', '/', 'n', 'r', 't', 'f', 'b':
+				case 'u':
+					_, n, err := parseUnicode(b[i+1:])
+					if err != nil {
+						return nil, b, false, err
+					}
+					i += n
+				default:
+					return nil, b, false, syntaxError(b, "invalid character '%c' in string escape code", b[i])
+				}
+			}
+
+		case '"':
+			return b[:i+1], b[i+1:], true, nil
+
+		default:
+			if b[i] < 0x20 {
+				return nil, b, false, syntaxError(b, "invalid character '%c' in string escape code", b[i])
+			}
+		}
+	}
+
+	return nil, b[len(b):], false, syntaxError(b, "missing '\"' at the end of a string value")
+}
+
+func parseString(b []byte) ([]byte, []byte, error) {
+	s, b, _, err := parseStringFast(b)
+	return s, b, err
+}
+
+func parseStringUnquote(b []byte, r []byte) ([]byte, []byte, bool, error) {
+	s, b, escaped, err := parseStringFast(b)
+	if err != nil {
+		return s, b, false, err
+	}
+
+	s = s[1 : len(s)-1] // trim the quotes
+
+	if !escaped {
+		return s, b, false, nil
+	}
+
+	if r == nil {
+		r = make([]byte, 0, len(s))
+	}
+
+	for len(s) != 0 {
+		i := bytes.IndexByte(s, '\\')
+
+		if i < 0 {
+			r = appendCoerceInvalidUTF8(r, s)
+			break
+		}
+
+		r = appendCoerceInvalidUTF8(r, s[:i])
+		s = s[i+1:]
+
+		c := s[0]
+		switch c {
+		case '"', '\\', '/':
+			// simple escaped character
+		case 'n':
+			c = '\n'
+
+		case 'r':
+			c = '\r'
+
+		case 't':
+			c = '\t'
+
+		case 'b':
+			c = '\b'
+
+		case 'f':
+			c = '\f'
+
+		case 'u':
+			s = s[1:]
+
+			r1, n1, err := parseUnicode(s)
+			if err != nil {
+				return r, b, true, err
+			}
+			s = s[n1:]
+
+			if utf16.IsSurrogate(r1) {
+				if !hasPrefix(s, `\u`) {
+					r1 = unicode.ReplacementChar
+				} else {
+					r2, n2, err := parseUnicode(s[2:])
+					if err != nil {
+						return r, b, true, err
+					}
+					if r1 = utf16.DecodeRune(r1, r2); r1 != unicode.ReplacementChar {
+						s = s[2+n2:]
+					}
+				}
+			}
+
+			r = appendRune(r, r1)
+			continue
+
+		default: // not sure what this escape sequence is
+			return r, b, false, syntaxError(s, "invalid character '%c' in string escape code", c)
+		}
+
+		r = append(r, c)
+		s = s[1:]
+	}
+
+	return r, b, true, nil
+}
+
+func appendRune(b []byte, r rune) []byte {
+	n := len(b)
+	b = append(b, 0, 0, 0, 0)
+	return b[:n+utf8.EncodeRune(b[n:], r)]
+}
+
+func appendCoerceInvalidUTF8(b []byte, s []byte) []byte {
+	c := [4]byte{}
+
+	for _, r := range string(s) {
+		b = append(b, c[:utf8.EncodeRune(c[:], r)]...)
+	}
+
+	return b
+}
+
+func parseObject(b []byte) ([]byte, []byte, error) {
+	if len(b) < 2 {
+		return nil, b[len(b):], unexpectedEOF(b)
+	}
+
+	if b[0] != '{' {
+		return nil, b, syntaxError(b, "expected '{' at the beginning of an object value")
+	}
+
+	var err error
+	var a = b
+	var n = len(b)
+	var i = 0
+
+	b = b[1:]
+	for {
+		b = skipSpaces(b)
+
+		if len(b) == 0 {
+			return nil, b, syntaxError(b, "cannot decode object from empty input")
+		}
+
+		if b[0] == '}' {
+			j := (n - len(b)) + 1
+			return a[:j], a[j:], nil
+		}
+
+		if i != 0 {
+			if len(b) == 0 {
+				return nil, b, syntaxError(b, "unexpected EOF after object field value")
+			}
+			if b[0] != ',' {
+				return nil, b, syntaxError(b, "expected ',' after object field value but found '%c'", b[0])
+			}
+			b = skipSpaces(b[1:])
+			if len(b) == 0 {
+				return nil, b, unexpectedEOF(b)
+			}
+			if b[0] == '}' {
+				return nil, b, syntaxError(b, "unexpected trailing comma after object field")
+			}
+		}
+
+		_, b, err = parseString(b)
+		if err != nil {
+			return nil, b, err
+		}
+		b = skipSpaces(b)
+
+		if len(b) == 0 {
+			return nil, b, syntaxError(b, "unexpected EOF after object field key")
+		}
+		if b[0] != ':' {
+			return nil, b, syntaxError(b, "expected ':' after object field key but found '%c'", b[0])
+		}
+		b = skipSpaces(b[1:])
+
+		_, b, err = parseValue(b)
+		if err != nil {
+			return nil, b, err
+		}
+
+		i++
+	}
+}
+
+func parseArray(b []byte) ([]byte, []byte, error) {
+	if len(b) < 2 {
+		return nil, b[len(b):], unexpectedEOF(b)
+	}
+
+	if b[0] != '[' {
+		return nil, b, syntaxError(b, "expected '[' at the beginning of array value")
+	}
+
+	var err error
+	var a = b
+	var n = len(b)
+	var i = 0
+
+	b = b[1:]
+	for {
+		b = skipSpaces(b)
+
+		if len(b) == 0 {
+			return nil, b, syntaxError(b, "missing closing ']' after array value")
+		}
+
+		if b[0] == ']' {
+			j := (n - len(b)) + 1
+			return a[:j], a[j:], nil
+		}
+
+		if i != 0 {
+			if len(b) == 0 {
+				return nil, b, syntaxError(b, "unexpected EOF after array element")
+			}
+			if b[0] != ',' {
+				return nil, b, syntaxError(b, "expected ',' after array element but found '%c'", b[0])
+			}
+			b = skipSpaces(b[1:])
+			if len(b) == 0 {
+				return nil, b, unexpectedEOF(b)
+			}
+			if b[0] == ']' {
+				return nil, b, syntaxError(b, "unexpected trailing comma after object field")
+			}
+		}
+
+		_, b, err = parseValue(b)
+		if err != nil {
+			return nil, b, err
+		}
+
+		i++
+	}
+}
+
+func parseValue(b []byte) ([]byte, []byte, error) {
+	if len(b) != 0 {
+		switch b[0] {
+		case '{':
+			return parseObject(b)
+		case '[':
+			return parseArray(b)
+		case '"':
+			return parseString(b)
+		case 'n':
+			return parseNull(b)
+		case 't':
+			return parseTrue(b)
+		case 'f':
+			return parseFalse(b)
+		case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+			return parseNumber(b)
+		default:
+			return nil, b, syntaxError(b, "invalid character '%c' looking for beginning of value", b[0])
+		}
+	}
+	return nil, b, syntaxError(b, "unexpected end of JSON input")
+}
+
+func hasNullPrefix(b []byte) bool {
+	return len(b) >= 4 && string(b[:4]) == "null"
+}
+
+func hasTruePrefix(b []byte) bool {
+	return len(b) >= 4 && string(b[:4]) == "true"
+}
+
+func hasFalsePrefix(b []byte) bool {
+	return len(b) >= 5 && string(b[:5]) == "false"
+}
+
+func hasPrefix(b []byte, s string) bool {
+	return len(b) >= len(s) && s == string(b[:len(s)])
+}
+
+func hasLeadingSign(b []byte) bool {
+	return len(b) > 0 && (b[0] == '+' || b[0] == '-')
+}
+
+func hasLeadingZeroes(b []byte) bool {
+	if hasLeadingSign(b) {
+		b = b[1:]
+	}
+	return len(b) > 1 && b[0] == '0' && '0' <= b[1] && b[1] <= '9'
+}
+
+func appendToLower(b, s []byte) []byte {
+	if ascii.Valid(s) { // fast path for ascii strings
+		i := 0
+
+		for j := range s {
+			c := s[j]
+
+			if 'A' <= c && c <= 'Z' {
+				b = append(b, s[i:j]...)
+				b = append(b, c+('a'-'A'))
+				i = j + 1
+			}
+		}
+
+		return append(b, s[i:]...)
+	}
+
+	for _, r := range string(s) {
+		b = appendRune(b, foldRune(r))
+	}
+
+	return b
+}
+
+func foldRune(r rune) rune {
+	if r = unicode.SimpleFold(r); 'A' <= r && r <= 'Z' {
+		r = r + ('a' - 'A')
+	}
+	return r
+}

cli/output/jsonw/internal/jcolorenc/parse_test.go

@@ -0,0 +1,174 @@
+package json
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+)
+
+func TestParseString(t *testing.T) {
+	tests := []struct {
+		in  string
+		out string
+		ext string
+	}{
+		{`""`, `""`, ``},
+		{`"1234567890"`, `"1234567890"`, ``},
+		{`"Hello World!"`, `"Hello World!"`, ``},
+		{`"Hello\"World!"`, `"Hello\"World!"`, ``},
+		{`"\\"`, `"\\"`, ``},
+	}
+
+	for _, test := range tests {
+		t.Run(test.in, func(t *testing.T) {
+			out, ext, err := parseString([]byte(test.in))
+
+			if err != nil {
+				t.Errorf("%s => %s", test.in, err)
+				return
+			}
+
+			if s := string(out); s != test.out {
+				t.Error("invalid output")
+				t.Logf("expected: %s", test.out)
+				t.Logf("found:    %s", s)
+			}
+
+			if s := string(ext); s != test.ext {
+				t.Error("invalid extra bytes")
+				t.Logf("expected: %s", test.ext)
+				t.Logf("found:    %s", s)
+			}
+		})
+	}
+}
+
+func TestAppendToLower(t *testing.T) {
+	tests := []string{
+		"",
+		"A",
+		"a",
+		"__segment_internal",
+		"someFieldWithALongBName",
+		"Hello World!",
+		"Hello\"World!",
+		"Hello\\World!",
+		"Hello\nWorld!",
+		"Hello\rWorld!",
+		"Hello\tWorld!",
+		"Hello\bWorld!",
+		"Hello\fWorld!",
+		"你好",
+		"<",
+		">",
+		"&",
+		"\u001944",
+		"\u00c2e>",
+		"\u00c2V?",
+		"\u000e=8",
+		"\u001944\u00c2e>\u00c2V?\u000e=8",
+		"ir\u001bQJ\u007f\u0007y\u0015)",
+	}
+
+	for _, test := range tests {
+		s1 := strings.ToLower(test)
+		s2 := string(appendToLower(nil, []byte(test)))
+
+		if s1 != s2 {
+			t.Error("lowercase values mismatch")
+			t.Log("expected:", s1)
+			t.Log("found:   ", s2)
+		}
+	}
+}
+
+func BenchmarkParseString(b *testing.B) {
+	s := []byte(`"__segment_internal"`)
+
+	for i := 0; i != b.N; i++ {
+		parseString(s)
+	}
+}
+
+func BenchmarkToLower(b *testing.B) {
+	s := []byte("someFieldWithALongName")
+
+	for i := 0; i != b.N; i++ {
+		bytes.ToLower(s)
+	}
+}
+
+func BenchmarkAppendToLower(b *testing.B) {
+	a := []byte(nil)
+	s := []byte("someFieldWithALongName")
+
+	for i := 0; i != b.N; i++ {
+		a = appendToLower(a[:0], s)
+	}
+}
+
+var benchmarkHasPrefixString = []byte("some random string")
+var benchmarkHasPrefixResult = false
+
+func BenchmarkHasPrefix(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		benchmarkHasPrefixResult = hasPrefix(benchmarkHasPrefixString, "null")
+	}
+}
+
+func BenchmarkHasNullPrefix(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		benchmarkHasPrefixResult = hasNullPrefix(benchmarkHasPrefixString)
+	}
+}
+
+func BenchmarkHasTruePrefix(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		benchmarkHasPrefixResult = hasTruePrefix(benchmarkHasPrefixString)
+	}
+}
+
+func BenchmarkHasFalsePrefix(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		benchmarkHasPrefixResult = hasFalsePrefix(benchmarkHasPrefixString)
+	}
+}
+
+func BenchmarkParseStringEscapeNone(b *testing.B) {
+	var j = []byte(`"` + strings.Repeat(`a`, 1000) + `"`)
+	var s string
+	b.SetBytes(int64(len(j)))
+
+	for i := 0; i < b.N; i++ {
+		if err := Unmarshal(j, &s); err != nil {
+			b.Fatal(err)
+		}
+		s = ""
+	}
+}
+
+func BenchmarkParseStringEscapeOne(b *testing.B) {
+	var j = []byte(`"` + strings.Repeat(`a`, 998) + `\n"`)
+	var s string
+	b.SetBytes(int64(len(j)))
+
+	for i := 0; i < b.N; i++ {
+		if err := Unmarshal(j, &s); err != nil {
+			b.Fatal(err)
+		}
+		s = ""
+	}
+}
+
+func BenchmarkParseStringEscapeAll(b *testing.B) {
+	var j = []byte(`"` + strings.Repeat(`\`, 1000) + `"`)
+	var s string
+	b.SetBytes(int64(len(j)))
+
+	for i := 0; i < b.N; i++ {
+		if err := Unmarshal(j, &s); err != nil {
+			b.Fatal(err)
+		}
+		s = ""
+	}
+}

cli/output/jsonw/internal/jcolorenc/reflect.go

@@ -0,0 +1,19 @@
+// +build go1.15
+
+package json
+
+import (
+	"reflect"
+	"unsafe"
+)
+
+func extendSlice(t reflect.Type, s *slice, n int) slice {
+	arrayType := reflect.ArrayOf(n, t.Elem())
+	arrayData := reflect.New(arrayType)
+	reflect.Copy(arrayData.Elem(), reflect.NewAt(t, unsafe.Pointer(s)).Elem())
+	return slice{
+		data: unsafe.Pointer(arrayData.Pointer()),
+		len:  s.len,
+		cap:  n,
+	}
+}

cli/output/jsonw/internal/jcolorenc/reflect_optimize.go

@@ -0,0 +1,29 @@
+// +build !go1.15
+
+package json
+
+import (
+	"reflect"
+	"unsafe"
+)
+
+//go:linkname unsafe_NewArray reflect.unsafe_NewArray
+func unsafe_NewArray(rtype unsafe.Pointer, length int) unsafe.Pointer
+
+//go:linkname typedslicecopy reflect.typedslicecopy
+//go:noescape
+func typedslicecopy(elemType unsafe.Pointer, dst, src slice) int
+
+func extendSlice(t reflect.Type, s *slice, n int) slice {
+	elemTypeRef := t.Elem()
+	elemTypePtr := ((*iface)(unsafe.Pointer(&elemTypeRef))).ptr
+
+	d := slice{
+		data: unsafe_NewArray(elemTypePtr, n),
+		len:  s.len,
+		cap:  n,
+	}
+
+	typedslicecopy(elemTypePtr, d, *s)
+	return d
+}

cli/output/jsonw/internal/jcolorenc/token.go

@@ -0,0 +1,286 @@
+package json
+
+// Tokenizer is an iterator-style type which can be used to progressively parse
+// through a json input.
+//
+// Tokenizing json is useful to build highly efficient parsing operations, for
+// example when doing tranformations on-the-fly where as the program reads the
+// input and produces the transformed json to an output buffer.
+//
+// Here is a common pattern to use a tokenizer:
+//
+//	for t := json.NewTokenizer(b); t.Next(); {
+//		switch t.Delim {
+//		case '{':
+//			...
+//		case '}':
+//			...
+//		case '[':
+//			...
+//		case ']':
+//			...
+//		case ':':
+//			...
+//		case ',':
+//			...
+//		}
+//
+//		switch {
+//		case t.Value.String():
+//			...
+//		case t.Value.Null():
+//			...
+//		case t.Value.True():
+//			...
+//		case t.Value.False():
+//			...
+//		case t.Value.Number():
+//			...
+//		}
+//	}
+//
+type Tokenizer struct {
+	// When the tokenizer is positioned on a json delimiter this field is not
+	// zero. In this case the possible values are '{', '}', '[', ']', ':', and
+	// ','.
+	Delim Delim
+
+	// This field contains the raw json token that the tokenizer is pointing at.
+	// When Delim is not zero, this field is a single-element byte slice
+	// continaing the delimiter value. Otherwise, this field holds values like
+	// null, true, false, numbers, or quoted strings.
+	Value RawValue
+
+	// When the tokenizer has encountered invalid content this field is not nil.
+	Err error
+
+	// When the value is in an array or an object, this field contains the depth
+	// at which it was found.
+	Depth int
+
+	// When the value is in an array or an object, this field contains the
+	// position at which it was found.
+	Index int
+
+	// This field is true when the value is the key of an object.
+	IsKey bool
+
+	// Tells whether the next value read from the tokenizer is a key.
+	isKey bool
+
+	// json input for the tokenizer, pointing at data right after the last token
+	// that was parsed.
+	json []byte
+
+	// Stack used to track entering and leaving arrays, objects, and keys. The
+	// buffer is used as a AppendPre-allocated space to
+	stack  []state
+	buffer [8]state
+}
+
+type state struct {
+	typ scope
+	len int
+}
+
+type scope int
+
+const (
+	inArray scope = iota
+	inObject
+)
+
+// NewTokenizer constructs a new Tokenizer which reads its json input from b.
+func NewTokenizer(b []byte) *Tokenizer { return &Tokenizer{json: b} }
+
+// Reset erases the state of t and re-initializes it with the json input from b.
+func (t *Tokenizer) Reset(b []byte) {
+	// This code is similar to:
+	//
+	//	*t = Tokenizer{json: b}
+	//
+	// However, it does not compile down to an invocation of duff-copy, which
+	// ends up being slower and prevents the code from being inlined.
+	t.Delim = 0
+	t.Value = nil
+	t.Err = nil
+	t.Depth = 0
+	t.Index = 0
+	t.IsKey = false
+	t.isKey = false
+	t.json = b
+	t.stack = nil
+}
+
+// Next returns a new tokenizer pointing at the next token, or the zero-value of
+// Tokenizer if the end of the json input has been reached.
+//
+// If the tokenizer encounters malformed json while reading the input the method
+// sets t.Err to an error describing the issue, and returns false. Once an error
+// has been encountered, the tokenizer will always fail until its input is
+// cleared by a call to its Reset method.
+func (t *Tokenizer) Next() bool {
+	if t.Err != nil {
+		return false
+	}
+
+	// Inlined code of the skipSpaces function, this give a ~15% speed boost.
+	i := 0
+skipLoop:
+	for _, c := range t.json {
+		switch c {
+		case sp, ht, nl, cr:
+			i++
+		default:
+			break skipLoop
+		}
+	}
+
+	if t.json = t.json[i:]; len(t.json) == 0 {
+		t.Reset(nil)
+		return false
+	}
+
+	var d Delim
+	var v []byte
+	var b []byte
+	var err error
+
+	switch t.json[0] {
+	case '"':
+		v, b, err = parseString(t.json)
+	case 'n':
+		v, b, err = parseNull(t.json)
+	case 't':
+		v, b, err = parseTrue(t.json)
+	case 'f':
+		v, b, err = parseFalse(t.json)
+	case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+		v, b, err = parseNumber(t.json)
+	case '{', '}', '[', ']', ':', ',':
+		d, v, b = Delim(t.json[0]), t.json[:1], t.json[1:]
+	default:
+		v, b, err = t.json[:1], t.json[1:], syntaxError(t.json, "expected token but found '%c'", t.json[0])
+	}
+
+	t.Delim = d
+	t.Value = RawValue(v)
+	t.Err = err
+	t.Depth = t.depth()
+	t.Index = t.index()
+	t.IsKey = d == 0 && t.isKey
+	t.json = b
+
+	if d != 0 {
+		switch d {
+		case '{':
+			t.isKey = true
+			t.push(inObject)
+		case '[':
+			t.push(inArray)
+		case '}':
+			err = t.pop(inObject)
+			t.Depth--
+			t.Index = t.index()
+		case ']':
+			err = t.pop(inArray)
+			t.Depth--
+			t.Index = t.index()
+		case ':':
+			t.isKey = false
+		case ',':
+			if t.is(inObject) {
+				t.isKey = true
+			}
+			t.stack[len(t.stack)-1].len++
+		}
+	}
+
+	return (d != 0 || len(v) != 0) && err == nil
+}
+
+func (t *Tokenizer) push(typ scope) {
+	if t.stack == nil {
+		t.stack = t.buffer[:0]
+	}
+	t.stack = append(t.stack, state{typ: typ, len: 1})
+}
+
+func (t *Tokenizer) pop(expect scope) error {
+	i := len(t.stack) - 1
+
+	if i < 0 {
+		return syntaxError(t.json, "found unexpected character while tokenizing json input")
+	}
+
+	if found := t.stack[i]; expect != found.typ {
+		return syntaxError(t.json, "found unexpected character while tokenizing json input")
+	}
+
+	t.stack = t.stack[:i]
+	return nil
+}
+
+func (t *Tokenizer) is(typ scope) bool {
+	return len(t.stack) != 0 && t.stack[len(t.stack)-1].typ == typ
+}
+
+func (t *Tokenizer) depth() int {
+	return len(t.stack)
+}
+
+func (t *Tokenizer) index() int {
+	if len(t.stack) == 0 {
+		return 0
+	}
+	return t.stack[len(t.stack)-1].len - 1
+}
+
+// RawValue represents a raw json value, it is intended to carry null, true,
+// false, number, and string values only.
+type RawValue []byte
+
+// String returns true if v contains a string value.
+func (v RawValue) String() bool { return len(v) != 0 && v[0] == '"' }
+
+// Null returns true if v contains a null value.
+func (v RawValue) Null() bool { return len(v) != 0 && v[0] == 'n' }
+
+// True returns true if v contains a true value.
+func (v RawValue) True() bool { return len(v) != 0 && v[0] == 't' }
+
+// False returns true if v contains a false value.
+func (v RawValue) False() bool { return len(v) != 0 && v[0] == 'f' }
+
+// Number returns true if v contains a number value.
+func (v RawValue) Number() bool {
+	if len(v) != 0 {
+		switch v[0] {
+		case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+			return true
+		}
+	}
+	return false
+}
+
+// AppendUnquote writes the unquoted version of the string value in v into b.
+func (v RawValue) AppendUnquote(b []byte) []byte {
+	s, r, new, err := parseStringUnquote([]byte(v), b)
+	if err != nil {
+		panic(err)
+	}
+	if len(r) != 0 {
+		panic(syntaxError(r, "unexpected trailing tokens after json value"))
+	}
+	if new {
+		b = s
+	} else {
+		b = append(b, s...)
+	}
+	return b
+}
+
+// Unquote returns the unquoted version of the string value in v.
+func (v RawValue) Unquote() []byte {
+	return v.AppendUnquote(nil)
+}

cli/output/jsonw/internal/jcolorenc/token_test.go

@@ -0,0 +1,287 @@
+package json
+
+import (
+	"reflect"
+	"testing"
+)
+
+type token struct {
+	delim Delim
+	value RawValue
+	err   error
+	depth int
+	index int
+	isKey bool
+}
+
+func delim(s string, depth, index int) token {
+	return token{
+		delim: Delim(s[0]),
+		value: RawValue(s),
+		depth: depth,
+		index: index,
+	}
+}
+
+func key(v string, depth, index int) token {
+	return token{
+		value: RawValue(v),
+		depth: depth,
+		index: index,
+		isKey: true,
+	}
+}
+
+func value(v string, depth, index int) token {
+	return token{
+		value: RawValue(v),
+		depth: depth,
+		index: index,
+	}
+}
+
+func tokenize(b []byte) (tokens []token) {
+	t := NewTokenizer(b)
+
+	for t.Next() {
+		tokens = append(tokens, token{
+			delim: t.Delim,
+			value: t.Value,
+			err:   t.Err,
+			depth: t.Depth,
+			index: t.Index,
+			isKey: t.IsKey,
+		})
+	}
+
+	if t.Err != nil {
+		panic(t.Err)
+	}
+
+	return
+}
+
+func TestTokenizer(t *testing.T) {
+	tests := []struct {
+		input  []byte
+		tokens []token
+	}{
+		{
+			input: []byte(`null`),
+			tokens: []token{
+				value(`null`, 0, 0),
+			},
+		},
+
+		{
+			input: []byte(`true`),
+			tokens: []token{
+				value(`true`, 0, 0),
+			},
+		},
+
+		{
+			input: []byte(`false`),
+			tokens: []token{
+				value(`false`, 0, 0),
+			},
+		},
+
+		{
+			input: []byte(`""`),
+			tokens: []token{
+				value(`""`, 0, 0),
+			},
+		},
+
+		{
+			input: []byte(`"Hello World!"`),
+			tokens: []token{
+				value(`"Hello World!"`, 0, 0),
+			},
+		},
+
+		{
+			input: []byte(`-0.1234`),
+			tokens: []token{
+				value(`-0.1234`, 0, 0),
+			},
+		},
+
+		{
+			input: []byte(` { } `),
+			tokens: []token{
+				delim(`{`, 0, 0),
+				delim(`}`, 0, 0),
+			},
+		},
+
+		{
+			input: []byte(`{ "answer": 42 }`),
+			tokens: []token{
+				delim(`{`, 0, 0),
+				key(`"answer"`, 1, 0),
+				delim(`:`, 1, 0),
+				value(`42`, 1, 0),
+				delim(`}`, 0, 0),
+			},
+		},
+
+		{
+			input: []byte(`{ "sub": { "key-A": 1, "key-B": 2, "key-C": 3 } }`),
+			tokens: []token{
+				delim(`{`, 0, 0),
+				key(`"sub"`, 1, 0),
+				delim(`:`, 1, 0),
+				delim(`{`, 1, 0),
+				key(`"key-A"`, 2, 0),
+				delim(`:`, 2, 0),
+				value(`1`, 2, 0),
+				delim(`,`, 2, 0),
+				key(`"key-B"`, 2, 1),
+				delim(`:`, 2, 1),
+				value(`2`, 2, 1),
+				delim(`,`, 2, 1),
+				key(`"key-C"`, 2, 2),
+				delim(`:`, 2, 2),
+				value(`3`, 2, 2),
+				delim(`}`, 1, 0),
+				delim(`}`, 0, 0),
+			},
+		},
+
+		{
+			input: []byte(` [ ] `),
+			tokens: []token{
+				delim(`[`, 0, 0),
+				delim(`]`, 0, 0),
+			},
+		},
+
+		{
+			input: []byte(`[1, 2, 3]`),
+			tokens: []token{
+				delim(`[`, 0, 0),
+				value(`1`, 1, 0),
+				delim(`,`, 1, 0),
+				value(`2`, 1, 1),
+				delim(`,`, 1, 1),
+				value(`3`, 1, 2),
+				delim(`]`, 0, 0),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(string(test.input), func(t *testing.T) {
+			tokens := tokenize(test.input)
+
+			if !reflect.DeepEqual(tokens, test.tokens) {
+				t.Error("tokens mismatch")
+				t.Logf("expected: %+v", test.tokens)
+				t.Logf("found:    %+v", tokens)
+			}
+		})
+	}
+}
+
+func BenchmarkTokenizer(b *testing.B) {
+	values := []struct {
+		scenario string
+		payload  []byte
+	}{
+		{
+			scenario: "null",
+			payload:  []byte(`null`),
+		},
+
+		{
+			scenario: "true",
+			payload:  []byte(`true`),
+		},
+
+		{
+			scenario: "false",
+			payload:  []byte(`false`),
+		},
+
+		{
+			scenario: "number",
+			payload:  []byte(`-1.23456789`),
+		},
+
+		{
+			scenario: "string",
+			payload:  []byte(`"1234567890"`),
+		},
+
+		{
+			scenario: "object",
+			payload: []byte(`{
+    "timestamp": "2019-01-09T18:59:57.456Z",
+    "channel": "server",
+    "type": "track",
+    "event": "Test",
+    "userId": "test-user-whatever",
+    "messageId": "test-message-whatever",
+    "integrations": {
+        "whatever": {
+            "debugMode": false
+        },
+        "myIntegration": {
+            "debugMode": true
+        }
+    },
+    "properties": {
+        "trait1": 1,
+        "trait2": "test",
+        "trait3": true
+    },
+    "settings": {
+        "apiKey": "1234567890",
+        "debugMode": false,
+        "directChannels": [
+            "server",
+            "client"
+        ],
+        "endpoint": "https://somewhere.com/v1/integrations/segment"
+    }
+}`),
+		},
+	}
+
+	benchmarks := []struct {
+		scenario string
+		function func(*testing.B, []byte)
+	}{
+		{
+			scenario: "github.com/segmentio/encoding/json",
+			function: func(b *testing.B, json []byte) {
+				t := NewTokenizer(nil)
+
+				for i := 0; i < b.N; i++ {
+					t.Reset(json)
+
+					for t.Next() {
+						// Does nothing other than iterating over each token to measure the
+						// CPU and memory footprint.
+					}
+
+					if t.Err != nil {
+						b.Error(t.Err)
+					}
+				}
+			},
+		},
+	}
+
+	for _, bechmark := range benchmarks {
+		b.Run(bechmark.scenario, func(b *testing.B) {
+			for _, value := range values {
+				b.Run(value.scenario, func(b *testing.B) {
+					bechmark.function(b, value.payload)
+				})
+			}
+		})
+	}
+}

cli/output/jsonw/jsonw.go

@@ -0,0 +1,435 @@
+// Package jsonw implements output writers for JSON.
+package jsonw
+
+import (
+	"bytes"
+	"io"
+	"strings"
+
+	"github.com/neilotoole/sq/cli/output"
+	"github.com/neilotoole/sq/cli/output/jsonw/internal"
+	"github.com/neilotoole/sq/libsq/errz"
+	"github.com/neilotoole/sq/libsq/sqlz"
+)
+
+// NewStdRecordWriter returns a record writer that outputs each
+// record as a JSON object that is an element of JSON array. This is
+// to say, standard JSON. For example:
+//
+//  [
+//    {
+//      "actor_id": 1,
+//      "first_name": "PENELOPE",
+//      "last_name": "GUINESS",
+//      "last_update": "2020-06-11T02:50:54Z"
+//    },
+//    {
+//      "actor_id": 2,
+//      "first_name": "NICK",
+//      "last_name": "WAHLBERG",
+//      "last_update": "2020-06-11T02:50:54Z"
+//    }
+//  ]
+func NewStdRecordWriter(out io.Writer, fm *output.Formatting) output.RecordWriter {
+	return &stdWriter{
+		out: out,
+		fm:  fm,
+	}
+}
+
+// stdWriter outputs records in standard JSON format.
+type stdWriter struct {
+	err error
+	out io.Writer
+	fm  *output.Formatting
+
+	// b is used as a buffer by writeRecord
+	b []byte
+
+	// outBuf is used to hold output prior to flushing.
+	outBuf *bytes.Buffer
+
+	recMeta     sqlz.RecordMeta
+	recsWritten bool
+
+	tpl       *stdTemplate
+	encodeFns []func(b []byte, v interface{}) ([]byte, error)
+}
+
+// Open implements output.RecordWriter.
+func (w *stdWriter) Open(recMeta sqlz.RecordMeta) error {
+	if w.err != nil {
+		return w.err
+	}
+
+	w.outBuf = &bytes.Buffer{}
+	w.recMeta = recMeta
+
+	if len(recMeta) == 0 {
+		// should never happen
+		return errz.New("empty record metadata")
+	}
+
+	w.encodeFns = getFieldEncoders(recMeta, w.fm)
+
+	tpl, err := newStdTemplate(recMeta, w.fm)
+	if err != nil {
+		w.err = err
+		return err
+	}
+
+	w.outBuf.Write(tpl.header)
+	w.tpl = tpl
+
+	return nil
+}
+
+// WriteRecords implements output.RecordWriter.
+func (w *stdWriter) WriteRecords(recs []sqlz.Record) error {
+	if w.err != nil {
+		return w.err
+	}
+
+	var err error
+	for i := range recs {
+		err = w.writeRecord(recs[i])
+		if err != nil {
+			w.err = err
+			return err
+		}
+	}
+	return nil
+}
+
+func (w *stdWriter) writeRecord(rec sqlz.Record) error {
+	var err error
+
+	if w.recsWritten {
+		// we need to add the separator
+		w.b = append(w.b, w.tpl.recSep...)
+	} else {
+		// This is the first record: we'll need the separator next time.
+		w.recsWritten = true
+	}
+
+	for i := range w.recMeta {
+		w.b = append(w.b, w.tpl.recTpl[i]...)
+		w.b, err = w.encodeFns[i](w.b, rec[i])
+		if err != nil {
+			return errz.Err(err)
+		}
+	}
+
+	w.b = append(w.b, w.tpl.recTpl[len(w.recMeta)]...)
+	w.outBuf.Write(w.b)
+	w.b = w.b[:0]
+
+	if w.outBuf.Len() > output.FlushThreshold {
+		return w.Flush()
+	}
+
+	return nil
+}
+
+// Flush implements output.RecordWriter.
+func (w *stdWriter) Flush() error {
+	if w.err != nil {
+		return w.err
+	}
+	_, err := w.outBuf.WriteTo(w.out)
+	if err != nil {
+		return errz.Err(err)
+	}
+	return nil
+}
+
+// Close implements output.RecordWriter.
+func (w *stdWriter) Close() error {
+	if w.err != nil {
+		return w.err
+	}
+
+	if w.recsWritten && w.fm.Pretty {
+		w.outBuf.WriteRune('\n')
+	}
+
+	w.outBuf.Write(w.tpl.footer)
+	return w.Flush()
+}
+
+// stdTemplate holds the various parts of the output template
+// used by stdWriter.
+type stdTemplate struct {
+	header []byte
+	recTpl [][]byte
+	recSep []byte
+	footer []byte
+}
+
+func newStdTemplate(recMeta sqlz.RecordMeta, fm *output.Formatting) (*stdTemplate, error) {
+	tpl := make([][]byte, len(recMeta)+1)
+	clrs := internal.NewColors(fm)
+	pnc := newPunc(fm)
+
+	fieldNames := make([][]byte, len(recMeta))
+
+	var err error
+	for i := range recMeta {
+		fieldNames[i], err = encodeString(nil, recMeta[i].Name(), false)
+		if err != nil {
+			return nil, errz.Err(err)
+		}
+		if !fm.IsMonochrome() {
+			fieldNames[i] = clrs.AppendKey(nil, fieldNames[i])
+		}
+	}
+
+	if !fm.Pretty {
+		tpl[0] = append(tpl[0], pnc.lBrace...)
+		tpl[0] = append(tpl[0], fieldNames[0]...)
+		tpl[0] = append(tpl[0], pnc.colon...)
+
+		for i := 1; i < len(fieldNames); i++ {
+			tpl[i] = append(tpl[i], pnc.comma...)
+			tpl[i] = append(tpl[i], fieldNames[i]...)
+			tpl[i] = append(tpl[i], pnc.colon...)
+		}
+
+		tpl[len(recMeta)] = append(tpl[len(recMeta)], pnc.rBrace...)
+
+		stdTpl := &stdTemplate{recTpl: tpl}
+
+		stdTpl.header = append(stdTpl.header, pnc.lBracket...)
+		stdTpl.footer = append(stdTpl.footer, pnc.rBracket...)
+		stdTpl.footer = append(stdTpl.footer, '\n')
+		stdTpl.recSep = append(stdTpl.recSep, pnc.comma...)
+
+		return stdTpl, nil
+	}
+
+	// Else we're doing pretty printing
+	tpl[0] = append(tpl[0], []byte("\n"+fm.Indent)...)
+	tpl[0] = append(tpl[0], pnc.lBrace...)
+	tpl[0] = append(tpl[0], '\n')
+	tpl[0] = append(tpl[0], strings.Repeat(fm.Indent, 2)...)
+
+	tpl[0] = append(tpl[0], fieldNames[0]...)
+	tpl[0] = append(tpl[0], pnc.colon...)
+	tpl[0] = append(tpl[0], ' ')
+
+	for i := 1; i < len(fieldNames); i++ {
+		tpl[i] = append(tpl[i], pnc.comma...)
+		tpl[i] = append(tpl[i], '\n')
+		tpl[i] = append(tpl[i], strings.Repeat(fm.Indent, 2)...)
+		tpl[i] = append(tpl[i], fieldNames[i]...)
+		tpl[i] = append(tpl[i], pnc.colon...)
+		tpl[i] = append(tpl[i], ' ')
+	}
+
+	last := len(recMeta)
+
+	tpl[last] = append(tpl[last], '\n')
+	tpl[last] = append(tpl[last], fm.Indent...)
+	tpl[last] = append(tpl[last], pnc.rBrace...)
+
+	stdTpl := &stdTemplate{recTpl: tpl}
+	stdTpl.header = append(stdTpl.header, pnc.lBracket...)
+	stdTpl.footer = append(stdTpl.footer, pnc.rBracket...)
+	stdTpl.footer = append(stdTpl.footer, '\n')
+	stdTpl.recSep = append(stdTpl.recSep, pnc.comma...)
+	return stdTpl, nil
+}
+
+// NewObjectRecordWriter writes out each record as a JSON object
+// on its own line. For example:
+//
+//  {"actor_id": 1, "first_name": "PENELOPE", "last_name": "GUINESS", "last_update": "2020-06-11T02:50:54Z"}
+//  {"actor_id": 2, "first_name": "NICK", "last_name": "WAHLBERG", "last_update": "2020-06-11T02:50:54Z"}
+func NewObjectRecordWriter(out io.Writer, fm *output.Formatting) output.RecordWriter {
+	return &lineRecordWriter{
+		out:      out,
+		fm:       fm,
+		newTplFn: newJSONObjectsTemplate,
+	}
+}
+
+// NewArrayRecordWriter returns a RecordWriter that outputs each
+// record as a JSON array on its own line. For example:
+//
+//  [1, "PENELOPE", "GUINESS", "2020-06-11T02:50:54Z"]
+//  [2, "NICK", "WAHLBERG", "2020-06-11T02:50:54Z"]
+func NewArrayRecordWriter(out io.Writer, fm *output.Formatting) output.RecordWriter {
+	return &lineRecordWriter{
+		out:      out,
+		fm:       fm,
+		newTplFn: newJSONArrayTemplate,
+	}
+}
+
+// lineRecordWriter is an output.RecordWriter that outputs each
+// record as its own line. This type is used to generate
+// both the "json array" and "json lines" output formats.
+//
+// For each element of a record, there is an encoding function in
+// w.encodeFns. So, there's len(rec) encode fns. The elements of w.tpl
+// surround the output of each encode func. Therefore there
+// are len(rec)+1 tpl elements.
+type lineRecordWriter struct {
+	err     error
+	out     io.Writer
+	fm      *output.Formatting
+	recMeta sqlz.RecordMeta
+
+	// outBuf holds the output of the writer prior to flushing.
+	outBuf *bytes.Buffer
+
+	// newTplFn is invoked during open to get the "template" for
+	// generating output.
+	newTplFn func(sqlz.RecordMeta, *output.Formatting) ([][]byte, error)
+
+	// tpl is a slice of []byte, where len(tpl) == len(recMeta) + 1.
+	tpl [][]byte
+
+	// encodeFns holds an encoder func for each element of the record.
+	encodeFns []func(b []byte, v interface{}) ([]byte, error)
+}
+
+// Open implements output.RecordWriter.
+func (w *lineRecordWriter) Open(recMeta sqlz.RecordMeta) error {
+	if w.err != nil {
+		return w.err
+	}
+
+	w.outBuf = &bytes.Buffer{}
+	w.recMeta = recMeta
+
+	w.tpl, w.err = w.newTplFn(recMeta, w.fm)
+	if w.err != nil {
+		return w.err
+	}
+	w.encodeFns = getFieldEncoders(recMeta, w.fm)
+	return nil
+}
+
+// WriteRecords implements output.RecordWriter.
+func (w *lineRecordWriter) WriteRecords(recs []sqlz.Record) error {
+	if w.err != nil {
+		return w.err
+	}
+
+	var err error
+	for i := range recs {
+		err = w.writeRecord(recs[i])
+		if err != nil {
+			w.err = err
+			return err
+		}
+	}
+	return nil
+}
+
+func (w *lineRecordWriter) writeRecord(rec sqlz.Record) error {
+	var err error
+	b := make([]byte, 0, 10)
+
+	for i := range w.recMeta {
+		b = append(b, w.tpl[i]...)
+		b, err = w.encodeFns[i](b, rec[i])
+		if err != nil {
+			return errz.Err(err)
+		}
+	}
+
+	b = append(b, w.tpl[len(w.recMeta)]...)
+	w.outBuf.Write(b)
+
+	if w.outBuf.Len() > output.FlushThreshold {
+		return w.Flush()
+	}
+
+	return nil
+}
+
+// Flush implements output.RecordWriter.
+func (w *lineRecordWriter) Flush() error {
+	if w.err != nil {
+		return w.err
+	}
+	_, err := w.outBuf.WriteTo(w.out)
+	if err != nil {
+		return errz.Err(err)
+	}
+	return nil
+}
+
+// Close implements output.RecordWriter.
+func (w *lineRecordWriter) Close() error {
+	if w.err != nil {
+		return w.err
+	}
+
+	return w.Flush()
+}
+
+func newJSONObjectsTemplate(recMeta sqlz.RecordMeta, fm *output.Formatting) ([][]byte, error) {
+	tpl := make([][]byte, len(recMeta)+1)
+	clrs := internal.NewColors(fm)
+	pnc := newPunc(fm)
+
+	fieldNames := make([][]byte, len(recMeta))
+	var err error
+	for i := range recMeta {
+		fieldNames[i], err = encodeString(nil, recMeta[i].Name(), false)
+		if err != nil {
+			return nil, errz.Err(err)
+		}
+		if !fm.IsMonochrome() {
+			fieldNames[i] = clrs.AppendKey(nil, fieldNames[i])
+		}
+	}
+
+	tpl[0] = append(tpl[0], pnc.lBrace...)
+	tpl[0] = append(tpl[0], fieldNames[0]...)
+	tpl[0] = append(tpl[0], pnc.colon...)
+	if fm.Pretty {
+		tpl[0] = append(tpl[0], ' ')
+	}
+
+	for i := 1; i < len(fieldNames); i++ {
+		tpl[i] = append(tpl[i], pnc.comma...)
+		if fm.Pretty {
+			tpl[i] = append(tpl[i], ' ')
+		}
+
+		tpl[i] = append(tpl[i], fieldNames[i]...)
+
+		tpl[i] = append(tpl[i], pnc.colon...)
+		if fm.Pretty {
+			tpl[i] = append(tpl[i], ' ')
+		}
+	}
+
+	tpl[len(recMeta)] = append(tpl[len(recMeta)], pnc.rBrace...)
+	tpl[len(recMeta)] = append(tpl[len(recMeta)], '\n')
+
+	return tpl, nil
+}
+
+func newJSONArrayTemplate(recMeta sqlz.RecordMeta, fm *output.Formatting) ([][]byte, error) {
+	tpl := make([][]byte, len(recMeta)+1)
+	pnc := newPunc(fm)
+
+	tpl[0] = append(tpl[0], pnc.lBracket...)
+
+	for i := 1; i < len(recMeta); i++ {
+		tpl[i] = append(tpl[i], pnc.comma...)
+		if fm.Pretty {
+			tpl[i] = append(tpl[i], ' ')
+		}
+	}
+
+	tpl[len(recMeta)] = append(tpl[len(recMeta)], pnc.rBracket...)
+	tpl[len(recMeta)] = append(tpl[len(recMeta)], '\n')
+
+	return tpl, nil
+}

cli/output/jsonw/jsonw_test.go

@@ -0,0 +1,214 @@
+package jsonw_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/neilotoole/lg/testlg"
+	"github.com/stretchr/testify/require"
+
+	"github.com/neilotoole/sq/cli/output"
+	"github.com/neilotoole/sq/cli/output/jsonw"
+	"github.com/neilotoole/sq/libsq/errz"
+	"github.com/neilotoole/sq/libsq/sqlz"
+	"github.com/neilotoole/sq/testh"
+	"github.com/neilotoole/sq/testh/fixt"
+)
+
+func TestRecordWriters(t *testing.T) {
+	const (
+		wantStdJSONNoPretty = `[{"col_int":64,"col_float":64.64,"col_decimal":"10000000000000000.99","col_bool":true,"col_text":"hello","col_datetime":"1970-01-01T00:00:00Z","col_date":"1970-01-01","col_time":"00:00:00","col_bytes":"aGVsbG8="},{"col_int":null,"col_float":null,"col_decimal":null,"col_bool":null,"col_text":null,"col_datetime":null,"col_date":null,"col_time":null,"col_bytes":null},{"col_int":64,"col_float":64.64,"col_decimal":"10000000000000000.99","col_bool":true,"col_text":"hello","col_datetime":"1970-01-01T00:00:00Z","col_date":"1970-01-01","col_time":"00:00:00","col_bytes":"aGVsbG8="}]
+`
+		wantStdJSONPretty = `[
+  {
+    "col_int": 64,
+    "col_float": 64.64,
+    "col_decimal": "10000000000000000.99",
+    "col_bool": true,
+    "col_text": "hello",
+    "col_datetime": "1970-01-01T00:00:00Z",
+    "col_date": "1970-01-01",
+    "col_time": "00:00:00",
+    "col_bytes": "aGVsbG8="
+  },
+  {
+    "col_int": null,
+    "col_float": null,
+    "col_decimal": null,
+    "col_bool": null,
+    "col_text": null,
+    "col_datetime": null,
+    "col_date": null,
+    "col_time": null,
+    "col_bytes": null
+  },
+  {
+    "col_int": 64,
+    "col_float": 64.64,
+    "col_decimal": "10000000000000000.99",
+    "col_bool": true,
+    "col_text": "hello",
+    "col_datetime": "1970-01-01T00:00:00Z",
+    "col_date": "1970-01-01",
+    "col_time": "00:00:00",
+    "col_bytes": "aGVsbG8="
+  }
+]
+`
+		wantArrayNoPretty = `[64,64.64,"10000000000000000.99",true,"hello","1970-01-01T00:00:00Z","1970-01-01","00:00:00","aGVsbG8="]
+[null,null,null,null,null,null,null,null,null]
+[64,64.64,"10000000000000000.99",true,"hello","1970-01-01T00:00:00Z","1970-01-01","00:00:00","aGVsbG8="]
+`
+		wantArrayPretty = `[64, 64.64, "10000000000000000.99", true, "hello", "1970-01-01T00:00:00Z", "1970-01-01", "00:00:00", "aGVsbG8="]
+[null, null, null, null, null, null, null, null, null]
+[64, 64.64, "10000000000000000.99", true, "hello", "1970-01-01T00:00:00Z", "1970-01-01", "00:00:00", "aGVsbG8="]
+`
+		wantObjectsNoPretty = `{"col_int":64,"col_float":64.64,"col_decimal":"10000000000000000.99","col_bool":true,"col_text":"hello","col_datetime":"1970-01-01T00:00:00Z","col_date":"1970-01-01","col_time":"00:00:00","col_bytes":"aGVsbG8="}
+{"col_int":null,"col_float":null,"col_decimal":null,"col_bool":null,"col_text":null,"col_datetime":null,"col_date":null,"col_time":null,"col_bytes":null}
+{"col_int":64,"col_float":64.64,"col_decimal":"10000000000000000.99","col_bool":true,"col_text":"hello","col_datetime":"1970-01-01T00:00:00Z","col_date":"1970-01-01","col_time":"00:00:00","col_bytes":"aGVsbG8="}
+`
+		wantObjectsPretty = `{"col_int": 64, "col_float": 64.64, "col_decimal": "10000000000000000.99", "col_bool": true, "col_text": "hello", "col_datetime": "1970-01-01T00:00:00Z", "col_date": "1970-01-01", "col_time": "00:00:00", "col_bytes": "aGVsbG8="}
+{"col_int": null, "col_float": null, "col_decimal": null, "col_bool": null, "col_text": null, "col_datetime": null, "col_date": null, "col_time": null, "col_bytes": null}
+{"col_int": 64, "col_float": 64.64, "col_decimal": "10000000000000000.99", "col_bool": true, "col_text": "hello", "col_datetime": "1970-01-01T00:00:00Z", "col_date": "1970-01-01", "col_time": "00:00:00", "col_bytes": "aGVsbG8="}
+`
+	)
+
+	testCases := []struct {
+		name      string
+		pretty    bool
+		color     bool
+		factoryFn func(io.Writer, *output.Formatting) output.RecordWriter
+		multiline bool
+		want      string
+	}{
+		{
+			name:      "std_no_pretty",
+			pretty:    false,
+			color:     false,
+			factoryFn: jsonw.NewStdRecordWriter,
+			want:      wantStdJSONNoPretty,
+		},
+		{
+			name:      "std_pretty",
+			pretty:    true,
+			color:     false,
+			factoryFn: jsonw.NewStdRecordWriter,
+			want:      wantStdJSONPretty,
+		},
+		{
+			name:      "array_no_pretty",
+			pretty:    false,
+			color:     false,
+			multiline: true,
+			factoryFn: jsonw.NewArrayRecordWriter,
+			want:      wantArrayNoPretty,
+		},
+		{
+			name:      "array_pretty",
+			pretty:    true,
+			color:     false,
+			multiline: true,
+			factoryFn: jsonw.NewArrayRecordWriter,
+			want:      wantArrayPretty,
+		},
+		{
+			name:      "object_no_pretty",
+			pretty:    false,
+			color:     false,
+			multiline: true,
+			factoryFn: jsonw.NewObjectRecordWriter,
+			want:      wantObjectsNoPretty,
+		},
+		{
+			name:      "object_pretty",
+			pretty:    true,
+			color:     false,
+			multiline: true,
+			factoryFn: jsonw.NewObjectRecordWriter,
+			want:      wantObjectsPretty,
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			colNames, kinds := fixt.ColNamePerKind(false, false, false)
+			recMeta := testh.NewRecordMeta(colNames, kinds)
+
+			v0, v1, v2, v3, v4, v5, v6, v7, v8 := int64(64), float64(64.64), "10000000000000000.99", true, "hello", time.Unix(0, 0).UTC(), time.Unix(0, 0).UTC(), time.Unix(0, 0).UTC(), []byte("hello")
+
+			recs := []sqlz.Record{
+				{&v0, &v1, &v2, &v3, &v4, &v5, &v6, &v7, &v8},
+				{nil, nil, nil, nil, nil, nil, nil, nil, nil},
+				{&v0, &v1, &v2, &v3, &v4, &v5, &v6, &v7, &v8},
+			}
+
+			buf := &bytes.Buffer{}
+			fm := output.NewFormatting()
+			fm.EnableColor(tc.color)
+			fm.Pretty = tc.pretty
+
+			w := tc.factoryFn(buf, fm)
+
+			require.NoError(t, w.Open(recMeta))
+			require.NoError(t, w.WriteRecords(recs))
+			require.NoError(t, w.Close())
+			require.Equal(t, tc.want, buf.String())
+
+			if !tc.multiline {
+				require.True(t, json.Valid(buf.Bytes()))
+				return
+			}
+
+			lines := strings.Split(strings.TrimSpace(buf.String()), "\n")
+			require.Equal(t, len(recs), len(lines))
+			for _, line := range lines {
+				require.True(t, json.Valid([]byte(line)))
+			}
+		})
+	}
+}
+
+func TestErrorWriter(t *testing.T) {
+	testCases := []struct {
+		name   string
+		pretty bool
+		color  bool
+		want   string
+	}{
+		{
+			name:   "no_pretty",
+			pretty: false,
+			color:  false,
+			want:   "{\"error\": \"err1\"}\n",
+		},
+		{
+			name:   "pretty",
+			pretty: true,
+			color:  false,
+			want:   "{\n  \"error\": \"err1\"\n}\n",
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(t.Name(), func(t *testing.T) {
+			buf := &bytes.Buffer{}
+			fm := output.NewFormatting()
+			fm.Pretty = tc.pretty
+			fm.EnableColor(tc.color)
+
+			errw := jsonw.NewErrorWriter(testlg.New(t), buf, fm)
+			errw.Error(errz.New("err1"))
+			got := buf.String()
+
+			require.Equal(t, tc.want, got)
+		})
+	}
+}

cli/output/jsonw/metadataw.go

@@ -0,0 +1,64 @@
+package jsonw
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+
+	"github.com/neilotoole/sq/cli/output"
+	"github.com/neilotoole/sq/cli/output/jsonw/internal"
+	jcolorenc "github.com/neilotoole/sq/cli/output/jsonw/internal/jcolorenc"
+	"github.com/neilotoole/sq/libsq/driver"
+	"github.com/neilotoole/sq/libsq/errz"
+	"github.com/neilotoole/sq/libsq/source"
+)
+
+// mdWriter implements output.MetadataWriter for JSON.
+type mdWriter struct {
+	out io.Writer
+	fm  *output.Formatting
+}
+
+// NewMetadataWriter returns a new output.MetadataWriter instance
+// that outputs metadata in JSON.
+func NewMetadataWriter(out io.Writer, fm *output.Formatting) output.MetadataWriter {
+	return &mdWriter{out: out, fm: fm}
+}
+
+func (w *mdWriter) write(v interface{}) error {
+	buf := &bytes.Buffer{}
+
+	enc := jcolorenc.NewEncoder(buf)
+	enc.SetColors(internal.NewColors(w.fm))
+	enc.SetEscapeHTML(false)
+	if w.fm.Pretty {
+		enc.SetIndent("", w.fm.Indent)
+	}
+
+	err := enc.Encode(v)
+	if err != nil {
+		return errz.Err(err)
+	}
+
+	_, err = fmt.Fprint(w.out, buf.String())
+	if err != nil {
+		return errz.Err(err)
+	}
+
+	return nil
+}
+
+// DriverMetadata implements output.MetadataWriter.
+func (w *mdWriter) DriverMetadata(md []driver.Metadata) error {
+	return w.write(md)
+}
+
+// TableMetadata implements output.MetadataWriter.
+func (w *mdWriter) TableMetadata(md *source.TableMetadata) error {
+	return w.write(md)
+}
+
+// SourceMetadata implements output.MetadataWriter.
+func (w *mdWriter) SourceMetadata(md *source.Metadata) error {
+	return w.write(md)
+}

cli/output/markdownw/markdownw.go

@@ -0,0 +1,130 @@
+// Package markdownw implements writers for Markdown.
+package markdownw
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"html"
+	"io"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/neilotoole/sq/libsq/sqlz"
+	"github.com/neilotoole/sq/libsq/stringz"
+)
+
+// RecordWriter implements output.RecordWriter.
+type RecordWriter struct {
+	recMeta sqlz.RecordMeta
+	out     io.Writer
+	buf     *bytes.Buffer
+}
+
+// NewRecordWriter returns a writer instance.
+func NewRecordWriter(out io.Writer) *RecordWriter {
+	return &RecordWriter{out: out}
+}
+
+// Open implements output.RecordWriter.
+func (w *RecordWriter) Open(recMeta sqlz.RecordMeta) error {
+	w.recMeta = recMeta
+	w.buf = &bytes.Buffer{}
+
+	// Write the header
+	for i, field := range recMeta {
+		w.buf.WriteString("| ")
+		w.buf.WriteString(field.Name() + " ")
+
+		if i == len(recMeta)-1 {
+			w.buf.WriteString("|\n")
+		}
+	}
+
+	// Write the header separator row
+	for i := range recMeta {
+		w.buf.WriteString("| --- ")
+
+		if i == len(recMeta)-1 {
+			w.buf.WriteString("|\n")
+		}
+	}
+
+	return nil
+}
+
+// Flush implements output.RecordWriter.
+func (w *RecordWriter) Flush() error {
+	_, err := w.buf.WriteTo(w.out) // resets buf
+	return err
+}
+
+// Close implements output.RecordWriter.
+func (w *RecordWriter) Close() error {
+	return w.Flush()
+}
+
+func (w *RecordWriter) writeRecord(rec sqlz.Record) error {
+	var s string
+	for i, field := range rec {
+		w.buf.WriteString("| ")
+
+		switch val := field.(type) {
+		default:
+			// should never happen
+			s = escapeMarkdown(fmt.Sprintf("%v", val))
+
+		case nil:
+			// nil is rendered as empty string, which this cell already is
+		case *int64:
+			s = strconv.FormatInt(*val, 10)
+		case *string:
+			s = escapeMarkdown(*val)
+		case *bool:
+			s = strconv.FormatBool(*val)
+		case *float64:
+			s = stringz.FormatFloat(*val)
+		case *[]byte:
+			s = base64.StdEncoding.EncodeToString(*val)
+		case *time.Time:
+			switch w.recMeta[i].Kind() {
+			default:
+				s = val.Format(stringz.DatetimeFormat)
+			case sqlz.KindTime:
+				s = val.Format(stringz.TimeFormat)
+			case sqlz.KindDate:
+				s = val.Format(stringz.DateFormat)
+			}
+		}
+
+		w.buf.WriteString(s + " ")
+	}
+
+	w.buf.WriteString("|\n")
+	return nil
+}
+
+// WriteRecords implements output.RecordWriter.
+func (w *RecordWriter) WriteRecords(recs []sqlz.Record) error {
+	var err error
+	for _, rec := range recs {
+		err = w.writeRecord(rec)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// escapeMarkdown is quick effort at escaping markdown
+// table cell text. It is not at all tested. Replace this
+// function with a real library call at the earliest opportunity.
+func escapeMarkdown(s string) string {
+	s = html.EscapeString(s)
+	s = strings.Replace(s, "|", "|", -1)
+	s = strings.Replace(s, "\r\n", "<br/>", -1)
+	s = strings.Replace(s, "\n", "<br/>", -1)
+	return s
+}

cli/output/markdownw/markdownw_test.go

@@ -0,0 +1,52 @@
+package markdownw_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/neilotoole/sq/cli/output/markdownw"
+	"github.com/neilotoole/sq/testh"
+	"github.com/neilotoole/sq/testh/sakila"
+)
+
+func TestRecordWriter(t *testing.T) {
+	const (
+		want0 = `| actor_id | first_name | last_name | last_update |
+| --- | --- | --- | --- |
+`
+		want3 = `| actor_id | first_name | last_name | last_update |
+| --- | --- | --- | --- |
+| 1 | PENELOPE | GUINESS | 2020-06-11T02:50:54Z |
+| 2 | NICK | WAHLBERG | 2020-06-11T02:50:54Z |
+| 3 | ED | CHASE | 2020-06-11T02:50:54Z |
+`
+	)
+
+	testCases := []struct {
+		name    string
+		numRecs int
+		want    string
+	}{
+		{name: "actor_0", numRecs: 0, want: want0},
+		{name: "actor_3", numRecs: 3, want: want3},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			recMeta, recs := testh.RecordsFromTbl(t, sakila.SL3, sakila.TblActor)
+			recs = recs[0:tc.numRecs]
+
+			buf := &bytes.Buffer{}
+			w := markdownw.NewRecordWriter(buf)
+			require.NoError(t, w.Open(recMeta))
+
+			require.NoError(t, w.WriteRecords(recs))
+			require.NoError(t, w.Close())
+			require.Equal(t, tc.want, buf.String())
+		})
+	}
+}

cli/output/raww/raww.go

@@ -0,0 +1,88 @@
+package raww
+
+import (
+	"fmt"
+	"io"
+	"strconv"
+	"time"
+
+	"github.com/neilotoole/sq/libsq/stringz"
+
+	"github.com/neilotoole/sq/cli/output"
+
+	"github.com/neilotoole/sq/libsq/sqlz"
+)
+
+// recordWriter implements output.RecordWriter for raw output.
+// This is typically used to output a single blob result, such
+// as a gif etc. The elements of each record are directly
+// written to the backing writer without any separator, or
+// encoding, etc.
+type recordWriter struct {
+	out     io.Writer
+	recMeta sqlz.RecordMeta
+}
+
+// NewRecordWriter returns an output.RecordWriter instance for
+// raw output. This is typically used to output a single blob result,
+// such as a gif etc. The elements of each record are directly
+// written to the backing writer without any separator, or
+// encoding, etc..
+func NewRecordWriter(out io.Writer) output.RecordWriter {
+	return &recordWriter{out: out}
+}
+
+// Open implements output.RecordWriter.
+func (w *recordWriter) Open(recMeta sqlz.RecordMeta) error {
+	w.recMeta = recMeta
+	return nil
+}
+
+// WriteRecords implements output.RecordWriter.
+func (w *recordWriter) WriteRecords(recs []sqlz.Record) error {
+	if len(recs) == 0 {
+		return nil
+	}
+
+	for _, rec := range recs {
+		for i, val := range rec {
+			switch val := val.(type) {
+			case nil:
+			case *[]byte:
+				_, _ = w.out.Write(*val)
+			case *string:
+				_, _ = w.out.Write([]byte(*val))
+			case *bool:
+				fmt.Fprint(w.out, strconv.FormatBool(*val))
+			case *int64:
+				fmt.Fprint(w.out, strconv.FormatInt(*val, 10))
+			case *float64:
+				fmt.Fprint(w.out, stringz.FormatFloat(*val))
+			case *time.Time:
+				switch w.recMeta[i].Kind() {
+				default:
+					fmt.Fprint(w.out, val.Format(stringz.DatetimeFormat))
+				case sqlz.KindTime:
+					fmt.Fprint(w.out, val.Format(stringz.TimeFormat))
+				case sqlz.KindDate:
+					fmt.Fprint(w.out, val.Format(stringz.DateFormat))
+				}
+			default:
+				// should never happen
+				fmt.Fprintf(w.out, "%s", val)
+			}
+		}
+	}
+
+	return nil
+}
+
+// Flush implements output.RecordWriter.
+func (w *recordWriter) Flush() error {
+	return nil
+}
+
+// Close implements output.RecordWriter.
+func (w *recordWriter) Close() error {
+	return nil
+}

cli/output/raww/raww_test.go

@@ -0,0 +1,67 @@
+package raww_test
+
+import (
+	"bytes"
+	"image/gif"
+	"testing"
+
+	"github.com/neilotoole/sq/testh/fixt"
+	"github.com/neilotoole/sq/testh/proj"
+
+	"github.com/neilotoole/lg"
+	"github.com/neilotoole/sq/testh/testsrc"
+
+	"github.com/neilotoole/sq/cli/output/raww"
+
+	"github.com/neilotoole/sq/testh"
+	"github.com/neilotoole/sq/testh/sakila"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRecordWriter_TblActor(t *testing.T) {
+	testCases := []struct {
+		name    string
+		numRecs int
+		want    []byte
+	}{
+		{name: "actor_0", numRecs: 0, want: nil},
+		{name: "actor_3", numRecs: 3, want: []byte("1PENELOPEGUINESS2020-06-11T02:50:54Z2NICKWAHLBERG2020-06-11T02:50:54Z3EDCHASE2020-06-11T02:50:54Z")},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			recMeta, recs := testh.RecordsFromTbl(t, sakila.SL3, sakila.TblActor)
+			recs = recs[0:tc.numRecs]
+
+			buf := &bytes.Buffer{}
+			w := raww.NewRecordWriter(buf)
+			require.NoError(t, w.Open(recMeta))
+			require.NoError(t, w.WriteRecords(recs))
+			require.NoError(t, w.Close())
+			require.Equal(t, tc.want, buf.Bytes())
+		})
+	}
+}
+
+func TestRecordWriter_TblBytes(t *testing.T) {
+	th := testh.New(t)
+	th.Log = lg.Discard()
+	src := th.Source(testsrc.MiscDB)
+	sink, err := th.QuerySQL(src, "SELECT col_bytes FROM tbl_bytes WHERE col_name='gopher.gif'")
+	require.NoError(t, err)
+	require.Equal(t, 1, len(sink.Recs))
+
+	fBytes := proj.ReadFile(fixt.GopherPath)
+
+	buf := &bytes.Buffer{}
+	w := raww.NewRecordWriter(buf)
+	require.NoError(t, w.Open(sink.RecMeta))
+	require.NoError(t, w.WriteRecords(sink.Recs))
+	require.NoError(t, w.Close())
+
+	require.Equal(t, fBytes, buf.Bytes())
+	_, err = gif.Decode(bytes.NewReader(buf.Bytes()))
+	require.NoError(t, err)
+}

cli/output/tablew/README.md

@@ -0,0 +1,15 @@
+# Package `tablew`
+
+Package `tablew` implements text table output writers.
+
+The actual rendering of the text table is handled by a modified version of
+[`olekukonko/tablewriter`](https://github.com/olekukonko/tablewriter),
+which can be found in the `internal` sub-package. At the time, `tablewriter`
+didn't provide all the functionality that sq required. However,
+that package has been significantly developed since then
+fork, and it may be possible that we could dispense with the forked
+version entirely and directly use a newer version of `tablewriter`.
+
+This entire package could use a rewrite, a lot has changed with sq
+since this package was first created. So, if you see code in here
+that doesn't make sense to you, you're probably judging it correctly.

cli/output/tablew/errorwriter.go

@@ -0,0 +1,25 @@
+package tablew
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/neilotoole/sq/cli/output"
+)
+
+// errorWriter implements output.ErrorWriter.
+type errorWriter struct {
+	w io.Writer
+	f *output.Formatting
+}
+
+// NewErrorWriter returns an output.ErrorWriter that
+// outputs in text format.
+func NewErrorWriter(w io.Writer, f *output.Formatting) output.ErrorWriter {
+	return &errorWriter{w: w, f: f}
+}
+
+// Error implements output.ErrorWriter.
+func (w *errorWriter) Error(err error) {
+	fmt.Fprintln(w.w, w.f.Error.Sprintf("sq: %v", err))
+}