#279: SQLite virtual tables (#304)

* sqlite: initial extensions support, including virtual tables and fts5
* sqlite: virtual table columns now report type

.github/workflows/main.yml

@@ -12,9 +12,11 @@ env:
   GO_VERSION: 1.21.0
   GORELEASER_VERSION: 1.20.0
   GOLANGCI_LINT_VERSION: v1.54.1
+  BUILD_TAGS: 'sqlite_vtable sqlite_stat4 sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_math_functions'
+  # Note that windows doesn't have sqlite_icu, as that fails to build. Needs investigation.
+  BUILD_TAGS_WIN: 'sqlite_vtable sqlite_stat4 sqlite_fts5 sqlite_introspect sqlite_json sqlite_math_functions'
 
 jobs:
-
   test-linux-darwin:
     strategy:
       matrix:
@@ -42,14 +44,14 @@ jobs:
 
 
       - name: Build
-        run: go build -v ./...
+        run: go build -tags '${{ env.BUILD_TAGS }}' -v ./...
 
       # Run tests with nice formatting. Save the original log in /tmp/gotest.log
       # https://github.com/GoTestTools/gotestfmt#github-actions
       - name: Run tests
         run: |
           set -euo pipefail
-          go test -json -v ./... 2>&1 | tee gotest.log | gotestfmt
+          go test -tags '${{ env.BUILD_TAGS }}' -json -v ./... 2>&1 | tee gotest.log | gotestfmt
 
       # Upload the original go test log as an artifact for later review.
       - name: Upload test log
@@ -63,6 +65,14 @@ jobs:
   test-windows:
     runs-on: windows-2022
     steps:
+    # Copied from https://github.com/mattn/go-sqlite3/blob/master/.github/workflows/go.yaml#L73
+#    - uses: msys2/setup-msys2@v2
+#      with:
+#        update: true
+#        install: mingw-w64-x86_64-toolchain mingw-w64-x86_64-sqlite3
+#        msystem: MINGW64
+#        path-type: inherit
+
     - name: Checkout
       uses: actions/checkout@v3
       with:
@@ -73,9 +83,14 @@ jobs:
       with:
         go-version: ${{ env.GO_VERSION }}
 
+    - name: Build
+      run: go build -tags '${{ env.BUILD_TAGS_WIN }}' -v ./...
+#      shell: msys2 {0}
+
     - name: Run tests
       run: |
-        go test -v ./...
+        go test -tags '${{ env.BUILD_TAGS_WIN }}' -v ./...
+#      shell: msys2 {0}
 
   go-lint:
     runs-on: ubuntu-22.04
@@ -95,32 +110,32 @@ jobs:
         with:
           version: ${{ env.GOLANGCI_LINT_VERSION }}
 
-  coverage:
+#  coverage:
-    runs-on: ubuntu-22.04
+#    runs-on: ubuntu-22.04
-    steps:
+#    steps:
-    - name: Checkout
+#    - name: Checkout
-      uses: actions/checkout@v3
+#      uses: actions/checkout@v3
-      with:
+#      with:
-        fetch-depth: 0
+#        fetch-depth: 0
-
+#
-    - name: Set up Go
+#    - name: Set up Go
-      uses: actions/setup-go@v3
+#      uses: actions/setup-go@v3
-      with:
+#      with:
-        go-version: ${{ env.GO_VERSION }}
+#        go-version: ${{ env.GO_VERSION }}
-
+#
-    - name: Test
+#    - name: Test
-      run: go test -v ./...
+#      run: go test -v ./...
-
+#
-    # https://github.com/ncruces/go-coverage-report
+#    # https://github.com/ncruces/go-coverage-report
-    - name: Update coverage report
+#    - name: Update coverage report
-      uses: ncruces/go-coverage-report@v0
+#      uses: ncruces/go-coverage-report@v0
-      with:
+#      with:
-        report: 'true'
+#        report: 'true'
-        chart: 'true'
+#        chart: 'true'
-        amend: 'false'
+#        amend: 'false'
-      if: |
+#      if: |
-        github.event_name == 'push'
+#        github.event_name == 'push'
-      continue-on-error: true
+#      continue-on-error: true
 
 
   binaries-darwin:

.goreleaser-darwin.yml

@@ -19,6 +19,14 @@ builds:
       - -X github.com/neilotoole/sq/cli/buildinfo.Version=v{{ .Version }}
       - -X github.com/neilotoole/sq/cli/buildinfo.Commit={{ .ShortCommit }}
       - -X github.com/neilotoole/sq/cli/buildinfo.Timestamp={{ .Date }}
+    tags:
+      - sqlite_vtable
+      - sqlite_stat4
+      - sqlite_fts5
+      - sqlite_icu
+      - sqlite_introspect
+      - sqlite_json
+      - sqlite_math_functions
 
 archives:
   - format: binary

.goreleaser-linux-amd64.yml

@@ -10,7 +10,7 @@ builds:
     binary: sq
     flags:
       - -a
-      - -tags=netgo
+#      - -tags=netgo
     env:
       - CGO_ENABLED=1
       - CGO_LDFLAGS=-static
@@ -23,6 +23,14 @@ builds:
       - -X github.com/neilotoole/sq/cli/buildinfo.Version=v{{ .Version }}
       - -X github.com/neilotoole/sq/cli/buildinfo.Commit={{ .ShortCommit }}
       - -X github.com/neilotoole/sq/cli/buildinfo.Timestamp={{ .Date }}
-
+    tags:
+      - netgo
+      - sqlite_vtable
+      - sqlite_stat4
+      - sqlite_fts5
+      - sqlite_icu
+      - sqlite_introspect
+      - sqlite_json
+      - sqlite_math_functions
 archives:
   - format: binary

.goreleaser-linux-arm64.yml

@@ -10,7 +10,7 @@ builds:
     binary: sq
     flags:
       - -a
-      - -tags=netgo
+#      - -tags=netgo
     env:
       - CGO_ENABLED=1
       - CGO_LDFLAGS=-static
@@ -26,6 +26,14 @@ builds:
       - -X github.com/neilotoole/sq/cli/buildinfo.Version=v{{ .Version }}
       - -X github.com/neilotoole/sq/cli/buildinfo.Commit={{ .ShortCommit }}
       - -X github.com/neilotoole/sq/cli/buildinfo.Timestamp={{ .Date }}
-
+    tags:
+      - netgo
+      - sqlite_vtable
+      - sqlite_stat4
+      - sqlite_fts5
+      - sqlite_icu
+      - sqlite_introspect
+      - sqlite_json
+      - sqlite_math_functions
 archives:
   - format: binary

.goreleaser-windows.yml

@@ -18,6 +18,12 @@ builds:
       - -X github.com/neilotoole/sq/cli/buildinfo.Version=v{{ .Version }}
       - -X github.com/neilotoole/sq/cli/buildinfo.Commit={{ .ShortCommit }}
       - -X github.com/neilotoole/sq/cli/buildinfo.Timestamp={{ .Date }}
-
+    tags:
+      - sqlite_vtable
+      - sqlite_stat4
+      - sqlite_fts5
+      - sqlite_introspect
+      - sqlite_json
+      - sqlite_math_functions
 archives:
   - format: binary

CHANGELOG.md

@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 Breaking changes are annotated with ☢️.
 
+## [v0.42.0] - 2023-08-21
+
+### Added
+
+- 🐥 [#279]: The SQLite [driver](https://sq.io/docs/drivers/sqlite) now has initial support
+  for several SQLite [extensions](https://sq.io/docs/drivers/sqlite#extensions)
+  baked in, including [Virtual Table](https://www.sqlite.org/vtab.html) and [FTS5](https://www.sqlite.org/fts5.html).
+  Note that this is an early access release of extensions support. Please [open an issue](https://github.com/neilotoole/sq/issues/new/choose) if
+  you discover something bad.
+
 ## [v0.41.1] - 2023-08-20
 
 ### Fixed
@@ -783,6 +793,7 @@ make working with lots of sources much easier.
 [#261]: https://github.com/neilotoole/sq/issues/261
 [#263]: https://github.com/neilotoole/sq/issues/263
 [#277]: https://github.com/neilotoole/sq/issues/277
+[#279]: https://github.com/neilotoole/sq/issues/279
 
 [v0.15.2]: https://github.com/neilotoole/sq/releases/tag/v0.15.2
 [v0.15.3]: https://github.com/neilotoole/sq/compare/v0.15.2...v0.15.3
@@ -828,3 +839,4 @@ make working with lots of sources much easier.
 [v0.40.0]: https://github.com/neilotoole/sq/compare/v0.39.1...v0.40.0
 [v0.41.0]: https://github.com/neilotoole/sq/compare/v0.40.0...v0.41.0
 [v0.41.1]: https://github.com/neilotoole/sq/compare/v0.41.0...v0.41.1
+[v0.42.0]: https://github.com/neilotoole/sq/compare/v0.41.1...v0.42.0

Makefile

@@ -4,15 +4,15 @@ BUILD_VERSION     	:= $(shell git describe --tags --always --dirty)
 BUILD_COMMIT      	:= $(shell git rev-parse HEAD)
 BUILD_TIMESTAMP		:= $(shell date -u '+%Y-%m-%dT%H:%M:%SZ')
 LDFLAGS				?= -s -w -X $(VERSION_PKG).Version=$(BUILD_VERSION) -X $(VERSION_PKG).Commit=$(BUILD_COMMIT) -X $(VERSION_PKG).Timestamp=$(BUILD_TIMESTAMP)
-
+BUILD_TAGS  		:= sqlite_vtable sqlite_stat4 sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_math_functions
 
 .PHONY: test
 test:
-	@go test ./...
+	@go test -tags "$(BUILD_TAGS)" ./...
 
 .PHONY: install
 install:
-	@go install -ldflags "$(LDFLAGS)"
+	@go install -ldflags "$(LDFLAGS)" -tags "$(BUILD_TAGS)"
 
 .PHONY: lint
 lint:

drivers/sqlite3/metadata.go

@@ -4,10 +4,13 @@ import "C"
 import (
 	"context"
 	"database/sql"
+	"errors"
 	"fmt"
 	"reflect"
 	"strings"
 
+	"github.com/neilotoole/sq/libsq/core/stringz"
+
 	"github.com/neilotoole/sq/libsq/driver"
 
 	"github.com/neilotoole/sq/libsq/core/record"
@@ -273,19 +276,24 @@ func getTableMetadata(ctx context.Context, db sqlz.DB, tblName string) (*source.
 	const tpl = `SELECT
 (SELECT COUNT(*) FROM %q),
 (SELECT type FROM sqlite_master WHERE name = %q LIMIT 1),
+(SELECT 1 FROM sqlite_master WHERE name = %q AND substr("sql",0,21) == 'CREATE VIRTUAL TABLE') AS is_virtual,
 (SELECT name FROM pragma_database_list ORDER BY seq LIMIT 1)`
+
 	var schema string
-	query := fmt.Sprintf(tpl, tblMeta.Name, tblMeta.Name)
+	var isVirtualTbl sql.NullBool
-	err := db.QueryRowContext(ctx, query).Scan(&tblMeta.RowCount, &tblMeta.DBTableType, &schema)
+	query := fmt.Sprintf(tpl, tblMeta.Name, tblMeta.Name, tblMeta.Name)
+	err := db.QueryRowContext(ctx, query).Scan(&tblMeta.RowCount, &tblMeta.DBTableType, &isVirtualTbl, &schema)
 	if err != nil {
 		return nil, errw(err)
 	}
 
-	switch tblMeta.DBTableType {
+	switch {
-	case "table":
+	case isVirtualTbl.Valid && isVirtualTbl.Bool:
-		tblMeta.TableType = sqlz.TableTypeTable
+		tblMeta.TableType = sqlz.TableTypeVirtual
-	case "view":
+	case tblMeta.DBTableType == sqlz.TableTypeView:
 		tblMeta.TableType = sqlz.TableTypeView
+	case tblMeta.DBTableType == sqlz.TableTypeTable:
+		tblMeta.TableType = sqlz.TableTypeTable
 	default:
 	}
 
@@ -312,6 +320,16 @@ func getTableMetadata(ctx context.Context, db sqlz.DB, tblName string) (*source.
 			return nil, errw(err)
 		}
 
+		if col.BaseType == "" {
+			// The TABLE_INFO pragma doesn't return column types for virtual tables.
+			//
+			// REVISIT: This logic should be pulled out into a separate query for
+			// all "untyped" columns, instead of invoking it for every untyped column.
+			if col.BaseType, err = getTypeOfColumn(ctx, db, tblMeta.Name, col.Name); err != nil {
+				return nil, err
+			}
+		}
+
 		col.PrimaryKey = pkValue.Int64 > 0 // pkVal can be 0,1,2 etc
 		col.ColumnType = col.BaseType
 		col.Nullable = notnull == 0
@@ -329,9 +347,11 @@ func getTableMetadata(ctx context.Context, db sqlz.DB, tblName string) (*source.
 	return tblMeta, nil
 }
 
-// getAllTblMeta gets metadata for each of the
+// getAllTableMetadata gets metadata for each of the
-// non-system tables in db.
+// non-system tables in db's schema. Arg schemaName is used to
-func getAllTblMeta(ctx context.Context, db sqlz.DB) ([]*source.TableMetadata, error) {
+// set TableMetadata.FQName; it is not used to select which schema
+// to introspect.
+func getAllTableMetadata(ctx context.Context, db sqlz.DB, schemaName string) ([]*source.TableMetadata, error) {
 	log := lg.FromContext(ctx)
 	// This query returns a row for each column of each table,
 	// order by table name then col id (ordinal).
@@ -350,7 +370,8 @@ func getAllTblMeta(ctx context.Context, db sqlz.DB) ([]*source.TableMetadata, er
 	// Note: dflt_value of col "address2" is the string "NULL", rather
 	// that NULL value itself.
 	const query = `
-SELECT m.name as table_name, m.type, p.cid, p.name, p.type, p.'notnull' as 'notnull', p.dflt_value, p.pk
+SELECT m.name as table_name, m.type, p.cid, p.name, p.type, p.'notnull' as 'notnull', p.dflt_value, p.pk,
+(substr(m.sql, 0, 21) == 'CREATE VIRTUAL TABLE') AS is_virtual
 FROM sqlite_master AS m JOIN pragma_table_info(m.name) AS p
 ORDER BY m.name, p.cid
 `
@@ -359,6 +380,7 @@ ORDER BY m.name, p.cid
 	var tblNames []string
 	var curTblName string
 	var curTblType string
+	var curTblIsVirtual bool
 	var curTblMeta *source.TableMetadata
 
 	rows, err := db.QueryContext(ctx, query)
@@ -379,7 +401,17 @@ ORDER BY m.name, p.cid
 		colDefault := &sql.NullString{}
 		pkValue := &sql.NullInt64{}
 
-		err = rows.Scan(&curTblName, &curTblType, &col.Position, &col.Name, &col.BaseType, &notnull, colDefault, pkValue)
+		err = rows.Scan(
+			&curTblName,
+			&curTblType,
+			&col.Position,
+			&col.Name,
+			&col.BaseType,
+			&notnull,
+			colDefault,
+			pkValue,
+			&curTblIsVirtual,
+		)
 		if err != nil {
 			return nil, errw(err)
 		}
@@ -389,19 +421,32 @@ ORDER BY m.name, p.cid
 			continue
 		}
 
+		if col.BaseType == "" {
+			// The TABLE_INFO pragma doesn't return column types for virtual tables.
+			//
+			// REVISIT: This logic should be pulled out into a separate query for
+			// all "untyped" columns, instead of invoking it for every untyped column.
+			if col.BaseType, err = getTypeOfColumn(ctx, db, curTblName, col.Name); err != nil {
+				return nil, err
+			}
+		}
+
 		if curTblMeta == nil || curTblMeta.Name != curTblName {
 			// On our first time encountering a new table name, we create a new TableMetadata
 			curTblMeta = &source.TableMetadata{
 				Name:        curTblName,
+				FQName:      schemaName + "." + curTblName,
 				Size:        nil, // No easy way of getting the storage size of a table
 				DBTableType: curTblType,
 			}
 
-			switch curTblMeta.DBTableType {
+			switch {
-			case "table":
+			case curTblIsVirtual:
-				curTblMeta.TableType = sqlz.TableTypeTable
+				curTblMeta.TableType = sqlz.TableTypeVirtual
-			case "view":
+			case curTblMeta.DBTableType == sqlz.TableTypeView:
 				curTblMeta.TableType = sqlz.TableTypeView
+			case curTblMeta.DBTableType == sqlz.TableTypeTable:
+				curTblMeta.TableType = sqlz.TableTypeTable
 			default:
 			}
 
@@ -514,3 +559,22 @@ func getTblRowCounts(ctx context.Context, db sqlz.DB, tblNames []string) ([]int6
 
 	return tblCounts, nil
 }
+
+// getTypeOfColumn executes "SELECT typeof(colName)", returning the first result.
+// Empty string is returned if there are no rows in that table, as SQLite determines
+// type on a per-cell basis, not per-column.
+func getTypeOfColumn(ctx context.Context, db sqlz.DB, tblName, colName string) (string, error) {
+	colTypeQuery := fmt.Sprintf(`SELECT typeof(%s) FROM %s LIMIT 1`,
+		stringz.DoubleQuote(colName), stringz.DoubleQuote(tblName))
+
+	var colType string
+	if err := db.QueryRowContext(ctx, colTypeQuery).Scan(&colType); err != nil {
+		if errors.Is(err, sql.ErrNoRows) {
+			return "", nil
+		}
+
+		return "", errw(err)
+	}
+
+	return colType, nil
+}

drivers/sqlite3/sqlite3.go

@@ -851,7 +851,7 @@ func (d *database) SourceMetadata(ctx context.Context, noSchema bool) (*source.M
 		return md, nil
 	}
 
-	md.Tables, err = getAllTblMeta(ctx, d.db)
+	md.Tables, err = getAllTableMetadata(ctx, d.db, md.Schema)
 	if err != nil {
 		return nil, err
 	}

drivers/sqlite3/sqlite3_test.go

@@ -5,6 +5,8 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/neilotoole/sq/libsq/core/kind"
+
 	_ "github.com/mattn/go-sqlite3"
 	"github.com/neilotoole/sq/testh/tutil"
 	"github.com/stretchr/testify/require"
@@ -309,3 +311,52 @@ func TestSQLQuery_Whitespace(t *testing.T) {
 	require.Equal(t, "last name", sink.RecMeta[2].Name())
 	require.Equal(t, "last name", sink.RecMeta[2].MungedName())
 }
+
+func TestExtension_fts5(t *testing.T) {
+	const tblActorFts = "actor_fts"
+
+	th := testh.New(t)
+	src := th.Add(&source.Source{
+		Handle:   "@fts",
+		Type:     sqlite3.Type,
+		Location: "sqlite3://" + tutil.MustAbsFilepath("testdata", "sakila_fts5.db"),
+	})
+
+	srcMeta, err := th.SourceMetadata(src)
+	require.NoError(t, err)
+	require.Equal(t, src.Handle, srcMeta.Handle)
+	tblMeta1 := srcMeta.Table(tblActorFts)
+	require.NotNil(t, tblMeta1)
+	require.Equal(t, tblActorFts, tblMeta1.Name)
+	require.Equal(t, sqlz.TableTypeVirtual, tblMeta1.TableType)
+
+	require.Equal(t, "actor_id", tblMeta1.Columns[0].Name)
+	require.Equal(t, "integer", tblMeta1.Columns[0].ColumnType)
+	require.Equal(t, "integer", tblMeta1.Columns[0].BaseType)
+	require.Equal(t, kind.Int, tblMeta1.Columns[0].Kind)
+	require.Equal(t, "first_name", tblMeta1.Columns[1].Name)
+	require.Equal(t, "text", tblMeta1.Columns[1].ColumnType)
+	require.Equal(t, "text", tblMeta1.Columns[1].BaseType)
+	require.Equal(t, kind.Text, tblMeta1.Columns[1].Kind)
+	require.Equal(t, "last_name", tblMeta1.Columns[2].Name)
+	require.Equal(t, "text", tblMeta1.Columns[2].ColumnType)
+	require.Equal(t, "text", tblMeta1.Columns[2].BaseType)
+	require.Equal(t, kind.Text, tblMeta1.Columns[2].Kind)
+	require.Equal(t, "last_update", tblMeta1.Columns[3].Name)
+	require.Equal(t, "text", tblMeta1.Columns[3].ColumnType)
+	require.Equal(t, "text", tblMeta1.Columns[3].BaseType)
+	require.Equal(t, kind.Text, tblMeta1.Columns[3].Kind)
+
+	tblMeta2, err := th.TableMetadata(src, tblActorFts)
+	require.NoError(t, err)
+	require.Equal(t, tblActorFts, tblMeta2.Name)
+	require.Equal(t, sqlz.TableTypeVirtual, tblMeta2.TableType)
+	require.EqualValues(t, *tblMeta1, *tblMeta2)
+
+	// Verify that the (non-virtual) "actor" table has its type set correctly.
+	actorMeta1 := srcMeta.Table(sakila.TblActor)
+	actorMeta2, err := th.TableMetadata(src, sakila.TblActor)
+	require.NoError(t, err)
+	require.Equal(t, actorMeta1.TableType, sqlz.TableTypeTable)
+	require.Equal(t, *actorMeta1, *actorMeta2)
+}

drivers/sqlite3/testdata/README.md

@@ -26,3 +26,13 @@ testing of `sq`'s ability to support such names. The mutated DB is achieved by
 applying [`sakila-whitespace-alter.sql`](./sakila-whitespace-alter.sql) to
 `sakila.db`. The changes can be reversed with
 [`sakila-whitespace-restore.sql](./sakila-whitespace-restore.sql).
+
+## sakila_fts5.db
+
+[`sakila_fts5.db`](./sakila_fts5.db) is based off [`sakila.db`](./sakila.db), but
+contains an FTS5 virtual table `actor_fts`. This table was created via the statement:
+
+```sql
+CREATE VIRTUAL TABLE actor_fts
+USING fts5(actor_id, first_name, last_name, last_update, content='actor', content_rowid='actor_id');
+```

libsq/core/sqlz/sqlz.go

@@ -59,8 +59,9 @@ func ExecAffected(ctx context.Context, db Execer, query string, args ...any) (af
 	return affected, nil
 }
 
-// Canonical driver-independent names for "table" and "view".
+// Canonical driver-independent names for table types.
 const (
-	TableTypeTable = "table"
+	TableTypeTable   = "table"
-	TableTypeView  = "view"
+	TableTypeView    = "view"
+	TableTypeVirtual = "virtual"
 )

testh/sakila/sakila.go

@@ -241,7 +241,6 @@ const (
 // Paths for sakila resources.
 const (
 	PathSL3              = "drivers/sqlite3/testdata/sakila.db"
-	PathSL3Whitespace    = "drivers/sqlite3/testdata/sakila-whitespace.db"
 	PathXLSX             = "drivers/xlsx/testdata/sakila.xlsx"
 	PathXLSXSubset       = "drivers/xlsx/testdata/sakila_subset.xlsx"
 	PathXLSXActorHeader  = "drivers/xlsx/testdata/actor_header.xlsx"

testh/tutil/tutil.go

@@ -368,3 +368,14 @@ func WriteTemp(t testing.TB, pattern string, b []byte, cleanup bool) (fpath stri
 	}
 	return fpath
 }
+
+// MustAbsFilepath invokes filepath.Join on elems, and then filepath.Abs
+// on the result. It panics on error.
+func MustAbsFilepath(elems ...string) string {
+	fp := filepath.Join(elems...)
+	s, err := filepath.Abs(fp)
+	if err != nil {
+		panic(err)
+	}
+	return s
+}