2020-08-06 20:58:47 +03:00
|
|
|
// Package testh (test helper) contains functionality
|
|
|
|
// for testing.
|
|
|
|
package testh
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2023-07-08 18:21:27 +03:00
|
|
|
"database/sql"
|
2020-08-06 20:58:47 +03:00
|
|
|
"io"
|
2023-08-12 21:54:14 +03:00
|
|
|
"log/slog"
|
2020-08-06 20:58:47 +03:00
|
|
|
"os"
|
|
|
|
"path/filepath"
|
|
|
|
"strings"
|
|
|
|
"sync"
|
|
|
|
"testing"
|
|
|
|
|
2023-11-19 03:05:48 +03:00
|
|
|
"github.com/samber/lo"
|
2023-11-20 04:06:36 +03:00
|
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
"github.com/stretchr/testify/require"
|
2023-11-19 03:05:48 +03:00
|
|
|
|
2023-04-26 18:16:42 +03:00
|
|
|
"github.com/neilotoole/sq/cli"
|
|
|
|
"github.com/neilotoole/sq/cli/buildinfo"
|
2020-08-06 20:58:47 +03:00
|
|
|
"github.com/neilotoole/sq/cli/config"
|
2023-11-20 04:06:36 +03:00
|
|
|
"github.com/neilotoole/sq/cli/config/yamlstore"
|
2020-08-06 20:58:47 +03:00
|
|
|
"github.com/neilotoole/sq/cli/output"
|
2023-11-20 04:06:36 +03:00
|
|
|
"github.com/neilotoole/sq/cli/run"
|
2020-08-06 20:58:47 +03:00
|
|
|
"github.com/neilotoole/sq/drivers/csv"
|
2020-08-23 13:42:15 +03:00
|
|
|
"github.com/neilotoole/sq/drivers/json"
|
2020-08-06 20:58:47 +03:00
|
|
|
"github.com/neilotoole/sq/drivers/mysql"
|
|
|
|
"github.com/neilotoole/sq/drivers/postgres"
|
|
|
|
"github.com/neilotoole/sq/drivers/sqlite3"
|
|
|
|
"github.com/neilotoole/sq/drivers/sqlserver"
|
|
|
|
"github.com/neilotoole/sq/drivers/userdriver"
|
|
|
|
"github.com/neilotoole/sq/drivers/userdriver/xmlud"
|
|
|
|
"github.com/neilotoole/sq/drivers/xlsx"
|
|
|
|
"github.com/neilotoole/sq/libsq"
|
2023-11-20 04:06:36 +03:00
|
|
|
"github.com/neilotoole/sq/libsq/ast"
|
2020-08-23 13:42:15 +03:00
|
|
|
"github.com/neilotoole/sq/libsq/core/cleanup"
|
2023-11-20 04:06:36 +03:00
|
|
|
"github.com/neilotoole/sq/libsq/core/ioz"
|
2024-01-15 04:45:34 +03:00
|
|
|
"github.com/neilotoole/sq/libsq/core/ioz/lockfile"
|
2023-11-20 04:06:36 +03:00
|
|
|
"github.com/neilotoole/sq/libsq/core/lg"
|
|
|
|
"github.com/neilotoole/sq/libsq/core/lg/lga"
|
|
|
|
"github.com/neilotoole/sq/libsq/core/lg/lgm"
|
2024-01-15 04:45:34 +03:00
|
|
|
"github.com/neilotoole/sq/libsq/core/lg/lgt"
|
2023-11-20 04:06:36 +03:00
|
|
|
"github.com/neilotoole/sq/libsq/core/options"
|
2024-01-25 10:42:51 +03:00
|
|
|
"github.com/neilotoole/sq/libsq/core/schema"
|
2023-11-20 04:06:36 +03:00
|
|
|
"github.com/neilotoole/sq/libsq/core/sqlz"
|
2020-08-23 13:42:15 +03:00
|
|
|
"github.com/neilotoole/sq/libsq/core/stringz"
|
2023-11-20 04:06:36 +03:00
|
|
|
"github.com/neilotoole/sq/libsq/core/tablefq"
|
2020-08-06 20:58:47 +03:00
|
|
|
"github.com/neilotoole/sq/libsq/driver"
|
2024-01-25 09:29:55 +03:00
|
|
|
"github.com/neilotoole/sq/libsq/files"
|
2020-08-06 20:58:47 +03:00
|
|
|
"github.com/neilotoole/sq/libsq/source"
|
2023-11-21 00:42:38 +03:00
|
|
|
"github.com/neilotoole/sq/libsq/source/drivertype"
|
|
|
|
"github.com/neilotoole/sq/libsq/source/metadata"
|
2020-08-06 20:58:47 +03:00
|
|
|
"github.com/neilotoole/sq/testh/proj"
|
|
|
|
"github.com/neilotoole/sq/testh/sakila"
|
|
|
|
"github.com/neilotoole/sq/testh/testsrc"
|
2024-01-15 04:45:34 +03:00
|
|
|
"github.com/neilotoole/sq/testh/tu"
|
2020-08-06 20:58:47 +03:00
|
|
|
)
|
|
|
|
|
2023-04-19 08:28:09 +03:00
|
|
|
// Option is a functional option type used with New to
|
|
|
|
// configure the helper.
|
|
|
|
type Option func(h *Helper)
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
// OptCaching enables or disables ingest caching.
|
|
|
|
func OptCaching(enable bool) Option {
|
2023-04-19 08:28:09 +03:00
|
|
|
return func(h *Helper) {
|
2024-01-15 04:45:34 +03:00
|
|
|
o := options.FromContext(h.Context)
|
|
|
|
if o == nil {
|
|
|
|
o = options.Options{driver.OptIngestCache.Key(): enable}
|
|
|
|
h.Context = options.NewContext(h.Context, o)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
o[driver.OptIngestCache.Key()] = enable
|
2023-04-19 08:28:09 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-08-06 20:58:47 +03:00
|
|
|
// Helper encapsulates a test helper session.
|
|
|
|
type Helper struct {
|
2020-08-23 13:42:15 +03:00
|
|
|
mu sync.Mutex
|
|
|
|
|
|
|
|
T testing.TB
|
2023-04-02 22:49:45 +03:00
|
|
|
Log *slog.Logger
|
2020-08-23 13:42:15 +03:00
|
|
|
|
2023-11-19 05:21:14 +03:00
|
|
|
registry *driver.Registry
|
2024-01-25 09:29:55 +03:00
|
|
|
files *files.Files
|
2024-01-15 04:45:34 +03:00
|
|
|
grips *driver.Grips
|
2023-11-19 05:21:14 +03:00
|
|
|
run *run.Run
|
2020-08-23 13:42:15 +03:00
|
|
|
|
2020-08-16 00:06:40 +03:00
|
|
|
initOnce sync.Once
|
2020-08-23 13:42:15 +03:00
|
|
|
|
2023-04-19 08:28:09 +03:00
|
|
|
coll *source.Collection
|
2020-08-06 20:58:47 +03:00
|
|
|
srcCache map[string]*source.Source
|
2020-08-23 13:42:15 +03:00
|
|
|
|
2020-08-06 20:58:47 +03:00
|
|
|
Context context.Context
|
|
|
|
cancelFn context.CancelFunc
|
2020-08-09 00:23:30 +03:00
|
|
|
|
|
|
|
Cleanup *cleanup.Cleanup
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
// New returns a new Helper. The helper's Close func will be
|
|
|
|
// automatically invoked via t.Cleanup.
|
2023-04-19 08:28:09 +03:00
|
|
|
func New(t testing.TB, opts ...Option) *Helper {
|
2020-08-06 20:58:47 +03:00
|
|
|
h := &Helper{
|
2024-01-15 04:45:34 +03:00
|
|
|
T: t,
|
|
|
|
Log: lgt.New(t),
|
|
|
|
Cleanup: cleanup.New(),
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
ctx, cancelFn := context.WithCancel(context.Background())
|
|
|
|
h.cancelFn = cancelFn
|
|
|
|
|
2023-04-30 17:18:56 +03:00
|
|
|
h.Context = lg.NewContext(ctx, h.Log)
|
2020-08-06 20:58:47 +03:00
|
|
|
t.Cleanup(h.Close)
|
2024-01-15 04:45:34 +03:00
|
|
|
|
|
|
|
for _, opt := range opts {
|
|
|
|
opt(h)
|
|
|
|
}
|
|
|
|
|
2020-08-06 20:58:47 +03:00
|
|
|
return h
|
|
|
|
}
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
// NewWith is a convenience wrapper for New, that also returns
|
|
|
|
// the source.Source for handle, the driver.SQLDriver, driver.Grip,
|
2023-07-08 18:21:27 +03:00
|
|
|
// and the *sql.DB.
|
2024-01-15 04:45:34 +03:00
|
|
|
func NewWith(t testing.TB, handle string) (*Helper, *source.Source, //nolint:revive
|
|
|
|
driver.SQLDriver, driver.Grip, *sql.DB,
|
|
|
|
) {
|
2020-08-06 20:58:47 +03:00
|
|
|
th := New(t)
|
|
|
|
src := th.Source(handle)
|
2024-01-15 04:45:34 +03:00
|
|
|
grip := th.Open(src)
|
|
|
|
drvr := grip.SQLDriver()
|
|
|
|
db, err := grip.DB(th.Context)
|
2023-07-08 18:21:27 +03:00
|
|
|
require.NoError(t, err)
|
2020-08-06 20:58:47 +03:00
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
return th, src, drvr, grip, db
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
|
|
|
|
2020-08-16 00:06:40 +03:00
|
|
|
func (h *Helper) init() {
|
|
|
|
h.initOnce.Do(func() {
|
|
|
|
log := h.Log
|
2024-01-15 04:45:34 +03:00
|
|
|
|
|
|
|
optRegistry := &options.Registry{}
|
|
|
|
cli.RegisterDefaultOpts(optRegistry)
|
2020-08-23 13:42:15 +03:00
|
|
|
h.registry = driver.NewRegistry(log)
|
2020-08-16 00:06:40 +03:00
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
cfg := config.New()
|
2020-08-16 00:06:40 +03:00
|
|
|
var err error
|
2024-01-15 04:45:34 +03:00
|
|
|
|
2024-01-25 09:29:55 +03:00
|
|
|
h.files, err = files.New(
|
2024-01-15 04:45:34 +03:00
|
|
|
h.Context,
|
|
|
|
optRegistry,
|
|
|
|
TempLockFunc(h.T),
|
|
|
|
tu.TempDir(h.T, false),
|
|
|
|
tu.TempDir(h.T, false),
|
|
|
|
)
|
2020-08-16 00:06:40 +03:00
|
|
|
require.NoError(h.T, err)
|
2020-12-30 21:57:58 +03:00
|
|
|
|
|
|
|
h.Cleanup.Add(func() {
|
|
|
|
h.T.Logf("Executing outer Files cleanup")
|
|
|
|
err := h.files.Close()
|
|
|
|
assert.NoError(h.T, err)
|
|
|
|
})
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
h.grips = driver.NewGrips(h.registry, h.files, sqlite3.NewScratchSource)
|
|
|
|
h.Cleanup.AddC(h.grips)
|
2020-08-23 13:42:15 +03:00
|
|
|
|
2024-01-25 09:29:55 +03:00
|
|
|
h.registry.AddProvider(drivertype.SQLite, &sqlite3.Provider{Log: log})
|
|
|
|
h.registry.AddProvider(drivertype.Pg, &postgres.Provider{Log: log})
|
|
|
|
h.registry.AddProvider(drivertype.MSSQL, &sqlserver.Provider{Log: log})
|
|
|
|
h.registry.AddProvider(drivertype.MySQL, &mysql.Provider{Log: log})
|
2020-08-16 00:06:40 +03:00
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
csvp := &csv.Provider{Log: log, Ingester: h.grips, Files: h.files}
|
2024-01-25 09:29:55 +03:00
|
|
|
h.registry.AddProvider(drivertype.CSV, csvp)
|
|
|
|
h.registry.AddProvider(drivertype.TSV, csvp)
|
2023-04-22 06:36:32 +03:00
|
|
|
h.files.AddDriverDetectors(csv.DetectCSV, csv.DetectTSV)
|
2020-08-16 00:06:40 +03:00
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
jsonp := &json.Provider{Log: log, Ingester: h.grips, Files: h.files}
|
2024-01-25 09:29:55 +03:00
|
|
|
h.registry.AddProvider(drivertype.JSON, jsonp)
|
|
|
|
h.registry.AddProvider(drivertype.JSONA, jsonp)
|
|
|
|
h.registry.AddProvider(drivertype.JSONL, jsonp)
|
2023-05-03 15:36:10 +03:00
|
|
|
h.files.AddDriverDetectors(
|
2023-07-04 20:31:47 +03:00
|
|
|
json.DetectJSON(driver.OptIngestSampleSize.Get(nil)),
|
|
|
|
json.DetectJSONA(driver.OptIngestSampleSize.Get(nil)),
|
|
|
|
json.DetectJSONL(driver.OptIngestSampleSize.Get(nil)),
|
2023-05-03 15:36:10 +03:00
|
|
|
)
|
2020-08-23 13:42:15 +03:00
|
|
|
|
2024-01-25 09:29:55 +03:00
|
|
|
h.registry.AddProvider(drivertype.XLSX, &xlsx.Provider{Log: log, Ingester: h.grips, Files: h.files})
|
2023-04-22 06:36:32 +03:00
|
|
|
h.files.AddDriverDetectors(xlsx.DetectXLSX)
|
2020-08-16 00:06:40 +03:00
|
|
|
|
|
|
|
h.addUserDrivers()
|
2023-06-13 19:06:18 +03:00
|
|
|
|
|
|
|
h.run = &run.Run{
|
|
|
|
Stdin: os.Stdin,
|
|
|
|
Out: os.Stdout,
|
|
|
|
ErrOut: os.Stdin,
|
2024-01-15 04:45:34 +03:00
|
|
|
Config: cfg,
|
2023-06-13 19:06:18 +03:00
|
|
|
ConfigStore: config.DiscardStore{},
|
2024-01-15 04:45:34 +03:00
|
|
|
OptionsRegistry: optRegistry,
|
2023-06-13 19:06:18 +03:00
|
|
|
DriverRegistry: h.registry,
|
|
|
|
}
|
2020-08-16 00:06:40 +03:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2020-08-06 20:58:47 +03:00
|
|
|
// Close runs any cleanup tasks, failing h's testing.T if any cleanup
|
|
|
|
// error occurs. Close is automatically invoked via t.Cleanup, it does
|
|
|
|
// not need to be explicitly invoked unless desired.
|
|
|
|
func (h *Helper) Close() {
|
|
|
|
err := h.Cleanup.Run()
|
2023-04-02 22:49:45 +03:00
|
|
|
lg.WarnIfError(h.Log, "helper cleanup", err)
|
2020-08-06 20:58:47 +03:00
|
|
|
assert.NoError(h.T, err)
|
|
|
|
h.cancelFn()
|
|
|
|
}
|
|
|
|
|
2023-08-16 18:09:50 +03:00
|
|
|
// Add adds src to the helper's collection.
|
|
|
|
func (h *Helper) Add(src *source.Source) *source.Source {
|
|
|
|
// This is a bit of a hack to ensure that internals are loaded: we
|
|
|
|
// load a known source. The loading mechanism should be refactored
|
|
|
|
// to not require this.
|
2024-01-15 04:45:34 +03:00
|
|
|
_ = h.Source(sakila.SL3)
|
2023-08-16 18:09:50 +03:00
|
|
|
|
|
|
|
h.mu.Lock()
|
|
|
|
defer h.mu.Unlock()
|
|
|
|
|
|
|
|
require.False(h.T, h.coll.IsExistingSource(src.Handle),
|
|
|
|
"source {%s} already exists", src.Handle)
|
|
|
|
|
|
|
|
require.NoError(h.T, h.coll.Add(src))
|
|
|
|
|
|
|
|
h.srcCache[src.Handle] = src
|
|
|
|
|
|
|
|
// envDiffDB is the name of the envar that controls whether the testing
|
|
|
|
// diffdb mechanism is executed automatically by Helper.Source.
|
|
|
|
const envDiffDB = "SQ_TEST_DIFFDB"
|
|
|
|
|
|
|
|
if proj.BoolEnvar(envDiffDB) {
|
|
|
|
h.DiffDB(src)
|
|
|
|
}
|
|
|
|
|
|
|
|
return src
|
|
|
|
}
|
|
|
|
|
|
|
|
// Source returns a test Source with the given handle. The standard test
|
|
|
|
// source collection is loaded from the sq config file at TestSourcesConfigPath,
|
|
|
|
// (but additional sources can be added via Helper.Add). Variables
|
2020-08-06 20:58:47 +03:00
|
|
|
// such as ${SQ_ROOT} in the config file are expanded. The same
|
|
|
|
// instance of *source.Source will be returned for multiple invocations
|
|
|
|
// of this method on the same Helper instance.
|
|
|
|
//
|
2023-04-22 06:36:32 +03:00
|
|
|
// For certain file-based driver types, the returned src's Location
|
2020-08-06 20:58:47 +03:00
|
|
|
// may point to a copy of the file. This helps avoid tests dirtying
|
|
|
|
// a version-controlled data file.
|
|
|
|
//
|
|
|
|
// Any external database source (that is, any SQL source other than SQLite3)
|
|
|
|
// will have its location determined from an envar. Given a source @sakila_pg12,
|
|
|
|
// its location is derived from an envar SQ_TEST_SRC__SAKILA_PG12. If that envar
|
|
|
|
// is not set, the test calling this method will be skipped.
|
2020-08-16 00:06:40 +03:00
|
|
|
//
|
|
|
|
// If envar SQ_TEST_DIFFDB is true, DiffDB is run on every SQL source
|
|
|
|
// returned by Source.
|
2020-08-06 20:58:47 +03:00
|
|
|
func (h *Helper) Source(handle string) *source.Source {
|
|
|
|
h.mu.Lock()
|
|
|
|
defer h.mu.Unlock()
|
|
|
|
t := h.T
|
|
|
|
|
2022-12-24 07:05:26 +03:00
|
|
|
// invoke h.init to ensure that its cleanup side effects
|
2020-08-09 00:23:30 +03:00
|
|
|
// happen in the correct order (files get cleaned after
|
|
|
|
// databases, etc.).
|
2020-08-16 00:06:40 +03:00
|
|
|
h.init()
|
2020-08-09 00:23:30 +03:00
|
|
|
|
2020-08-06 20:58:47 +03:00
|
|
|
// If the handle refers to an external database, we will skip
|
|
|
|
// the test if the envar for the handle is not set.
|
2020-08-09 17:40:46 +03:00
|
|
|
if stringz.InSlice(sakila.SQLAllExternal(), handle) {
|
2020-08-06 20:58:47 +03:00
|
|
|
// Skip the test if the envar for the handle is not set
|
|
|
|
handleEnvar := "SQ_TEST_SRC__" + strings.ToUpper(strings.TrimPrefix(handle, "@"))
|
|
|
|
if envar, ok := os.LookupEnv(handleEnvar); !ok || strings.TrimSpace(envar) == "" {
|
2023-04-02 22:49:45 +03:00
|
|
|
h.T.Skipf("Skip test %s because envar {%s} for {%s} is not set",
|
2020-08-06 20:58:47 +03:00
|
|
|
h.T.Name(), handleEnvar, handle)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-04-19 08:28:09 +03:00
|
|
|
if h.coll == nil {
|
2020-08-06 20:58:47 +03:00
|
|
|
// It might be expected that we would simply use the
|
2023-04-19 08:28:09 +03:00
|
|
|
// collection (h.coll) to return the source, but this
|
2020-08-06 20:58:47 +03:00
|
|
|
// method also uses a cache. This is because this
|
2024-01-15 04:45:34 +03:00
|
|
|
// method makes a copy of the data file of file-based sources
|
2020-08-06 20:58:47 +03:00
|
|
|
// as mentioned in the method godoc.
|
2023-04-22 16:37:07 +03:00
|
|
|
h.coll = mustLoadCollection(h.Context, t)
|
2020-08-06 20:58:47 +03:00
|
|
|
h.srcCache = map[string]*source.Source{}
|
|
|
|
}
|
|
|
|
|
|
|
|
// If it's already in the cache, return it.
|
|
|
|
src, ok := h.srcCache[handle]
|
|
|
|
if ok {
|
|
|
|
return src
|
|
|
|
}
|
|
|
|
|
2023-04-19 08:28:09 +03:00
|
|
|
src, err := h.coll.Get(handle)
|
2020-08-06 20:58:47 +03:00
|
|
|
require.NoError(t, err,
|
|
|
|
"source %s was not found in %s", handle, testsrc.PathSrcsConfig)
|
|
|
|
|
2024-01-25 09:29:55 +03:00
|
|
|
if src.Type == drivertype.SQLite {
|
2020-08-06 20:58:47 +03:00
|
|
|
// This could be easily generalized for CSV/XLSX etc.
|
2020-08-07 22:51:30 +03:00
|
|
|
fpath, err := sqlite3.PathFromLocation(src)
|
2020-08-06 20:58:47 +03:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
srcFile, err := os.Open(fpath)
|
|
|
|
require.NoError(t, err)
|
2020-08-09 00:23:30 +03:00
|
|
|
defer func() {
|
|
|
|
assert.NoError(t, srcFile.Close())
|
|
|
|
}()
|
2020-08-06 20:58:47 +03:00
|
|
|
|
2024-01-25 09:29:55 +03:00
|
|
|
destFile, err := h.files.CreateTemp("*_"+filepath.Base(src.Location), true)
|
2020-08-06 20:58:47 +03:00
|
|
|
require.NoError(t, err)
|
2020-08-09 00:23:30 +03:00
|
|
|
defer func() {
|
|
|
|
assert.NoError(t, destFile.Close())
|
|
|
|
}()
|
2020-08-08 08:10:41 +03:00
|
|
|
|
|
|
|
destFileName := destFile.Name()
|
|
|
|
|
2020-08-06 20:58:47 +03:00
|
|
|
_, err = io.Copy(destFile, srcFile)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
2020-08-08 08:10:41 +03:00
|
|
|
src.Location = sqlite3.Prefix + destFileName
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
2022-12-18 09:07:38 +03:00
|
|
|
|
2020-08-06 20:58:47 +03:00
|
|
|
h.srcCache[handle] = src
|
2020-08-16 00:06:40 +03:00
|
|
|
|
|
|
|
// envDiffDB is the name of the envar that controls whether the testing
|
|
|
|
// diffdb mechanism is executed automatically by Helper.Source.
|
|
|
|
const envDiffDB = "SQ_TEST_DIFFDB"
|
|
|
|
|
|
|
|
if proj.BoolEnvar(envDiffDB) {
|
|
|
|
h.DiffDB(src)
|
|
|
|
}
|
|
|
|
|
2020-08-06 20:58:47 +03:00
|
|
|
return src
|
|
|
|
}
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
// SourceConfigured returns true if the source is configured. Note
|
|
|
|
// that Helper.Source skips the test if the source is not configured: that
|
|
|
|
// is to say, if the source location requires population via an envar, and
|
|
|
|
// the envar is not set. For example, for the PostgreSQL source @sakila_pg12,
|
|
|
|
// the envar SQ_TEST_SRC__SAKILA_PG12 is required. SourceConfigured tests
|
|
|
|
// if that envar is set.
|
|
|
|
func (h *Helper) SourceConfigured(handle string) bool {
|
|
|
|
h.mu.Lock()
|
|
|
|
defer h.mu.Unlock()
|
|
|
|
|
|
|
|
if !stringz.InSlice(sakila.SQLAllExternal(), handle) {
|
|
|
|
// Non-SQL and SQLite sources are always available.
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
handleEnvar := "SQ_TEST_SRC__" + strings.ToUpper(strings.TrimPrefix(handle, "@"))
|
|
|
|
if envar, ok := os.LookupEnv(handleEnvar); !ok || strings.TrimSpace(envar) == "" {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2023-04-19 08:28:09 +03:00
|
|
|
// NewCollection is a convenience function for building a
|
|
|
|
// new *source.Collection incorporating the supplied handles. See
|
2021-03-08 09:27:35 +03:00
|
|
|
// Helper.Source for more on the behavior.
|
2023-04-19 08:28:09 +03:00
|
|
|
func (h *Helper) NewCollection(handles ...string) *source.Collection {
|
|
|
|
coll := &source.Collection{}
|
2021-03-08 09:27:35 +03:00
|
|
|
for _, handle := range handles {
|
2023-04-19 08:28:09 +03:00
|
|
|
require.NoError(h.T, coll.Add(h.Source(handle)))
|
2021-03-08 09:27:35 +03:00
|
|
|
}
|
2023-04-19 08:28:09 +03:00
|
|
|
return coll
|
2021-03-08 09:27:35 +03:00
|
|
|
}
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
// Open opens a driver.Grip for src via h's internal Grips
|
2020-08-06 20:58:47 +03:00
|
|
|
// instance: thus subsequent calls to Open may return the
|
2024-01-15 04:45:34 +03:00
|
|
|
// same driver.Grip instance. The opened driver.Grip will be closed
|
2020-08-06 20:58:47 +03:00
|
|
|
// during h.Close.
|
2024-01-15 04:45:34 +03:00
|
|
|
func (h *Helper) Open(src *source.Source) driver.Grip {
|
|
|
|
ctx := h.Context
|
|
|
|
grip, err := h.Grips().Open(ctx, src)
|
2020-08-06 20:58:47 +03:00
|
|
|
require.NoError(h.T, err)
|
2023-04-16 01:28:51 +03:00
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
db, err := grip.DB(ctx)
|
2023-07-08 18:21:27 +03:00
|
|
|
require.NoError(h.T, err)
|
|
|
|
|
|
|
|
require.NoError(h.T, db.PingContext(ctx))
|
2024-01-15 04:45:34 +03:00
|
|
|
return grip
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
|
|
|
|
2023-07-08 18:21:27 +03:00
|
|
|
// OpenDB is a convenience method for getting the sql.DB for src.
|
|
|
|
// The returned sql.DB is closed during h.Close, via the closing
|
2024-01-15 04:45:34 +03:00
|
|
|
// of its parent driver.Grip.
|
2023-07-08 18:21:27 +03:00
|
|
|
func (h *Helper) OpenDB(src *source.Source) *sql.DB {
|
2024-01-15 04:45:34 +03:00
|
|
|
grip := h.Open(src)
|
|
|
|
db, err := grip.DB(h.Context)
|
2023-07-08 18:21:27 +03:00
|
|
|
require.NoError(h.T, err)
|
|
|
|
return db
|
|
|
|
}
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
// openNew opens a new driver.Grip. It is the caller's responsibility
|
|
|
|
// to close the returned Grip. Unlike method Open, this method
|
2020-08-06 20:58:47 +03:00
|
|
|
// will always invoke the driver's Open method.
|
|
|
|
//
|
|
|
|
// Some of Helper's methods (e.g. DropTable) need to use openNew rather
|
2024-01-15 04:45:34 +03:00
|
|
|
// than Open, as the Grip returned by Open can be closed by test code,
|
2020-08-06 20:58:47 +03:00
|
|
|
// potentially causing problems during Cleanup.
|
2024-01-15 04:45:34 +03:00
|
|
|
func (h *Helper) openNew(src *source.Source) driver.Grip {
|
2023-04-30 17:18:56 +03:00
|
|
|
h.Log.Debug("openNew", lga.Src, src)
|
2020-08-06 20:58:47 +03:00
|
|
|
reg := h.Registry()
|
|
|
|
drvr, err := reg.DriverFor(src.Type)
|
|
|
|
require.NoError(h.T, err)
|
2024-01-15 04:45:34 +03:00
|
|
|
grip, err := drvr.Open(h.Context, src)
|
2020-08-06 20:58:47 +03:00
|
|
|
require.NoError(h.T, err)
|
2024-01-15 04:45:34 +03:00
|
|
|
return grip
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
// SQLDriverFor is a convenience method to get src's driver.SQLDriver.
|
|
|
|
func (h *Helper) SQLDriverFor(src *source.Source) driver.SQLDriver {
|
|
|
|
reg := h.Registry()
|
|
|
|
drvr, err := reg.DriverFor(src.Type)
|
|
|
|
require.NoError(h.T, err)
|
|
|
|
sqlDrvr, ok := drvr.(driver.SQLDriver)
|
2022-12-18 05:43:53 +03:00
|
|
|
require.True(h.T, ok,
|
|
|
|
"driver %T is not a driver.SQLDriver: ensure that the src passed to SQLDriverFor implements driver.SQLDriver",
|
|
|
|
drvr)
|
2020-08-06 20:58:47 +03:00
|
|
|
return sqlDrvr
|
|
|
|
}
|
|
|
|
|
|
|
|
// DriverFor is a convenience method to get src's driver.Driver.
|
|
|
|
func (h *Helper) DriverFor(src *source.Source) driver.Driver {
|
|
|
|
reg := h.Registry()
|
|
|
|
drvr, err := reg.DriverFor(src.Type)
|
|
|
|
require.NoError(h.T, err)
|
|
|
|
return drvr
|
|
|
|
}
|
|
|
|
|
|
|
|
// RowCount returns the result of "SELECT COUNT(*) FROM tbl",
|
|
|
|
// failing h's test on any error.
|
|
|
|
func (h *Helper) RowCount(src *source.Source, tbl string) int64 {
|
2024-01-15 04:45:34 +03:00
|
|
|
grip := h.openNew(src)
|
|
|
|
defer lg.WarnIfCloseError(h.Log, lgm.CloseDB, grip)
|
2020-08-06 20:58:47 +03:00
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
query := "SELECT COUNT(*) FROM " + grip.SQLDriver().Dialect().Enquote(tbl)
|
2020-08-06 20:58:47 +03:00
|
|
|
var count int64
|
2024-01-15 04:45:34 +03:00
|
|
|
db, err := grip.DB(h.Context)
|
2023-07-08 18:21:27 +03:00
|
|
|
require.NoError(h.T, err)
|
|
|
|
|
|
|
|
require.NoError(h.T, db.QueryRowContext(h.Context, query).Scan(&count))
|
2020-08-06 20:58:47 +03:00
|
|
|
return count
|
|
|
|
}
|
|
|
|
|
|
|
|
// CreateTable creates a new table in src, and inserts data, returning
|
|
|
|
// the number of data rows inserted. If dropAfter is true, the created
|
|
|
|
// table is dropped when t.Cleanup is run.
|
2024-01-25 10:42:51 +03:00
|
|
|
func (h *Helper) CreateTable(dropAfter bool, src *source.Source, tblDef *schema.Table,
|
2022-12-18 11:35:59 +03:00
|
|
|
data ...[]any,
|
|
|
|
) (affected int64) {
|
2024-01-15 04:45:34 +03:00
|
|
|
grip := h.openNew(src)
|
|
|
|
defer lg.WarnIfCloseError(h.Log, lgm.CloseDB, grip)
|
2020-08-06 20:58:47 +03:00
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
db, err := grip.DB(h.Context)
|
2023-07-08 18:21:27 +03:00
|
|
|
require.NoError(h.T, err)
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
require.NoError(h.T, grip.SQLDriver().CreateTable(h.Context, db, tblDef))
|
2020-08-06 20:58:47 +03:00
|
|
|
h.T.Logf("Created table %s.%s", src.Handle, tblDef.Name)
|
|
|
|
|
|
|
|
if dropAfter {
|
2023-11-19 03:05:48 +03:00
|
|
|
h.Cleanup.Add(func() { h.DropTable(src, tablefq.From(tblDef.Name)) })
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
if len(data) == 0 {
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
|
|
|
|
return h.Insert(src, tblDef.Name, tblDef.ColNames(), data...)
|
|
|
|
}
|
|
|
|
|
2020-08-12 21:24:01 +03:00
|
|
|
// Insert inserts records for cols into src.tbl, returning the number of
|
|
|
|
// records inserted. Note that the records arg may be mutated by src's
|
|
|
|
// driver InsertMungeFunc.
|
2022-12-17 02:34:33 +03:00
|
|
|
func (h *Helper) Insert(src *source.Source, tbl string, cols []string, records ...[]any) (affected int64) {
|
2020-08-12 21:24:01 +03:00
|
|
|
if len(records) == 0 {
|
2020-08-06 20:58:47 +03:00
|
|
|
return 0
|
|
|
|
}
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
grip := h.openNew(src)
|
|
|
|
defer lg.WarnIfCloseError(h.Log, lgm.CloseDB, grip)
|
2020-08-06 20:58:47 +03:00
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
drvr := grip.SQLDriver()
|
|
|
|
db, err := grip.DB(h.Context)
|
2023-07-08 18:21:27 +03:00
|
|
|
require.NoError(h.T, err)
|
2020-08-12 21:24:01 +03:00
|
|
|
|
2023-07-08 18:21:27 +03:00
|
|
|
conn, err := db.Conn(h.Context)
|
2020-08-12 21:24:01 +03:00
|
|
|
require.NoError(h.T, err)
|
2023-04-02 22:49:45 +03:00
|
|
|
defer lg.WarnIfCloseError(h.Log, lgm.CloseDB, conn)
|
2020-08-12 21:24:01 +03:00
|
|
|
|
|
|
|
batchSize := driver.MaxBatchRows(drvr, len(cols))
|
2024-01-15 04:45:34 +03:00
|
|
|
bi, err := driver.NewBatchInsert(
|
|
|
|
h.Context,
|
|
|
|
libsq.MsgIngestRecords,
|
|
|
|
drvr,
|
|
|
|
conn,
|
|
|
|
tbl,
|
|
|
|
cols,
|
|
|
|
batchSize,
|
|
|
|
)
|
2020-08-06 20:58:47 +03:00
|
|
|
require.NoError(h.T, err)
|
|
|
|
|
2020-08-12 21:24:01 +03:00
|
|
|
for _, rec := range records {
|
|
|
|
require.NoError(h.T, bi.Munge(rec))
|
|
|
|
|
|
|
|
select {
|
|
|
|
case <-h.Context.Done():
|
|
|
|
// Should not happen
|
|
|
|
close(bi.RecordCh)
|
|
|
|
require.NoError(h.T, h.Context.Err())
|
|
|
|
|
|
|
|
case err = <-bi.ErrCh:
|
|
|
|
// Should not happen
|
|
|
|
if err != nil {
|
|
|
|
close(bi.RecordCh)
|
|
|
|
require.NoError(h.T, err)
|
|
|
|
} else {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
|
|
|
case bi.RecordCh <- rec:
|
|
|
|
}
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
|
|
|
|
2020-08-12 21:24:01 +03:00
|
|
|
close(bi.RecordCh) // Indicate that we're finished writing records
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
err = <-bi.ErrCh // Stop for bi to complete
|
2020-08-12 21:24:01 +03:00
|
|
|
require.NoError(h.T, err)
|
|
|
|
|
|
|
|
h.T.Logf("Inserted %d rows to %s.%s", bi.Written(), src.Handle, tbl)
|
|
|
|
return bi.Written()
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
// CopyTable copies fromTable into a new table toTable. If
|
2023-11-19 03:05:48 +03:00
|
|
|
// toTable is empty, a unique table name is generated based on
|
2020-08-06 20:58:47 +03:00
|
|
|
// fromTable. The table name used is returned.
|
|
|
|
// If dropAfter is true, the table is dropped when t.Cleanup is run.
|
|
|
|
// If copyData is true, fromTable's data is also copied.
|
|
|
|
// Constraints (keys, defaults etc.) may not be copied.
|
2023-11-19 03:05:48 +03:00
|
|
|
//
|
|
|
|
// TODO: CopyTable should return tablefq.T instead of string.
|
|
|
|
func (h *Helper) CopyTable(
|
|
|
|
dropAfter bool,
|
|
|
|
src *source.Source,
|
|
|
|
fromTable, toTable tablefq.T,
|
|
|
|
copyData bool,
|
|
|
|
) string {
|
|
|
|
if lo.IsEmpty(toTable) {
|
|
|
|
toTable = fromTable
|
|
|
|
toTable.Table = stringz.UniqTableName(fromTable.Table)
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
grip := h.openNew(src)
|
|
|
|
defer lg.WarnIfCloseError(h.Log, lgm.CloseDB, grip)
|
2020-08-06 20:58:47 +03:00
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
db, err := grip.DB(h.Context)
|
2023-07-08 18:21:27 +03:00
|
|
|
require.NoError(h.T, err)
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
copied, err := grip.SQLDriver().CopyTable(
|
2023-11-19 03:05:48 +03:00
|
|
|
h.Context,
|
|
|
|
db,
|
|
|
|
fromTable,
|
|
|
|
toTable,
|
|
|
|
copyData,
|
|
|
|
)
|
2020-08-06 20:58:47 +03:00
|
|
|
require.NoError(h.T, err)
|
|
|
|
if dropAfter {
|
|
|
|
h.Cleanup.Add(func() { h.DropTable(src, toTable) })
|
|
|
|
}
|
|
|
|
|
2023-04-02 22:49:45 +03:00
|
|
|
h.Log.Debug("Copied table",
|
2023-11-19 03:05:48 +03:00
|
|
|
lga.From, fromTable,
|
|
|
|
lga.To, toTable,
|
2023-04-02 22:49:45 +03:00
|
|
|
"copy_data", copyData,
|
|
|
|
lga.Count, copied,
|
|
|
|
"drop_after", dropAfter,
|
|
|
|
)
|
2023-11-19 03:05:48 +03:00
|
|
|
return toTable.Table
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
// DropTable drops tbl from src.
|
2023-11-19 03:05:48 +03:00
|
|
|
func (h *Helper) DropTable(src *source.Source, tbl tablefq.T) {
|
2024-01-15 04:45:34 +03:00
|
|
|
grip := h.openNew(src)
|
|
|
|
defer lg.WarnIfCloseError(h.Log, lgm.CloseDB, grip)
|
2020-08-06 20:58:47 +03:00
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
db, err := grip.DB(h.Context)
|
2023-07-08 18:21:27 +03:00
|
|
|
require.NoError(h.T, err)
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
require.NoError(h.T, grip.SQLDriver().DropTable(h.Context, db, tbl, true))
|
2023-11-19 03:05:48 +03:00
|
|
|
h.Log.Debug("Dropped table", lga.Target, source.Target(src, tbl.Table))
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
// QuerySQL uses libsq.QuerySQL to execute SQL query
|
|
|
|
// against src, returning a sink to which all records have
|
2023-11-19 03:05:48 +03:00
|
|
|
// been written. Typically the db arg is nil, and QuerySQL uses the
|
2024-01-15 04:45:34 +03:00
|
|
|
// same driver.Grip instance as returned by Helper.Open. If db
|
2023-11-19 03:05:48 +03:00
|
|
|
// is non-nil, it is passed to libsq.QuerySQL (e.g. the query needs to
|
|
|
|
// execute against a sql.Tx), and the caller is responsible for closing db.
|
|
|
|
func (h *Helper) QuerySQL(src *source.Source, db sqlz.DB, query string, args ...any) (*RecordSink, error) {
|
2024-01-15 04:45:34 +03:00
|
|
|
grip := h.Open(src)
|
2020-08-06 20:58:47 +03:00
|
|
|
|
|
|
|
sink := &RecordSink{}
|
2023-05-22 18:08:14 +03:00
|
|
|
recw := output.NewRecordWriterAdapter(h.Context, sink)
|
2024-01-15 04:45:34 +03:00
|
|
|
err := libsq.QuerySQL(h.Context, grip, db, recw, query, args...)
|
2020-08-06 20:58:47 +03:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
_, err = recw.Wait()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return sink, nil
|
|
|
|
}
|
|
|
|
|
2023-04-07 11:00:49 +03:00
|
|
|
// QuerySLQ executes the SLQ query. Args are predefined variables for
|
|
|
|
// substitution.
|
|
|
|
func (h *Helper) QuerySLQ(query string, args map[string]string) (*RecordSink, error) {
|
2023-03-27 05:03:40 +03:00
|
|
|
// We need to ensure that each of the handles in the query is loaded.
|
|
|
|
a, err := ast.Parse(h.Log, query)
|
|
|
|
require.NoError(h.T, err)
|
|
|
|
|
2023-04-07 11:00:49 +03:00
|
|
|
for _, handle := range ast.NewInspector(a).FindHandles() {
|
2023-03-27 05:03:40 +03:00
|
|
|
// This triggers handle loading
|
|
|
|
_ = h.Source(handle)
|
|
|
|
}
|
|
|
|
|
2023-04-01 12:48:24 +03:00
|
|
|
qc := &libsq.QueryContext{
|
2024-01-15 04:45:34 +03:00
|
|
|
Collection: h.coll,
|
|
|
|
Grips: h.grips,
|
|
|
|
Args: args,
|
2023-04-01 12:48:24 +03:00
|
|
|
}
|
|
|
|
|
2023-03-27 05:03:40 +03:00
|
|
|
sink := &RecordSink{}
|
2023-05-22 18:08:14 +03:00
|
|
|
recw := output.NewRecordWriterAdapter(h.Context, sink)
|
2023-03-27 05:03:40 +03:00
|
|
|
|
2023-04-02 22:49:45 +03:00
|
|
|
err = libsq.ExecuteSLQ(h.Context, qc, query, recw)
|
2023-03-27 05:03:40 +03:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
if _, err = recw.Wait(); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return sink, nil
|
|
|
|
}
|
|
|
|
|
2020-08-06 20:58:47 +03:00
|
|
|
// ExecSQL is a convenience wrapper for sql.DB.Exec that returns the
|
|
|
|
// rows affected, failing on any error. Note that ExecSQL uses the
|
2024-01-15 04:45:34 +03:00
|
|
|
// same Grip instance as returned by h.Open.
|
2022-12-17 02:34:33 +03:00
|
|
|
func (h *Helper) ExecSQL(src *source.Source, query string, args ...any) (affected int64) {
|
2023-07-08 18:21:27 +03:00
|
|
|
db := h.OpenDB(src)
|
2020-08-06 20:58:47 +03:00
|
|
|
|
2023-07-08 18:21:27 +03:00
|
|
|
res, err := db.ExecContext(h.Context, query, args...)
|
2020-08-06 20:58:47 +03:00
|
|
|
|
|
|
|
require.NoError(h.T, err)
|
|
|
|
|
|
|
|
affected, err = res.RowsAffected()
|
|
|
|
require.NoError(h.T, err)
|
|
|
|
|
|
|
|
return affected
|
|
|
|
}
|
|
|
|
|
|
|
|
// InsertDefaultRow executes the equivalent
|
|
|
|
// of INSERT INTO tbl DEFAULT VALUES.
|
|
|
|
// It fails if a row was not inserted.
|
|
|
|
//
|
|
|
|
// Note that for some driver types, the driver does not
|
|
|
|
// support DEFAULT values for some col types. For example
|
|
|
|
// this method may fail for a MySQL column "col_text TEXT NOT NULL",
|
|
|
|
// as TEXT and BLOB cannot have a DEFAULT value.
|
|
|
|
func (h *Helper) InsertDefaultRow(src *source.Source, tbl string) {
|
|
|
|
drvr := h.SQLDriverFor(src)
|
|
|
|
var query string
|
|
|
|
|
2024-01-25 09:29:55 +03:00
|
|
|
if src.Type == drivertype.MySQL {
|
2020-08-06 20:58:47 +03:00
|
|
|
// One driver had to be different...
|
|
|
|
// We could push this mysql-specific logic down to the driver impl
|
|
|
|
// but prob not worth the effort just for one driver.
|
|
|
|
query = "INSERT INTO " + drvr.Dialect().Enquote(tbl) + " () VALUES ()"
|
|
|
|
} else {
|
|
|
|
query = "INSERT INTO " + drvr.Dialect().Enquote(tbl) + " DEFAULT VALUES"
|
|
|
|
}
|
|
|
|
|
|
|
|
affected := h.ExecSQL(src, query)
|
|
|
|
require.Equal(h.T, int64(1), affected)
|
|
|
|
}
|
|
|
|
|
|
|
|
// TruncateTable truncates tbl in src.
|
|
|
|
func (h *Helper) TruncateTable(src *source.Source, tbl string) (affected int64) {
|
2024-01-15 04:45:34 +03:00
|
|
|
grip := h.openNew(src)
|
|
|
|
defer lg.WarnIfCloseError(h.Log, lgm.CloseDB, grip)
|
2020-08-06 20:58:47 +03:00
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
drvr := h.SQLDriverFor(src)
|
|
|
|
require.NotNil(h.T, drvr, "not a SQL driver")
|
|
|
|
|
|
|
|
affected, err := drvr.Truncate(h.Context, src, tbl, true)
|
2020-08-06 20:58:47 +03:00
|
|
|
require.NoError(h.T, err)
|
|
|
|
return affected
|
|
|
|
}
|
|
|
|
|
|
|
|
// Registry returns the helper's registry instance,
|
2020-08-16 00:06:40 +03:00
|
|
|
// configured with standard providers.
|
2020-08-06 20:58:47 +03:00
|
|
|
func (h *Helper) Registry() *driver.Registry {
|
2020-08-16 00:06:40 +03:00
|
|
|
h.init()
|
2020-08-23 13:42:15 +03:00
|
|
|
return h.registry
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
|
|
|
|
2023-06-13 19:06:18 +03:00
|
|
|
// Run returns the helper's run instance.
|
|
|
|
func (h *Helper) Run() *run.Run {
|
|
|
|
h.init()
|
|
|
|
return h.run
|
|
|
|
}
|
|
|
|
|
2020-08-06 20:58:47 +03:00
|
|
|
// addUserDrivers adds some user drivers to the registry.
|
|
|
|
func (h *Helper) addUserDrivers() {
|
|
|
|
userDriverDefs := DriverDefsFrom(h.T, testsrc.PathDriverDefPpl, testsrc.PathDriverDefRSS)
|
|
|
|
|
|
|
|
// One day we may have more supported user driver genres.
|
2024-01-15 04:45:34 +03:00
|
|
|
userDriverImporters := map[string]userdriver.IngestFunc{
|
|
|
|
xmlud.Genre: xmlud.Ingest,
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, userDriverDef := range userDriverDefs {
|
|
|
|
userDriverDef := userDriverDef
|
|
|
|
|
|
|
|
errs := userdriver.ValidateDriverDef(userDriverDef)
|
|
|
|
require.Empty(h.T, errs)
|
|
|
|
|
|
|
|
importFn, ok := userDriverImporters[userDriverDef.Genre]
|
2023-04-02 22:49:45 +03:00
|
|
|
require.True(h.T, ok, "unsupported genre {%s} for user driver {%s} specified via config",
|
2020-08-06 20:58:47 +03:00
|
|
|
userDriverDef.Genre, userDriverDef.Name)
|
|
|
|
|
|
|
|
// For each user driver definition, we register a
|
|
|
|
// distinct userdriver.Provider instance.
|
|
|
|
udp := &userdriver.Provider{
|
|
|
|
Log: h.Log,
|
|
|
|
DriverDef: userDriverDef,
|
2024-01-15 04:45:34 +03:00
|
|
|
IngestFn: importFn,
|
|
|
|
Ingester: h.grips,
|
2020-08-06 20:58:47 +03:00
|
|
|
Files: h.files,
|
|
|
|
}
|
|
|
|
|
2023-11-21 00:42:38 +03:00
|
|
|
h.registry.AddProvider(drivertype.Type(userDriverDef.Name), udp)
|
2023-04-22 06:36:32 +03:00
|
|
|
h.files.AddDriverDetectors(udp.Detectors()...)
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// IsMonotable returns true if src's driver is monotable.
|
|
|
|
func (h *Helper) IsMonotable(src *source.Source) bool {
|
|
|
|
return h.DriverFor(src).DriverMetadata().Monotable
|
|
|
|
}
|
|
|
|
|
2024-01-25 07:01:24 +03:00
|
|
|
// Grips returns the helper's driver.Grips instance.
|
2024-01-15 04:45:34 +03:00
|
|
|
func (h *Helper) Grips() *driver.Grips {
|
2020-08-16 00:06:40 +03:00
|
|
|
h.init()
|
2024-01-15 04:45:34 +03:00
|
|
|
return h.grips
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
|
|
|
|
2024-01-25 07:01:24 +03:00
|
|
|
// Files returns the helper's source.Files instance.
|
2024-01-25 09:29:55 +03:00
|
|
|
func (h *Helper) Files() *files.Files {
|
2020-08-16 00:06:40 +03:00
|
|
|
h.init()
|
2020-08-06 20:58:47 +03:00
|
|
|
return h.files
|
|
|
|
}
|
|
|
|
|
2023-08-16 18:09:50 +03:00
|
|
|
// SourceMetadata returns metadata for src.
|
2023-11-21 00:42:38 +03:00
|
|
|
func (h *Helper) SourceMetadata(src *source.Source) (*metadata.Source, error) {
|
2024-01-15 04:45:34 +03:00
|
|
|
grip, err := h.Grips().Open(h.Context, src)
|
2023-08-16 18:09:50 +03:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
return grip.SourceMetadata(h.Context, false)
|
2023-08-16 18:09:50 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
// TableMetadata returns metadata for src's table.
|
2023-11-21 00:42:38 +03:00
|
|
|
func (h *Helper) TableMetadata(src *source.Source, tbl string) (*metadata.Table, error) {
|
2024-01-15 04:45:34 +03:00
|
|
|
grip, err := h.Grips().Open(h.Context, src)
|
2023-08-16 18:09:50 +03:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
return grip.TableMetadata(h.Context, tbl)
|
2023-08-16 18:09:50 +03:00
|
|
|
}
|
|
|
|
|
2020-08-16 00:06:40 +03:00
|
|
|
// DiffDB fails the test if src's metadata is substantially different
|
|
|
|
// when t.Cleanup runs vs when DiffDB is invoked. Effectively DiffDB
|
2020-08-06 20:58:47 +03:00
|
|
|
// takes before and after snapshots of src's metadata, and compares
|
|
|
|
// various elements such as the number of tables, table row counts, etc.
|
2020-08-16 00:06:40 +03:00
|
|
|
// DiffDB is useful for verifying that tests are leaving the database
|
2020-08-06 20:58:47 +03:00
|
|
|
// as they found it.
|
2020-08-16 00:06:40 +03:00
|
|
|
//
|
|
|
|
// Note that DiffDB adds considerable overhead to test runtime.
|
|
|
|
//
|
|
|
|
// If envar SQ_TEST_DIFFDB is true, DiffDB is run on every SQL source
|
|
|
|
// returned by Helper.Source.
|
|
|
|
func (h *Helper) DiffDB(src *source.Source) {
|
|
|
|
if !h.DriverFor(src).DriverMetadata().IsSQL {
|
2023-04-22 06:36:32 +03:00
|
|
|
// SkipDiffDB for non-SQL driver types
|
2020-08-16 00:06:40 +03:00
|
|
|
return
|
|
|
|
}
|
2020-08-06 20:58:47 +03:00
|
|
|
|
2023-04-16 01:28:51 +03:00
|
|
|
h.T.Logf("Executing DiffDB for %s", src.Handle)
|
2020-08-16 00:06:40 +03:00
|
|
|
|
|
|
|
beforeDB := h.openNew(src)
|
2023-04-02 22:49:45 +03:00
|
|
|
defer lg.WarnIfCloseError(h.Log, lgm.CloseDB, beforeDB)
|
2020-08-06 20:58:47 +03:00
|
|
|
|
2023-06-22 08:48:58 +03:00
|
|
|
beforeMeta, err := beforeDB.SourceMetadata(h.Context, false)
|
2020-08-06 20:58:47 +03:00
|
|
|
require.NoError(h.T, err)
|
|
|
|
|
|
|
|
h.Cleanup.Add(func() {
|
2020-08-16 00:06:40 +03:00
|
|
|
// Currently DiffDB just checks if the tables and each
|
2020-08-06 20:58:47 +03:00
|
|
|
// table's row count match.
|
|
|
|
|
2020-08-16 00:06:40 +03:00
|
|
|
afterDB := h.openNew(src)
|
2023-04-02 22:49:45 +03:00
|
|
|
defer lg.WarnIfCloseError(h.Log, lgm.CloseDB, afterDB)
|
2020-08-06 20:58:47 +03:00
|
|
|
|
2023-06-22 08:48:58 +03:00
|
|
|
afterMeta, err := afterDB.SourceMetadata(h.Context, false)
|
2020-08-06 20:58:47 +03:00
|
|
|
require.NoError(h.T, err)
|
2020-08-16 00:06:40 +03:00
|
|
|
require.Equal(h.T, beforeMeta.TableNames(), afterMeta.TableNames(),
|
|
|
|
"diffdb: should have the same set of tables before and after")
|
2020-08-06 20:58:47 +03:00
|
|
|
|
2020-08-16 00:06:40 +03:00
|
|
|
for i, beforeTbl := range beforeMeta.Tables {
|
|
|
|
assert.Equal(h.T, beforeTbl.RowCount, afterMeta.Tables[i].RowCount,
|
2023-04-02 22:49:45 +03:00
|
|
|
"diffdb: %s: row count for {%s} is expected to be %d but got %d", src.Handle, beforeTbl.Name,
|
2022-12-18 05:43:53 +03:00
|
|
|
beforeTbl.RowCount, afterMeta.Tables[i].RowCount)
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2023-04-22 16:37:07 +03:00
|
|
|
func mustLoadCollection(ctx context.Context, t testing.TB) *source.Collection {
|
2020-08-06 20:58:47 +03:00
|
|
|
hookExpand := func(data []byte) ([]byte, error) {
|
|
|
|
// expand vars such as "${SQ_ROOT}"
|
|
|
|
return []byte(proj.Expand(string(data))), nil
|
|
|
|
}
|
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
store := &yamlstore.Store{
|
2023-05-01 06:59:34 +03:00
|
|
|
Path: proj.Rel(testsrc.PathSrcsConfig),
|
|
|
|
OptionsRegistry: &options.Registry{},
|
|
|
|
HookLoad: hookExpand,
|
|
|
|
}
|
2024-01-15 04:45:34 +03:00
|
|
|
cli.RegisterDefaultOpts(store.OptionsRegistry)
|
2023-04-26 18:16:42 +03:00
|
|
|
|
2024-01-15 04:45:34 +03:00
|
|
|
cfg, err := store.Load(ctx)
|
2020-08-06 20:58:47 +03:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.NotNil(t, cfg)
|
2023-04-19 08:28:09 +03:00
|
|
|
require.NotNil(t, cfg.Collection)
|
2020-08-06 20:58:47 +03:00
|
|
|
|
2023-04-19 08:28:09 +03:00
|
|
|
return cfg.Collection
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
// DriverDefsFrom builds DriverDef values from cfg files.
|
|
|
|
func DriverDefsFrom(t testing.TB, cfgFiles ...string) []*userdriver.DriverDef {
|
|
|
|
var userDriverDefs []*userdriver.DriverDef
|
|
|
|
for _, f := range cfgFiles {
|
|
|
|
ext := &config.Ext{}
|
2023-04-26 18:16:42 +03:00
|
|
|
require.NoError(t, ioz.UnmarshallYAML(proj.ReadFile(f), ext))
|
2020-08-06 20:58:47 +03:00
|
|
|
userDriverDefs = append(userDriverDefs, ext.UserDrivers...)
|
|
|
|
}
|
|
|
|
return userDriverDefs
|
|
|
|
}
|
|
|
|
|
2023-04-22 06:36:32 +03:00
|
|
|
// DriverDetectors returns the common set of TypeDetectorFuncs.
|
2024-01-25 09:29:55 +03:00
|
|
|
func DriverDetectors() []files.TypeDetectFunc {
|
|
|
|
return []files.TypeDetectFunc{
|
|
|
|
files.DetectMagicNumber,
|
2020-10-20 18:05:43 +03:00
|
|
|
xlsx.DetectXLSX,
|
2024-01-15 04:45:34 +03:00
|
|
|
csv.DetectCSV,
|
|
|
|
csv.DetectTSV,
|
|
|
|
json.DetectJSON(1000),
|
|
|
|
json.DetectJSONA(1000),
|
|
|
|
json.DetectJSONL(1000),
|
2020-10-20 18:05:43 +03:00
|
|
|
}
|
2020-08-06 20:58:47 +03:00
|
|
|
}
|
2023-04-26 18:16:42 +03:00
|
|
|
|
|
|
|
// SetBuildVersion sets the build version for the lifecycle
|
|
|
|
// of test t.
|
|
|
|
func SetBuildVersion(t testing.TB, vers string) {
|
|
|
|
prevVers := buildinfo.Version
|
|
|
|
t.Setenv(buildinfo.EnvOverrideVersion, vers)
|
|
|
|
buildinfo.Version = vers
|
|
|
|
t.Cleanup(func() {
|
|
|
|
buildinfo.Version = prevVers
|
|
|
|
})
|
|
|
|
}
|
2024-01-15 04:45:34 +03:00
|
|
|
|
|
|
|
// TempLockfile returns a lockfile.Lockfile that uses a temp file.
|
|
|
|
func TempLockfile(t testing.TB) lockfile.Lockfile {
|
|
|
|
return lockfile.Lockfile(tu.TempFile(t, "pid.lock", false))
|
|
|
|
}
|
|
|
|
|
|
|
|
// TempLockFunc returns a lockfile.LockFunc that uses a temp file.
|
|
|
|
func TempLockFunc(t testing.TB) lockfile.LockFunc {
|
|
|
|
return func(ctx context.Context) (unlock func(), err error) {
|
|
|
|
lock := TempLockfile(t)
|
|
|
|
timeout := config.OptConfigLockTimeout.Default()
|
|
|
|
if err = lock.Lock(ctx, timeout); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return func() {
|
|
|
|
if err := lock.Unlock(); err != nil {
|
|
|
|
t.Logf("failed to release temp lock: %v", err)
|
|
|
|
}
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// ExtractHandlesFromQuery returns all handles mentioned in the query.
|
|
|
|
// If failOnErr is true, the test will fail on any parse error; otherwise,
|
|
|
|
// the test will log the error and return an empty slice.
|
|
|
|
func ExtractHandlesFromQuery(t testing.TB, query string, failOnErr bool) []string {
|
|
|
|
a, err := ast.Parse(lg.Discard(), query)
|
|
|
|
if err != nil {
|
|
|
|
if failOnErr {
|
|
|
|
require.NoError(t, err)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
t.Logf("Failed to parse query: >> %s << : %v", query, err)
|
|
|
|
return []string{}
|
|
|
|
}
|
|
|
|
|
|
|
|
return ast.ExtractHandles(a)
|
|
|
|
}
|