sq/drivers/sqlite3/metadata_test.go
Neil O'Toole 58ccfc9ded
Json driver; refactoring of core packages (#66)
* Type Detector refactor

* json driver impl; refactoring of source.Files reader et al

* working on kind detector

* significant switcheroo of packages

* partway throught refactoring Kind

* major package switcheroo for Kind
2020-08-23 04:42:15 -06:00

304 lines
10 KiB
Go

package sqlite3_test
import (
"context"
"fmt"
"reflect"
"strings"
"testing"
"github.com/neilotoole/lg"
"github.com/neilotoole/lg/testlg"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/neilotoole/sq/drivers/sqlite3"
"github.com/neilotoole/sq/libsq/core/errz"
"github.com/neilotoole/sq/libsq/core/kind"
"github.com/neilotoole/sq/libsq/core/sqlz"
"github.com/neilotoole/sq/libsq/source"
"github.com/neilotoole/sq/testh"
"github.com/neilotoole/sq/testh/sakila"
"github.com/neilotoole/sq/testh/testsrc"
)
func TestKindFromDBTypeName(t *testing.T) {
t.Parallel()
testCases := map[string]kind.Kind{
"": kind.Bytes,
"NUMERIC": kind.Decimal,
"INT": kind.Int,
"INTEGER": kind.Int,
"TINYINT": kind.Int,
"SMALLINT": kind.Int,
"MEDIUMINT": kind.Int,
"BIGINT": kind.Int,
"UNSIGNED BIG INT": kind.Int,
"INT2": kind.Int,
"INT8": kind.Int,
"CHARACTER(20)": kind.Text,
"VARCHAR(255)": kind.Text,
"VARYING CHARACTER(255)": kind.Text,
"NCHAR(55)": kind.Text,
"NATIVE CHARACTER(70)": kind.Text,
"NVARCHAR(100)": kind.Text,
"TEXT": kind.Text,
"CLOB": kind.Text,
"REAL": kind.Float,
"DOUBLE": kind.Float,
"DOUBLE PRECISION": kind.Float,
"FLOAT": kind.Float,
"DECIMAL(10,5)": kind.Decimal,
"BOOLEAN": kind.Bool,
"DATETIME": kind.Datetime,
"TIMESTAMP": kind.Datetime,
"DATE": kind.Date,
"TIME": kind.Time,
}
log := testlg.New(t)
for dbTypeName, wantKind := range testCases {
gotKind := sqlite3.KindFromDBTypeName(log, "col", dbTypeName, nil)
require.Equal(t, wantKind, gotKind, "%s should produce %s but got %s", dbTypeName)
}
}
func TestRecordMetadata(t *testing.T) {
t.Parallel()
testCases := []struct {
tbl string
rowCount int64
colNames []string
colKinds []kind.Kind
scanTypes []reflect.Type
colsMeta []*source.ColMetadata
}{
{
tbl: sakila.TblActor,
rowCount: sakila.TblActorCount,
colNames: sakila.TblActorCols(),
colKinds: []kind.Kind{kind.Int, kind.Text, kind.Text, kind.Datetime},
scanTypes: []reflect.Type{sqlz.RTypeNullInt64, sqlz.RTypeNullString, sqlz.RTypeNullString, sqlz.RTypeNullTime},
colsMeta: []*source.ColMetadata{
{Name: "actor_id", Position: 0, PrimaryKey: true, BaseType: "INTEGER", ColumnType: "INTEGER", Kind: kind.Int, Nullable: false},
{Name: "first_name", Position: 1, BaseType: "VARCHAR(45)", ColumnType: "VARCHAR(45)", Kind: kind.Text, Nullable: false},
{Name: "last_name", Position: 2, BaseType: "VARCHAR(45)", ColumnType: "VARCHAR(45)", Kind: kind.Text, Nullable: false},
{Name: "last_update", Position: 3, BaseType: "TIMESTAMP", ColumnType: "TIMESTAMP", Kind: kind.Datetime, Nullable: false, DefaultValue: "CURRENT_TIMESTAMP"},
},
},
{
tbl: sakila.TblFilmActor,
rowCount: sakila.TblFilmActorCount,
colNames: sakila.TblFilmActorCols(),
colKinds: []kind.Kind{kind.Int, kind.Int, kind.Datetime},
scanTypes: []reflect.Type{sqlz.RTypeNullInt64, sqlz.RTypeNullInt64, sqlz.RTypeNullTime},
colsMeta: []*source.ColMetadata{
{Name: "actor_id", Position: 0, PrimaryKey: true, BaseType: "INT", ColumnType: "INT", Kind: kind.Int, Nullable: false},
{Name: "film_id", Position: 1, PrimaryKey: true, BaseType: "INT", ColumnType: "INT", Kind: kind.Int, Nullable: false},
{Name: "last_update", Position: 2, BaseType: "TIMESTAMP", ColumnType: "TIMESTAMP", Kind: kind.Datetime, Nullable: false},
},
},
{
tbl: sakila.TblPayment,
rowCount: sakila.TblPaymentCount,
colNames: sakila.TblPaymentCols(),
colKinds: []kind.Kind{kind.Int, kind.Int, kind.Int, kind.Int, kind.Decimal, kind.Datetime, kind.Datetime},
scanTypes: []reflect.Type{sqlz.RTypeNullInt64, sqlz.RTypeNullInt64, sqlz.RTypeNullInt64, sqlz.RTypeNullInt64, sqlz.RTypeNullString, sqlz.RTypeNullTime, sqlz.RTypeNullTime},
colsMeta: []*source.ColMetadata{
{Name: "payment_id", Position: 0, PrimaryKey: true, BaseType: "INT", ColumnType: "INT", Kind: kind.Int, Nullable: false},
{Name: "customer_id", Position: 1, BaseType: "INT", ColumnType: "INT", Kind: kind.Int, Nullable: false},
{Name: "staff_id", Position: 2, BaseType: "SMALLINT", ColumnType: "SMALLINT", Kind: kind.Int, Nullable: false},
{Name: "rental_id", Position: 3, BaseType: "INT", ColumnType: "INT", Kind: kind.Int, Nullable: true, DefaultValue: "NULL"},
{Name: "amount", Position: 4, BaseType: "DECIMAL(5,2)", ColumnType: "DECIMAL(5,2)", Kind: kind.Decimal, Nullable: false},
{Name: "payment_date", Position: 5, BaseType: "TIMESTAMP", ColumnType: "TIMESTAMP", Kind: kind.Datetime, Nullable: false},
{Name: "last_update", Position: 6, BaseType: "TIMESTAMP", ColumnType: "TIMESTAMP", Kind: kind.Datetime, Nullable: false},
},
},
}
for _, tc := range testCases {
tc := tc
t.Run(tc.tbl, func(t *testing.T) {
t.Parallel()
th := testh.New(t)
src := th.Source(sakila.SL3)
dbase := th.Open(src)
query := fmt.Sprintf("SELECT %s FROM %s", strings.Join(tc.colNames, ", "), tc.tbl)
rows, err := dbase.DB().QueryContext(th.Context, query) // nolint:rowserrcheck
require.NoError(t, err)
t.Cleanup(func() { assert.NoError(t, rows.Close()) })
hasNext := rows.Next() // invoke rows.Next before invoking RecordMeta
colTypes, err := rows.ColumnTypes()
require.NoError(t, err)
recMeta, _, err := th.SQLDriverFor(src).RecordMeta(colTypes)
require.NoError(t, err)
require.Equal(t, len(tc.colNames), len(recMeta))
scanDests := recMeta.NewScanRow()
for hasNext {
// Scan rows to verify scan dests are ok
require.NoError(t, rows.Scan(scanDests...))
hasNext = rows.Next()
}
require.NoError(t, rows.Err())
gotScanTypes := recMeta.ScanTypes()
require.Equal(t, len(tc.scanTypes), len(gotScanTypes))
for i := range tc.scanTypes {
require.Equal(t, tc.scanTypes[i], gotScanTypes[i])
}
// Now check our table metadata
gotTblMeta, err := dbase.TableMetadata(th.Context, tc.tbl)
require.NoError(t, err)
require.Equal(t, tc.tbl, gotTblMeta.Name)
require.Equal(t, tc.rowCount, gotTblMeta.RowCount)
require.Equal(t, len(tc.colsMeta), len(gotTblMeta.Columns))
for i := range tc.colsMeta {
require.Equal(t, *tc.colsMeta[i], *gotTblMeta.Columns[i])
}
})
}
}
func TestPayments(t *testing.T) {
t.Parallel()
th := testh.New(t)
src := th.Source(sakila.SL3)
sink, err := th.QuerySQL(src, "SELECT * FROM payment")
require.NoError(t, err)
require.Equal(t, sakila.TblPaymentCount, len(sink.Recs))
}
// TestAggregateFuncsQuery performs a smoke test of executing
// a query with aggregate funcs to verify that
// column type info is being correctly determined.
func TestAggregateFuncsQuery(t *testing.T) {
t.Parallel()
const query = `SELECT COUNT(*), SUM(rental_rate), TOTAL(rental_rate),
AVG(rental_rate), MAX(rental_rate), MIN(rental_rate),
MAX(title), MAX(last_update), GROUP_CONCAT(rating,',')
FROM film`
th := testh.New(t)
src := th.Source(sakila.SL3)
sink, err := th.QuerySQL(src, query)
require.NoError(t, err)
require.Equal(t, 1, len(sink.Recs))
}
// TestScalarFuncsQuery performs a smoke test of executing
// a query with some scalar funcs to verify that
// column type info is being correctly determined.
func TestScalarFuncsQuery(t *testing.T) {
t.Parallel()
const query = `SELECT NULL, ABS(film_id), LOWER(rating), LAST_INSERT_ROWID(),
MAX(rental_rate, replacement_cost)
FROM film`
wantKinds := []kind.Kind{kind.Bytes, kind.Int, kind.Text, kind.Int, kind.Float}
th := testh.New(t)
src := th.Source(sakila.SL3)
sink, err := th.QuerySQL(src, query)
require.NoError(t, err)
require.Equal(t, sakila.TblFilmCount, len(sink.Recs))
require.Equal(t, wantKinds, sink.RecMeta.Kinds())
}
func BenchmarkDatabase_SourceMetadata(b *testing.B) {
const numTables = 1000
th, src, dbase, drvr := testh.NewWith(b, testsrc.MiscDB)
db := dbase.DB()
tblNames := createTypeTestTbls(th, src, numTables, true)
b.ResetTimer()
for n := 0; n < b.N; n++ {
srcMeta, err := dbase.SourceMetadata(th.Context)
require.NoError(b, err)
require.True(b, len(srcMeta.Tables) > len(tblNames))
}
b.StopTimer()
for _, tblName := range tblNames {
require.NoError(b, drvr.DropTable(th.Context, db, tblName, true))
}
}
func TestGetTblRowCounts(t *testing.T) {
const numTables = 10
th, src, dbase, _ := testh.NewWith(t, testsrc.MiscDB)
db := dbase.DB()
tblNames := createTypeTestTbls(th, src, numTables, true)
counts, err := sqlite3.GetTblRowCounts(th.Context, th.Log, db, tblNames)
require.NoError(t, err)
require.Equal(t, len(tblNames), len(counts))
}
func BenchmarkGetTblRowCounts(b *testing.B) {
const numTables = 1300
th, src, dbase, drvr := testh.NewWith(b, testsrc.MiscDB)
db := dbase.DB()
tblNames := createTypeTestTbls(th, src, numTables, true)
testCases := []struct {
name string
fn func(ctx context.Context, log lg.Log, db sqlz.DB, tblNames []string) ([]int64, error)
}{
{name: "benchGetTblRowCountsBaseline", fn: benchGetTblRowCountsBaseline},
{name: "getTblRowCounts", fn: sqlite3.GetTblRowCounts},
}
for _, tc := range testCases {
tc := tc
b.Run(tc.name, func(b *testing.B) {
log := testlg.New(b)
for n := 0; n < b.N; n++ {
counts, err := tc.fn(th.Context, log, db, tblNames)
require.NoError(b, err)
require.Len(b, counts, len(tblNames))
}
})
}
for _, tblName := range tblNames {
require.NoError(b, drvr.DropTable(th.Context, db, tblName, true))
}
}
// benchGetTblRowCountsBaseline is a baseline impl of getTblRowCounts
// for benchmark comparision.
func benchGetTblRowCountsBaseline(ctx context.Context, log lg.Log, db sqlz.DB, tblNames []string) ([]int64, error) {
tblCounts := make([]int64, len(tblNames))
for i := range tblNames {
row := db.QueryRowContext(ctx, fmt.Sprintf("SELECT COUNT(*) FROM %q", tblNames[i]))
err := row.Scan(&tblCounts[i])
if err != nil {
return nil, errz.Err(err)
}
}
return tblCounts, nil
}