package source_test import ( "context" "io" "os" "path/filepath" "testing" "github.com/neilotoole/sq/libsq/core/lg" "github.com/neilotoole/slogt" "github.com/neilotoole/sq/testh/tutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" "github.com/neilotoole/sq/drivers/csv" "github.com/neilotoole/sq/drivers/mysql" "github.com/neilotoole/sq/drivers/postgres" "github.com/neilotoole/sq/drivers/sqlite3" "github.com/neilotoole/sq/drivers/sqlserver" "github.com/neilotoole/sq/drivers/xlsx" "github.com/neilotoole/sq/libsq/core/stringz" "github.com/neilotoole/sq/libsq/source" "github.com/neilotoole/sq/testh" "github.com/neilotoole/sq/testh/proj" "github.com/neilotoole/sq/testh/sakila" "github.com/neilotoole/sq/testh/testsrc" ) func TestFiles_Type(t *testing.T) { testCases := []struct { loc string wantType source.DriverType wantErr bool }{ {loc: proj.Expand("sqlite3://${SQ_ROOT}/drivers/sqlite3/testdata/sakila.db"), wantType: sqlite3.Type}, {loc: proj.Abs(sakila.PathSL3), wantType: sqlite3.Type}, {loc: proj.Abs("drivers/sqlite3/testdata/sakila_db"), wantType: sqlite3.Type}, {loc: "sqlserver://sakila:p_ssW0rd@localhost?database=sakila", wantType: sqlserver.Type}, {loc: "postgres://sakila:p_ssW0rd@localhost/sakila?sslmode=disable", wantType: postgres.Type}, {loc: "mysql://sakila:p_ssW0rd@localhost/sakila", wantType: mysql.Type}, {loc: proj.Abs(testsrc.PathXLSXTestHeader), wantType: xlsx.Type}, {loc: proj.Abs("drivers/xlsx/testdata/test_header_xlsx"), wantType: xlsx.Type}, {loc: sakila.URLSubsetXLSX, wantType: xlsx.Type}, {loc: proj.Abs(sakila.PathCSVActor), wantType: csv.TypeCSV}, {loc: proj.Abs("drivers/csv/testdata/person_csv"), wantType: csv.TypeCSV}, {loc: sakila.URLActorCSV, wantType: csv.TypeCSV}, {loc: proj.Abs("drivers/csv/testdata/person_tsv"), wantType: csv.TypeTSV}, {loc: proj.Abs(sakila.PathTSVActor), wantType: csv.TypeTSV}, } for _, tc := range testCases { tc := tc t.Run(tc.loc, func(t *testing.T) { ctx := lg.NewContext(context.Background(), slogt.New(t)) fs, err := source.NewFiles(ctx) require.NoError(t, err) fs.AddDriverDetectors(testh.DriverDetectors()...) gotType, gotErr := fs.DriverType(context.Background(), tc.loc) if tc.wantErr { require.Error(t, gotErr) return } require.NoError(t, gotErr) require.Equal(t, tc.wantType, gotType) }) } } func TestFiles_DetectType(t *testing.T) { testCases := []struct { loc string wantType source.DriverType wantOK bool wantErr bool }{ {loc: proj.Abs(sakila.PathSL3), wantType: sqlite3.Type, wantOK: true}, {loc: proj.Abs("drivers/sqlite3/testdata/sakila_db"), wantType: sqlite3.Type, wantOK: true}, {loc: proj.Abs(testsrc.PathXLSXTestHeader), wantType: xlsx.Type, wantOK: true}, {loc: proj.Abs("drivers/xlsx/testdata/test_header_xlsx"), wantType: xlsx.Type, wantOK: true}, {loc: proj.Abs("drivers/xlsx/testdata/test_noheader.xlsx"), wantType: xlsx.Type, wantOK: true}, {loc: proj.Abs("drivers/csv/testdata/person.csv"), wantType: csv.TypeCSV, wantOK: true}, {loc: proj.Abs("drivers/csv/testdata/person_noheader.csv"), wantType: csv.TypeCSV, wantOK: true}, {loc: proj.Abs("drivers/csv/testdata/person_csv"), wantType: csv.TypeCSV, wantOK: true}, {loc: proj.Abs("drivers/csv/testdata/person.tsv"), wantType: csv.TypeTSV, wantOK: true}, {loc: proj.Abs("drivers/csv/testdata/person_noheader.tsv"), wantType: csv.TypeTSV, wantOK: true}, {loc: proj.Abs("drivers/csv/testdata/person_tsv"), wantType: csv.TypeTSV, wantOK: true}, {loc: proj.Abs("README.md"), wantType: source.TypeNone, wantOK: false}, } for _, tc := range testCases { tc := tc t.Run(filepath.Base(tc.loc), func(t *testing.T) { ctx := lg.NewContext(context.Background(), slogt.New(t)) fs, err := source.NewFiles(ctx) require.NoError(t, err) fs.AddDriverDetectors(testh.DriverDetectors()...) typ, ok, err := source.FilesDetectTypeFn(fs, ctx, tc.loc) if tc.wantErr { require.Error(t, err) } else { require.NoError(t, err) } require.Equal(t, tc.wantOK, ok) require.Equal(t, tc.wantType, typ) }) } } func TestDetectMagicNumber(t *testing.T) { testCases := []struct { loc string wantType source.DriverType wantScore float32 wantErr bool }{ {loc: proj.Abs(sakila.PathSL3), wantType: sqlite3.Type, wantScore: 1.0}, {loc: proj.Abs("drivers/sqlite3/testdata/sakila_db"), wantType: sqlite3.Type, wantScore: 1.0}, } for _, tc := range testCases { tc := tc t.Run(filepath.Base(tc.loc), func(t *testing.T) { rFn := func() (io.ReadCloser, error) { return os.Open(tc.loc) } ctx := lg.NewContext(context.Background(), slogt.New(t)) typ, score, err := source.DetectMagicNumber(ctx, rFn) if tc.wantErr { require.Error(t, err) return } require.NoError(t, err) require.Equal(t, tc.wantType, typ) require.Equal(t, tc.wantScore, score) }) } } func TestFiles_NewReader(t *testing.T) { ctx := lg.NewContext(context.Background(), slogt.New(t)) fpath := sakila.PathCSVActor wantBytes := proj.ReadFile(fpath) src := &source.Source{ Handle: "@test_" + stringz.Uniq8(), Type: csv.TypeCSV, Location: proj.Abs(fpath), } fs, err := source.NewFiles(ctx) require.NoError(t, err) g := &errgroup.Group{} for i := 0; i < 1000; i++ { g.Go(func() error { r, gErr := fs.Open(src) require.NoError(t, gErr) b, gErr := io.ReadAll(r) require.NoError(t, gErr) require.Equal(t, wantBytes, b) return nil }) } err = g.Wait() require.NoError(t, err) } func TestFiles_Stdin(t *testing.T) { testCases := []struct { fpath string wantType source.DriverType wantErr bool }{ {fpath: proj.Abs(sakila.PathCSVActor), wantType: csv.TypeCSV}, {fpath: proj.Abs(sakila.PathTSVActor), wantType: csv.TypeTSV}, {fpath: proj.Abs(sakila.PathXLSX), wantType: xlsx.Type}, } for _, tc := range testCases { tc := tc t.Run(tutil.Name(tc.fpath), func(t *testing.T) { th := testh.New(t) fs := th.Files() f, err := os.Open(tc.fpath) require.NoError(t, err) err = fs.AddStdin(f) // f is closed by AddStdin require.NoError(t, err) typ, err := fs.TypeStdin(th.Context) if tc.wantErr { require.Error(t, err) return } require.Equal(t, tc.wantType, typ) }) } } func TestFiles_Stdin_ErrorWrongOrder(t *testing.T) { th := testh.New(t) fs := th.Files() typ, err := fs.TypeStdin(th.Context) require.Error(t, err, "should error because AddStdin not yet invoked") require.Equal(t, source.TypeNone, typ) f, err := os.Open(proj.Abs(sakila.PathCSVActor)) require.NoError(t, err) require.NoError(t, fs.AddStdin(f)) // AddStdin closes f typ, err = fs.TypeStdin(th.Context) require.NoError(t, err) require.Equal(t, csv.TypeCSV, typ) } func TestFiles_Size(t *testing.T) { f, err := os.Open(proj.Abs(sakila.PathCSVActor)) require.NoError(t, err) t.Cleanup(func() { assert.NoError(t, f.Close()) }) fi, err := os.Stat(f.Name()) require.NoError(t, err) wantSize := fi.Size() th := testh.New(t) fs := th.Files() gotSize, err := fs.Size(&source.Source{ Handle: stringz.UniqSuffix("@h"), Location: f.Name(), }) require.NoError(t, err) require.Equal(t, wantSize, gotSize) f2, err := os.Open(proj.Abs(sakila.PathCSVActor)) require.NoError(t, err) // Verify that this works with @stdin as well require.NoError(t, fs.AddStdin(f2)) gotSize2, err := fs.Size(&source.Source{ Handle: "@stdin", Location: "@stdin", }) require.NoError(t, err) require.Equal(t, wantSize, gotSize2) }