mirror of
https://github.com/neilotoole/sq.git
synced 2024-12-11 09:25:50 +03:00
a92b9abf34
* implementation work for json importers * json driver checkpoint * working on json.ParseObjectsInArray * json.ParseObjectsInArray seems to be working * checkpoint while tidying up ParseObjectsInArray * more tidy checkpoint * more tidy checkpoint 2 * tidying up ParseObjectsInArray * tidy up * code/docs cleanup * more cleanup of json driver * more cleanup of json driver * flat json import seemingly working * improvements to json driver * json writer now prints empty [] for postgres empty tables
92 lines
2.0 KiB
Go
92 lines
2.0 KiB
Go
package json
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/neilotoole/sq/libsq/core/kind"
|
|
"github.com/neilotoole/sq/libsq/driver"
|
|
"github.com/neilotoole/sq/libsq/source"
|
|
"github.com/neilotoole/sq/testh/sakila"
|
|
)
|
|
|
|
// export for testing
|
|
var (
|
|
ImportJSON = importJSON
|
|
ImportJSONA = importJSONA
|
|
ImportJSONL = importJSONL
|
|
ColumnOrderFlat = columnOrderFlat
|
|
)
|
|
|
|
// NewImportJob is a constructor for the unexported importJob type.
|
|
// If sampleSize <= 0, a default value is used.
|
|
func NewImportJob(fromSrc *source.Source, openFn source.FileOpenFunc, destDB driver.Database, sampleSize int, flatten bool) importJob {
|
|
if sampleSize <= 0 {
|
|
sampleSize = driver.Tuning.SampleSize
|
|
}
|
|
|
|
return importJob{
|
|
fromSrc: fromSrc,
|
|
openFn: openFn,
|
|
destDB: destDB,
|
|
sampleSize: sampleSize,
|
|
flatten: flatten,
|
|
}
|
|
}
|
|
|
|
func TestDetectColKindsJSONA(t *testing.T) {
|
|
testCases := []struct {
|
|
tbl string
|
|
wantKinds []kind.Kind
|
|
}{
|
|
{tbl: sakila.TblActor, wantKinds: sakila.TblActorColKinds()},
|
|
{tbl: sakila.TblFilmActor, wantKinds: sakila.TblFilmActorColKinds()},
|
|
{tbl: sakila.TblPayment, wantKinds: sakila.TblPaymentColKinds()},
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
tc := tc
|
|
|
|
t.Run(tc.tbl, func(t *testing.T) {
|
|
f, err := os.Open(fmt.Sprintf("testdata/%s.jsona", tc.tbl))
|
|
require.NoError(t, err)
|
|
t.Cleanup(func() { require.NoError(t, f.Close()) })
|
|
|
|
kinds, _, err := detectColKindsJSONA(context.Background(), f)
|
|
require.NoError(t, err)
|
|
require.Equal(t, tc.wantKinds, kinds)
|
|
})
|
|
}
|
|
}
|
|
|
|
// ScanObjectsInArray is a convenience function
|
|
// for objectsInArrayScanner.
|
|
func ScanObjectsInArray(r io.Reader) (objs []map[string]interface{}, chunks [][]byte, err error) {
|
|
sc := newObjectInArrayScanner(r)
|
|
|
|
for {
|
|
var obj map[string]interface{}
|
|
var chunk []byte
|
|
|
|
obj, chunk, err = sc.next()
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
if obj == nil {
|
|
// No more objects to be scanned
|
|
break
|
|
}
|
|
|
|
objs = append(objs, obj)
|
|
chunks = append(chunks, chunk)
|
|
}
|
|
|
|
return objs, chunks, nil
|
|
}
|