diff --git a/CHANGELOG.md b/CHANGELOG.md index d33e20accad..1af9e7980f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,29 @@ Support for this is now added through the `add_computed_field` API. Read more about the session argument for computed fields in the [docs](https://hasura.io/docs/1.0/graphql/manual/api-reference/schema-metadata-api/computed-field.html). +### Manage seed migrations as SQL files +A new `seeds` command is introduced in CLI, this will allow managing seed migrations as SQL files + +#### Creating seed +``` +# create a new seed file and use editor to add SQL content +hasura seed create new_table_seed + +# create a new seed by exporting data from tables already present in the database +hasura seed create table1_seed --from-table table1 + +# create from data in multiple tables: +hasura seed create tables_seed --from-table table1 --from-table table2 +``` +#### Applying seed +``` +# apply all seeds on the database: +hasura seed apply + +# apply only a particular seed +hasura seed apply --file 1234_add_some_seed_data.sql +``` + ### Bug fixes and improvements (Add entries here in the order of: server, console, cli, docs, others) diff --git a/cli/cli.go b/cli/cli.go index e52aa5594e1..6d582bbfc36 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -52,6 +52,10 @@ const ( // Name of the cli extension plugin CLIExtPluginName = "cli-ext" + + DefaultMigrationsDirectory = "migrations" + DefaultMetadataDirectory = "metadata" + DefaultSeedsDirectory = "seeds" ) const ( @@ -290,6 +294,8 @@ type Config struct { MetadataDirectory string `yaml:"metadata_directory,omitempty"` // MigrationsDirectory defines the directory where the migration files were stored. MigrationsDirectory string `yaml:"migrations_directory,omitempty"` + // SeedsDirectory defines the directory where seed files will be stored + SeedsDirectory string `yaml:"seeds_directory,omitempty"` // ActionConfig defines the config required to create or generate codegen for an action. ActionConfig *types.ActionExecutionConfig `yaml:"actions,omitempty"` } @@ -322,6 +328,8 @@ type ExecutionContext struct { MigrationDir string // MetadataDir is the name of directory where metadata files are stored. MetadataDir string + // Seed directory -- directory in which seed files are to be stored + SeedsDirectory string // ConfigFile is the file where endpoint etc. are stored. ConfigFile string // HGE Headers, are the custom headers which can be passed to HGE API @@ -548,6 +556,14 @@ func (ec *ExecutionContext) Validate() error { } } + ec.SeedsDirectory = filepath.Join(ec.ExecutionDirectory, ec.Config.SeedsDirectory) + if _, err := os.Stat(ec.SeedsDirectory); os.IsNotExist(err) { + err = os.MkdirAll(ec.SeedsDirectory, os.ModePerm) + if err != nil { + return errors.Wrap(err, "cannot create seeds directory") + } + } + if ec.Config.Version == V2 && ec.Config.MetadataDirectory != "" { // set name of metadata directory ec.MetadataDir = filepath.Join(ec.ExecutionDirectory, ec.Config.MetadataDirectory) @@ -638,7 +654,8 @@ func (ec *ExecutionContext) readConfig() error { v.SetDefault("api_paths.pg_dump", "v1alpha1/pg_dump") v.SetDefault("api_paths.version", "v1/version") v.SetDefault("metadata_directory", "") - v.SetDefault("migrations_directory", "migrations") + v.SetDefault("migrations_directory", DefaultMigrationsDirectory) + v.SetDefault("seeds_directory", DefaultSeedsDirectory) v.SetDefault("actions.kind", "synchronous") v.SetDefault("actions.handler_webhook_baseurl", "http://localhost:3000") v.SetDefault("actions.codegen.framework", "") @@ -671,6 +688,7 @@ func (ec *ExecutionContext) readConfig() error { }, MetadataDirectory: v.GetString("metadata_directory"), MigrationsDirectory: v.GetString("migrations_directory"), + SeedsDirectory: v.GetString("seeds_directory"), ActionConfig: &types.ActionExecutionConfig{ Kind: v.GetString("actions.kind"), HandlerWebhookBaseURL: v.GetString("actions.handler_webhook_baseurl"), diff --git a/cli/commands/help.go b/cli/commands/help.go index 74f8641450b..effa1e7c9c5 100644 --- a/cli/commands/help.go +++ b/cli/commands/help.go @@ -49,6 +49,7 @@ func (o *helpOptions) run() { NewMetadataCmd(o.EC), NewConsoleCmd(o.EC), NewActionsCmd(o.EC), + NewSeedCmd(o.EC), }, }, { diff --git a/cli/commands/init.go b/cli/commands/init.go index e6ae76ca927..0b7e0848733 100644 --- a/cli/commands/init.go +++ b/cli/commands/init.go @@ -230,7 +230,7 @@ func (o *InitOptions) createFiles() error { } // create migrations directory - o.EC.MigrationDir = filepath.Join(o.EC.ExecutionDirectory, "migrations") + o.EC.MigrationDir = filepath.Join(o.EC.ExecutionDirectory, cli.DefaultMigrationsDirectory) err = os.MkdirAll(o.EC.MigrationDir, os.ModePerm) if err != nil { return errors.Wrap(err, "cannot write migration directory") @@ -238,7 +238,7 @@ func (o *InitOptions) createFiles() error { if config.Version == cli.V2 { // create metadata directory - o.EC.MetadataDir = filepath.Join(o.EC.ExecutionDirectory, "metadata") + o.EC.MetadataDir = filepath.Join(o.EC.ExecutionDirectory, cli.DefaultMetadataDirectory) err = os.MkdirAll(o.EC.MetadataDir, os.ModePerm) if err != nil { return errors.Wrap(err, "cannot write migration directory") @@ -261,6 +261,13 @@ func (o *InitOptions) createFiles() error { } } } + + // create seeds directory + o.EC.SeedsDirectory = filepath.Join(o.EC.ExecutionDirectory, cli.DefaultSeedsDirectory) + err = os.MkdirAll(o.EC.SeedsDirectory, os.ModePerm) + if err != nil { + return errors.Wrap(err, "cannot write seeds directory") + } return nil } diff --git a/cli/commands/root.go b/cli/commands/root.go index 85f6dc4d2a9..f93e8570967 100644 --- a/cli/commands/root.go +++ b/cli/commands/root.go @@ -63,6 +63,7 @@ func init() { NewConsoleCmd(ec), NewMetadataCmd(ec), NewMigrateCmd(ec), + NewSeedCmd(ec), NewActionsCmd(ec), NewPluginsCmd(ec), NewVersionCmd(ec), diff --git a/cli/commands/seed.go b/cli/commands/seed.go new file mode 100644 index 00000000000..201726e13fd --- /dev/null +++ b/cli/commands/seed.go @@ -0,0 +1,41 @@ +package commands + +import ( + "github.com/hasura/graphql-engine/cli" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// NewSeedCmd will return the seed command +func NewSeedCmd(ec *cli.ExecutionContext) *cobra.Command { + v := viper.New() + ec.Viper = v + seedCmd := &cobra.Command{ + Use: "seeds", + Aliases: []string{"sd"}, + Short: "Manage seed data", + SilenceUsage: true, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + err := ec.Prepare() + if err != nil { + return err + } + return ec.Validate() + }, + } + + seedCmd.AddCommand( + newSeedCreateCmd(ec), + newSeedApplyCmd(ec), + ) + seedCmd.PersistentFlags().String("endpoint", "", "http(s) endpoint for Hasura GraphQL Engine") + seedCmd.PersistentFlags().String("admin-secret", "", "admin secret for Hasura GraphQL Engine") + seedCmd.PersistentFlags().String("access-key", "", "access key for Hasura GraphQL Engine") + seedCmd.PersistentFlags().MarkDeprecated("access-key", "use --admin-secret instead") + + v.BindPFlag("endpoint", seedCmd.PersistentFlags().Lookup("endpoint")) + v.BindPFlag("admin_secret", seedCmd.PersistentFlags().Lookup("admin-secret")) + v.BindPFlag("access_key", seedCmd.PersistentFlags().Lookup("access-key")) + + return seedCmd +} diff --git a/cli/commands/seed_apply.go b/cli/commands/seed_apply.go new file mode 100644 index 00000000000..8d8f09833f4 --- /dev/null +++ b/cli/commands/seed_apply.go @@ -0,0 +1,57 @@ +package commands + +import ( + "github.com/hasura/graphql-engine/cli/migrate" + "github.com/spf13/afero" + "github.com/spf13/cobra" + + "github.com/hasura/graphql-engine/cli" + "github.com/hasura/graphql-engine/cli/seed" +) + +type SeedApplyOptions struct { + EC *cli.ExecutionContext + + // seed file to apply + FileNames []string +} + +func newSeedApplyCmd(ec *cli.ExecutionContext) *cobra.Command { + opts := SeedApplyOptions{ + EC: ec, + } + cmd := &cobra.Command{ + Use: "apply", + Short: "Apply seed data", + Example: ` # Apply all seeds on the database: + hasura seed apply + + # Apply only a particular file: + hasura seed apply --file seeds/1234_add_some_seed_data.sql`, + SilenceUsage: false, + PreRunE: func(cmd *cobra.Command, args []string) error { + return ec.Validate() + }, + RunE: func(cmd *cobra.Command, args []string) error { + opts.EC.Spin("Applying seeds...") + err := opts.Run() + opts.EC.Spinner.Stop() + if err != nil { + return err + } + opts.EC.Logger.Info("Seeds planted") + return nil + }, + } + cmd.Flags().StringArrayVarP(&opts.FileNames, "file", "f", []string{}, "seed file to apply") + return cmd +} + +func (o *SeedApplyOptions) Run() error { + migrateDriver, err := migrate.NewMigrate(o.EC, true) + if err != nil { + return err + } + fs := afero.NewOsFs() + return seed.ApplySeedsToDatabase(o.EC, fs, migrateDriver, o.FileNames) +} diff --git a/cli/commands/seed_create.go b/cli/commands/seed_create.go new file mode 100644 index 00000000000..353fd778b55 --- /dev/null +++ b/cli/commands/seed_create.go @@ -0,0 +1,104 @@ +package commands + +import ( + "bytes" + + "github.com/hasura/graphql-engine/cli/migrate" + "github.com/pkg/errors" + "github.com/spf13/afero" + "github.com/spf13/cobra" + + "github.com/hasura/graphql-engine/cli" + "github.com/hasura/graphql-engine/cli/metadata/actions/editor" + "github.com/hasura/graphql-engine/cli/seed" +) + +type SeedNewOptions struct { + EC *cli.ExecutionContext + + // filename for the new seed file + SeedName string + // table name if seed file has to be created from a database table + FromTableNames []string + + // seed file that was created + FilePath string +} + +func newSeedCreateCmd(ec *cli.ExecutionContext) *cobra.Command { + opts := SeedNewOptions{ + EC: ec, + } + cmd := &cobra.Command{ + Use: "create seed_name", + Short: "create a new seed file", + Example: ` # Create a new seed file and use editor to add SQL: + hasura seed create new_table_seed + + # Create a new seed by exporting data from tables already present in the database: + hasura seed create table1_seed --from-table table1 + + # Export data from multiple tables: + hasura seed create tables_seed --from-table table1 --from-table table2`, + Args: cobra.ExactArgs(1), + SilenceUsage: false, + PreRunE: func(cmd *cobra.Command, args []string) error { + return ec.Validate() + }, + RunE: func(cmd *cobra.Command, args []string) error { + opts.SeedName = args[0] + err := opts.Run() + if err != nil { + return err + } + ec.Logger.WithField("file", opts.FilePath).Info("created seed file successfully") + return nil + }, + } + + cmd.Flags().StringArrayVar(&opts.FromTableNames, "from-table", []string{}, "name of table from which seed file has to be initialized") + + return cmd +} + +func (o *SeedNewOptions) Run() error { + createSeedOpts := seed.CreateSeedOpts{ + UserProvidedSeedName: o.SeedName, + DirectoryPath: o.EC.SeedsDirectory, + } + + // If we are initializing from a database table + // create a hasura client and add table name opts + if createSeedOpts.Data == nil { + if len(o.FromTableNames) > 0 { + migrateDriver, err := migrate.NewMigrate(ec, true) + if err != nil { + return errors.Wrap(err, "cannot initialize migrate driver") + } + // Send the query + body, err := migrateDriver.ExportDataDump(o.FromTableNames) + if err != nil { + return errors.Wrap(err, "exporting seed data") + } + + createSeedOpts.Data = bytes.NewReader(body) + } else { + const defaultText = "" + data, err := editor.CaptureInputFromEditor(editor.GetPreferredEditorFromEnvironment, defaultText, "*.sql") + if err != nil { + return errors.Wrap(err, "cannot find default editor from env") + } + createSeedOpts.Data = bytes.NewReader(data) + } + } + + fs := afero.NewOsFs() + filepath, err := seed.CreateSeedFile(fs, createSeedOpts) + if err != nil || filepath == nil { + return errors.Wrap(err, "failed to create seed file") + } + + o.FilePath = *filepath + + return nil +} diff --git a/cli/go.sum b/cli/go.sum index 253ded2062b..95096812b13 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -260,12 +260,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= diff --git a/cli/integration_test/cli_test.go b/cli/integration_test/cli_test.go index c2aadcc98a4..fe9c6c02e43 100644 --- a/cli/integration_test/cli_test.go +++ b/cli/integration_test/cli_test.go @@ -132,6 +132,16 @@ func TestCommands(t *testing.T) { t.Run("metadata commands", func(t *testing.T) { v2.TestMetadataCmd(t, ec) }) + + skip(t) + t.Run("seed create command", func(t *testing.T) { + v2.TestSeedsCreateCmd(t, ec) + }) + + skip(t) + t.Run("seed apply commands", func(t *testing.T) { + v2.TestSeedsApplyCmd(t, ec) + }) }) } diff --git a/cli/integration_test/v1/seeds/1591867862409_test.sql b/cli/integration_test/v1/seeds/1591867862409_test.sql new file mode 100644 index 00000000000..3160b7115fa --- /dev/null +++ b/cli/integration_test/v1/seeds/1591867862409_test.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS account( + user_id serial PRIMARY KEY, + username VARCHAR (50) UNIQUE NOT NULL, + password VARCHAR (50) NOT NULL, + email VARCHAR (355) UNIQUE NOT NULL, + created_on TIMESTAMP NOT NULL, + last_login TIMESTAMP +); diff --git a/cli/integration_test/v1/seeds/1591867862419_test2.sql b/cli/integration_test/v1/seeds/1591867862419_test2.sql new file mode 100644 index 00000000000..24ca6422db8 --- /dev/null +++ b/cli/integration_test/v1/seeds/1591867862419_test2.sql @@ -0,0 +1,8 @@ +CREATE TABLE account2( + user_id serial PRIMARY KEY, + username VARCHAR (50) UNIQUE NOT NULL, + password VARCHAR (50) NOT NULL, + email VARCHAR (355) UNIQUE NOT NULL, + created_on TIMESTAMP NOT NULL, + last_login TIMESTAMP +); diff --git a/cli/integration_test/v1/seeds/seeds.go b/cli/integration_test/v1/seeds/seeds.go new file mode 100644 index 00000000000..9a17563f6cd --- /dev/null +++ b/cli/integration_test/v1/seeds/seeds.go @@ -0,0 +1,157 @@ +package v2 + +import ( + "bytes" + "io" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/hasura/graphql-engine/cli/commands" + + "github.com/spf13/afero" + + "github.com/hasura/graphql-engine/cli/seed" + + "github.com/hasura/graphql-engine/cli" + "github.com/hasura/graphql-engine/cli/util" + "github.com/stretchr/testify/assert" +) + +type seedCreateInterface interface { + Run() error +} + +func TestSeedsCreateCmd(t *testing.T, ec *cli.ExecutionContext) { + // copy migrations to ec.Execution.Directory/migrations + os.RemoveAll(ec.SeedsDirectory) + currDir, _ := os.Getwd() + err := util.CopyDir(filepath.Join(currDir, "v2/seeds"), ec.SeedsDirectory) + if err != nil { + t.Fatalf("unable to copy migrations directory %v", err) + } + type args struct { + fs afero.Fs + opts seed.CreateSeedOpts + } + tt := []struct { + name string + args args + wantErr bool + wantFilepath *string + }{ + { + "can create a seed a file", + args{ + fs: afero.NewMemMapFs(), + opts: seed.CreateSeedOpts{ + DirectoryPath: "seeds/", + Data: strings.NewReader("INSERT INTO account1 (username, password, email) values ('scriptonist', 'no you cant guess it', 'hello@drogon.com');"), + UserProvidedSeedName: "can_we_create_seed_files", + }, + }, + false, + func() *string { s := "test/test"; return &s }(), + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + var inData bytes.Buffer + tc.args.opts.Data = io.TeeReader(tc.args.opts.Data, &inData) + gotFilename, err := seed.CreateSeedFile(tc.args.fs, tc.args.opts) + if (err != nil) && !tc.wantErr { + t.Errorf("CreateSeedFile() error = %v, wantErr %v", err, tc.wantErr) + return + } + + if gotFilename == nil { + return + } + // Do a regex match for filename returned + // check if it is in required format + var re = regexp.MustCompile(`^([a-z]+\/)([0-9]+)\_(.+)(\.sql)$`) + regexGroups := re.FindStringSubmatch(*gotFilename) + + // Since filename has to be in form + // dirname/21212_filename.sql + // regexGroups should have 5 elements + // element 0: whole string + // element 1: dirname + // element 2: timestamp + // element 3: filename + // element 4: extension + if len(regexGroups) != 5 { + t.Fatalf("CreateSeedFile() = %v, but want filepath of form"+` [a-z]+\/[0-9]+\_[a-zA-Z]+\.sql`, *gotFilename) + } + gotDirectoryPath := regexGroups[1] + gotUserProvidedFilename := regexGroups[3] + gotFileExtension := regexGroups[4] + + assert.Equal(t, gotDirectoryPath, tc.args.opts.DirectoryPath) + assert.Equal(t, gotUserProvidedFilename, tc.args.opts.UserProvidedSeedName) + assert.Equal(t, gotFileExtension, ".sql") + + // test if a filewith the filename was created + if s, err := tc.args.fs.Stat(*gotFilename); err != nil { + if s.IsDir() { + t.Fatalf("expected to get a file with name %v", *gotFilename) + } + } + + // check if the contents match + gotBytes, err := afero.ReadFile(tc.args.fs, *gotFilename) + assert.NoError(t, err) + assert.Equal(t, string(gotBytes), string(inData.Bytes())) + }) + } +} +func TestSeedsApplyCmd(t *testing.T, ec *cli.ExecutionContext) { + // copy migrations to ec.Execution.Directory/migrations + os.RemoveAll(ec.SeedsDirectory) + currDir, _ := os.Getwd() + err := util.CopyDir(filepath.Join(currDir, "v2/seeds"), ec.SeedsDirectory) + if err != nil { + t.Fatalf("unable to copy migrations directory %v", err) + } + tt := []struct { + name string + opts seedCreateInterface + wantErr bool + }{ + { + "can apply all seeds", + &commands.SeedApplyOptions{ + EC: ec, + }, + false, + }, + { + "can apply single file", + &commands.SeedApplyOptions{ + EC: ec, + FileNames: []string{"1591867862409_test.sql"}, + }, + false, + }, + { + "throws error when applying no idempotent operations", + &commands.SeedApplyOptions{ + EC: ec, + FileNames: []string{"1591867862419_test2.sql"}, + }, + true, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + err := tc.opts.Run() + if (err != nil) && (tc.wantErr == false) { + t.Fatalf("%s: expected no error got %v", tc.name, err) + } + }) + } +} diff --git a/cli/integration_test/v2/seeds.go b/cli/integration_test/v2/seeds.go new file mode 100644 index 00000000000..9a17563f6cd --- /dev/null +++ b/cli/integration_test/v2/seeds.go @@ -0,0 +1,157 @@ +package v2 + +import ( + "bytes" + "io" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/hasura/graphql-engine/cli/commands" + + "github.com/spf13/afero" + + "github.com/hasura/graphql-engine/cli/seed" + + "github.com/hasura/graphql-engine/cli" + "github.com/hasura/graphql-engine/cli/util" + "github.com/stretchr/testify/assert" +) + +type seedCreateInterface interface { + Run() error +} + +func TestSeedsCreateCmd(t *testing.T, ec *cli.ExecutionContext) { + // copy migrations to ec.Execution.Directory/migrations + os.RemoveAll(ec.SeedsDirectory) + currDir, _ := os.Getwd() + err := util.CopyDir(filepath.Join(currDir, "v2/seeds"), ec.SeedsDirectory) + if err != nil { + t.Fatalf("unable to copy migrations directory %v", err) + } + type args struct { + fs afero.Fs + opts seed.CreateSeedOpts + } + tt := []struct { + name string + args args + wantErr bool + wantFilepath *string + }{ + { + "can create a seed a file", + args{ + fs: afero.NewMemMapFs(), + opts: seed.CreateSeedOpts{ + DirectoryPath: "seeds/", + Data: strings.NewReader("INSERT INTO account1 (username, password, email) values ('scriptonist', 'no you cant guess it', 'hello@drogon.com');"), + UserProvidedSeedName: "can_we_create_seed_files", + }, + }, + false, + func() *string { s := "test/test"; return &s }(), + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + var inData bytes.Buffer + tc.args.opts.Data = io.TeeReader(tc.args.opts.Data, &inData) + gotFilename, err := seed.CreateSeedFile(tc.args.fs, tc.args.opts) + if (err != nil) && !tc.wantErr { + t.Errorf("CreateSeedFile() error = %v, wantErr %v", err, tc.wantErr) + return + } + + if gotFilename == nil { + return + } + // Do a regex match for filename returned + // check if it is in required format + var re = regexp.MustCompile(`^([a-z]+\/)([0-9]+)\_(.+)(\.sql)$`) + regexGroups := re.FindStringSubmatch(*gotFilename) + + // Since filename has to be in form + // dirname/21212_filename.sql + // regexGroups should have 5 elements + // element 0: whole string + // element 1: dirname + // element 2: timestamp + // element 3: filename + // element 4: extension + if len(regexGroups) != 5 { + t.Fatalf("CreateSeedFile() = %v, but want filepath of form"+` [a-z]+\/[0-9]+\_[a-zA-Z]+\.sql`, *gotFilename) + } + gotDirectoryPath := regexGroups[1] + gotUserProvidedFilename := regexGroups[3] + gotFileExtension := regexGroups[4] + + assert.Equal(t, gotDirectoryPath, tc.args.opts.DirectoryPath) + assert.Equal(t, gotUserProvidedFilename, tc.args.opts.UserProvidedSeedName) + assert.Equal(t, gotFileExtension, ".sql") + + // test if a filewith the filename was created + if s, err := tc.args.fs.Stat(*gotFilename); err != nil { + if s.IsDir() { + t.Fatalf("expected to get a file with name %v", *gotFilename) + } + } + + // check if the contents match + gotBytes, err := afero.ReadFile(tc.args.fs, *gotFilename) + assert.NoError(t, err) + assert.Equal(t, string(gotBytes), string(inData.Bytes())) + }) + } +} +func TestSeedsApplyCmd(t *testing.T, ec *cli.ExecutionContext) { + // copy migrations to ec.Execution.Directory/migrations + os.RemoveAll(ec.SeedsDirectory) + currDir, _ := os.Getwd() + err := util.CopyDir(filepath.Join(currDir, "v2/seeds"), ec.SeedsDirectory) + if err != nil { + t.Fatalf("unable to copy migrations directory %v", err) + } + tt := []struct { + name string + opts seedCreateInterface + wantErr bool + }{ + { + "can apply all seeds", + &commands.SeedApplyOptions{ + EC: ec, + }, + false, + }, + { + "can apply single file", + &commands.SeedApplyOptions{ + EC: ec, + FileNames: []string{"1591867862409_test.sql"}, + }, + false, + }, + { + "throws error when applying no idempotent operations", + &commands.SeedApplyOptions{ + EC: ec, + FileNames: []string{"1591867862419_test2.sql"}, + }, + true, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + err := tc.opts.Run() + if (err != nil) && (tc.wantErr == false) { + t.Fatalf("%s: expected no error got %v", tc.name, err) + } + }) + } +} diff --git a/cli/integration_test/v2/seeds/1591867862409_test.sql b/cli/integration_test/v2/seeds/1591867862409_test.sql new file mode 100644 index 00000000000..3160b7115fa --- /dev/null +++ b/cli/integration_test/v2/seeds/1591867862409_test.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS account( + user_id serial PRIMARY KEY, + username VARCHAR (50) UNIQUE NOT NULL, + password VARCHAR (50) NOT NULL, + email VARCHAR (355) UNIQUE NOT NULL, + created_on TIMESTAMP NOT NULL, + last_login TIMESTAMP +); diff --git a/cli/integration_test/v2/seeds/1591867862419_test2.sql b/cli/integration_test/v2/seeds/1591867862419_test2.sql new file mode 100644 index 00000000000..24ca6422db8 --- /dev/null +++ b/cli/integration_test/v2/seeds/1591867862419_test2.sql @@ -0,0 +1,8 @@ +CREATE TABLE account2( + user_id serial PRIMARY KEY, + username VARCHAR (50) UNIQUE NOT NULL, + password VARCHAR (50) NOT NULL, + email VARCHAR (355) UNIQUE NOT NULL, + created_on TIMESTAMP NOT NULL, + last_login TIMESTAMP +); diff --git a/cli/metadata/actions/actions.go b/cli/metadata/actions/actions.go index bab7bc7d6ff..182b32fcec0 100644 --- a/cli/metadata/actions/actions.go +++ b/cli/metadata/actions/actions.go @@ -100,7 +100,7 @@ input SampleInput { defaultSDL = sdlToResp.SDL.Complete } graphqlFileContent = defaultSDL + "\n" + graphqlFileContent - data, err := editor.CaptureInputFromEditor(editor.GetPreferredEditorFromEnvironment, graphqlFileContent) + data, err := editor.CaptureInputFromEditor(editor.GetPreferredEditorFromEnvironment, graphqlFileContent, "graphql") if err != nil { return errors.Wrap(err, "error in getting input from editor") } diff --git a/cli/metadata/actions/editor/editor.go b/cli/metadata/actions/editor/editor.go index 5f9fdd88ff7..3d38ebeca3b 100644 --- a/cli/metadata/actions/editor/editor.go +++ b/cli/metadata/actions/editor/editor.go @@ -73,8 +73,8 @@ func OpenFileInEditor(filename string, resolveEditor PreferredEditorResolver) er // CaptureInputFromEditor opens a temporary file in a text editor and returns // the written bytes on success or an error on failure. It handles deletion // of the temporary file behind the scenes. -func CaptureInputFromEditor(resolveEditor PreferredEditorResolver, text string) ([]byte, error) { - file, err := ioutil.TempFile(os.TempDir(), "*.graphql") +func CaptureInputFromEditor(resolveEditor PreferredEditorResolver, text, extension string) ([]byte, error) { + file, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("*.%s", extension)) if err != nil { return []byte{}, err } diff --git a/cli/migrate/database/driver.go b/cli/migrate/database/driver.go index da22e05861e..1c2337cd285 100644 --- a/cli/migrate/database/driver.go +++ b/cli/migrate/database/driver.go @@ -113,6 +113,8 @@ type Driver interface { GraphQLDriver SchemaDriver + + SeedDriver } // Open returns a new driver instance. diff --git a/cli/migrate/database/driver_test.go b/cli/migrate/database/driver_test.go index e31a54303fb..e48ccfe71c9 100644 --- a/cli/migrate/database/driver_test.go +++ b/cli/migrate/database/driver_test.go @@ -147,6 +147,13 @@ func (m *mockDriver) UpdateSetting(name string, value string) error { return nil } +func (m *mockDriver) ApplySeed(interface{}) error { + return nil +} +func (m *mockDriver) ExportDataDump([]string) ([]byte, error) { + return nil, nil +} + func TestRegisterTwice(t *testing.T) { Register("mock", &mockDriver{}) diff --git a/cli/migrate/database/hasuradb/seed.go b/cli/migrate/database/hasuradb/seed.go new file mode 100644 index 00000000000..17e29684fae --- /dev/null +++ b/cli/migrate/database/hasuradb/seed.go @@ -0,0 +1,40 @@ +package hasuradb + +import ( + "net/http" +) + +func (h *HasuraDB) ApplySeed(m interface{}) error { + resp, body, err := h.sendv1Query(m) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return NewHasuraError(body, h.config.isCMD) + } + return nil +} + +func (h *HasuraDB) ExportDataDump(fromTables []string) ([]byte, error) { + pgDumpOpts := []string{"--no-owner", "--no-acl", "--data-only", "--column-inserts"} + for _, table := range fromTables { + pgDumpOpts = append(pgDumpOpts, "--table", table) + } + query := SchemaDump{ + Opts: pgDumpOpts, + CleanOutput: true, + } + + resp, body, err := h.sendSchemaDumpQuery(query) + if err != nil { + h.logger.Debug(err) + return nil, err + } + h.logger.Debug("exporting data: ", string(body)) + + if resp.StatusCode != http.StatusOK { + return nil, NewHasuraError(body, h.config.isCMD) + } + + return body, nil +} diff --git a/cli/migrate/database/seed.go b/cli/migrate/database/seed.go new file mode 100644 index 00000000000..8027503e5c6 --- /dev/null +++ b/cli/migrate/database/seed.go @@ -0,0 +1,6 @@ +package database + +type SeedDriver interface { + ApplySeed(m interface{}) error + ExportDataDump(tableNames []string) ([]byte, error) +} diff --git a/cli/migrate/migrate.go b/cli/migrate/migrate.go index 62751b87eb1..d497209cefc 100644 --- a/cli/migrate/migrate.go +++ b/cli/migrate/migrate.go @@ -1842,6 +1842,14 @@ func (m *Migrate) readDownFromVersion(from int64, to int64, ret chan<- interface } } +func (m *Migrate) ApplySeed(q interface{}) error { + return m.databaseDrv.ApplySeed(q) +} + +func (m *Migrate) ExportDataDump(tableNames []string) ([]byte, error) { + return m.databaseDrv.ExportDataDump(tableNames) +} + func printDryRunStatus(migrations []*Migration) *bytes.Buffer { out := new(tabwriter.Writer) buf := &bytes.Buffer{} diff --git a/cli/seed/apply.go b/cli/seed/apply.go new file mode 100644 index 00000000000..ab5fd88352e --- /dev/null +++ b/cli/seed/apply.go @@ -0,0 +1,69 @@ +package seed + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/hasura/graphql-engine/cli" + + "github.com/hasura/graphql-engine/cli/migrate" + "github.com/hasura/graphql-engine/cli/migrate/database/hasuradb" + "github.com/pkg/errors" + + "github.com/spf13/afero" +) + +// ApplySeedsToDatabase will read all .sql files in the given +// directory and apply it to hasura +func ApplySeedsToDatabase(ec *cli.ExecutionContext, fs afero.Fs, m *migrate.Migrate, filenames []string) error { + seedQuery := hasuradb.HasuraInterfaceBulk{ + Type: "bulk", + Args: make([]interface{}, 0), + } + + if len(filenames) > 0 { + for _, filename := range filenames { + absFilename := filepath.Join(ec.SeedsDirectory, filename) + b, err := afero.ReadFile(fs, absFilename) + if err != nil { + return errors.Wrap(err, "error opening file") + } + q := hasuradb.HasuraInterfaceQuery{ + Type: "run_sql", + Args: hasuradb.HasuraArgs{ + SQL: string(b), + }, + } + seedQuery.Args = append(seedQuery.Args, q) + } + } else { + err := afero.Walk(fs, ec.SeedsDirectory, func(path string, file os.FileInfo, err error) error { + if file == nil || err != nil { + return err + } + if !file.IsDir() && filepath.Ext(file.Name()) == ".sql" { + b, err := afero.ReadFile(fs, path) + if err != nil { + return errors.Wrap(err, "error opening file") + } + q := hasuradb.HasuraInterfaceQuery{ + Type: "run_sql", + Args: hasuradb.HasuraArgs{ + SQL: string(b), + }, + } + seedQuery.Args = append(seedQuery.Args, q) + } + + return nil + }) + if err != nil { + return errors.Wrap(err, "error walking the directory path") + } + } + if len(seedQuery.Args) == 0 { + return fmt.Errorf("no SQL files found in %s", ec.SeedsDirectory) + } + return m.ApplySeed(seedQuery) +} diff --git a/cli/seed/create.go b/cli/seed/create.go new file mode 100644 index 00000000000..7498b4e819b --- /dev/null +++ b/cli/seed/create.go @@ -0,0 +1,49 @@ +package seed + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/spf13/afero" +) + +// CreateSeedOpts has the list of options required +// to create a seed file +type CreateSeedOpts struct { + UserProvidedSeedName string + // DirectoryPath in which seed file should be created + DirectoryPath string + Data io.Reader +} + +// CreateSeedFile creates a .sql file according to the arguments +// it'll return full filepath and an error if any +func CreateSeedFile(fs afero.Fs, opts CreateSeedOpts) (*string, error) { + if opts.Data == nil { + return nil, errors.New("no data provided") + } + const fileExtension = "sql" + + timestamp := strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10) + // filename will be in format _.sql + filenameWithTimeStamp := fmt.Sprintf("%s_%s.%s", timestamp, opts.UserProvidedSeedName, fileExtension) + fullFilePath := filepath.Join(opts.DirectoryPath, filenameWithTimeStamp) + + // Write contents to file + file, err := fs.OpenFile(fullFilePath, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return nil, err + } + defer file.Close() + + r := bufio.NewReader(opts.Data) + io.Copy(file, r) + + return &fullFilePath, nil +} diff --git a/rfcs/seed-data.md b/rfcs/seed-data.md new file mode 100644 index 00000000000..87d128d015b --- /dev/null +++ b/rfcs/seed-data.md @@ -0,0 +1,39 @@ +## How should we model the interface to add seed data to the database ? + +### Motivation + +There is an ongoing discussion in [#2431](https://github.com/hasura/graphql-engine/issues/2431) about how we should model an interface for adding seed data. Two approaches have come to light from the discussions which are described below. + +#### Approach 1: Add a new "seed" command to CLI + +In this approach, the user has to write the corresponding SQL for seeds and everything else taken care by the CLI. But here the user is limited to writing SQL. To explain this situation let's take the case of a person coming from a ruby background, they will be used to writing seed migrations in Ruby DSL and might find difficulty in adapting to the new change. + +A PR [#3614](https://github.com/hasura/graphql-engine/pull/3614) has been submitted which implements this approach. + +#### Approach 2: Delegate adding seed data completely to the user + +A user can use whatever interface they want to communicate with the database, let it be GraphQL mutations or any ORM. From this [comment](https://github.com/hasura/graphql-engine/issues/2431#issuecomment-566033630) it is evident that atleast some users have this in mind. + +In this approach, everything is left to the user, from connecting to the underlying database to writing and managing seeds. + +An example for this approach would be Prisma. Prisma CLI has a seed command and they have a `prisma.yml` file which can be populated as follows. + +``` +seed: + run: node ./data/seed.js +``` + +On running `prisma seed`, will in turn run `node ./data/seed.js`, The user is delegated the responsibility of managing seeds. + +## Proposed change +From the discussions in the RFC the majority feels like we should go ahead with approach 1. +This will introduce a new command `hasura seed` and two new subcommands `create` and `apply`. + +- `hasura seed create ` will create a new seed file and will open it in the default editor. +- `hasura seed apply` will try to apply all migrations files in the `seeds/` directory. + +## Implementation + +PR [#3614](https://github.com/hasura/graphql-engine/pull/3614) is very similar implementation of the proposed changes, but it is tied to the schema migrations implementation, which as per discussion around [comment1](https://github.com/hasura/graphql-engine/pull/3763#issuecomment-578011460) and [comment2](https://github.com/hasura/graphql-engine/pull/3763#issuecomment-578071739) feels like something to avoid. + +I've a draft implementation (part of the RFC), which implements the proposed changes. Also adding an API endpoint which can probably be used by console (similar to migration endpoints).