From 040bef2fd56916079ce52afd1f095b0786af5362 Mon Sep 17 00:00:00 2001 From: Aravind Shankar Date: Mon, 18 Mar 2019 22:10:04 +0530 Subject: [PATCH] add options to create migration from files from sql and server (close #1699) (#1761) --- cli/commands/migrate_apply.go | 11 +- cli/commands/migrate_create.go | 107 +++++++++++++++++- cli/commands/migrate_create_test.go | 4 +- cli/commands/migrate_test.go | 2 +- cli/migrate/cmd/commands.go | 74 +++++++++--- cli/migrate/migrate.go | 8 +- .../hasura-cli/hasura_migrate_apply.rst | 8 +- .../hasura-cli/hasura_migrate_create.rst | 12 +- 8 files changed, 194 insertions(+), 32 deletions(-) diff --git a/cli/commands/migrate_apply.go b/cli/commands/migrate_apply.go index 6f704dc43f1..8600e0512a9 100644 --- a/cli/commands/migrate_apply.go +++ b/cli/commands/migrate_apply.go @@ -34,6 +34,7 @@ func newMigrateApplyCmd(ec *cli.ExecutionContext) *cobra.Command { f.StringVar(&opts.downMigration, "down", "", "apply all or N down migration steps") f.StringVar(&opts.versionMigration, "version", "", "migrate the database to a specific version") f.StringVar(&opts.migrationType, "type", "up", "type of migration (up, down) to be used with version flag") + f.BoolVar(&opts.skipExecution, "skip-execution", false, "skip executing the migration action, but mark them as applied") f.String("endpoint", "", "http(s) endpoint for Hasura GraphQL Engine") f.String("admin-secret", "", "admin secret for Hasura GraphQL Engine") @@ -54,10 +55,11 @@ type migrateApplyOptions struct { downMigration string versionMigration string migrationType string + skipExecution bool } func (o *migrateApplyOptions) run() error { - migrationType, step, err := getMigrationTypeAndStep(o.upMigration, o.downMigration, o.versionMigration, o.migrationType) + migrationType, step, err := getMigrationTypeAndStep(o.upMigration, o.downMigration, o.versionMigration, o.migrationType, o.skipExecution) if err != nil { return errors.Wrap(err, "error validating flags") } @@ -66,6 +68,7 @@ func (o *migrateApplyOptions) run() error { if err != nil { return err } + migrateDrv.SkipExecution = o.skipExecution err = ExecuteMigration(migrationType, migrateDrv, step) if err != nil { @@ -88,7 +91,7 @@ func (o *migrateApplyOptions) run() error { // Only one flag out of up, down and version can be set at a time. This function // checks whether that is the case and returns an error is not -func getMigrationTypeAndStep(upMigration, downMigration, versionMigration, migrationType string) (string, int64, error) { +func getMigrationTypeAndStep(upMigration, downMigration, versionMigration, migrationType string, skipExecution bool) (string, int64, error) { var flagCount = 0 var stepString = "all" var migrationName = "up" @@ -114,6 +117,10 @@ func getMigrationTypeAndStep(upMigration, downMigration, versionMigration, migra return "", 0, errors.New("Only one migration type can be applied at a time (--up, --down or --goto)") } + if migrationName != "version" && skipExecution { + return "", 0, errors.New("--skip-execution flag can be set only with --version flag") + } + if stepString == "all" && migrationName != "version" { return migrationName, -1, nil } diff --git a/cli/commands/migrate_create.go b/cli/commands/migrate_create.go index 3071c5f3f7d..73c92ef11ad 100644 --- a/cli/commands/migrate_create.go +++ b/cli/commands/migrate_create.go @@ -1,12 +1,17 @@ package commands import ( + "io/ioutil" + "os" "time" + "github.com/ghodss/yaml" "github.com/hasura/graphql-engine/cli" mig "github.com/hasura/graphql-engine/cli/migrate/cmd" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/spf13/pflag" "github.com/spf13/viper" ) @@ -31,6 +36,22 @@ func newMigrateCreateCmd(ec *cli.ExecutionContext) *cobra.Command { return opts.run() }, } + f := migrateCreateCmd.Flags() + opts.flags = f + f.StringVar(&opts.sqlFile, "sql-from-file", "", "path to an sql file which contains the up actions") + f.StringVar(&opts.metaDataFile, "metadata-from-file", "", "path to a hasura metadata file to be used for up actions") + f.BoolVar(&opts.metaDataServer, "metadata-from-server", false, "take metadata from the server and write it as an up migration file") + f.String("endpoint", "", "http(s) endpoint for Hasura GraphQL Engine") + f.String("admin-secret", "", "admin secret for Hasura GraphQL Engine") + f.String("access-key", "", "access key for Hasura GraphQL Engine") + f.MarkDeprecated("access-key", "use --admin-secret instead") + migrateCreateCmd.MarkFlagFilename("sql-from-file") + migrateCreateCmd.MarkFlagFilename("metadata-from-file") + + // need to create a new viper because https://github.com/spf13/viper/issues/233 + v.BindPFlag("endpoint", f.Lookup("endpoint")) + v.BindPFlag("admin_secret", f.Lookup("admin-secret")) + v.BindPFlag("access_key", f.Lookup("access-key")) return migrateCreateCmd } @@ -38,18 +59,94 @@ func newMigrateCreateCmd(ec *cli.ExecutionContext) *cobra.Command { type migrateCreateOptions struct { EC *cli.ExecutionContext - name string + name string + flags *pflag.FlagSet + + // Flags + sqlFile string + metaDataFile string + metaDataServer bool } -func (o *migrateCreateOptions) run() error { +func (o *migrateCreateOptions) run() (err error) { timestamp := getTime() createOptions := mig.New(timestamp, o.name, o.EC.MigrationDir) - createOptions.IsCMD = true - err := createOptions.Create() + + if o.flags.Changed("sql-from-file") { + // sql-file flag is set + err := createOptions.SetSQLUpFromFile(o.sqlFile) + if err != nil { + return errors.Wrap(err, "cannot set sql file") + } + } + + if o.flags.Changed("metadata-from-file") && o.metaDataServer { + return errors.New("only one metadata type can be set") + } + + if o.flags.Changed("metadata-from-file") { + // metadata-file flag is set + err := createOptions.SetMetaUpFromFile(o.metaDataFile) + if err != nil { + return errors.Wrap(err, "cannot set metadata file") + } + } + + if o.metaDataServer { + // create new migrate instance + migrateDrv, err := newMigrate(o.EC.MigrationDir, o.EC.ServerConfig.ParsedEndpoint, o.EC.ServerConfig.AdminSecret, o.EC.Logger, o.EC.Version) + if err != nil { + return errors.Wrap(err, "cannot create migrate instance") + } + + // fetch metadata from server + metaData, err := migrateDrv.ExportMetadata() + if err != nil { + return errors.Wrap(err, "cannot fetch metadata from server") + } + + tmpfile, err := ioutil.TempFile("", "metadata") + if err != nil { + return errors.Wrap(err, "cannot create tempfile") + } + defer os.Remove(tmpfile.Name()) + + t, err := yaml.Marshal(metaData) + if err != nil { + return errors.Wrap(err, "cannot marshal metadata") + } + if _, err := tmpfile.Write(t); err != nil { + return errors.Wrap(err, "cannot write to temp file") + } + if err := tmpfile.Close(); err != nil { + return errors.Wrap(err, "cannot close tmp file") + } + + err = createOptions.SetMetaUpFromFile(tmpfile.Name()) + if err != nil { + return errors.Wrap(err, "cannot parse metadata from the server") + } + } + + if !o.flags.Changed("sql-from-file") && !o.flags.Changed("metadata-from-file") && !o.metaDataServer { + // Set empty data for [up|down].yaml + createOptions.MetaUp = []byte(`[]`) + createOptions.MetaDown = []byte(`[]`) + } + + defer func() { + if err != nil { + createOptions.Delete() + } + }() + err = createOptions.Create() if err != nil { return errors.Wrap(err, "error creating migration files") } - o.EC.Logger.Infof("Migration files created with version %d_%s.[up|down].[yaml|sql]", timestamp, o.name) + o.EC.Logger.WithFields(log.Fields{ + "version": timestamp, + "name": o.name, + }).Info("Migrations files created") return nil } diff --git a/cli/commands/migrate_create_test.go b/cli/commands/migrate_create_test.go index bd1c102bc0d..43d924267fc 100644 --- a/cli/commands/migrate_create_test.go +++ b/cli/commands/migrate_create_test.go @@ -11,6 +11,7 @@ import ( "github.com/briandowns/spinner" "github.com/hasura/graphql-engine/cli" "github.com/sirupsen/logrus/hooks/test" + "github.com/spf13/pflag" ) func TestMigrateCreateCmd(t *testing.T) { @@ -21,7 +22,8 @@ func TestMigrateCreateCmd(t *testing.T) { Spinner: spinner.New(spinner.CharSets[7], 100*time.Millisecond), MigrationDir: filepath.Join(os.TempDir(), "hasura-cli-test-"+strconv.Itoa(rand.Intn(1000))), }, - name: "create_article", + name: "create_article", + flags: pflag.NewFlagSet("migrate-create-test", pflag.ContinueOnError), } err := opts.run() diff --git a/cli/commands/migrate_test.go b/cli/commands/migrate_test.go index 3dcc2a9b4e1..83c4699b6b2 100644 --- a/cli/commands/migrate_test.go +++ b/cli/commands/migrate_test.go @@ -230,5 +230,5 @@ func compareMetadata(t testing.TB, metadataFile string, actualData []byte) { if err != nil { t.Fatalf("error reading metadata %s", err) } - assert.Equal(t, actualData, data) + assert.Equal(t, string(actualData), string(data)) } diff --git a/cli/migrate/cmd/commands.go b/cli/migrate/cmd/commands.go index 08df60cbe69..0b46a63b4c5 100644 --- a/cli/migrate/cmd/commands.go +++ b/cli/migrate/cmd/commands.go @@ -25,7 +25,6 @@ type CreateOptions struct { Version int64 Directory string Name string - IsCMD bool MetaUp []byte MetaDown []byte SQLUp []byte @@ -40,10 +39,6 @@ func New(version int64, name, directory string) *CreateOptions { Version: version, Directory: directory, Name: name, - MetaUp: []byte(`[]`), - MetaDown: []byte(`[]`), - SQLUp: []byte{}, - SQLDown: []byte{}, } } @@ -60,6 +55,29 @@ func (c *CreateOptions) SetMetaUp(data interface{}) error { return nil } +func (c *CreateOptions) SetMetaUpFromFile(filePath string) error { + data, err := ioutil.ReadFile(filePath) + if err != nil { + return err + } + + var metadata []interface{} + var q interface{} + err = yaml.Unmarshal(data, &q) + if err != nil { + return err + } + + metadata = append( + metadata, + map[string]interface{}{ + "type": "replace_metadata", + "args": q, + }, + ) + return c.SetMetaUp(metadata) +} + func (c *CreateOptions) SetMetaDown(data interface{}) error { t, err := json.Marshal(data) if err != nil { @@ -78,6 +96,16 @@ func (c *CreateOptions) SetSQLUp(data string) error { return nil } +func (c *CreateOptions) SetSQLUpFromFile(filePath string) error { + data, err := ioutil.ReadFile(filePath) + if err != nil { + return err + } + + c.SQLUp = data + return nil +} + func (c *CreateOptions) SetSQLDown(data string) error { c.SQLDown = []byte(data) return nil @@ -90,22 +118,38 @@ func (c *CreateOptions) Create() error { if err != nil { return err } - // Create MetaUp - err = createFile(base+"up.yaml", c.MetaUp) - if err != nil { - return err - } - // Create MetaDown - err = createFile(base+"down.yaml", c.MetaDown) - if err != nil { - return err + + // Check if data has been set in one of the files + if c.MetaUp == nil && c.MetaDown == nil && c.SQLUp == nil && c.SQLDown == nil { + return errors.New("none of the files has been set with data") } - if c.IsCMD { + if c.MetaUp != nil { + // Create MetaUp + err = createFile(base+"up.yaml", c.MetaUp) + if err != nil { + return err + } + } + + if c.MetaDown != nil { + // Create MetaDown + err = createFile(base+"down.yaml", c.MetaDown) + if err != nil { + return err + } + } + + if c.SQLUp != nil { + // Create SQLUp err = createFile(base+"up.sql", c.SQLUp) if err != nil { return err } + } + + if c.SQLDown != nil { + // Create SQLDown err = createFile(base+"down.sql", c.SQLDown) if err != nil { return err diff --git a/cli/migrate/migrate.go b/cli/migrate/migrate.go index a4519ed7e9d..80b31667d88 100644 --- a/cli/migrate/migrate.go +++ b/cli/migrate/migrate.go @@ -91,6 +91,8 @@ type Migrate struct { isCMD bool status *Status + + SkipExecution bool } // New returns a new Migrate instance from a source URL and a database URL. @@ -801,8 +803,10 @@ func (m *Migrate) runMigrations(ret <-chan interface{}) error { case *Migration: migr := r.(*Migration) if migr.Body != nil { - if err := m.databaseDrv.Run(migr.BufferedBody, migr.FileType, migr.FileName); err != nil { - return err + if !m.SkipExecution { + if err := m.databaseDrv.Run(migr.BufferedBody, migr.FileType, migr.FileName); err != nil { + return err + } } version := int64(migr.Version) diff --git a/docs/graphql/manual/hasura-cli/hasura_migrate_apply.rst b/docs/graphql/manual/hasura-cli/hasura_migrate_apply.rst index af50cd3d6df..f23b6a07cd8 100644 --- a/docs/graphql/manual/hasura-cli/hasura_migrate_apply.rst +++ b/docs/graphql/manual/hasura-cli/hasura_migrate_apply.rst @@ -20,10 +20,11 @@ Options :: - --admin-secret string admin secret key for Hasura GraphQL Engine + --admin-secret string admin secret for Hasura GraphQL Engine --down string apply all or N down migration steps --endpoint string http(s) endpoint for Hasura GraphQL Engine -h, --help help for apply + --skip-execution skip executing the migration action, but mark them as applied --type string type of migration (up, down) to be used with version flag (default "up") --up string apply all or N up migration steps --version string migrate the database to a specific version @@ -33,8 +34,9 @@ Options inherited from parent commands :: - --log-level string log level (DEBUG, INFO, WARN, ERROR, FATAL) (default "INFO") - --project string directory where commands are executed. (default: current dir) + --log-level string log level (DEBUG, INFO, WARN, ERROR, FATAL) (default "INFO") + --project string directory where commands are executed (default: current dir) + --skip-update-check Skip automatic update check on command execution SEE ALSO ~~~~~~~~ diff --git a/docs/graphql/manual/hasura-cli/hasura_migrate_create.rst b/docs/graphql/manual/hasura-cli/hasura_migrate_create.rst index 562cb50ebb5..1d38ef77ae7 100644 --- a/docs/graphql/manual/hasura-cli/hasura_migrate_create.rst +++ b/docs/graphql/manual/hasura-cli/hasura_migrate_create.rst @@ -20,15 +20,21 @@ Options :: - -h, --help help for create + --admin-secret string admin secret for Hasura GraphQL Engine + --endpoint string http(s) endpoint for Hasura GraphQL Engine + -h, --help help for create + --metadata-from-file string path to a hasura metadata file to be used for up actions + --metadata-from-server take metadata from the server and write it as an up migration file + --sql-from-file string path to an sql file which contains the up actions Options inherited from parent commands ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :: - --log-level string log level (DEBUG, INFO, WARN, ERROR, FATAL) (default "INFO") - --project string directory where commands are executed. (default: current dir) + --log-level string log level (DEBUG, INFO, WARN, ERROR, FATAL) (default "INFO") + --project string directory where commands are executed (default: current dir) + --skip-update-check Skip automatic update check on command execution SEE ALSO ~~~~~~~~