add options to create migration from files from sql and server (close #1699) (#1761)

cli/commands/migrate_apply.go

@@ -34,6 +34,7 @@ func newMigrateApplyCmd(ec *cli.ExecutionContext) *cobra.Command {
 	f.StringVar(&opts.downMigration, "down", "", "apply all or N down migration steps")
 	f.StringVar(&opts.versionMigration, "version", "", "migrate the database to a specific version")
 	f.StringVar(&opts.migrationType, "type", "up", "type of migration (up, down) to be used with version flag")
+	f.BoolVar(&opts.skipExecution, "skip-execution", false, "skip executing the migration action, but mark them as applied")
 
 	f.String("endpoint", "", "http(s) endpoint for Hasura GraphQL Engine")
 	f.String("admin-secret", "", "admin secret for Hasura GraphQL Engine")
@@ -54,10 +55,11 @@ type migrateApplyOptions struct {
 	downMigration    string
 	versionMigration string
 	migrationType    string
+	skipExecution    bool
 }
 
 func (o *migrateApplyOptions) run() error {
-	migrationType, step, err := getMigrationTypeAndStep(o.upMigration, o.downMigration, o.versionMigration, o.migrationType)
+	migrationType, step, err := getMigrationTypeAndStep(o.upMigration, o.downMigration, o.versionMigration, o.migrationType, o.skipExecution)
 	if err != nil {
 		return errors.Wrap(err, "error validating flags")
 	}
@@ -66,6 +68,7 @@ func (o *migrateApplyOptions) run() error {
 	if err != nil {
 		return err
 	}
+	migrateDrv.SkipExecution = o.skipExecution
 
 	err = ExecuteMigration(migrationType, migrateDrv, step)
 	if err != nil {
@@ -88,7 +91,7 @@ func (o *migrateApplyOptions) run() error {
 
 // Only one flag out of up, down and version can be set at a time. This function
 // checks whether that is the case and returns an error is not
-func getMigrationTypeAndStep(upMigration, downMigration, versionMigration, migrationType string) (string, int64, error) {
+func getMigrationTypeAndStep(upMigration, downMigration, versionMigration, migrationType string, skipExecution bool) (string, int64, error) {
 	var flagCount = 0
 	var stepString = "all"
 	var migrationName = "up"
@@ -114,6 +117,10 @@ func getMigrationTypeAndStep(upMigration, downMigration, versionMigration, migra
 		return "", 0, errors.New("Only one migration type can be applied at a time (--up, --down or --goto)")
 	}
 
+	if migrationName != "version" && skipExecution {
+		return "", 0, errors.New("--skip-execution flag can be set only with --version flag")
+	}
+
 	if stepString == "all" && migrationName != "version" {
 		return migrationName, -1, nil
 	}

cli/commands/migrate_create.go

@@ -1,12 +1,17 @@
 package commands
 
 import (
+	"io/ioutil"
+	"os"
 	"time"
 
+	"github.com/ghodss/yaml"
 	"github.com/hasura/graphql-engine/cli"
 	mig "github.com/hasura/graphql-engine/cli/migrate/cmd"
 	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 )
 
@@ -31,6 +36,22 @@ func newMigrateCreateCmd(ec *cli.ExecutionContext) *cobra.Command {
 			return opts.run()
 		},
 	}
+	f := migrateCreateCmd.Flags()
+	opts.flags = f
+	f.StringVar(&opts.sqlFile, "sql-from-file", "", "path to an sql file which contains the up actions")
+	f.StringVar(&opts.metaDataFile, "metadata-from-file", "", "path to a hasura metadata file to be used for up actions")
+	f.BoolVar(&opts.metaDataServer, "metadata-from-server", false, "take metadata from the server and write it as an up migration file")
+	f.String("endpoint", "", "http(s) endpoint for Hasura GraphQL Engine")
+	f.String("admin-secret", "", "admin secret for Hasura GraphQL Engine")
+	f.String("access-key", "", "access key for Hasura GraphQL Engine")
+	f.MarkDeprecated("access-key", "use --admin-secret instead")
+	migrateCreateCmd.MarkFlagFilename("sql-from-file")
+	migrateCreateCmd.MarkFlagFilename("metadata-from-file")
+
+	// need to create a new viper because https://github.com/spf13/viper/issues/233
+	v.BindPFlag("endpoint", f.Lookup("endpoint"))
+	v.BindPFlag("admin_secret", f.Lookup("admin-secret"))
+	v.BindPFlag("access_key", f.Lookup("access-key"))
 
 	return migrateCreateCmd
 }
@@ -39,17 +60,93 @@ type migrateCreateOptions struct {
 	EC *cli.ExecutionContext
 
 	name  string
+	flags *pflag.FlagSet
+
+	// Flags
+	sqlFile        string
+	metaDataFile   string
+	metaDataServer bool
 }
 
-func (o *migrateCreateOptions) run() error {
+func (o *migrateCreateOptions) run() (err error) {
 	timestamp := getTime()
 	createOptions := mig.New(timestamp, o.name, o.EC.MigrationDir)
-	createOptions.IsCMD = true
-	err := createOptions.Create()
+
+	if o.flags.Changed("sql-from-file") {
+		// sql-file flag is set
+		err := createOptions.SetSQLUpFromFile(o.sqlFile)
+		if err != nil {
+			return errors.Wrap(err, "cannot set sql file")
+		}
+	}
+
+	if o.flags.Changed("metadata-from-file") && o.metaDataServer {
+		return errors.New("only one metadata type can be set")
+	}
+
+	if o.flags.Changed("metadata-from-file") {
+		// metadata-file flag is set
+		err := createOptions.SetMetaUpFromFile(o.metaDataFile)
+		if err != nil {
+			return errors.Wrap(err, "cannot set metadata file")
+		}
+	}
+
+	if o.metaDataServer {
+		// create new migrate instance
+		migrateDrv, err := newMigrate(o.EC.MigrationDir, o.EC.ServerConfig.ParsedEndpoint, o.EC.ServerConfig.AdminSecret, o.EC.Logger, o.EC.Version)
+		if err != nil {
+			return errors.Wrap(err, "cannot create migrate instance")
+		}
+
+		// fetch metadata from server
+		metaData, err := migrateDrv.ExportMetadata()
+		if err != nil {
+			return errors.Wrap(err, "cannot fetch metadata from server")
+		}
+
+		tmpfile, err := ioutil.TempFile("", "metadata")
+		if err != nil {
+			return errors.Wrap(err, "cannot create tempfile")
+		}
+		defer os.Remove(tmpfile.Name())
+
+		t, err := yaml.Marshal(metaData)
+		if err != nil {
+			return errors.Wrap(err, "cannot marshal metadata")
+		}
+		if _, err := tmpfile.Write(t); err != nil {
+			return errors.Wrap(err, "cannot write to temp file")
+		}
+		if err := tmpfile.Close(); err != nil {
+			return errors.Wrap(err, "cannot close tmp file")
+		}
+
+		err = createOptions.SetMetaUpFromFile(tmpfile.Name())
+		if err != nil {
+			return errors.Wrap(err, "cannot parse metadata from the server")
+		}
+	}
+
+	if !o.flags.Changed("sql-from-file") && !o.flags.Changed("metadata-from-file") && !o.metaDataServer {
+		// Set empty data for [up|down].yaml
+		createOptions.MetaUp = []byte(`[]`)
+		createOptions.MetaDown = []byte(`[]`)
+	}
+
+	defer func() {
+		if err != nil {
+			createOptions.Delete()
+		}
+	}()
+	err = createOptions.Create()
 	if err != nil {
 		return errors.Wrap(err, "error creating migration files")
 	}
-	o.EC.Logger.Infof("Migration files created with version %d_%s.[up|down].[yaml|sql]", timestamp, o.name)
+	o.EC.Logger.WithFields(log.Fields{
+		"version": timestamp,
+		"name":    o.name,
+	}).Info("Migrations files created")
 	return nil
 }
 

cli/commands/migrate_create_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/briandowns/spinner"
 	"github.com/hasura/graphql-engine/cli"
 	"github.com/sirupsen/logrus/hooks/test"
+	"github.com/spf13/pflag"
 )
 
 func TestMigrateCreateCmd(t *testing.T) {
@@ -22,6 +23,7 @@ func TestMigrateCreateCmd(t *testing.T) {
 			MigrationDir: filepath.Join(os.TempDir(), "hasura-cli-test-"+strconv.Itoa(rand.Intn(1000))),
 		},
 		name:  "create_article",
+		flags: pflag.NewFlagSet("migrate-create-test", pflag.ContinueOnError),
 	}
 
 	err := opts.run()

cli/commands/migrate_test.go

@@ -230,5 +230,5 @@ func compareMetadata(t testing.TB, metadataFile string, actualData []byte) {
 	if err != nil {
 		t.Fatalf("error reading metadata %s", err)
 	}
-	assert.Equal(t, actualData, data)
+	assert.Equal(t, string(actualData), string(data))
 }

cli/migrate/cmd/commands.go

@@ -25,7 +25,6 @@ type CreateOptions struct {
 	Version   int64
 	Directory string
 	Name      string
-	IsCMD     bool
 	MetaUp    []byte
 	MetaDown  []byte
 	SQLUp     []byte
@@ -40,10 +39,6 @@ func New(version int64, name, directory string) *CreateOptions {
 		Version:   version,
 		Directory: directory,
 		Name:      name,
-		MetaUp:    []byte(`[]`),
-		MetaDown:  []byte(`[]`),
-		SQLUp:     []byte{},
-		SQLDown:   []byte{},
 	}
 }
 
@@ -60,6 +55,29 @@ func (c *CreateOptions) SetMetaUp(data interface{}) error {
 	return nil
 }
 
+func (c *CreateOptions) SetMetaUpFromFile(filePath string) error {
+	data, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return err
+	}
+
+	var metadata []interface{}
+	var q interface{}
+	err = yaml.Unmarshal(data, &q)
+	if err != nil {
+		return err
+	}
+
+	metadata = append(
+		metadata,
+		map[string]interface{}{
+			"type": "replace_metadata",
+			"args": q,
+		},
+	)
+	return c.SetMetaUp(metadata)
+}
+
 func (c *CreateOptions) SetMetaDown(data interface{}) error {
 	t, err := json.Marshal(data)
 	if err != nil {
@@ -78,6 +96,16 @@ func (c *CreateOptions) SetSQLUp(data string) error {
 	return nil
 }
 
+func (c *CreateOptions) SetSQLUpFromFile(filePath string) error {
+	data, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return err
+	}
+
+	c.SQLUp = data
+	return nil
+}
+
 func (c *CreateOptions) SetSQLDown(data string) error {
 	c.SQLDown = []byte(data)
 	return nil
@@ -90,22 +118,38 @@ func (c *CreateOptions) Create() error {
 	if err != nil {
 		return err
 	}
+
+	// Check if data has been set in one of the files
+	if c.MetaUp == nil && c.MetaDown == nil && c.SQLUp == nil && c.SQLDown == nil {
+		return errors.New("none of the files has been set with data")
+	}
+
+	if c.MetaUp != nil {
 		// Create MetaUp
 		err = createFile(base+"up.yaml", c.MetaUp)
 		if err != nil {
 			return err
 		}
+	}
+
+	if c.MetaDown != nil {
 		// Create MetaDown
 		err = createFile(base+"down.yaml", c.MetaDown)
 		if err != nil {
 			return err
 		}
+	}
 
-	if c.IsCMD {
+	if c.SQLUp != nil {
+		// Create SQLUp
 		err = createFile(base+"up.sql", c.SQLUp)
 		if err != nil {
 			return err
 		}
+	}
+
+	if c.SQLDown != nil {
+		// Create SQLDown
 		err = createFile(base+"down.sql", c.SQLDown)
 		if err != nil {
 			return err

cli/migrate/migrate.go

@@ -91,6 +91,8 @@ type Migrate struct {
 	isCMD bool
 
 	status *Status
+
+	SkipExecution bool
 }
 
 // New returns a new Migrate instance from a source URL and a database URL.
@@ -801,9 +803,11 @@ func (m *Migrate) runMigrations(ret <-chan interface{}) error {
 		case *Migration:
 			migr := r.(*Migration)
 			if migr.Body != nil {
+				if !m.SkipExecution {
 					if err := m.databaseDrv.Run(migr.BufferedBody, migr.FileType, migr.FileName); err != nil {
 						return err
 					}
+				}
 
 				version := int64(migr.Version)
 				if version == migr.TargetVersion {

docs/graphql/manual/hasura-cli/hasura_migrate_apply.rst

@@ -20,10 +20,11 @@ Options
 
 ::
 
-      --admin-secret string   admin secret key for Hasura GraphQL Engine
+      --admin-secret string   admin secret for Hasura GraphQL Engine
       --down string           apply all or N down migration steps
       --endpoint string       http(s) endpoint for Hasura GraphQL Engine
   -h, --help                  help for apply
+      --skip-execution        skip executing the migration action, but mark them as applied
       --type string           type of migration (up, down) to be used with version flag (default "up")
       --up string             apply all or N up migration steps
       --version string        migrate the database to a specific version
@@ -34,7 +35,8 @@ Options inherited from parent commands
 ::
 
       --log-level string    log level (DEBUG, INFO, WARN, ERROR, FATAL) (default "INFO")
-      --project string     directory where commands are executed. (default: current dir)
+      --project string      directory where commands are executed (default: current dir)
+      --skip-update-check   Skip automatic update check on command execution
 
 SEE ALSO
 ~~~~~~~~

docs/graphql/manual/hasura-cli/hasura_migrate_create.rst

@@ -20,7 +20,12 @@ Options
 
 ::
 
+      --admin-secret string         admin secret for Hasura GraphQL Engine
+      --endpoint string             http(s) endpoint for Hasura GraphQL Engine
   -h, --help                        help for create
+      --metadata-from-file string   path to a hasura metadata file to be used for up actions
+      --metadata-from-server        take metadata from the server and write it as an up migration file
+      --sql-from-file string        path to an sql file which contains the up actions
 
 Options inherited from parent commands
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -28,7 +33,8 @@ Options inherited from parent commands
 ::
 
       --log-level string    log level (DEBUG, INFO, WARN, ERROR, FATAL) (default "INFO")
-      --project string     directory where commands are executed. (default: current dir)
+      --project string      directory where commands are executed (default: current dir)
+      --skip-update-check   Skip automatic update check on command execution
 
 SEE ALSO
 ~~~~~~~~