add options to create migration from files from sql and server (close #1699) (#1761)

This commit is contained in:
Aravind Shankar 2019-03-18 22:10:04 +05:30 committed by Shahidh K Muhammed
parent 24dcefb142
commit 040bef2fd5
8 changed files with 194 additions and 32 deletions

View File

@ -34,6 +34,7 @@ func newMigrateApplyCmd(ec *cli.ExecutionContext) *cobra.Command {
f.StringVar(&opts.downMigration, "down", "", "apply all or N down migration steps") f.StringVar(&opts.downMigration, "down", "", "apply all or N down migration steps")
f.StringVar(&opts.versionMigration, "version", "", "migrate the database to a specific version") f.StringVar(&opts.versionMigration, "version", "", "migrate the database to a specific version")
f.StringVar(&opts.migrationType, "type", "up", "type of migration (up, down) to be used with version flag") f.StringVar(&opts.migrationType, "type", "up", "type of migration (up, down) to be used with version flag")
f.BoolVar(&opts.skipExecution, "skip-execution", false, "skip executing the migration action, but mark them as applied")
f.String("endpoint", "", "http(s) endpoint for Hasura GraphQL Engine") f.String("endpoint", "", "http(s) endpoint for Hasura GraphQL Engine")
f.String("admin-secret", "", "admin secret for Hasura GraphQL Engine") f.String("admin-secret", "", "admin secret for Hasura GraphQL Engine")
@ -54,10 +55,11 @@ type migrateApplyOptions struct {
downMigration string downMigration string
versionMigration string versionMigration string
migrationType string migrationType string
skipExecution bool
} }
func (o *migrateApplyOptions) run() error { func (o *migrateApplyOptions) run() error {
migrationType, step, err := getMigrationTypeAndStep(o.upMigration, o.downMigration, o.versionMigration, o.migrationType) migrationType, step, err := getMigrationTypeAndStep(o.upMigration, o.downMigration, o.versionMigration, o.migrationType, o.skipExecution)
if err != nil { if err != nil {
return errors.Wrap(err, "error validating flags") return errors.Wrap(err, "error validating flags")
} }
@ -66,6 +68,7 @@ func (o *migrateApplyOptions) run() error {
if err != nil { if err != nil {
return err return err
} }
migrateDrv.SkipExecution = o.skipExecution
err = ExecuteMigration(migrationType, migrateDrv, step) err = ExecuteMigration(migrationType, migrateDrv, step)
if err != nil { if err != nil {
@ -88,7 +91,7 @@ func (o *migrateApplyOptions) run() error {
// Only one flag out of up, down and version can be set at a time. This function // Only one flag out of up, down and version can be set at a time. This function
// checks whether that is the case and returns an error is not // checks whether that is the case and returns an error is not
func getMigrationTypeAndStep(upMigration, downMigration, versionMigration, migrationType string) (string, int64, error) { func getMigrationTypeAndStep(upMigration, downMigration, versionMigration, migrationType string, skipExecution bool) (string, int64, error) {
var flagCount = 0 var flagCount = 0
var stepString = "all" var stepString = "all"
var migrationName = "up" var migrationName = "up"
@ -114,6 +117,10 @@ func getMigrationTypeAndStep(upMigration, downMigration, versionMigration, migra
return "", 0, errors.New("Only one migration type can be applied at a time (--up, --down or --goto)") return "", 0, errors.New("Only one migration type can be applied at a time (--up, --down or --goto)")
} }
if migrationName != "version" && skipExecution {
return "", 0, errors.New("--skip-execution flag can be set only with --version flag")
}
if stepString == "all" && migrationName != "version" { if stepString == "all" && migrationName != "version" {
return migrationName, -1, nil return migrationName, -1, nil
} }

View File

@ -1,12 +1,17 @@
package commands package commands
import ( import (
"io/ioutil"
"os"
"time" "time"
"github.com/ghodss/yaml"
"github.com/hasura/graphql-engine/cli" "github.com/hasura/graphql-engine/cli"
mig "github.com/hasura/graphql-engine/cli/migrate/cmd" mig "github.com/hasura/graphql-engine/cli/migrate/cmd"
"github.com/pkg/errors" "github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/spf13/viper" "github.com/spf13/viper"
) )
@ -31,6 +36,22 @@ func newMigrateCreateCmd(ec *cli.ExecutionContext) *cobra.Command {
return opts.run() return opts.run()
}, },
} }
f := migrateCreateCmd.Flags()
opts.flags = f
f.StringVar(&opts.sqlFile, "sql-from-file", "", "path to an sql file which contains the up actions")
f.StringVar(&opts.metaDataFile, "metadata-from-file", "", "path to a hasura metadata file to be used for up actions")
f.BoolVar(&opts.metaDataServer, "metadata-from-server", false, "take metadata from the server and write it as an up migration file")
f.String("endpoint", "", "http(s) endpoint for Hasura GraphQL Engine")
f.String("admin-secret", "", "admin secret for Hasura GraphQL Engine")
f.String("access-key", "", "access key for Hasura GraphQL Engine")
f.MarkDeprecated("access-key", "use --admin-secret instead")
migrateCreateCmd.MarkFlagFilename("sql-from-file")
migrateCreateCmd.MarkFlagFilename("metadata-from-file")
// need to create a new viper because https://github.com/spf13/viper/issues/233
v.BindPFlag("endpoint", f.Lookup("endpoint"))
v.BindPFlag("admin_secret", f.Lookup("admin-secret"))
v.BindPFlag("access_key", f.Lookup("access-key"))
return migrateCreateCmd return migrateCreateCmd
} }
@ -39,17 +60,93 @@ type migrateCreateOptions struct {
EC *cli.ExecutionContext EC *cli.ExecutionContext
name string name string
flags *pflag.FlagSet
// Flags
sqlFile string
metaDataFile string
metaDataServer bool
} }
func (o *migrateCreateOptions) run() error { func (o *migrateCreateOptions) run() (err error) {
timestamp := getTime() timestamp := getTime()
createOptions := mig.New(timestamp, o.name, o.EC.MigrationDir) createOptions := mig.New(timestamp, o.name, o.EC.MigrationDir)
createOptions.IsCMD = true
err := createOptions.Create() if o.flags.Changed("sql-from-file") {
// sql-file flag is set
err := createOptions.SetSQLUpFromFile(o.sqlFile)
if err != nil {
return errors.Wrap(err, "cannot set sql file")
}
}
if o.flags.Changed("metadata-from-file") && o.metaDataServer {
return errors.New("only one metadata type can be set")
}
if o.flags.Changed("metadata-from-file") {
// metadata-file flag is set
err := createOptions.SetMetaUpFromFile(o.metaDataFile)
if err != nil {
return errors.Wrap(err, "cannot set metadata file")
}
}
if o.metaDataServer {
// create new migrate instance
migrateDrv, err := newMigrate(o.EC.MigrationDir, o.EC.ServerConfig.ParsedEndpoint, o.EC.ServerConfig.AdminSecret, o.EC.Logger, o.EC.Version)
if err != nil {
return errors.Wrap(err, "cannot create migrate instance")
}
// fetch metadata from server
metaData, err := migrateDrv.ExportMetadata()
if err != nil {
return errors.Wrap(err, "cannot fetch metadata from server")
}
tmpfile, err := ioutil.TempFile("", "metadata")
if err != nil {
return errors.Wrap(err, "cannot create tempfile")
}
defer os.Remove(tmpfile.Name())
t, err := yaml.Marshal(metaData)
if err != nil {
return errors.Wrap(err, "cannot marshal metadata")
}
if _, err := tmpfile.Write(t); err != nil {
return errors.Wrap(err, "cannot write to temp file")
}
if err := tmpfile.Close(); err != nil {
return errors.Wrap(err, "cannot close tmp file")
}
err = createOptions.SetMetaUpFromFile(tmpfile.Name())
if err != nil {
return errors.Wrap(err, "cannot parse metadata from the server")
}
}
if !o.flags.Changed("sql-from-file") && !o.flags.Changed("metadata-from-file") && !o.metaDataServer {
// Set empty data for [up|down].yaml
createOptions.MetaUp = []byte(`[]`)
createOptions.MetaDown = []byte(`[]`)
}
defer func() {
if err != nil {
createOptions.Delete()
}
}()
err = createOptions.Create()
if err != nil { if err != nil {
return errors.Wrap(err, "error creating migration files") return errors.Wrap(err, "error creating migration files")
} }
o.EC.Logger.Infof("Migration files created with version %d_%s.[up|down].[yaml|sql]", timestamp, o.name) o.EC.Logger.WithFields(log.Fields{
"version": timestamp,
"name": o.name,
}).Info("Migrations files created")
return nil return nil
} }

View File

@ -11,6 +11,7 @@ import (
"github.com/briandowns/spinner" "github.com/briandowns/spinner"
"github.com/hasura/graphql-engine/cli" "github.com/hasura/graphql-engine/cli"
"github.com/sirupsen/logrus/hooks/test" "github.com/sirupsen/logrus/hooks/test"
"github.com/spf13/pflag"
) )
func TestMigrateCreateCmd(t *testing.T) { func TestMigrateCreateCmd(t *testing.T) {
@ -22,6 +23,7 @@ func TestMigrateCreateCmd(t *testing.T) {
MigrationDir: filepath.Join(os.TempDir(), "hasura-cli-test-"+strconv.Itoa(rand.Intn(1000))), MigrationDir: filepath.Join(os.TempDir(), "hasura-cli-test-"+strconv.Itoa(rand.Intn(1000))),
}, },
name: "create_article", name: "create_article",
flags: pflag.NewFlagSet("migrate-create-test", pflag.ContinueOnError),
} }
err := opts.run() err := opts.run()

View File

@ -230,5 +230,5 @@ func compareMetadata(t testing.TB, metadataFile string, actualData []byte) {
if err != nil { if err != nil {
t.Fatalf("error reading metadata %s", err) t.Fatalf("error reading metadata %s", err)
} }
assert.Equal(t, actualData, data) assert.Equal(t, string(actualData), string(data))
} }

View File

@ -25,7 +25,6 @@ type CreateOptions struct {
Version int64 Version int64
Directory string Directory string
Name string Name string
IsCMD bool
MetaUp []byte MetaUp []byte
MetaDown []byte MetaDown []byte
SQLUp []byte SQLUp []byte
@ -40,10 +39,6 @@ func New(version int64, name, directory string) *CreateOptions {
Version: version, Version: version,
Directory: directory, Directory: directory,
Name: name, Name: name,
MetaUp: []byte(`[]`),
MetaDown: []byte(`[]`),
SQLUp: []byte{},
SQLDown: []byte{},
} }
} }
@ -60,6 +55,29 @@ func (c *CreateOptions) SetMetaUp(data interface{}) error {
return nil return nil
} }
func (c *CreateOptions) SetMetaUpFromFile(filePath string) error {
data, err := ioutil.ReadFile(filePath)
if err != nil {
return err
}
var metadata []interface{}
var q interface{}
err = yaml.Unmarshal(data, &q)
if err != nil {
return err
}
metadata = append(
metadata,
map[string]interface{}{
"type": "replace_metadata",
"args": q,
},
)
return c.SetMetaUp(metadata)
}
func (c *CreateOptions) SetMetaDown(data interface{}) error { func (c *CreateOptions) SetMetaDown(data interface{}) error {
t, err := json.Marshal(data) t, err := json.Marshal(data)
if err != nil { if err != nil {
@ -78,6 +96,16 @@ func (c *CreateOptions) SetSQLUp(data string) error {
return nil return nil
} }
func (c *CreateOptions) SetSQLUpFromFile(filePath string) error {
data, err := ioutil.ReadFile(filePath)
if err != nil {
return err
}
c.SQLUp = data
return nil
}
func (c *CreateOptions) SetSQLDown(data string) error { func (c *CreateOptions) SetSQLDown(data string) error {
c.SQLDown = []byte(data) c.SQLDown = []byte(data)
return nil return nil
@ -90,22 +118,38 @@ func (c *CreateOptions) Create() error {
if err != nil { if err != nil {
return err return err
} }
// Check if data has been set in one of the files
if c.MetaUp == nil && c.MetaDown == nil && c.SQLUp == nil && c.SQLDown == nil {
return errors.New("none of the files has been set with data")
}
if c.MetaUp != nil {
// Create MetaUp // Create MetaUp
err = createFile(base+"up.yaml", c.MetaUp) err = createFile(base+"up.yaml", c.MetaUp)
if err != nil { if err != nil {
return err return err
} }
}
if c.MetaDown != nil {
// Create MetaDown // Create MetaDown
err = createFile(base+"down.yaml", c.MetaDown) err = createFile(base+"down.yaml", c.MetaDown)
if err != nil { if err != nil {
return err return err
} }
}
if c.IsCMD { if c.SQLUp != nil {
// Create SQLUp
err = createFile(base+"up.sql", c.SQLUp) err = createFile(base+"up.sql", c.SQLUp)
if err != nil { if err != nil {
return err return err
} }
}
if c.SQLDown != nil {
// Create SQLDown
err = createFile(base+"down.sql", c.SQLDown) err = createFile(base+"down.sql", c.SQLDown)
if err != nil { if err != nil {
return err return err

View File

@ -91,6 +91,8 @@ type Migrate struct {
isCMD bool isCMD bool
status *Status status *Status
SkipExecution bool
} }
// New returns a new Migrate instance from a source URL and a database URL. // New returns a new Migrate instance from a source URL and a database URL.
@ -801,9 +803,11 @@ func (m *Migrate) runMigrations(ret <-chan interface{}) error {
case *Migration: case *Migration:
migr := r.(*Migration) migr := r.(*Migration)
if migr.Body != nil { if migr.Body != nil {
if !m.SkipExecution {
if err := m.databaseDrv.Run(migr.BufferedBody, migr.FileType, migr.FileName); err != nil { if err := m.databaseDrv.Run(migr.BufferedBody, migr.FileType, migr.FileName); err != nil {
return err return err
} }
}
version := int64(migr.Version) version := int64(migr.Version)
if version == migr.TargetVersion { if version == migr.TargetVersion {

View File

@ -20,10 +20,11 @@ Options
:: ::
--admin-secret string admin secret key for Hasura GraphQL Engine --admin-secret string admin secret for Hasura GraphQL Engine
--down string apply all or N down migration steps --down string apply all or N down migration steps
--endpoint string http(s) endpoint for Hasura GraphQL Engine --endpoint string http(s) endpoint for Hasura GraphQL Engine
-h, --help help for apply -h, --help help for apply
--skip-execution skip executing the migration action, but mark them as applied
--type string type of migration (up, down) to be used with version flag (default "up") --type string type of migration (up, down) to be used with version flag (default "up")
--up string apply all or N up migration steps --up string apply all or N up migration steps
--version string migrate the database to a specific version --version string migrate the database to a specific version
@ -34,7 +35,8 @@ Options inherited from parent commands
:: ::
--log-level string log level (DEBUG, INFO, WARN, ERROR, FATAL) (default "INFO") --log-level string log level (DEBUG, INFO, WARN, ERROR, FATAL) (default "INFO")
--project string directory where commands are executed. (default: current dir) --project string directory where commands are executed (default: current dir)
--skip-update-check Skip automatic update check on command execution
SEE ALSO SEE ALSO
~~~~~~~~ ~~~~~~~~

View File

@ -20,7 +20,12 @@ Options
:: ::
--admin-secret string admin secret for Hasura GraphQL Engine
--endpoint string http(s) endpoint for Hasura GraphQL Engine
-h, --help help for create -h, --help help for create
--metadata-from-file string path to a hasura metadata file to be used for up actions
--metadata-from-server take metadata from the server and write it as an up migration file
--sql-from-file string path to an sql file which contains the up actions
Options inherited from parent commands Options inherited from parent commands
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -28,7 +33,8 @@ Options inherited from parent commands
:: ::
--log-level string log level (DEBUG, INFO, WARN, ERROR, FATAL) (default "INFO") --log-level string log level (DEBUG, INFO, WARN, ERROR, FATAL) (default "INFO")
--project string directory where commands are executed. (default: current dir) --project string directory where commands are executed (default: current dir)
--skip-update-check Skip automatic update check on command execution
SEE ALSO SEE ALSO
~~~~~~~~ ~~~~~~~~