From dece69c5a9341234ad406a59554d954673846563 Mon Sep 17 00:00:00 2001 From: Pradeep Murugesan Date: Wed, 25 Dec 2019 08:33:06 +0000 Subject: [PATCH] cli: add commands to manage inconsistent metadata (close #2766) (#2973) Co-authored-by: Aravind Shankar Co-authored-by: Shahidh K Muhammed --- cli/commands/metadata.go | 3 + cli/commands/metadata_inconsistency.go | 22 ++++ cli/commands/metadata_inconsistency_drop.go | 60 +++++++++ .../metadata_inconsistency_drop_test.go | 45 +++++++ cli/commands/metadata_inconsistency_list.go | 105 +++++++++++++++ cli/commands/metadata_inconsistency_status.go | 52 ++++++++ cli/commands/migrate_test.go | 4 +- cli/migrate/database/hasuradb/metadata.go | 62 +++++++++ cli/migrate/database/hasuradb/types.go | 121 +++++++++++++++++- cli/migrate/database/metadata.go | 11 ++ cli/migrate/migrate.go | 8 ++ 11 files changed, 489 insertions(+), 4 deletions(-) create mode 100644 cli/commands/metadata_inconsistency.go create mode 100644 cli/commands/metadata_inconsistency_drop.go create mode 100644 cli/commands/metadata_inconsistency_drop_test.go create mode 100644 cli/commands/metadata_inconsistency_list.go create mode 100644 cli/commands/metadata_inconsistency_status.go diff --git a/cli/commands/metadata.go b/cli/commands/metadata.go index 86eb8d8f2dd..2cb3508dbf0 100644 --- a/cli/commands/metadata.go +++ b/cli/commands/metadata.go @@ -12,9 +12,11 @@ import ( v2yaml "gopkg.in/yaml.v2" ) +// NewMetadataCmd returns the metadata command func NewMetadataCmd(ec *cli.ExecutionContext) *cobra.Command { metadataCmd := &cobra.Command{ Use: "metadata", + Aliases: []string{"md"}, Short: "Manage Hasura GraphQL Engine metadata saved in the database", SilenceUsage: true, } @@ -24,6 +26,7 @@ func NewMetadataCmd(ec *cli.ExecutionContext) *cobra.Command { newMetadataClearCmd(ec), newMetadataReloadCmd(ec), newMetadataApplyCmd(ec), + newMetadataInconsistencyCmd(ec), ) return metadataCmd } diff --git a/cli/commands/metadata_inconsistency.go b/cli/commands/metadata_inconsistency.go new file mode 100644 index 00000000000..0c06e7487b9 --- /dev/null +++ b/cli/commands/metadata_inconsistency.go @@ -0,0 +1,22 @@ +package commands + +import ( + "github.com/hasura/graphql-engine/cli" + "github.com/spf13/cobra" +) + +func newMetadataInconsistencyCmd(ec *cli.ExecutionContext) *cobra.Command { + metadataInconsistencyCmd := &cobra.Command{ + Use: "inconsistency", + Short: "Manage inconsistent objects in Hasura Metadata", + Aliases: []string{"inconsistencies", "ic"}, + SilenceUsage: true, + } + + metadataInconsistencyCmd.AddCommand( + newMetadataInconsistencyListCmd(ec), + newMetadataInconsistencyDropCmd(ec), + newMetadataInconsistencyStatusCmd(ec), + ) + return metadataInconsistencyCmd +} diff --git a/cli/commands/metadata_inconsistency_drop.go b/cli/commands/metadata_inconsistency_drop.go new file mode 100644 index 00000000000..dda9f512a50 --- /dev/null +++ b/cli/commands/metadata_inconsistency_drop.go @@ -0,0 +1,60 @@ +package commands + +import ( + "github.com/hasura/graphql-engine/cli" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +func newMetadataInconsistencyDropCmd(ec *cli.ExecutionContext) *cobra.Command { + v := viper.New() + opts := &metadataInconsistencyDropOptions{ + EC: ec, + } + + metadataInconsistencyDropCmd := &cobra.Command{ + Use: "drop", + Short: "Drop inconsistent objects from the metadata", + SilenceUsage: true, + PreRunE: func(cmd *cobra.Command, args []string) error { + ec.Viper = v + return ec.Validate() + }, + RunE: func(cmd *cobra.Command, args []string) error { + opts.EC.Spin("Dropping inconsistent metadata...") + err := opts.run() + opts.EC.Spinner.Stop() + if err != nil { + return errors.Wrap(err, "failed to drop inconsistent metadata") + } + opts.EC.Logger.Info("all inconsistent objects removed from metadata") + return nil + }, + } + + f := metadataInconsistencyDropCmd.Flags() + f.String("endpoint", "", "http(s) endpoint for Hasura GraphQL Engine") + f.String("admin-secret", "", "admin secret for Hasura GraphQL Engine") + f.String("access-key", "", "access key for Hasura GraphQL Engine") + f.MarkDeprecated("access-key", "use --admin-secret instead") + + // need to create a new viper because https://github.com/spf13/viper/issues/233 + v.BindPFlag("endpoint", f.Lookup("endpoint")) + v.BindPFlag("admin_secret", f.Lookup("admin-secret")) + v.BindPFlag("access_key", f.Lookup("access-key")) + + return metadataInconsistencyDropCmd +} + +type metadataInconsistencyDropOptions struct { + EC *cli.ExecutionContext +} + +func (o *metadataInconsistencyDropOptions) run() error { + d, err := newMigrate(o.EC.MigrationDir, o.EC.ServerConfig.ParsedEndpoint, o.EC.ServerConfig.AdminSecret, o.EC.Logger, o.EC.Version, true) + if err != nil { + return err + } + return d.DropInconsistentMetadata() +} diff --git a/cli/commands/metadata_inconsistency_drop_test.go b/cli/commands/metadata_inconsistency_drop_test.go new file mode 100644 index 00000000000..69669bdc10c --- /dev/null +++ b/cli/commands/metadata_inconsistency_drop_test.go @@ -0,0 +1,45 @@ +package commands + +import ( + "net/url" + "os" + "testing" + "time" + + "github.com/briandowns/spinner" + "github.com/sirupsen/logrus/hooks/test" + + "github.com/hasura/graphql-engine/cli" + "github.com/hasura/graphql-engine/cli/version" +) + +func testMetadataInconsistencyDropCmd(t *testing.T, migrationsDir string, metadataFile string, endpoint *url.URL) { + logger, _ := test.NewNullLogger() + opts := &metadataInconsistencyDropOptions{ + EC: &cli.ExecutionContext{ + Logger: logger, + Spinner: spinner.New(spinner.CharSets[7], 100*time.Millisecond), + MetadataFile: []string{metadataFile}, + ServerConfig: &cli.ServerConfig{ + Endpoint: endpoint.String(), + AdminSecret: os.Getenv("HASURA_GRAPHQL_TEST_ADMIN_SECRET"), + ParsedEndpoint: endpoint, + }, + MigrationDir: migrationsDir, + }, + } + + opts.EC.Version = version.New() + v, err := version.FetchServerVersion(opts.EC.ServerConfig.Endpoint) + if err != nil { + t.Fatalf("getting server version failed: %v", err) + } + opts.EC.Version.SetServerVersion(v) + + err = opts.run() + if err != nil { + t.Fatalf("failed dropping the inconsistency: %v", err) + } + + os.RemoveAll(opts.EC.MigrationDir) +} diff --git a/cli/commands/metadata_inconsistency_list.go b/cli/commands/metadata_inconsistency_list.go new file mode 100644 index 00000000000..e964c52d4ac --- /dev/null +++ b/cli/commands/metadata_inconsistency_list.go @@ -0,0 +1,105 @@ +package commands + +import ( + "bytes" + "fmt" + "text/tabwriter" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/hasura/graphql-engine/cli" + "github.com/hasura/graphql-engine/cli/migrate/database" + "github.com/hasura/graphql-engine/cli/util" +) + +func newMetadataInconsistencyListCmd(ec *cli.ExecutionContext) *cobra.Command { + v := viper.New() + opts := &metadataInconsistencyListOptions{ + EC: ec, + } + + metadataInconsistencyListCmd := &cobra.Command{ + Use: "list", + Aliases: []string{"ls"}, + Short: "List all inconsistent objects from the metadata", + SilenceUsage: true, + PreRunE: func(cmd *cobra.Command, args []string) error { + ec.Viper = v + return ec.Validate() + }, + RunE: func(cmd *cobra.Command, args []string) error { + err := opts.run() + opts.EC.Spinner.Stop() + if err != nil { + return errors.Wrap(err, "failed to list inconsistent metadata") + } + if opts.isConsistent { + opts.EC.Logger.Println("metadata is consistent") + } + return nil + }, + } + + f := metadataInconsistencyListCmd.Flags() + f.String("endpoint", "", "http(s) endpoint for Hasura GraphQL Engine") + f.String("admin-secret", "", "admin secret for Hasura GraphQL Engine") + f.String("access-key", "", "access key for Hasura GraphQL Engine") + f.MarkDeprecated("access-key", "use --admin-secret instead") + + // need to create a new viper because https://github.com/spf13/viper/issues/233 + v.BindPFlag("endpoint", f.Lookup("endpoint")) + v.BindPFlag("admin_secret", f.Lookup("admin-secret")) + v.BindPFlag("access_key", f.Lookup("access-key")) + + return metadataInconsistencyListCmd +} + +type metadataInconsistencyListOptions struct { + EC *cli.ExecutionContext + + isConsistent bool + inconsistentObjects []database.InconsistentMetadataInterface +} + +func (o *metadataInconsistencyListOptions) read() error { + d, err := newMigrate(o.EC.MigrationDir, o.EC.ServerConfig.ParsedEndpoint, o.EC.ServerConfig.AdminSecret, o.EC.Logger, o.EC.Version, true) + if err != nil { + return err + } + o.isConsistent, o.inconsistentObjects, err = d.GetInconsistentMetadata() + if err != nil { + return err + } + return nil +} + +func (o *metadataInconsistencyListOptions) run() error { + o.EC.Spin("Getting inconsistent metadata...") + + err := o.read() + if err != nil { + return err + } + if o.isConsistent { + return nil + } + out := new(tabwriter.Writer) + buf := &bytes.Buffer{} + out.Init(buf, 0, 8, 2, ' ', 0) + w := util.NewPrefixWriter(out) + w.Write(util.LEVEL_0, "NAME\tTYPE\tDESCRIPTION\tREASON\n") + for _, obj := range o.inconsistentObjects { + w.Write(util.LEVEL_0, "%s\t%s\t%s\t%s\n", + obj.GetName(), + obj.GetType(), + obj.GetDescription(), + obj.GetReason(), + ) + } + out.Flush() + o.EC.Spinner.Stop() + fmt.Println(buf.String()) + return nil +} diff --git a/cli/commands/metadata_inconsistency_status.go b/cli/commands/metadata_inconsistency_status.go new file mode 100644 index 00000000000..b9a3b26ed3d --- /dev/null +++ b/cli/commands/metadata_inconsistency_status.go @@ -0,0 +1,52 @@ +package commands + +import ( + "github.com/hasura/graphql-engine/cli" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +func newMetadataInconsistencyStatusCmd(ec *cli.ExecutionContext) *cobra.Command { + v := viper.New() + opts := &metadataInconsistencyListOptions{ + EC: ec, + } + + metadataInconsistencyStatusCmd := &cobra.Command{ + Use: "status", + Short: "Check if the metadata is inconsistent or not", + SilenceUsage: true, + PreRunE: func(cmd *cobra.Command, args []string) error { + ec.Viper = v + return ec.Validate() + }, + RunE: func(cmd *cobra.Command, args []string) error { + opts.EC.Spin("reading metadata status...") + err := opts.read() + opts.EC.Spinner.Stop() + if err != nil { + return errors.Wrap(err, "failed to read metadata status") + } + if opts.isConsistent { + opts.EC.Logger.Println("metadata is consistent") + } else { + return errors.New("metadata is inconsistent, use list command to see the objects") + } + return nil + }, + } + + f := metadataInconsistencyStatusCmd.Flags() + f.String("endpoint", "", "http(s) endpoint for Hasura GraphQL Engine") + f.String("admin-secret", "", "admin secret for Hasura GraphQL Engine") + f.String("access-key", "", "access key for Hasura GraphQL Engine") + f.MarkDeprecated("access-key", "use --admin-secret instead") + + // need to create a new viper because https://github.com/spf13/viper/issues/233 + v.BindPFlag("endpoint", f.Lookup("endpoint")) + v.BindPFlag("admin_secret", f.Lookup("admin-secret")) + v.BindPFlag("access_key", f.Lookup("access-key")) + + return metadataInconsistencyStatusCmd +} diff --git a/cli/commands/migrate_test.go b/cli/commands/migrate_test.go index 1b77328aaee..454dadb2b18 100644 --- a/cli/commands/migrate_test.go +++ b/cli/commands/migrate_test.go @@ -4,6 +4,7 @@ import ( "database/sql" sqldriver "database/sql/driver" "fmt" + "github.com/hasura/graphql-engine/cli/migrate" "io" "io/ioutil" "net/url" @@ -14,7 +15,6 @@ import ( "github.com/Masterminds/semver" - "github.com/hasura/graphql-engine/cli/migrate" mt "github.com/hasura/graphql-engine/cli/migrate/testing" "github.com/hasura/graphql-engine/cli/version" _ "github.com/lib/pq" @@ -262,6 +262,8 @@ func testMigrate(t *testing.T, endpoint *url.URL, migrationsDir string) { testMetadataReset(t, metadataFile, endpoint) testMetadataExport(t, metadataFile, endpoint) compareMetadata(t, metadataFile, "empty-metadata", versionCtx.ServerSemver) + + testMetadataInconsistencyDropCmd(t, migrationsDir, metadataFile, endpoint) } func mustWriteFile(t testing.TB, dir, file string, body string) { diff --git a/cli/migrate/database/hasuradb/metadata.go b/cli/migrate/database/hasuradb/metadata.go index d651f103060..54ee5d13cd1 100644 --- a/cli/migrate/database/hasuradb/metadata.go +++ b/cli/migrate/database/hasuradb/metadata.go @@ -4,6 +4,8 @@ import ( "encoding/json" "net/http" + "github.com/hasura/graphql-engine/cli/migrate/database" + "github.com/oliveagle/jsonpath" v2yaml "gopkg.in/yaml.v2" ) @@ -91,6 +93,66 @@ func (h *HasuraDB) ReloadMetadata() error { return nil } +func (h *HasuraDB) GetInconsistentMetadata() (bool, []database.InconsistentMetadataInterface, error) { + query := HasuraInterfaceQuery{ + Type: "get_inconsistent_metadata", + Args: HasuraArgs{}, + } + + resp, body, err := h.sendv1Query(query) + if err != nil { + h.logger.Debug(err) + return false, nil, err + } + h.logger.Debug("response: ", string(body)) + + var horror HasuraError + if resp.StatusCode != http.StatusOK { + err = json.Unmarshal(body, &horror) + if err != nil { + h.logger.Debug(err) + return false, nil, err + } + return false, nil, horror.Error(h.config.isCMD) + } + + var inMet InconsistentMetadata + err = json.Unmarshal(body, &inMet) + if err != nil { + return false, nil, err + } + inMetInterface := make([]database.InconsistentMetadataInterface, 0) + for _, obj := range inMet.InConsistentObjects { + inMetInterface = append(inMetInterface, database.InconsistentMetadataInterface(obj)) + } + return inMet.IsConsistent, inMetInterface, nil +} + +func (h *HasuraDB) DropInconsistentMetadata() error { + query := HasuraInterfaceQuery{ + Type: "drop_inconsistent_metadata", + Args: HasuraArgs{}, + } + + resp, body, err := h.sendv1Query(query) + if err != nil { + h.logger.Debug(err) + return err + } + h.logger.Debug("response: ", string(body)) + + var horror HasuraError + if resp.StatusCode != http.StatusOK { + err = json.Unmarshal(body, &horror) + if err != nil { + h.logger.Debug(err) + return err + } + return horror.Error(h.config.isCMD) + } + return nil +} + func (h *HasuraDB) ApplyMetadata(data interface{}) error { query := HasuraInterfaceBulk{ Type: "bulk", diff --git a/cli/migrate/database/hasuradb/types.go b/cli/migrate/database/hasuradb/types.go index 5f50f07ffb1..d77a979fc0e 100644 --- a/cli/migrate/database/hasuradb/types.go +++ b/cli/migrate/database/hasuradb/types.go @@ -523,9 +523,9 @@ type deleteEventTriggerInput struct { } type addRemoteSchemaInput struct { - Name string `json:"name" yaml:"name"` - Definition interface{} `json:"definition" yaml:"definition"` - Comment *string `json:"comment,omitempty" yaml:"comment,omitempty"` + Name string `json:"name" yaml:"name"` + Definition map[string]interface{} `json:"definition" yaml:"definition"` + Comment *string `json:"comment,omitempty" yaml:"comment,omitempty"` } type removeRemoteSchemaInput struct { @@ -727,6 +727,121 @@ func (rmi *replaceMetadataInput) convertToMetadataActions(l *database.CustomList } } +type InconsistentMetadata struct { + IsConsistent bool `json:"is_consistent"` + InConsistentObjects []InconsistentMeatadataObject `json:"inconsistent_objects"` +} + +type InconsistentMeatadataObject struct { + Type string `json:"type"` + Reason string `json:"reason"` + Definition interface{} `json:"definition"` +} + +func (i *InconsistentMeatadataObject) UnmarshalJSON(b []byte) error { + type t InconsistentMeatadataObject + var q t + if err := json.Unmarshal(b, &q); err != nil { + return err + } + defBody, err := json.Marshal(q.Definition) + if err != nil { + return err + } + switch q.Type { + case "object_relation": + q.Definition = &createObjectRelationshipInput{} + case "array_relation": + q.Definition = &createArrayRelationshipInput{} + case "select_permission": + q.Definition = &createSelectPermissionInput{} + case "update_permission": + q.Definition = &createUpdatePermissionInput{} + case "insert_permission": + q.Definition = &createInsertPermissionInput{} + case "delete_permission": + q.Definition = &createDeletePermissionInput{} + case "table": + q.Definition = &trackTableInput{} + case "function": + q.Definition = &trackFunctionInput{} + case "event_trigger": + q.Definition = &createEventTriggerInput{} + case "remote_schema": + q.Definition = &addRemoteSchemaInput{} + } + if err := json.Unmarshal(defBody, &q.Definition); err != nil { + return err + } + *i = InconsistentMeatadataObject(q) + return nil +} + +func (i InconsistentMeatadataObject) GetType() string { + return i.Type +} + +func (i InconsistentMeatadataObject) GetName() string { + switch defType := i.Definition.(type) { + case *createObjectRelationshipInput: + return defType.Name + case *createArrayRelationshipInput: + return defType.Name + case *createSelectPermissionInput: + return fmt.Sprintf("%s-permission", defType.Role) + case *createUpdatePermissionInput: + return fmt.Sprintf("%s-permission", defType.Role) + case *createInsertPermissionInput: + return fmt.Sprintf("%s-permission", defType.Role) + case *createDeletePermissionInput: + return fmt.Sprintf("%s-permission", defType.Role) + case *trackTableInput: + return defType.Name + case *trackFunctionInput: + return defType.Name + case *createEventTriggerInput: + return defType.Name + case *addRemoteSchemaInput: + return defType.Name + } + return "N/A" +} + +func (i InconsistentMeatadataObject) GetDescription() string { + switch defType := i.Definition.(type) { + case *createObjectRelationshipInput: + return fmt.Sprintf("relationship of table %s in %s schema", defType.Table.Name, defType.Table.Schema) + case *createArrayRelationshipInput: + return fmt.Sprintf("relationship of table %s in %s schema", defType.Table.Name, defType.Table.Schema) + case *createSelectPermissionInput: + return fmt.Sprintf("%s on table %s in %s schema", i.Type, defType.Table.Name, defType.Table.Schema) + case *createUpdatePermissionInput: + return fmt.Sprintf("%s on table %s in %s schema", i.Type, defType.Table.Name, defType.Table.Schema) + case *createInsertPermissionInput: + return fmt.Sprintf("%s on table %s in %s schema", i.Type, defType.Table.Name, defType.Table.Schema) + case *createDeletePermissionInput: + return fmt.Sprintf("%s on table %s in %s schema", i.Type, defType.Table.Name, defType.Table.Schema) + case *trackTableInput: + return fmt.Sprintf("table %s in %s schema", defType.tableSchema.Name, defType.tableSchema.Schema) + case *trackFunctionInput: + return fmt.Sprintf("function %s in %s schema", defType.Name, defType.Schema) + case *createEventTriggerInput: + return fmt.Sprintf("event trigger %s on table %s in %s schema", defType.Name, defType.Table.Name, defType.Table.Schema) + case *addRemoteSchemaInput: + url := defType.Definition["url"] + urlFromEnv, ok := defType.Definition["url_from_env"] + if ok { + url = fmt.Sprintf("the url from the value of env var %s", urlFromEnv) + } + return fmt.Sprintf("remote schema %s at %s", defType.Name, url) + } + return "N/A" +} + +func (i InconsistentMeatadataObject) GetReason() string { + return i.Reason +} + type runSQLInput struct { SQL string `json:"sql" yaml:"sql"` } diff --git a/cli/migrate/database/metadata.go b/cli/migrate/database/metadata.go index 598d362c581..6753dc00dbd 100644 --- a/cli/migrate/database/metadata.go +++ b/cli/migrate/database/metadata.go @@ -7,7 +7,18 @@ type MetadataDriver interface { ReloadMetadata() error + GetInconsistentMetadata() (bool, []InconsistentMetadataInterface, error) + + DropInconsistentMetadata() error + ApplyMetadata(data interface{}) error Query(data []interface{}) error } + +type InconsistentMetadataInterface interface { + GetType() string + GetName() string + GetDescription() string + GetReason() string +} diff --git a/cli/migrate/migrate.go b/cli/migrate/migrate.go index 60b512ba2cd..118cf2db160 100644 --- a/cli/migrate/migrate.go +++ b/cli/migrate/migrate.go @@ -320,6 +320,14 @@ func (m *Migrate) ReloadMetadata() error { return m.databaseDrv.ReloadMetadata() } +func (m *Migrate) GetInconsistentMetadata() (bool, []database.InconsistentMetadataInterface, error) { + return m.databaseDrv.GetInconsistentMetadata() +} + +func (m *Migrate) DropInconsistentMetadata() error { + return m.databaseDrv.DropInconsistentMetadata() +} + func (m *Migrate) ApplyMetadata(data interface{}) error { return m.databaseDrv.ApplyMetadata(data) }