cli: add commands to manage inconsistent metadata (close #2766) (#2973)

Co-authored-by: Aravind Shankar <face11301@gmail.com>
Co-authored-by: Shahidh K Muhammed <muhammedshahid.k@gmail.com>

cli/commands/metadata.go

@@ -12,9 +12,11 @@ import (
 	v2yaml "gopkg.in/yaml.v2"
 )
 
+// NewMetadataCmd returns the metadata command
 func NewMetadataCmd(ec *cli.ExecutionContext) *cobra.Command {
 	metadataCmd := &cobra.Command{
 		Use:          "metadata",
+		Aliases:      []string{"md"},
 		Short:        "Manage Hasura GraphQL Engine metadata saved in the database",
 		SilenceUsage: true,
 	}
@@ -24,6 +26,7 @@ func NewMetadataCmd(ec *cli.ExecutionContext) *cobra.Command {
 		newMetadataClearCmd(ec),
 		newMetadataReloadCmd(ec),
 		newMetadataApplyCmd(ec),
+		newMetadataInconsistencyCmd(ec),
 	)
 	return metadataCmd
 }

cli/commands/metadata_inconsistency.go

@@ -0,0 +1,22 @@
+package commands
+
+import (
+	"github.com/hasura/graphql-engine/cli"
+	"github.com/spf13/cobra"
+)
+
+func newMetadataInconsistencyCmd(ec *cli.ExecutionContext) *cobra.Command {
+	metadataInconsistencyCmd := &cobra.Command{
+		Use:          "inconsistency",
+		Short:        "Manage inconsistent objects in Hasura Metadata",
+		Aliases:      []string{"inconsistencies", "ic"},
+		SilenceUsage: true,
+	}
+
+	metadataInconsistencyCmd.AddCommand(
+		newMetadataInconsistencyListCmd(ec),
+		newMetadataInconsistencyDropCmd(ec),
+		newMetadataInconsistencyStatusCmd(ec),
+	)
+	return metadataInconsistencyCmd
+}

cli/commands/metadata_inconsistency_drop.go

@@ -0,0 +1,60 @@
+package commands
+
+import (
+	"github.com/hasura/graphql-engine/cli"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func newMetadataInconsistencyDropCmd(ec *cli.ExecutionContext) *cobra.Command {
+	v := viper.New()
+	opts := &metadataInconsistencyDropOptions{
+		EC: ec,
+	}
+
+	metadataInconsistencyDropCmd := &cobra.Command{
+		Use:          "drop",
+		Short:        "Drop inconsistent objects from the metadata",
+		SilenceUsage: true,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			ec.Viper = v
+			return ec.Validate()
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			opts.EC.Spin("Dropping inconsistent metadata...")
+			err := opts.run()
+			opts.EC.Spinner.Stop()
+			if err != nil {
+				return errors.Wrap(err, "failed to drop inconsistent metadata")
+			}
+			opts.EC.Logger.Info("all inconsistent objects removed from metadata")
+			return nil
+		},
+	}
+
+	f := metadataInconsistencyDropCmd.Flags()
+	f.String("endpoint", "", "http(s) endpoint for Hasura GraphQL Engine")
+	f.String("admin-secret", "", "admin secret for Hasura GraphQL Engine")
+	f.String("access-key", "", "access key for Hasura GraphQL Engine")
+	f.MarkDeprecated("access-key", "use --admin-secret instead")
+
+	// need to create a new viper because https://github.com/spf13/viper/issues/233
+	v.BindPFlag("endpoint", f.Lookup("endpoint"))
+	v.BindPFlag("admin_secret", f.Lookup("admin-secret"))
+	v.BindPFlag("access_key", f.Lookup("access-key"))
+
+	return metadataInconsistencyDropCmd
+}
+
+type metadataInconsistencyDropOptions struct {
+	EC *cli.ExecutionContext
+}
+
+func (o *metadataInconsistencyDropOptions) run() error {
+	d, err := newMigrate(o.EC.MigrationDir, o.EC.ServerConfig.ParsedEndpoint, o.EC.ServerConfig.AdminSecret, o.EC.Logger, o.EC.Version, true)
+	if err != nil {
+		return err
+	}
+	return d.DropInconsistentMetadata()
+}

cli/commands/metadata_inconsistency_drop_test.go

@@ -0,0 +1,45 @@
+package commands
+
+import (
+	"net/url"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/briandowns/spinner"
+	"github.com/sirupsen/logrus/hooks/test"
+
+	"github.com/hasura/graphql-engine/cli"
+	"github.com/hasura/graphql-engine/cli/version"
+)
+
+func testMetadataInconsistencyDropCmd(t *testing.T, migrationsDir string, metadataFile string, endpoint *url.URL) {
+	logger, _ := test.NewNullLogger()
+	opts := &metadataInconsistencyDropOptions{
+		EC: &cli.ExecutionContext{
+			Logger:       logger,
+			Spinner:      spinner.New(spinner.CharSets[7], 100*time.Millisecond),
+			MetadataFile: []string{metadataFile},
+			ServerConfig: &cli.ServerConfig{
+				Endpoint:       endpoint.String(),
+				AdminSecret:    os.Getenv("HASURA_GRAPHQL_TEST_ADMIN_SECRET"),
+				ParsedEndpoint: endpoint,
+			},
+			MigrationDir: migrationsDir,
+		},
+	}
+
+	opts.EC.Version = version.New()
+	v, err := version.FetchServerVersion(opts.EC.ServerConfig.Endpoint)
+	if err != nil {
+		t.Fatalf("getting server version failed: %v", err)
+	}
+	opts.EC.Version.SetServerVersion(v)
+
+	err = opts.run()
+	if err != nil {
+		t.Fatalf("failed dropping the inconsistency: %v", err)
+	}
+
+	os.RemoveAll(opts.EC.MigrationDir)
+}

cli/commands/metadata_inconsistency_list.go

@@ -0,0 +1,105 @@
+package commands
+
+import (
+	"bytes"
+	"fmt"
+	"text/tabwriter"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/hasura/graphql-engine/cli"
+	"github.com/hasura/graphql-engine/cli/migrate/database"
+	"github.com/hasura/graphql-engine/cli/util"
+)
+
+func newMetadataInconsistencyListCmd(ec *cli.ExecutionContext) *cobra.Command {
+	v := viper.New()
+	opts := &metadataInconsistencyListOptions{
+		EC: ec,
+	}
+
+	metadataInconsistencyListCmd := &cobra.Command{
+		Use:          "list",
+		Aliases:      []string{"ls"},
+		Short:        "List all inconsistent objects from the metadata",
+		SilenceUsage: true,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			ec.Viper = v
+			return ec.Validate()
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			err := opts.run()
+			opts.EC.Spinner.Stop()
+			if err != nil {
+				return errors.Wrap(err, "failed to list inconsistent metadata")
+			}
+			if opts.isConsistent {
+				opts.EC.Logger.Println("metadata is consistent")
+			}
+			return nil
+		},
+	}
+
+	f := metadataInconsistencyListCmd.Flags()
+	f.String("endpoint", "", "http(s) endpoint for Hasura GraphQL Engine")
+	f.String("admin-secret", "", "admin secret for Hasura GraphQL Engine")
+	f.String("access-key", "", "access key for Hasura GraphQL Engine")
+	f.MarkDeprecated("access-key", "use --admin-secret instead")
+
+	// need to create a new viper because https://github.com/spf13/viper/issues/233
+	v.BindPFlag("endpoint", f.Lookup("endpoint"))
+	v.BindPFlag("admin_secret", f.Lookup("admin-secret"))
+	v.BindPFlag("access_key", f.Lookup("access-key"))
+
+	return metadataInconsistencyListCmd
+}
+
+type metadataInconsistencyListOptions struct {
+	EC *cli.ExecutionContext
+
+	isConsistent        bool
+	inconsistentObjects []database.InconsistentMetadataInterface
+}
+
+func (o *metadataInconsistencyListOptions) read() error {
+	d, err := newMigrate(o.EC.MigrationDir, o.EC.ServerConfig.ParsedEndpoint, o.EC.ServerConfig.AdminSecret, o.EC.Logger, o.EC.Version, true)
+	if err != nil {
+		return err
+	}
+	o.isConsistent, o.inconsistentObjects, err = d.GetInconsistentMetadata()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (o *metadataInconsistencyListOptions) run() error {
+	o.EC.Spin("Getting inconsistent metadata...")
+
+	err := o.read()
+	if err != nil {
+		return err
+	}
+	if o.isConsistent {
+		return nil
+	}
+	out := new(tabwriter.Writer)
+	buf := &bytes.Buffer{}
+	out.Init(buf, 0, 8, 2, ' ', 0)
+	w := util.NewPrefixWriter(out)
+	w.Write(util.LEVEL_0, "NAME\tTYPE\tDESCRIPTION\tREASON\n")
+	for _, obj := range o.inconsistentObjects {
+		w.Write(util.LEVEL_0, "%s\t%s\t%s\t%s\n",
+			obj.GetName(),
+			obj.GetType(),
+			obj.GetDescription(),
+			obj.GetReason(),
+		)
+	}
+	out.Flush()
+	o.EC.Spinner.Stop()
+	fmt.Println(buf.String())
+	return nil
+}

cli/commands/metadata_inconsistency_status.go

@@ -0,0 +1,52 @@
+package commands
+
+import (
+	"github.com/hasura/graphql-engine/cli"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func newMetadataInconsistencyStatusCmd(ec *cli.ExecutionContext) *cobra.Command {
+	v := viper.New()
+	opts := &metadataInconsistencyListOptions{
+		EC: ec,
+	}
+
+	metadataInconsistencyStatusCmd := &cobra.Command{
+		Use:          "status",
+		Short:        "Check if the metadata is inconsistent or not",
+		SilenceUsage: true,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			ec.Viper = v
+			return ec.Validate()
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			opts.EC.Spin("reading metadata status...")
+			err := opts.read()
+			opts.EC.Spinner.Stop()
+			if err != nil {
+				return errors.Wrap(err, "failed to read metadata status")
+			}
+			if opts.isConsistent {
+				opts.EC.Logger.Println("metadata is consistent")
+			} else {
+				return errors.New("metadata is inconsistent, use list command to see the objects")
+			}
+			return nil
+		},
+	}
+
+	f := metadataInconsistencyStatusCmd.Flags()
+	f.String("endpoint", "", "http(s) endpoint for Hasura GraphQL Engine")
+	f.String("admin-secret", "", "admin secret for Hasura GraphQL Engine")
+	f.String("access-key", "", "access key for Hasura GraphQL Engine")
+	f.MarkDeprecated("access-key", "use --admin-secret instead")
+
+	// need to create a new viper because https://github.com/spf13/viper/issues/233
+	v.BindPFlag("endpoint", f.Lookup("endpoint"))
+	v.BindPFlag("admin_secret", f.Lookup("admin-secret"))
+	v.BindPFlag("access_key", f.Lookup("access-key"))
+
+	return metadataInconsistencyStatusCmd
+}

cli/commands/migrate_test.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	sqldriver "database/sql/driver"
 	"fmt"
+	"github.com/hasura/graphql-engine/cli/migrate"
 	"io"
 	"io/ioutil"
 	"net/url"
@@ -14,7 +15,6 @@ import (
 
 	"github.com/Masterminds/semver"
 
-	"github.com/hasura/graphql-engine/cli/migrate"
 	mt "github.com/hasura/graphql-engine/cli/migrate/testing"
 	"github.com/hasura/graphql-engine/cli/version"
 	_ "github.com/lib/pq"
@@ -262,6 +262,8 @@ func testMigrate(t *testing.T, endpoint *url.URL, migrationsDir string) {
 	testMetadataReset(t, metadataFile, endpoint)
 	testMetadataExport(t, metadataFile, endpoint)
 	compareMetadata(t, metadataFile, "empty-metadata", versionCtx.ServerSemver)
+
+	testMetadataInconsistencyDropCmd(t, migrationsDir, metadataFile, endpoint)
 }
 
 func mustWriteFile(t testing.TB, dir, file string, body string) {

cli/migrate/database/hasuradb/metadata.go

@@ -4,6 +4,8 @@ import (
 	"encoding/json"
 	"net/http"
 
+	"github.com/hasura/graphql-engine/cli/migrate/database"
+
 	"github.com/oliveagle/jsonpath"
 	v2yaml "gopkg.in/yaml.v2"
 )
@@ -91,6 +93,66 @@ func (h *HasuraDB) ReloadMetadata() error {
 	return nil
 }
 
+func (h *HasuraDB) GetInconsistentMetadata() (bool, []database.InconsistentMetadataInterface, error) {
+	query := HasuraInterfaceQuery{
+		Type: "get_inconsistent_metadata",
+		Args: HasuraArgs{},
+	}
+
+	resp, body, err := h.sendv1Query(query)
+	if err != nil {
+		h.logger.Debug(err)
+		return false, nil, err
+	}
+	h.logger.Debug("response: ", string(body))
+
+	var horror HasuraError
+	if resp.StatusCode != http.StatusOK {
+		err = json.Unmarshal(body, &horror)
+		if err != nil {
+			h.logger.Debug(err)
+			return false, nil, err
+		}
+		return false, nil, horror.Error(h.config.isCMD)
+	}
+
+	var inMet InconsistentMetadata
+	err = json.Unmarshal(body, &inMet)
+	if err != nil {
+		return false, nil, err
+	}
+	inMetInterface := make([]database.InconsistentMetadataInterface, 0)
+	for _, obj := range inMet.InConsistentObjects {
+		inMetInterface = append(inMetInterface, database.InconsistentMetadataInterface(obj))
+	}
+	return inMet.IsConsistent, inMetInterface, nil
+}
+
+func (h *HasuraDB) DropInconsistentMetadata() error {
+	query := HasuraInterfaceQuery{
+		Type: "drop_inconsistent_metadata",
+		Args: HasuraArgs{},
+	}
+
+	resp, body, err := h.sendv1Query(query)
+	if err != nil {
+		h.logger.Debug(err)
+		return err
+	}
+	h.logger.Debug("response: ", string(body))
+
+	var horror HasuraError
+	if resp.StatusCode != http.StatusOK {
+		err = json.Unmarshal(body, &horror)
+		if err != nil {
+			h.logger.Debug(err)
+			return err
+		}
+		return horror.Error(h.config.isCMD)
+	}
+	return nil
+}
+
 func (h *HasuraDB) ApplyMetadata(data interface{}) error {
 	query := HasuraInterfaceBulk{
 		Type: "bulk",

cli/migrate/database/hasuradb/types.go

@@ -523,9 +523,9 @@ type deleteEventTriggerInput struct {
 }
 
 type addRemoteSchemaInput struct {
-	Name       string      `json:"name" yaml:"name"`
-	Definition interface{} `json:"definition" yaml:"definition"`
-	Comment    *string     `json:"comment,omitempty" yaml:"comment,omitempty"`
+	Name       string                 `json:"name" yaml:"name"`
+	Definition map[string]interface{} `json:"definition" yaml:"definition"`
+	Comment    *string                `json:"comment,omitempty" yaml:"comment,omitempty"`
 }
 
 type removeRemoteSchemaInput struct {
@@ -727,6 +727,121 @@ func (rmi *replaceMetadataInput) convertToMetadataActions(l *database.CustomList
 	}
 }
 
+type InconsistentMetadata struct {
+	IsConsistent        bool                          `json:"is_consistent"`
+	InConsistentObjects []InconsistentMeatadataObject `json:"inconsistent_objects"`
+}
+
+type InconsistentMeatadataObject struct {
+	Type       string      `json:"type"`
+	Reason     string      `json:"reason"`
+	Definition interface{} `json:"definition"`
+}
+
+func (i *InconsistentMeatadataObject) UnmarshalJSON(b []byte) error {
+	type t InconsistentMeatadataObject
+	var q t
+	if err := json.Unmarshal(b, &q); err != nil {
+		return err
+	}
+	defBody, err := json.Marshal(q.Definition)
+	if err != nil {
+		return err
+	}
+	switch q.Type {
+	case "object_relation":
+		q.Definition = &createObjectRelationshipInput{}
+	case "array_relation":
+		q.Definition = &createArrayRelationshipInput{}
+	case "select_permission":
+		q.Definition = &createSelectPermissionInput{}
+	case "update_permission":
+		q.Definition = &createUpdatePermissionInput{}
+	case "insert_permission":
+		q.Definition = &createInsertPermissionInput{}
+	case "delete_permission":
+		q.Definition = &createDeletePermissionInput{}
+	case "table":
+		q.Definition = &trackTableInput{}
+	case "function":
+		q.Definition = &trackFunctionInput{}
+	case "event_trigger":
+		q.Definition = &createEventTriggerInput{}
+	case "remote_schema":
+		q.Definition = &addRemoteSchemaInput{}
+	}
+	if err := json.Unmarshal(defBody, &q.Definition); err != nil {
+		return err
+	}
+	*i = InconsistentMeatadataObject(q)
+	return nil
+}
+
+func (i InconsistentMeatadataObject) GetType() string {
+	return i.Type
+}
+
+func (i InconsistentMeatadataObject) GetName() string {
+	switch defType := i.Definition.(type) {
+	case *createObjectRelationshipInput:
+		return defType.Name
+	case *createArrayRelationshipInput:
+		return defType.Name
+	case *createSelectPermissionInput:
+		return fmt.Sprintf("%s-permission", defType.Role)
+	case *createUpdatePermissionInput:
+		return fmt.Sprintf("%s-permission", defType.Role)
+	case *createInsertPermissionInput:
+		return fmt.Sprintf("%s-permission", defType.Role)
+	case *createDeletePermissionInput:
+		return fmt.Sprintf("%s-permission", defType.Role)
+	case *trackTableInput:
+		return defType.Name
+	case *trackFunctionInput:
+		return defType.Name
+	case *createEventTriggerInput:
+		return defType.Name
+	case *addRemoteSchemaInput:
+		return defType.Name
+	}
+	return "N/A"
+}
+
+func (i InconsistentMeatadataObject) GetDescription() string {
+	switch defType := i.Definition.(type) {
+	case *createObjectRelationshipInput:
+		return fmt.Sprintf("relationship of table %s in %s schema", defType.Table.Name, defType.Table.Schema)
+	case *createArrayRelationshipInput:
+		return fmt.Sprintf("relationship of table %s in %s schema", defType.Table.Name, defType.Table.Schema)
+	case *createSelectPermissionInput:
+		return fmt.Sprintf("%s on table %s in %s schema", i.Type, defType.Table.Name, defType.Table.Schema)
+	case *createUpdatePermissionInput:
+		return fmt.Sprintf("%s on table %s in %s schema", i.Type, defType.Table.Name, defType.Table.Schema)
+	case *createInsertPermissionInput:
+		return fmt.Sprintf("%s on table %s in %s schema", i.Type, defType.Table.Name, defType.Table.Schema)
+	case *createDeletePermissionInput:
+		return fmt.Sprintf("%s on table %s in %s schema", i.Type, defType.Table.Name, defType.Table.Schema)
+	case *trackTableInput:
+		return fmt.Sprintf("table %s in %s schema", defType.tableSchema.Name, defType.tableSchema.Schema)
+	case *trackFunctionInput:
+		return fmt.Sprintf("function %s in %s schema", defType.Name, defType.Schema)
+	case *createEventTriggerInput:
+		return fmt.Sprintf("event trigger %s on table %s in %s schema", defType.Name, defType.Table.Name, defType.Table.Schema)
+	case *addRemoteSchemaInput:
+		url := defType.Definition["url"]
+		urlFromEnv, ok := defType.Definition["url_from_env"]
+		if ok {
+			url = fmt.Sprintf("the url from the value of env var %s", urlFromEnv)
+		}
+		return fmt.Sprintf("remote schema %s at %s", defType.Name, url)
+	}
+	return "N/A"
+}
+
+func (i InconsistentMeatadataObject) GetReason() string {
+	return i.Reason
+}
+
 type runSQLInput struct {
 	SQL string `json:"sql" yaml:"sql"`
 }

cli/migrate/database/metadata.go

@@ -7,7 +7,18 @@ type MetadataDriver interface {
 
 	ReloadMetadata() error
 
+	GetInconsistentMetadata() (bool, []InconsistentMetadataInterface, error)
+
+	DropInconsistentMetadata() error
+
 	ApplyMetadata(data interface{}) error
 
 	Query(data []interface{}) error
 }
+
+type InconsistentMetadataInterface interface {
+	GetType() string
+	GetName() string
+	GetDescription() string
+	GetReason() string
+}

cli/migrate/migrate.go

@@ -320,6 +320,14 @@ func (m *Migrate) ReloadMetadata() error {
 	return m.databaseDrv.ReloadMetadata()
 }
 
+func (m *Migrate) GetInconsistentMetadata() (bool, []database.InconsistentMetadataInterface, error) {
+	return m.databaseDrv.GetInconsistentMetadata()
+}
+
+func (m *Migrate) DropInconsistentMetadata() error {
+	return m.databaseDrv.DropInconsistentMetadata()
+}
+
 func (m *Migrate) ApplyMetadata(data interface{}) error {
 	return m.databaseDrv.ApplyMetadata(data)
 }