From 2ee7f7d76e27f135963aa04472c5d7afc46f8a52 Mon Sep 17 00:00:00 2001 From: Jeff Sieu Date: Wed, 30 Oct 2019 21:54:22 +0800 Subject: [PATCH] cli(metadata): add diff command and dry-run flag (#3157) ### Description Adds a `metadata diff` command to show comparisons between two different sets of Hasura metadata. ``` # Show changes between server metadata and the exported metadata file: hasura metadata diff # Show changes between server metadata and that in local_metadata.yaml: hasura metadata diff local_metadata.yaml # Show changes between metadata from metadata.yaml and metadata_old.yaml: hasura metadata diff metadata.yaml metadata_old.yaml ``` Also adds a `--dry-run` flag to `metadata apply` command which will print the diff and exit rather than actually applying the metadata. ### Affected components - CLI - Docs ### Related Issues Close #3126, Close #3127 ### Solution and Design - Added `metadata_diff.go` and `metadata_diff_test.go` ### Steps to test and verify ``` hasura metadata export # Make changes to migrations/metadata.yaml hasura metadata diff ``` ### Limitations, known bugs & workarounds This is just a general-purpose diff. A more contextual diff with the understanding of metadata can be added once #3072 is merged. --- cli/Gopkg.lock | 19 ++++ cli/cli.go | 20 ++++ cli/commands/metadata.go | 1 + cli/commands/metadata_apply.go | 18 ++++ cli/commands/metadata_diff.go | 151 +++++++++++++++++++++++++++++ cli/commands/metadata_diff_test.go | 125 ++++++++++++++++++++++++ 6 files changed, 334 insertions(+) create mode 100644 cli/commands/metadata_diff.go create mode 100644 cli/commands/metadata_diff_test.go diff --git a/cli/Gopkg.lock b/cli/Gopkg.lock index 16abd1fed34..88ca5e123bc 100644 --- a/cli/Gopkg.lock +++ b/cli/Gopkg.lock @@ -17,6 +17,14 @@ revision = "7da180ee92d8bd8bb8c37fc560e673e6557c392f" version = "v0.4.7" +[[projects]] + branch = "master" + digest = "1:67f5c19d64f788aa79a8e612eefa1cc68dca46c3436c93b84af9c95bd7e1a556" + name = "github.com/aryann/difflib" + packages = ["."] + pruneopts = "UT" + revision = "e206f873d14a916d3d26c40ab667bca123f365a3" + [[projects]] digest = "1:889290ee5c1f1888baa7caa2b4cdfa8a6abcfb86dd772fe6470ad7925cc44bff" name = "github.com/briandowns/spinner" @@ -297,6 +305,14 @@ revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" version = "v0.0.3" +[[projects]] + branch = "master" + digest = "1:2b32af4d2a529083275afc192d1067d8126b578c7a9613b26600e4df9c735155" + name = "github.com/mgutz/ansi" + packages = ["."] + pruneopts = "UT" + revision = "9520e82c474b0a04dd04f8a40959027271bab992" + [[projects]] branch = "master" digest = "1:8eb17c2ec4df79193ae65b621cd1c0c4697db3bc317fe6afdc76d7f2746abd05" @@ -535,6 +551,7 @@ analyzer-version = 1 input-imports = [ "github.com/Masterminds/semver", + "github.com/aryann/difflib", "github.com/briandowns/spinner", "github.com/docker/docker/api/types", "github.com/docker/docker/api/types/container", @@ -553,6 +570,7 @@ "github.com/lib/pq", "github.com/manifoldco/promptui", "github.com/mattn/go-colorable", + "github.com/mgutz/ansi", "github.com/mitchellh/go-homedir", "github.com/oliveagle/jsonpath", "github.com/parnurzeal/gorequest", @@ -562,6 +580,7 @@ "github.com/skratchdot/open-golang/open", "github.com/spf13/cobra", "github.com/spf13/cobra/doc", + "github.com/spf13/pflag", "github.com/spf13/viper", "github.com/stretchr/testify/assert", ] diff --git a/cli/cli.go b/cli/cli.go index a7fd975616b..b78ac401bdf 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -396,3 +396,23 @@ func (ec *ExecutionContext) GetMetadataFilePath(format string) (string, error) { } return "", errors.New("unsupported file type") } + +// GetExistingMetadataFile returns the path to the default metadata file that +// also exists, json or yaml +func (ec *ExecutionContext) GetExistingMetadataFile() (string, error) { + filename := "" + for _, format := range []string{"yaml", "json"} { + f, err := ec.GetMetadataFilePath(format) + if err != nil { + return "", errors.Wrap(err, "cannot get metadata file") + } + + filename = f + if _, err := os.Stat(filename); os.IsNotExist(err) { + continue + } + break + } + + return filename, nil +} diff --git a/cli/commands/metadata.go b/cli/commands/metadata.go index 43801c2d775..6347b720586 100644 --- a/cli/commands/metadata.go +++ b/cli/commands/metadata.go @@ -19,6 +19,7 @@ func NewMetadataCmd(ec *cli.ExecutionContext) *cobra.Command { SilenceUsage: true, } metadataCmd.AddCommand( + newMetadataDiffCmd(ec), newMetadataExportCmd(ec), newMetadataClearCmd(ec), newMetadataReloadCmd(ec), diff --git a/cli/commands/metadata_apply.go b/cli/commands/metadata_apply.go index 027f0a2060e..3e4fc384783 100644 --- a/cli/commands/metadata_apply.go +++ b/cli/commands/metadata_apply.go @@ -1,6 +1,8 @@ package commands import ( + "os" + "github.com/hasura/graphql-engine/cli" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -25,6 +27,18 @@ func newMetadataApplyCmd(ec *cli.ExecutionContext) *cobra.Command { return ec.Validate() }, RunE: func(cmd *cobra.Command, args []string) error { + if opts.dryRun { + o := &metadataDiffOptions{ + EC: ec, + output: os.Stdout, + } + filename, err := ec.GetExistingMetadataFile() + if err != nil { + return errors.Wrap(err, "failed getting metadata file") + } + o.metadata[0] = filename + return o.run() + } opts.EC.Spin("Applying metadata...") err := opts.run() opts.EC.Spinner.Stop() @@ -42,6 +56,8 @@ func newMetadataApplyCmd(ec *cli.ExecutionContext) *cobra.Command { f.String("access-key", "", "access key for Hasura GraphQL Engine") f.MarkDeprecated("access-key", "use --admin-secret instead") + f.BoolVar(&opts.dryRun, "dry-run", false, "show a diff instead of applying the metadata") + // need to create a new viper because https://github.com/spf13/viper/issues/233 v.BindPFlag("endpoint", f.Lookup("endpoint")) v.BindPFlag("admin_secret", f.Lookup("admin-secret")) @@ -54,6 +70,8 @@ type metadataApplyOptions struct { EC *cli.ExecutionContext actionType string + + dryRun bool } func (o *metadataApplyOptions) run() error { diff --git a/cli/commands/metadata_diff.go b/cli/commands/metadata_diff.go new file mode 100644 index 00000000000..6cff82bfb03 --- /dev/null +++ b/cli/commands/metadata_diff.go @@ -0,0 +1,151 @@ +package commands + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "strings" + + "github.com/aryann/difflib" + "github.com/ghodss/yaml" + "github.com/hasura/graphql-engine/cli" + "github.com/mgutz/ansi" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +type metadataDiffOptions struct { + EC *cli.ExecutionContext + output io.Writer + + // two metadata to diff, 2nd is server if it's empty + metadata [2]string +} + +func newMetadataDiffCmd(ec *cli.ExecutionContext) *cobra.Command { + v := viper.New() + opts := &metadataDiffOptions{ + EC: ec, + output: os.Stdout, + } + + metadataDiffCmd := &cobra.Command{ + Use: "diff [file1] [file2]", + Short: "(PREVIEW) Show a highlighted diff of Hasura metadata", + Long: `(PREVIEW) Show changes between two different sets of Hasura metadata. +By default, shows changes between exported metadata file and server metadata.`, + Example: ` # NOTE: This command is in preview, usage and diff format may change. + + # Show changes between server metadata and the exported metadata file: + hasura metadata diff + + # Show changes between server metadata and that in local_metadata.yaml: + hasura metadata diff local_metadata.yaml + + # Show changes between metadata from metadata.yaml and metadata_old.yaml: + hasura metadata diff metadata.yaml metadata_old.yaml`, + Args: cobra.MaximumNArgs(2), + PreRunE: func(cmd *cobra.Command, args []string) error { + ec.Viper = v + return ec.Validate() + }, + RunE: func(cmd *cobra.Command, args []string) error { + messageFormat := "Showing diff between %s and %s..." + message := "" + + switch len(args) { + case 0: + // no args, diff exported metadata and metadata on server + filename, err := ec.GetExistingMetadataFile() + if err != nil { + return errors.Wrap(err, "failed getting metadata file") + } + opts.metadata[0] = filename + message = fmt.Sprintf(messageFormat, filename, "the server") + case 1: + // 1 arg, diff given filename and the metadata on server + opts.metadata[0] = args[0] + message = fmt.Sprintf(messageFormat, args[0], "the server") + case 2: + // 2 args, diff given filenames + opts.metadata[0] = args[0] + opts.metadata[1] = args[1] + message = fmt.Sprintf(messageFormat, args[0], args[1]) + } + + opts.EC.Logger.Info(message) + err := opts.run() + if err != nil { + return errors.Wrap(err, "failed to show metadata diff") + } + return nil + }, + } + + f := metadataDiffCmd.Flags() + f.String("endpoint", "", "http(s) endpoint for Hasura GraphQL Engine") + f.String("admin-secret", "", "admin secret for Hasura GraphQL Engine") + f.String("access-key", "", "access key for Hasura GraphQL Engine") + f.MarkDeprecated("access-key", "use --admin-secret instead") + + // need to create a new viper because https://github.com/spf13/viper/issues/233 + v.BindPFlag("endpoint", f.Lookup("endpoint")) + v.BindPFlag("admin_secret", f.Lookup("admin-secret")) + v.BindPFlag("access_key", f.Lookup("access-key")) + + return metadataDiffCmd +} + +func (o *metadataDiffOptions) run() error { + var oldYaml, newYaml []byte + var err error + migrateDrv, err := newMigrate(o.EC.MigrationDir, o.EC.ServerConfig.ParsedEndpoint, o.EC.ServerConfig.AdminSecret, o.EC.Logger, o.EC.Version, true) + if err != nil { + return err + } + + if o.metadata[1] == "" { + // get metadata from server + m, err := migrateDrv.ExportMetadata() + if err != nil { + return errors.Wrap(err, "cannot fetch metadata from server") + } + + newYaml, err = yaml.Marshal(m) + if err != nil { + return errors.Wrap(err, "cannot convert metadata from server to yaml") + } + } else { + newYaml, err = ioutil.ReadFile(o.metadata[1]) + if err != nil { + return errors.Wrap(err, "cannot read file") + } + } + + oldYaml, err = ioutil.ReadFile(o.metadata[0]) + if err != nil { + return errors.Wrap(err, "cannot read file") + } + + printDiff(string(oldYaml), string(newYaml), o.output) + return nil +} + +func printDiff(before, after string, to io.Writer) { + diffs := difflib.Diff(strings.Split(before, "\n"), strings.Split(after, "\n")) + + for _, diff := range diffs { + text := diff.Payload + + switch diff.Delta { + case difflib.RightOnly: + fmt.Fprintf(to, "%s\n", ansi.Color(text, "green")) + case difflib.LeftOnly: + fmt.Fprintf(to, "%s\n", ansi.Color(text, "red")) + case difflib.Common: + fmt.Fprintf(to, "%s\n", text) + } + } +} diff --git a/cli/commands/metadata_diff_test.go b/cli/commands/metadata_diff_test.go new file mode 100644 index 00000000000..f5475fbd135 --- /dev/null +++ b/cli/commands/metadata_diff_test.go @@ -0,0 +1,125 @@ +package commands + +import ( + "bytes" + "io/ioutil" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + "github.com/briandowns/spinner" + "github.com/hasura/graphql-engine/cli" + "github.com/hasura/graphql-engine/cli/version" + "github.com/sirupsen/logrus/hooks/test" +) + +var testMetadata1 = `allowlist: [] +functions: [] +query_collections: [] +remote_schemas: [] +tables: +- array_relationships: [] + delete_permissions: [] + event_triggers: [] + insert_permissions: [] + is_enum: false + object_relationships: [] + select_permissions: [] + table: test + update_permissions: [] +` + +var testMetadata2 = `allowlist: [] +functions: [] +query_collections: [] +remote_schemas: [] +tables: +- array_relationships: [] + configuration: + custom_column_names: {} + custom_root_fields: + delete: null + insert: null + select: null + select_aggregate: null + select_by_pk: null + update: null + delete_permissions: [] + event_triggers: [] + insert_permissions: [] + is_enum: false + object_relationships: [] + select_permissions: [] + table: test + update_permissions: [] +` + +func TestMetadataDiffCmd(t *testing.T) { + endpointURL, err := url.Parse(os.Getenv("HASURA_GRAPHQL_TEST_ENDPOINT")) + if err != nil { + t.Fatal(err) + } + + // Create migration Dir + migrationsDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(migrationsDir) + + metadataFile := filepath.Join(migrationsDir, "metadata.yaml") + testMetadataFile1 := filepath.Join(migrationsDir, "testmetadata1.yaml") + testMetadataFile2 := filepath.Join(migrationsDir, "testmetadata2.yaml") + + mustWriteFile(t, "", metadataFile, testMetadata1) + mustWriteFile(t, "", testMetadataFile1, testMetadata1) + mustWriteFile(t, "", testMetadataFile2, testMetadata2) + + logger, _ := test.NewNullLogger() + outputFile := new(bytes.Buffer) + opts := &metadataDiffOptions{ + EC: &cli.ExecutionContext{ + Logger: logger, + Spinner: spinner.New(spinner.CharSets[7], 100*time.Millisecond), + MetadataFile: []string{metadataFile}, + ServerConfig: &cli.ServerConfig{ + Endpoint: endpointURL.String(), + AdminSecret: os.Getenv("HASURA_GRAPHQL_TEST_ADMIN_SECRET"), + ParsedEndpoint: endpointURL, + }, + }, + output: outputFile, + } + + opts.EC.Version = version.New() + v, err := version.FetchServerVersion(opts.EC.ServerConfig.Endpoint) + if err != nil { + t.Fatalf("getting server version failed: %v", err) + } + opts.EC.Version.SetServerVersion(v) + + // Run without args + opts.metadata[0] = metadataFile + err = opts.run() + if err != nil { + t.Fatalf("failed diffing metadata: %v", err) + } + + // Run with one arg + opts.metadata = [2]string{testMetadataFile1, ""} + + err = opts.run() + if err != nil { + t.Fatalf("failed diffing metadata: %v", err) + } + + // Run with two args + opts.metadata = [2]string{testMetadataFile1, testMetadataFile2} + + err = opts.run() + if err != nil { + t.Fatalf("failed diffing metadata: %v", err) + } +}