cli: when applying metadata, assume empty metadata plugin files if they don't exist (close #5163) (#5170)

CHANGELOG.md

@@ -10,6 +10,7 @@
 - server: fix introspection when multiple actions defined with Postgres scalar types (fix #5166) (#5173)
 - console: allow manual edit of column types and handle array data types (close #2544, #3335, #2583) (#4546)
 - console: add the ability to delete a role in permissions summary page (close #3353) (#4987)
+- cli: handle missing files during metadata apply (close #5163) (#5170)
 - docs: add page on Relay schema (close #4912) (#5150)
 
 ## `v1.3.0-beta.2`
@@ -99,7 +100,7 @@ Read more about the session argument for computed fields in the [docs](https://h
 A new `seeds` command is introduced in CLI, this will allow managing seed migrations as SQL files
 
 #### Creating seed
-```
+```                                                        
 # create a new seed file and use editor to add SQL content
 hasura seed create new_table_seed
 

cli/go.sum

@@ -47,8 +47,10 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
 github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=

cli/integration_test/cli_test.go

@@ -143,6 +143,56 @@ func TestCommands(t *testing.T) {
 			v2.TestSeedsApplyCmd(t, ec)
 		})
 	})
+	t.Run("config=v2/incomplete_metadata_dir", func(t *testing.T) {
+		ec := cli.NewExecutionContext()
+		ec.Config = &cli.Config{}
+		logger, _ := test.NewNullLogger()
+		ec.Logger = logger
+		ec.Spinner = spinner.New(spinner.CharSets[7], 100*time.Millisecond)
+		ec.Spinner.Writer = ioutil.Discard
+		ec.Viper = viper.New()
+
+		initDir := filepath.Join(os.TempDir(), "hasura-cli-test-"+strconv.Itoa(rand.Intn(1000)))
+		defer os.RemoveAll(initDir)
+
+		// This will prepare the execution context, so no need to run ec.Prepare() on all the other tests
+		t.Run("prepare", func(t *testing.T) {
+			integrationtest.TestPrepare(t, ec)
+		})
+
+		skip(t)
+		// This will init the project dir
+		t.Run("init command", func(t *testing.T) {
+			v2.TestInitCmd(t, ec, initDir)
+		})
+
+		skip(t)
+		// This will validate the project dir
+		t.Run("validate", func(t *testing.T) {
+			integrationtest.TestValidate(t, ec)
+		})
+
+		skip(t)
+		if cliExtManifestFilePath := os.Getenv("HASURA_GRAPHQL_TEST_CLI_EXT_MANIFEST_FILE_PATH"); cliExtManifestFilePath != "" {
+			t.Run("cli-ext-plugin-install", func(t *testing.T) {
+				installOpts := &commands.PluginInstallOptions{
+					EC:           ec,
+					Name:         cli.CLIExtPluginName,
+					ManifestFile: cliExtManifestFilePath,
+				}
+				err := installOpts.Run()
+				if err != nil {
+					t.Fatalf("unable to install %s plugin, got %v", cli.CLIExtPluginName, err)
+				}
+			})
+		}
+
+		skip(t)
+		t.Run("metadata apply", func(t *testing.T) {
+			v2.TestIncompleteMetadataDir(t, ec)
+		})
+
+	})
 }
 
 func skip(t *testing.T) {

cli/integration_test/testdata/incomplete_metadata/t1/from-server.golden

@@ -0,0 +1,4 @@
+{
+  "version": 2,
+  "tables": []
+}

cli/integration_test/testdata/incomplete_metadata/t1/metadata/actions.graphql

@@ -0,0 +1,18 @@
+type Mutation {
+  actionName (
+    arg1: SampleInput!
+  ): SampleOutput
+}
+
+
+
+
+input SampleInput {
+  username : String!
+  password : String!
+}
+
+type SampleOutput {
+  accessToken : String!
+}
+

cli/integration_test/testdata/incomplete_metadata/t1/metadata/allow_list.yaml

@@ -0,0 +1 @@
+[]

cli/integration_test/testdata/incomplete_metadata/t1/metadata/functions.yaml

@@ -0,0 +1 @@
+[]

cli/integration_test/testdata/incomplete_metadata/t1/metadata/query_collections.yaml

@@ -0,0 +1 @@
+[]

cli/integration_test/testdata/incomplete_metadata/t1/metadata/remote_schemas.yaml

@@ -0,0 +1 @@
+[]

cli/integration_test/testdata/incomplete_metadata/t1/metadata/tables.yaml

@@ -0,0 +1 @@
+[]

cli/integration_test/testdata/incomplete_metadata/t1/metadata/version.yaml

@@ -0,0 +1 @@
+version: 2

cli/integration_test/testdata/incomplete_metadata/t2/from-server.golden

@@ -0,0 +1,49 @@
+{
+  "version": 2,
+  "tables": [],
+  "actions": [
+    {
+      "name": "actionName",
+      "definition": {
+        "handler": "http://localhost:3000",
+        "output_type": "SampleOutput",
+        "arguments": [
+          {
+            "name": "arg1",
+            "type": "SampleInput!"
+          }
+        ],
+        "type": "mutation",
+        "kind": "synchronous"
+      }
+    }
+  ],
+  "custom_types": {
+    "input_objects": [
+      {
+        "name": "SampleInput",
+        "fields": [
+          {
+            "name": "username",
+            "type": "String!"
+          },
+          {
+            "name": "password",
+            "type": "String!"
+          }
+        ]
+      }
+    ],
+    "objects": [
+      {
+        "name": "SampleOutput",
+        "fields": [
+          {
+            "name": "accessToken",
+            "type": "String!"
+          }
+        ]
+      }
+    ]
+  }
+}

cli/integration_test/testdata/incomplete_metadata/t2/metadata/actions.graphql

@@ -0,0 +1,18 @@
+type Mutation {
+  actionName (
+    arg1: SampleInput!
+  ): SampleOutput
+}
+
+
+
+
+input SampleInput {
+  username : String!
+  password : String!
+}
+
+type SampleOutput {
+  accessToken : String!
+}
+

cli/integration_test/testdata/incomplete_metadata/t2/metadata/actions.yaml

@@ -0,0 +1,12 @@
+actions:
+- name: actionName
+  definition:
+    kind: synchronous
+    handler: http://localhost:3000
+custom_types:
+  enums: []
+  input_objects:
+  - name: SampleInput
+  objects:
+  - name: SampleOutput
+  scalars: []

cli/integration_test/testdata/incomplete_metadata/t2/metadata/allow_list.yaml

@@ -0,0 +1 @@
+[]

cli/integration_test/testdata/incomplete_metadata/t2/metadata/cron_triggers.yaml

@@ -0,0 +1 @@
+[]

cli/integration_test/testdata/incomplete_metadata/t2/metadata/functions.yaml

@@ -0,0 +1 @@
+[]

cli/integration_test/testdata/incomplete_metadata/t2/metadata/query_collections.yaml

@@ -0,0 +1 @@
+[]

cli/integration_test/testdata/incomplete_metadata/t2/metadata/remote_schemas.yaml

@@ -0,0 +1 @@
+[]

cli/integration_test/testdata/incomplete_metadata/t2/metadata/tables.yaml

@@ -0,0 +1 @@
+[]

cli/integration_test/testdata/incomplete_metadata/t2/metadata/version.yaml

@@ -0,0 +1 @@
+version: 2

cli/integration_test/v2/metadata.go

@@ -2,7 +2,10 @@ package v2
 
 import (
 	"bytes"
+	"encoding/json"
+	"fmt"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path/filepath"
 	"testing"
@@ -138,3 +141,94 @@ func TestMetadataCmd(t *testing.T, ec *cli.ExecutionContext) {
 		})
 	}
 }
+
+func TestIncompleteMetadataDir(t *testing.T, ec *cli.ExecutionContext) {
+	currDir, _ := os.Getwd()
+	tt := []struct {
+		name               string
+		opts               metadataInterface
+		err                error
+		copyMetadataFolder string
+		goldenFile         string
+	}{
+		{
+			"t1",
+			&commands.MetadataExportOptions{
+				EC:         ec,
+				ActionType: "apply",
+			},
+			nil,
+			filepath.Join(currDir, "testdata/incomplete_metadata/t1/metadata"),
+			filepath.Join(currDir, "testdata/incomplete_metadata/t1/from-server.golden"),
+		},
+		{
+			"t2",
+			&commands.MetadataExportOptions{
+				EC:         ec,
+				ActionType: "apply",
+			},
+			nil,
+			filepath.Join(currDir, "testdata/incomplete_metadata/t2/metadata"),
+			filepath.Join(currDir, "testdata/incomplete_metadata/t2/from-server.golden"),
+		},
+	}
+
+	for _, tc := range tt {
+		t.Run(tc.name, func(t *testing.T) {
+			if tc.copyMetadataFolder != "" {
+				err := os.RemoveAll(ec.MetadataDir)
+				if err != nil {
+					t.Fatalf("%s: unable to remove metadata directory, got %v", tc.name, err)
+				}
+				err = util.CopyDir(tc.copyMetadataFolder, ec.MetadataDir)
+				if err != nil {
+					t.Fatalf("%s: unable to copy metadata file, got %v", tc.name, err)
+				}
+			}
+			err := tc.opts.Run()
+			if err != tc.err {
+				t.Fatalf("%s: expected %v, got %v", tc.name, tc.err, err)
+			}
+			// get metadata from server
+			testServerUrl := os.Getenv("HASURA_GRAPHQL_TEST_ENDPOINT")
+			if testServerUrl == "" {
+				testServerUrl = "http://localhost:8080"
+			}
+			url := fmt.Sprintf("%s/%s", testServerUrl, "v1/query")
+			var body = []byte(`
+{
+    "type" : "export_metadata",
+    "args": {}
+}
+`)
+			req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
+			assert.NoError(t, err)
+			if ec.Config.AdminSecret != "" {
+				req.Header.Set(cli.XHasuraAdminSecret, ec.Config.AdminSecret)
+			}
+			c := http.Client{}
+			resp, err := c.Do(req)
+
+			defer resp.Body.Close()
+			assert.Equal(t, resp.StatusCode, http.StatusOK)
+			got, err := ioutil.ReadAll(resp.Body)
+			assert.NoError(t, err)
+
+			want, err := ioutil.ReadFile(tc.goldenFile)
+			assert.NoError(t, err)
+			var wantM, gotM map[string]interface{}
+
+			err = json.Unmarshal(want, &wantM)
+			assert.NoError(t, err)
+			err = json.Unmarshal(got, &gotM)
+			assert.NoError(t, err)
+
+			wantB, err := json.Marshal(wantM)
+			assert.NoError(t, err)
+			gotB, err := json.Marshal(gotM)
+			assert.NoError(t, err)
+
+			assert.Equal(t, string(wantB), string(gotB))
+		})
+	}
+}

cli/migrate/database/hasuradb/metadata.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"os"
 
 	gyaml "github.com/ghodss/yaml"
 	"github.com/hasura/graphql-engine/cli/metadata/types"
@@ -151,6 +152,10 @@ func (h *HasuraDB) BuildMetadata() (yaml.MapSlice, error) {
 	for _, plg := range h.config.Plugins {
 		err := plg.Build(&tmpMeta)
 		if err != nil {
+			if os.IsNotExist(errors.Cause(err)) {
+				h.logger.Debugf("metadata file for %s was not found, assuming an empty file", plg.Name())
+				continue
+			}
 			return tmpMeta, errors.Wrap(err, fmt.Sprintf("cannot build %s from metadata", plg.Name()))
 		}
 	}

scripts/cli-migrations/Makefile

@@ -40,6 +40,7 @@ build-cli-migrations-v2:
 test-cli-migrations-v2:
 	cd v2/test
 	./test.sh
+	./test-upgrade-from-latest-release.sh
 
 .PHONY: all
 all: load-server-image build-cli-migrations-v1 build-cli-migrations-v2 test-cli-migrations-v1 test-cli-migrations-v2

scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -evo pipefail
+IFS=$'\n\t'
+ROOT="$(readlink -f ${BASH_SOURCE[0]%/*}/../../)"
+
+wait_for_server() {
+    echo "waiting for server"
+    for _ in $(seq 1 60);
+    do
+      docker run --network container:graphql-engine appropriate/curl http://127.0.0.1:8080/v1/version && return
+      echo -n .
+      sleep 1
+    done
+    echo "Failed waiting for server" && exit 1
+}
+
+
+# get previous stable version docker-compose file
+curl -L https://raw.githubusercontent.com/hasura/graphql-engine/stable/install-manifests/docker-compose/docker-compose.yaml -o docker-compose-latest.yaml
+sed -i '/hasura\/graphql-engine:/ s/$/.cli-migrations-v2\n    container_name: graphql-engine/' docker-compose-latest.yaml
+# start postgres
+docker-compose -f docker-compose-latest.yaml up --no-start graphql-engine
+# copy migrations directory to /hasura-migrations
+docker cp migrations/. graphql-engine:/hasura-migrations
+# copy metadata directory to /hasura-metadata
+docker cp metadata/. graphql-engine:/hasura-metadata
+# start graphql-engine
+docker-compose -f docker-compose-latest.yaml up -d --no-recreate graphql-engine
+wait_for_server
+# export metadata and run diff with validation/metadata.json
+docker run --network container:graphql-engine appropriate/curl -s -f   -d'{"type" : "export_metadata", "args" : {} }' localhost:8080/v1/query | jq -j '.' | diff validation/metadata.json -
+# get list of migrations applied from graphql-engine server
+docker run --network container:graphql-engine appropriate/curl -s -f   -d'{"type" : "run_sql", "args" : {"sql": "select * from hdb_catalog.schema_migrations"} }' localhost:8080/v1/query | jq -j '.' | diff validation/schema_migrations.json -
+
+# use the current build to start container
+docker-compose up -d
+wait_for_server
+# export metadata and run diff with validation/metadata.json
+docker run --network container:graphql-engine appropriate/curl -s -f   -d'{"type" : "export_metadata", "args" : {} }' localhost:8080/v1/query | jq -j '.' | diff validation/metadata.json -
+# get list of migrations applied from graphql-engine server
+docker run --network container:graphql-engine appropriate/curl -s -f   -d'{"type" : "run_sql", "args" : {"sql": "select * from hdb_catalog.schema_migrations"} }' localhost:8080/v1/query | jq -j '.' | diff validation/schema_migrations.json -
+# delete postgres and graphql-engine
+docker-compose down -v