cli: config v3 project metadata directory

GitOrigin-RevId: c2dce9ca1f37688eecb7eb78c97944ac96d81d54
2024-12-14 08:02:15 +03:00 · 2021-02-16 11:55:26 +05:30 · 2021-02-16 11:55:26 +05:30 · 8ec92cc198
commit 8ec92cc198
parent 7290a57308
29 changed files with 792 additions and 52 deletions
--- a/cli/go.mod
+++ b/cli/go.mod
@ -12,12 +12,13 @@ require (
 	github.com/briandowns/spinner v1.8.0
 	github.com/disintegration/imaging v1.6.2 // indirect
 	github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 // indirect
-	github.com/fatih/color v1.7.0
+	github.com/fatih/color v1.10.0
 	github.com/ghodss/yaml v1.0.0
 	github.com/gin-contrib/cors v1.3.0
 	github.com/gin-contrib/static v0.0.0-20191128031702-f81c604d8ac2
 	github.com/gin-gonic/contrib v0.0.0-20191209060500-d6e26eeaa607
 	github.com/gin-gonic/gin v1.5.0
+	github.com/goccy/go-yaml v1.8.8
 	github.com/gofrs/uuid v3.2.0+incompatible
 	github.com/gorilla/sessions v1.2.0 // indirect
 	github.com/gosimple/slug v1.9.0 // indirect
@ -60,10 +61,11 @@ require (
 	github.com/theplant/htmltestingutils v0.0.0-20190423050759-0e06de7b6967 // indirect
 	github.com/theplant/testingutils v0.0.0-20190603093022-26d8b4d95c61 // indirect
 	github.com/yosssi/gohtml v0.0.0-20190915184251-7ff6f235ecaf // indirect
-	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550
+	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
 	golang.org/x/net v0.0.0-20200226121028-0de0cce0169b // indirect
 	golang.org/x/sys v0.0.0-20201117222635-ba5294a509c7 // indirect
 	gopkg.in/src-d/go-git.v4 v4.13.1
 	gopkg.in/yaml.v2 v2.2.7
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	moul.io/http2curl v1.0.0 // indirect
 )
--- a/cli/go.sum
+++ b/cli/go.sum
@ -77,6 +77,8 @@ github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DP
 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
@ -100,15 +102,25 @@ github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aev
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
+github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc=
 github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM=
+github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM=
 github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
+github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
+github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gobuffalo/here v0.6.0 h1:hYrd0a6gDmWxBM4TnrGw8mQg24iSVoIkHEk7FodQcBI=
 github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
+github.com/goccy/go-yaml v1.8.8 h1:MGfRB1GeSn/hWXYWS2Pt67iC2GJNnebdIro01ddyucA=
+github.com/goccy/go-yaml v1.8.8/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
 github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@ -194,6 +206,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8=
 github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
+github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
+github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a h1:weJVJJRzAJBFRlAiJQROKQs8oC9vOxvm4rZmBBk0ONw=
@ -372,8 +386,8 @@ golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0F
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@ -447,6 +461,8 @@ golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a h1:mEQZbbaBjWyLNy0tmZmgEuQAR8XOQ3hL8GYi3J/NG64=
 golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
@ -486,6 +502,8 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
--- a/cli/integration_test/v3/init.go
+++ b/cli/integration_test/v3/init.go
@ -29,7 +29,6 @@ func TestInitCmd(t *testing.T, ec *cli.ExecutionContext, initDir string) {
 			if err != tc.err {
 				t.Fatalf("%s: expected %v, got %v", tc.name, tc.err, err)
 			}
-			// TODO: (shahidhk) need to verify the contents of the spec generated
 		})
 	}
 }
--- a/cli/integration_test/v3/metadata.go
+++ b/cli/integration_test/v3/metadata.go
@ -8,6 +8,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"

 	"github.com/hasura/graphql-engine/cli"
@ -121,23 +122,21 @@ func TestMetadataCmd(t *testing.T, ec *cli.ExecutionContext) {
 			}
 			if tc.expectedMetadataFolder != "" {
 				assert.DirExists(t, ec.MetadataDir)
-				files, err := ioutil.ReadDir(tc.expectedMetadataFolder)
-				if err != nil {
-					t.Fatalf("%s: unable to read expected metadata directory, got %v", tc.name, err)
-				}
-
-				for _, file := range files {
-					name := file.Name()
-					expectedByt, err := ioutil.ReadFile(filepath.Join(tc.expectedMetadataFolder, name))
-					if err != nil {
-						t.Fatalf("%s: unable to read expected metadata file %s, got %v", tc.name, name, err)
+				filepath.Walk(filepath.Join(tc.expectedMetadataFolder), func(path string, info os.FileInfo, err error) error {
+					if !info.IsDir() {
+						name := info.Name()
+						expectedByt, err := ioutil.ReadFile(path)
+						if err != nil {
+							t.Fatalf("%s: unable to read expected metadata file %s, got %v", tc.name, name, err)
+						}
+						actualByt, err := ioutil.ReadFile(strings.Replace(path, tc.expectedMetadataFolder, ec.MetadataDir, 1))
+						if err != nil {
+							t.Fatalf("%s: unable to read actual metadata file %s, got %v", tc.name, name, err)
+						}
+						assert.Equal(t, string(expectedByt), string(actualByt))
 					}
-					actualByt, err := ioutil.ReadFile(filepath.Join(ec.MetadataDir, name))
-					if err != nil {
-						t.Fatalf("%s: unable to read actual metadata file %s, got %v", tc.name, name, err)
-					}
-					assert.Equal(t, string(expectedByt), string(actualByt))
-				}
+					return nil
+				})
 			}
 		})
 	}
--- a/cli/integration_test/v3/metadata/latest/1_metadata/sources/sources.yaml
+++ b/cli/integration_test/v3/metadata/latest/1_metadata/sources/sources.yaml
@ -1,10 +1,11 @@
 - name: default
-  tables: []
  configuration:
    connection_info:
      database_url:
        from_env: HASURA_GRAPHQL_DATABASE_URL
      pool_settings:
-        retries: 1
        idle_timeout: 180
        max_connections: 50
+        retries: 1
+  tables: []
+  functions: []
--- a/cli/integration_test/v3/metadata/latest/2_metadata/sources/default/tables/public_test.yaml
+++ b/cli/integration_test/v3/metadata/latest/2_metadata/sources/default/tables/public_test.yaml
@ -0,0 +1,3 @@
+table:
+  name: test
+  schema: public
--- a/cli/integration_test/v3/metadata/latest/2_metadata/sources/sources.yaml
+++ b/cli/integration_test/v3/metadata/latest/2_metadata/sources/sources.yaml
@ -1,13 +1,12 @@
 - name: default
-  tables:
-  - table:
-      schema: public
-      name: test
  configuration:
    connection_info:
      database_url:
        from_env: HASURA_GRAPHQL_DATABASE_URL
      pool_settings:
-        retries: 1
        idle_timeout: 180
        max_connections: 50
+        retries: 1
+  tables:
+  - "!include public_test.yaml"
+  functions: []
--- a/cli/metadata/sources/sources.go
+++ b/cli/metadata/sources/sources.go
@ -1,6 +1,8 @@
 package sources

 import (
+	"bytes"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@ -9,14 +11,29 @@ import (

 	"github.com/sirupsen/logrus"

+	"github.com/goccy/go-yaml"
 	"github.com/hasura/graphql-engine/cli"
-	"gopkg.in/yaml.v2"
+	goyaml "gopkg.in/yaml.v2"
+	v3yaml "gopkg.in/yaml.v3"
 )

 const (
-	fileName string = "sources.yaml"
+	fileName           string = "sources.yaml"
+	sourcesDirectory   string = "sources"
+	functionsDirectory string = "functions"
+	tablesDirectory    string = "tables"
 )

+type SourceWithNormalFields struct {
+	Name          string      `yaml:"name"`
+	Configuration interface{} `yaml:"configuration"`
+}
+type Source struct {
+	SourceWithNormalFields `yaml:",inline"`
+	Tables                 interface{} `yaml:"tables"`
+	Functions              interface{} `yaml:"functions"`
+}
+
 type SourceConfig struct {
 	MetadataDir string

@ -47,39 +64,186 @@ func (t *SourceConfig) CreateFiles() error {
 	return nil
 }

-func (t *SourceConfig) Build(metadata *yaml.MapSlice) error {
-	data, err := ioutil.ReadFile(filepath.Join(t.MetadataDir, fileName))
+func (t *SourceConfig) Build(metadata *goyaml.MapSlice) error {
+	sourceFile := filepath.Join(t.MetadataDir, sourcesDirectory, fileName)
+	sourcesBytes, err := ioutil.ReadFile(sourceFile)
 	if err != nil {
 		return err
 	}
-	item := yaml.MapItem{
+	// unmarshal everything else except tables and functions
+	var sourceNormalFields []SourceWithNormalFields
+	if err := yaml.Unmarshal(sourcesBytes, &sourceNormalFields); err != nil {
+		return err
+	}
+	var sources []Source
+	for idx, minisource := range sourceNormalFields {
+		source := Source{
+			SourceWithNormalFields: minisource,
+		}
+
+		// get tables node
+		tablepath, err := yaml.PathString(fmt.Sprintf("$[%d].tables", idx))
+		if err != nil {
+			return err
+		}
+		tableNode, err := tablepath.ReadNode(bytes.NewReader(sourcesBytes))
+		if err == nil {
+			tableNodeBytes, err := ioutil.ReadAll(tableNode)
+			if err != nil {
+				return err
+			}
+			var tablesKey interface{}
+			err = v3yaml.Unmarshal(tableNodeBytes, newSourcesYamlDecoder(
+				sourcesYamlDecoderOpts{
+					IncludeTagBaseDirectory: filepath.Join(t.MetadataDir, sourcesDirectory, source.Name, tablesDirectory),
+				},
+				&tablesKey,
+			))
+			if err != nil {
+				return err
+			}
+			source.Tables = tablesKey
+		} else {
+			t.logger.Debugf("building metadata: table node not found for %s", source.Name)
+		}
+
+		// get functions node
+		functionsPath, err := yaml.PathString(fmt.Sprintf("$[%d].functions", idx))
+		if err != nil {
+			return err
+		}
+		functionsNode, err := functionsPath.ReadNode(bytes.NewReader(sourcesBytes))
+		if err == nil {
+			functionsNodeBytes, err := ioutil.ReadAll(functionsNode)
+			if err != nil {
+				return err
+			}
+			var functionsKey interface{}
+			err = v3yaml.Unmarshal(functionsNodeBytes, newSourcesYamlDecoder(
+				sourcesYamlDecoderOpts{
+					IncludeTagBaseDirectory: filepath.Join(t.MetadataDir, sourcesDirectory, source.Name, functionsDirectory),
+				},
+				&functionsKey,
+			))
+			if err != nil {
+				return err
+			}
+			source.Functions = functionsKey
+		} else {
+			t.logger.Debugf("building metadata: functions node not found for %s", source.Name)
+		}
+
+		sources = append(sources, source)
+	}
+
+	sourcesStructBytes, err := goyaml.Marshal(sources)
+	if err != nil {
+		return err
+	}
+	var item = goyaml.MapItem{
 		Key:   "sources",
 		Value: []yaml.MapSlice{},
 	}
-	err = yaml.Unmarshal(data, &item.Value)
-	if err != nil {
+	if err := goyaml.Unmarshal(sourcesStructBytes, &item.Value); err != nil {
 		return err
 	}
 	*metadata = append(*metadata, item)
 	return nil
 }

-func (t *SourceConfig) Export(metadata yaml.MapSlice) (map[string][]byte, error) {
-	var sources interface{}
-	for _, item := range metadata {
-		k, ok := item.Key.(string)
-		if !ok || k != "sources" {
-			continue
-		}
-		sources = item.Value
-	}
-	if sources == nil {
-		sources = make([]interface{}, 0)
-	}
-	data, err := yaml.Marshal(sources)
+func (t *SourceConfig) Export(metadata goyaml.MapSlice) (map[string][]byte, error) {
+	metadataBytes, err := goyaml.Marshal(metadata)
 	if err != nil {
 		return nil, err
 	}
+	files := map[string][]byte{}
+	// Build sources.yaml
+	// sources.yaml
+	var sources []*Source
+	sourcePath, err := yaml.PathString("$.sources")
+	if err != nil {
+		return nil, err
+	}
+	if err := sourcePath.Read(bytes.NewReader(metadataBytes), &sources); err != nil {
+		return nil, err
+	}
+	for idx, source := range sources {
+		var tableTags []string
+		var functionTags []string
+
+		// populate !include syntax
+		var tablesKey []struct {
+			Table struct {
+				Name   string `yaml:"name"`
+				Schema string `yaml:"schema"`
+			} `yaml:"table"`
+		}
+		path := fmt.Sprintf("$.sources[%d].tables", idx)
+		tablesPath, err := yaml.PathString(path)
+		if err != nil {
+			return nil, err
+		}
+		if err := tablesPath.Read(bytes.NewReader(metadataBytes), &tablesKey); err != nil {
+			t.logger.Debug("reading functions node from metadata", err)
+		}
+
+		var rawTables []interface{}
+		if err := tablesPath.Read(bytes.NewReader(metadataBytes), &rawTables); err != nil {
+			t.logger.Debug("reading tables node from metadata", err)
+		}
+		for idx, table := range tablesKey {
+			tableFileName := fmt.Sprintf("%s_%s.yaml", table.Table.Schema, table.Table.Name)
+			tableIncludeTag := fmt.Sprintf(fmt.Sprintf("%s %s", "!include", tableFileName))
+			tableTags = append(tableTags, tableIncludeTag)
+
+			// build <source>/tables/<table_primary_key>.yaml
+			b, err := yaml.Marshal(rawTables[idx])
+			if err != nil {
+				return nil, err
+			}
+			tableFilePath := filepath.Join(t.MetadataDir, sourcesDirectory, source.Name, tablesDirectory, tableFileName)
+			files[tableFilePath] = b
+		}
+
+		var functions []struct {
+			Function struct {
+				Name   string `yaml:"name"`
+				Schema string `yaml:"schema"`
+			} `yaml:"function"`
+		}
+		functionsPath, err := yaml.PathString(fmt.Sprintf("$.sources[%d].functions", idx))
+		if err != nil {
+			return nil, err
+		}
+		if err := functionsPath.Read(bytes.NewReader(metadataBytes), &functions); err != nil {
+			t.logger.Debug("reading functions node from metadata", err)
+		}
+		var rawFunctions []interface{}
+		if err := functionsPath.Read(bytes.NewReader(metadataBytes), &rawFunctions); err != nil {
+			t.logger.Debug("reading functions node from metadata", err)
+		}
+		for idx, function := range functions {
+			functionFileName := fmt.Sprintf("%s_%s.yaml", function.Function.Schema, function.Function.Name)
+			includeTag := fmt.Sprintf(fmt.Sprintf("%s %s", "!include", functionFileName))
+			functionTags = append(functionTags, includeTag)
+
+			// build <source>/functions/<function_primary_key>.yaml
+			b, err := yaml.Marshal(rawFunctions[idx])
+			if err != nil {
+				return nil, err
+			}
+			functionFilePath := filepath.Join(t.MetadataDir, sourcesDirectory, source.Name, functionsDirectory, functionFileName)
+			files[functionFilePath] = b
+		}
+		source.Tables = tableTags
+		source.Functions = functionTags
+	}
+
+	sourcesYamlBytes, err := yaml.Marshal(sources)
+	if err != nil {
+		return nil, err
+	}
+	files[filepath.Join(t.MetadataDir, sourcesDirectory, fileName)] = sourcesYamlBytes

 	// clear old tables.yaml and functions.yaml files if exists
 	if f, _ := os.Stat(filepath.Join(t.MetadataDir, tables.MetadataFilename)); f != nil {
@ -89,9 +253,7 @@ func (t *SourceConfig) Export(metadata yaml.MapSlice) (map[string][]byte, error)
 		os.Remove(filepath.Join(t.MetadataDir, tables.MetadataFilename))
 	}

-	return map[string][]byte{
-		filepath.Join(t.MetadataDir, fileName): data,
-	}, nil
+	return files, nil
 }

 func (t *SourceConfig) Name() string {
--- a/cli/metadata/sources/sources_test.go
+++ b/cli/metadata/sources/sources_test.go
@ -0,0 +1,193 @@
+package sources
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v2"
+
+	"github.com/sirupsen/logrus"
+)
+
+func TestSourceConfig_Export(t *testing.T) {
+	type fields struct {
+		MetadataDir string
+		logger      *logrus.Logger
+	}
+	type args struct {
+		metadata yaml.MapSlice
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		want    map[string][]byte
+		wantErr bool
+	}{
+		{
+			"can create sources metadata representation",
+			fields{
+				MetadataDir: "./metadata",
+				logger:      logrus.New(),
+			},
+			args{
+				metadata: func() yaml.MapSlice {
+					var metadata yaml.MapSlice
+					jsonb, err := ioutil.ReadFile("testdata/metadata.json")
+					assert.NoError(t, err)
+					assert.NoError(t, yaml.Unmarshal(jsonb, &metadata))
+					return metadata
+				}(),
+			},
+			map[string][]byte{
+				"metadata/sources/sources.yaml": []byte(`- name: default
+  configuration:
+    connection_info:
+      database_url:
+        from_env: HASURA_GRAPHQL_DATABASE_URL
+      pool_settings:
+        idle_timeout: 180
+        max_connections: 50
+        retries: 1
+  tables:
+  - "!include public_t1.yaml"
+  - "!include public_t2.yaml"
+  functions:
+  - "!include public_get_t1.yaml"
+  - "!include public_get_t2.yaml"
+`),
+				"metadata/sources/default/tables/public_t1.yaml": []byte(`table:
+  name: t1
+  schema: public
+`),
+				"metadata/sources/default/tables/public_t2.yaml": []byte(`table:
+  name: t2
+  schema: public
+`),
+				"metadata/sources/default/functions/public_get_t1.yaml": []byte(`function:
+  name: get_t1
+  schema: public
+`),
+				"metadata/sources/default/functions/public_get_t2.yaml": []byte(`function:
+  name: get_t2
+  schema: public
+`),
+			},
+			false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tc := &SourceConfig{
+				MetadataDir: tt.fields.MetadataDir,
+				logger:      tt.fields.logger,
+			}
+			got, err := tc.Export(tt.args.metadata)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("Export() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			var wantContent = map[string]string{}
+			var gotContent = map[string]string{}
+			for k, v := range got {
+				gotContent[k] = string(v)
+			}
+			for k, v := range tt.want {
+				wantContent[k] = string(v)
+			}
+			assert.Equal(t, wantContent, gotContent)
+		})
+	}
+}
+
+func TestSourceConfig_Build(t *testing.T) {
+	type fields struct {
+		MetadataDir string
+		logger      *logrus.Logger
+	}
+	type args struct {
+		metadata *yaml.MapSlice
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		want    string
+		wantErr bool
+	}{
+		{
+			"can build metadata from file",
+			fields{
+				MetadataDir: "testdata/metadata",
+				logger:      logrus.New(),
+			},
+			args{
+				metadata: new(yaml.MapSlice),
+			},
+			`sources:
+- configuration:
+    connection_info:
+      database_url:
+        from_env: HASURA_GRAPHQL_DATABASE_URL
+      pool_settings:
+        idle_timeout: 180
+        max_connections: 50
+        retries: 1
+  functions:
+  - function:
+      name: get_t1
+      schema: public
+  - function:
+      name: get_t2
+      schema: public
+  name: s1
+  tables:
+  - table:
+      name: t1
+      schema: public
+  - table:
+      name: t2
+      schema: public
+- configuration:
+    connection_info:
+      database_url:
+        from_env: HASURA_GRAPHQL_DATABASE_URL
+      pool_settings:
+        idle_timeout: 180
+        max_connections: 50
+        retries: 1
+  functions:
+  - function:
+      name: get_t1
+      schema: public
+  - function:
+      name: get_t2
+      schema: public
+  name: s2
+  tables:
+  - table:
+      name: t1
+      schema: public
+  - table:
+      name: t2
+      schema: public
+`,
+			false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tc := &SourceConfig{
+				MetadataDir: tt.fields.MetadataDir,
+				logger:      tt.fields.logger,
+			}
+			if err := tc.Build(tt.args.metadata); (err != nil) != tt.wantErr {
+				t.Fatalf("Build() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			b, err := yaml.Marshal(tt.args.metadata)
+			assert.NoError(t, err)
+			assert.Equal(t, tt.want, string(b))
+		})
+	}
+}
--- a/cli/metadata/sources/testdata/metadata.json
+++ b/cli/metadata/sources/testdata/metadata.json
@ -0,0 +1,118 @@
+{
+  "version": 3,
+  "sources": [
+    {
+      "name": "default",
+      "tables": [
+        {
+          "table": {
+            "schema": "public",
+            "name": "t1"
+          }
+        },
+        {
+          "table": {
+            "schema": "public",
+            "name": "t2"
+          }
+        }
+      ],
+      "functions": [
+        {
+          "function": {
+            "schema": "public",
+            "name": "get_t1"
+          }
+        },
+        {
+          "function": {
+            "schema": "public",
+            "name": "get_t2"
+          }
+        }
+      ],
+      "configuration": {
+        "connection_info": {
+          "database_url": {
+            "from_env": "HASURA_GRAPHQL_DATABASE_URL"
+          },
+          "pool_settings": {
+            "retries": 1,
+            "idle_timeout": 180,
+            "max_connections": 50
+          }
+        }
+      }
+    }
+  ],
+  "remote_schemas": [
+    {
+      "name": "countries",
+      "definition": {
+        "url": "https://countries.trevorblades.com/",
+        "timeout_seconds": 60,
+        "forward_client_headers": true
+      }
+    }
+  ],
+  "actions": [
+    {
+      "name": "action1",
+      "definition": {
+        "handler": "http://localhost:3000",
+        "output_type": "SampleOutput",
+        "arguments": [
+          {
+            "name": "arg1",
+            "type": "SampleInput!"
+          }
+        ],
+        "type": "mutation",
+        "kind": "synchronous"
+      }
+    },
+    {
+      "name": "action2",
+      "definition": {
+        "handler": "http://localhost:3000",
+        "output_type": "SampleOutput",
+        "arguments": [
+          {
+            "name": "arg1",
+            "type": "SampleInput!"
+          }
+        ],
+        "type": "mutation",
+        "kind": "synchronous"
+      }
+    }
+  ],
+  "custom_types": {
+    "input_objects": [
+      {
+        "name": "SampleInput",
+        "fields": [
+          {
+            "name": "username",
+            "type": "String!"
+          },
+          {
+            "name": "password",
+            "type": "String!"
+          }
+        ]
+      }
+    ],
+    "objects": [
+      {
+        "name": "SampleOutput",
+        "fields": [
+          {
+            "name": "accessToken",
+            "type": "String!"
+          }
+        ]
+      }
+    ]
+  }
+}
--- a/cli/metadata/sources/testdata/metadata/actions.graphql
+++ b/cli/metadata/sources/testdata/metadata/actions.graphql
@ -0,0 +1,2 @@
+
+
--- a/cli/metadata/sources/testdata/metadata/actions.yaml
+++ b/cli/metadata/sources/testdata/metadata/actions.yaml
@ -0,0 +1,6 @@
+actions: []
+custom_types:
+  enums: []
+  input_objects: []
+  objects: []
+  scalars: []
--- a/cli/metadata/sources/testdata/metadata/allow_list.yaml
+++ b/cli/metadata/sources/testdata/metadata/allow_list.yaml
@ -0,0 +1 @@
+[]
--- a/cli/metadata/sources/testdata/metadata/cron_triggers.yaml
+++ b/cli/metadata/sources/testdata/metadata/cron_triggers.yaml
@ -0,0 +1 @@
+[]
--- a/cli/metadata/sources/testdata/metadata/query_collections.yaml
+++ b/cli/metadata/sources/testdata/metadata/query_collections.yaml
@ -0,0 +1 @@
+[]
--- a/cli/metadata/sources/testdata/metadata/remote_schemas.yaml
+++ b/cli/metadata/sources/testdata/metadata/remote_schemas.yaml
@ -0,0 +1 @@
+[]
--- a/cli/metadata/sources/testdata/metadata/sources/s1/functions/public_get_t1.yaml
+++ b/cli/metadata/sources/testdata/metadata/sources/s1/functions/public_get_t1.yaml
@ -0,0 +1,3 @@
+function:
+  name: get_t1
+  schema: public
--- a/cli/metadata/sources/testdata/metadata/sources/s1/functions/public_get_t2.yaml
+++ b/cli/metadata/sources/testdata/metadata/sources/s1/functions/public_get_t2.yaml
@ -0,0 +1,3 @@
+function:
+  name: get_t2
+  schema: public
--- a/cli/metadata/sources/testdata/metadata/sources/s1/tables/public_t1.yaml
+++ b/cli/metadata/sources/testdata/metadata/sources/s1/tables/public_t1.yaml
@ -0,0 +1,3 @@
+table:
+  name: t1
+  schema: public
--- a/cli/metadata/sources/testdata/metadata/sources/s1/tables/public_t2.yaml
+++ b/cli/metadata/sources/testdata/metadata/sources/s1/tables/public_t2.yaml
@ -0,0 +1,3 @@
+table:
+  name: t2
+  schema: public
--- a/cli/metadata/sources/testdata/metadata/sources/s2/functions/public_get_t1.yaml
+++ b/cli/metadata/sources/testdata/metadata/sources/s2/functions/public_get_t1.yaml
@ -0,0 +1,3 @@
+function:
+  name: get_t1
+  schema: public
--- a/cli/metadata/sources/testdata/metadata/sources/s2/functions/public_get_t2.yaml
+++ b/cli/metadata/sources/testdata/metadata/sources/s2/functions/public_get_t2.yaml
@ -0,0 +1,3 @@
+function:
+  name: get_t2
+  schema: public
--- a/cli/metadata/sources/testdata/metadata/sources/s2/tables/public_t1.yaml
+++ b/cli/metadata/sources/testdata/metadata/sources/s2/tables/public_t1.yaml
@ -0,0 +1,3 @@
+table:
+  name: t1
+  schema: public
--- a/cli/metadata/sources/testdata/metadata/sources/s2/tables/public_t2.yaml
+++ b/cli/metadata/sources/testdata/metadata/sources/s2/tables/public_t2.yaml
@ -0,0 +1,3 @@
+table:
+  name: t2
+  schema: public
--- a/cli/metadata/sources/testdata/metadata/sources/sources.yaml
+++ b/cli/metadata/sources/testdata/metadata/sources/sources.yaml
@ -0,0 +1,30 @@
+- name: s1
+  configuration:
+    connection_info:
+      database_url:
+        from_env: HASURA_GRAPHQL_DATABASE_URL
+      pool_settings:
+        idle_timeout: 180
+        max_connections: 50
+        retries: 1
+  tables:
+  - !include "public_t1.yaml"
+  - !include "public_t2.yaml"
+  functions:
+  - !include "public_get_t1.yaml"
+  - !include "public_get_t2.yaml"
+- name: s2
+  configuration:
+    connection_info:
+      database_url:
+        from_env: HASURA_GRAPHQL_DATABASE_URL
+      pool_settings:
+        idle_timeout: 180
+        max_connections: 50
+        retries: 1
+  tables:
+    - !include "public_t1.yaml"
+    - !include "public_t2.yaml"
+  functions:
+    - !include "public_get_t1.yaml"
+    - !include "public_get_t2.yaml"
--- a/cli/metadata/sources/testdata/metadata/version.yaml
+++ b/cli/metadata/sources/testdata/metadata/version.yaml
@ -0,0 +1 @@
+version: 3
--- a/cli/metadata/sources/yaml.go
+++ b/cli/metadata/sources/yaml.go
@ -0,0 +1,106 @@
+package sources
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+const includeTag = "!include"
+
+type sourcesYamlDecoderOpts struct {
+	// directory which is to be used as the parent directory to look for filenames
+	// specified in !include tag
+	IncludeTagBaseDirectory string
+}
+type sourcesYamlDecoder struct {
+	destination interface{}
+	opts        sourcesYamlDecoderOpts
+}
+
+func newSourcesYamlDecoder(opts sourcesYamlDecoderOpts, destination interface{}) *sourcesYamlDecoder {
+	return &sourcesYamlDecoder{destination, opts}
+}
+
+func (s *sourcesYamlDecoder) UnmarshalYAML(value *yaml.Node) error {
+	ctx := map[string]string{}
+	ctx[includeTag] = s.opts.IncludeTagBaseDirectory
+
+	resolved, err := resolveTags(ctx, value)
+	if err != nil {
+		return err
+	}
+	return resolved.Decode(s.destination)
+}
+
+type Fragment struct {
+	ctx     map[string]string
+	content *yaml.Node
+}
+
+func newFragment(ctx map[string]string) *Fragment {
+	f := new(Fragment)
+	f.ctx = ctx
+	return f
+}
+func (f *Fragment) UnmarshalYAML(value *yaml.Node) error {
+	var err error
+	// process includes in fragments
+	f.content, err = resolveTags(f.ctx, value)
+	return err
+}
+
+func resolveTags(ctx map[string]string, node *yaml.Node) (*yaml.Node, error) {
+	resolve := func(node *yaml.Node) (*yaml.Node, error) {
+		if node.Kind != yaml.ScalarNode {
+			return nil, fmt.Errorf("found %s on scalar node", includeTag)
+		}
+		baseDir, ok := ctx[includeTag]
+		if !ok {
+			return nil, fmt.Errorf("parser errror: base directory for !include tag not specified")
+		}
+		file, err := ioutil.ReadFile(filepath.Join(baseDir, node.Value))
+		if err != nil {
+			return nil, err
+		}
+		var f = newFragment(ctx)
+		err = yaml.Unmarshal(file, f)
+		if err != nil {
+			return nil, err
+		}
+		return f.content, err
+	}
+	switch node.Tag {
+	case includeTag:
+		return resolve(node)
+	case "!!str":
+		if strings.Contains(node.Value, includeTag) {
+			node.Tag = includeTag
+			parts := strings.Split(node.Value, " ")
+			if len(parts) == 2 {
+				node.Value = strings.Trim(parts[1], "\"")
+				return resolve(node)
+			}
+		}
+
+	}
+
+	switch node.Kind {
+	case yaml.DocumentNode, yaml.SequenceNode, yaml.MappingNode:
+		var err error
+		for idx := range node.Content {
+			node.Content[idx], err = resolveTags(ctx, node.Content[idx])
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	return node, nil
+}
+
+type IncludeTagVisitor struct {
+	baseDir string
+}
--- a/cli/metadata/sources/yaml_test.go
+++ b/cli/metadata/sources/yaml_test.go
@ -0,0 +1,67 @@
+package sources
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"gopkg.in/yaml.v3"
+)
+
+func Test_resolveTags(t *testing.T) {
+	type args struct {
+		ctx  map[string]string
+		node *yaml.Node
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    string
+		wantErr bool
+	}{
+		{
+			"can resolve !include tags",
+			args{
+				ctx: map[string]string{includeTag: "testdata/metadata/"},
+				node: func() *yaml.Node {
+					v := new(yaml.Node)
+					b := []byte(`
+actions: !include "actions.yaml"
+`)
+					assert.NoError(t, yaml.Unmarshal(b, v))
+					return v
+				}(),
+			},
+			"actions:\n    actions: []\n    custom_types:\n        enums: []\n        input_objects: []\n        objects: []\n        scalars: []\n",
+			false,
+		},
+		{
+			"can resolve !include tags in strings",
+			args{
+				ctx: map[string]string{includeTag: "testdata/metadata/"},
+				node: func() *yaml.Node {
+					v := new(yaml.Node)
+					b := []byte(`
+actions: '!include "actions.yaml"'
+`)
+					assert.NoError(t, yaml.Unmarshal(b, v))
+					return v
+				}(),
+			},
+			"actions:\n    actions: []\n    custom_types:\n        enums: []\n        input_objects: []\n        objects: []\n        scalars: []\n",
+			false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := resolveTags(tt.args.ctx, tt.args.node)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("resolveTags() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			b, err := yaml.Marshal(got)
+			assert.NoError(t, err)
+			assert.Equal(t, tt.want, string(b))
+		})
+	}
+}
--- a/cli/migrate/source/file/file.go
+++ b/cli/migrate/source/file/file.go
@ -11,6 +11,8 @@ import (
 	"runtime"
 	"strings"

+	"github.com/spf13/afero"
+
 	"github.com/hasura/graphql-engine/cli/migrate/source"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
@ -232,7 +234,11 @@ func (f *File) ReadName(version uint64) (name string) {

 func (f *File) WriteMetadata(files map[string][]byte) error {
 	for name, content := range files {
-		err := ioutil.WriteFile(name, content, 0644)
+		fs := afero.NewOsFs()
+		if err := fs.MkdirAll(filepath.Dir(name), os.ModePerm); err != nil {
+			return err
+		}
+		err := afero.WriteFile(fs, name, content, 0644)
 		if err != nil {
 			return errors.Wrapf(err, "creating metadata file %s failed", name)
 		}