graphql-engine/cli/internal/scripts/update-project-v3.go

package scripts

import (
	"io/ioutil"
	"path/filepath"
	"regexp"

	"github.com/fatih/color"
	"github.com/goccy/go-yaml"
	"github.com/goccy/go-yaml/parser"
	"github.com/hasura/graphql-engine/cli/internal/hasura"

	"github.com/hasura/graphql-engine/cli/migrate"

	"github.com/hasura/graphql-engine/cli/internal/statestore"
	"github.com/hasura/graphql-engine/cli/internal/statestore/migrations"
	"github.com/hasura/graphql-engine/cli/internal/statestore/settings"

	"github.com/hasura/graphql-engine/cli"

	"fmt"

	"github.com/hasura/graphql-engine/cli/util"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/afero"
)

type UpgradeToMuUpgradeProjectToMultipleSourcesOpts struct {
	EC *cli.ExecutionContext
	Fs afero.Fs
	// Path to project directory
	ProjectDirectory string
	// Directory in which migrations are stored
	MigrationsAbsDirectoryPath string
	SeedsAbsDirectoryPath      string
	Logger                     *logrus.Logger
}

// UpgradeProjectToMultipleSources will help a project directory move from a single
// datasource structure to multiple datasource
// The project is expected to be in Config V2
func UpgradeProjectToMultipleSources(opts UpgradeToMuUpgradeProjectToMultipleSourcesOpts) error {
	/* New flow
		Config V2 -> Config V3
		- Warn user about creating a backup
		- Ask user for the name of datasource to migrate to
	  	- copy state from hdb_tables to catalog state
		- Move current migration directories to a new source directory
		- Move seeds belonging to the source to a new directory
		- Update config file and version
	*/

	// Validate config version is
	if opts.EC.Config.Version != cli.V2 {
		return fmt.Errorf("project should be using config V2 to be able to update to V3")
	}

	opts.Logger.Warn("The upgrade process will make some changes to your project directory, It is advised to create a backup project directory before continuing")
	response, err := util.GetYesNoPrompt("continue?")
	if err != nil {
		return err
	}
	if response == "n" {
		return nil
	}
	// move migration child directories
	// get directory names to move
	targetDatasource, err := util.GetInputPrompt("what datasource does the current migrations / seeds belong to?")
	if err != nil {
		return err
	}
	opts.EC.Spinner.Start()
	opts.EC.Spin("updating project... ")
	// copy state
	// if a default datasource is setup copy state from it
	sources, err := ListDatasources(opts.EC.APIClient.V1Metadata)
	if err != nil {
		return err
	}
	if len(sources) >= 1 {
		if err := copyState(opts.EC, targetDatasource); err != nil {
			return err
		}
	}

	// move migration child directories
	// get directory names to move
	migrationDirectoriesToMove, err := getMigrationDirectoryNames(opts.Fs, opts.MigrationsAbsDirectoryPath)
	if err != nil {
		return errors.Wrap(err, "getting list of migrations to move")
	}
	// move seed child directories
	// get directory names to move
	seedFilesToMove, err := getSeedFiles(opts.Fs, opts.SeedsAbsDirectoryPath)
	if err != nil {
		return errors.Wrap(err, "getting list of seed files to move")
	}

	// create a new directory for TargetDatasource
	targetMigrationsDirectoryName := filepath.Join(opts.MigrationsAbsDirectoryPath, targetDatasource)
	if err = opts.Fs.Mkdir(targetMigrationsDirectoryName, 0755); err != nil {
		errors.Wrap(err, "creating target datasource name")
	}

	// create a new directory for TargetDatasource
	targetSeedsDirectoryName := filepath.Join(opts.SeedsAbsDirectoryPath, targetDatasource)
	if err = opts.Fs.Mkdir(targetSeedsDirectoryName, 0755); err != nil {
		errors.Wrap(err, "creating target datasource name")
	}

	// move migration directories to target datasource directory
	if err := copyMigrations(opts.Fs, migrationDirectoriesToMove, opts.MigrationsAbsDirectoryPath, targetMigrationsDirectoryName); err != nil {
		return errors.Wrap(err, "moving migrations to target datasource directory")
	}
	// move seed directories to target datasource directory
	if err := copyFiles(opts.Fs, seedFilesToMove, opts.SeedsAbsDirectoryPath, targetSeedsDirectoryName); err != nil {
		return errors.Wrap(err, "moving seeds to target datasource directory")
	}

	// write new config file
	newConfig := *opts.EC.Config
	newConfig.Version = cli.V3
	newConfig.DatasourcesConfig = []cli.DatasourceConfig{
		{
			Name:                targetDatasource,
			MigrationsDirectory: targetDatasource,
			SeedsDirectory:      targetDatasource,
		},
	}
	if err := opts.EC.WriteConfig(&newConfig); err != nil {
		return err
	}
	opts.EC.Config = &newConfig

	// delete original migrations
	if err := removeDirectories(opts.Fs, opts.MigrationsAbsDirectoryPath, migrationDirectoriesToMove); err != nil {
		return errors.Wrap(err, "removing up original migrations")
	}
	// delete original seeds
	if err := removeDirectories(opts.Fs, opts.SeedsAbsDirectoryPath, seedFilesToMove); err != nil {
		return errors.Wrap(err, "removing up original migrations")
	}
	// remove functions.yaml and tables.yaml files
	metadataFiles := []string{"functions.yaml", "tables.yaml"}
	if err := removeDirectories(opts.Fs, opts.EC.MetadataDir, metadataFiles); err != nil {
		return err
	}
	// do a metadata export
	m, err := migrate.NewMigrate(opts.EC, true, "")
	if err != nil {
		return err
	}
	files, err := m.ExportMetadata()
	if err != nil {
		return err
	}
	if err := m.WriteMetadata(files); err != nil {
		return err
	}
	opts.EC.Spinner.Stop()
	return nil
}

func removeDirectories(fs afero.Fs, parentDirectory string, dirNames []string) error {
	for _, d := range dirNames {
		if err := fs.RemoveAll(filepath.Join(parentDirectory, d)); err != nil {
			return err
		}
	}
	return nil
}

func copyMigrations(fs afero.Fs, dirs []string, parentDir, target string) error {
	for _, dir := range dirs {
		f, _ := fs.Stat(filepath.Join(parentDir, dir))
		if f != nil {
			if f.IsDir() {
				err := util.CopyDirAfero(fs, filepath.Join(parentDir, dir), filepath.Join(target, dir))
				if err != nil {
					return errors.Wrapf(err, "moving %s to %s", dir, target)
				}
			} else {
				err := util.CopyFileAfero(fs, filepath.Join(parentDir, dir), filepath.Join(target, dir))
				if err != nil {
					return errors.Wrapf(err, "moving %s to %s", dir, target)
				}
			}
		}

	}
	return nil
}

func copyFiles(fs afero.Fs, files []string, parentDir, target string) error {
	for _, dir := range files {
		err := util.CopyFileAfero(fs, filepath.Join(parentDir, dir), filepath.Join(target, dir))
		if err != nil {
			return errors.Wrapf(err, "moving %s to %s", dir, target)
		}
	}
	return nil
}

func getMigrationDirectoryNames(fs afero.Fs, rootMigrationsDir string) ([]string, error) {
	return getMatchingFilesAndDirs(fs, rootMigrationsDir, isHasuraCLIGeneratedMigration)
}

func getSeedFiles(fs afero.Fs, rootSeedDir string) ([]string, error) {
	// find migrations which are in the format <timestamp>_name
	var seedFiles []string
	dirs, err := afero.ReadDir(fs, rootSeedDir)
	if err != nil {
		return nil, err
	}
	for _, info := range dirs {
		if !info.IsDir() {
			seedFiles = append(seedFiles, filepath.Join(info.Name()))
		}

	}
	return seedFiles, nil
}

func getMatchingFilesAndDirs(fs afero.Fs, parentDir string, matcher func(string) (bool, error)) ([]string, error) {
	// find migrations which are in the format <timestamp>_name
	var migs []string
	dirs, err := afero.ReadDir(fs, parentDir)
	if err != nil {
		return nil, err
	}
	for _, info := range dirs {
		if ok, err := matcher(info.Name()); !ok || err != nil {
			if err != nil {
				return nil, err
			}
			continue
		}
		migs = append(migs, filepath.Join(info.Name()))

	}
	return migs, nil
}

func isHasuraCLIGeneratedMigration(dirPath string) (bool, error) {
	const regex = `^([0-9]{13})_(.*)$`
	return regexp.MatchString(regex, filepath.Base(dirPath))
}

func copyState(ec *cli.ExecutionContext, destdatasource string) error {
	// copy migrations state
	src := cli.GetMigrationsStateStore(ec)
	if err := src.PrepareMigrationsStateStore(); err != nil {
		return err
	}
	dst := migrations.NewCatalogStateStore(statestore.NewCLICatalogState(ec.APIClient.V1Metadata))
	if err := dst.PrepareMigrationsStateStore(); err != nil {
		return err
	}
	err := statestore.CopyMigrationState(src, dst, "", destdatasource)
	if err != nil {
		return err
	}
	// copy settings state
	srcSettingsStore := cli.GetSettingsStateStore(ec)
	if err := srcSettingsStore.PrepareSettingsDriver(); err != nil {
		return err
	}
	dstSettingsStore := settings.NewStateStoreCatalog(statestore.NewCLICatalogState(ec.APIClient.V1Metadata))
	if err := dstSettingsStore.PrepareSettingsDriver(); err != nil {
		return err
	}
	err = statestore.CopySettingsState(srcSettingsStore, dstSettingsStore)
	if err != nil {
		return err
	}
	return nil
}

func CheckIfUpdateToConfigV3IsRequired(ec *cli.ExecutionContext) error {
	// see if an update to config V3 is necessary
	if ec.Config.Version <= cli.V1 && ec.HasMetadataV3 {
		ec.Logger.Info("config v1 is deprecated from v1.4")
		return errors.New("please upgrade your project to a newer version.\ntip: use " + color.New(color.FgCyan).SprintFunc()("hasura scripts update-project-v2") + " to upgrade your project to config v2")
	}
	if ec.Config.Version < cli.V3 && ec.HasMetadataV3 {
		sources, err := ListDatasources(ec.APIClient.V1Metadata)
		if err != nil {
			return err
		}
		upgrade := func() error {
			// server is configured with a default datasource
			// and other datasources
			ec.Logger.Info("Looks like you are trying to use hasura with multiple datasources, which requires some changes on your project directory\n")
			ec.Logger.Info("please use " + color.New(color.FgCyan).SprintFunc()("hasura scripts update-project-v3") + " to make this change")
			return errors.New("update to config V3")
		}
		// if no sources are configured prompt and upgrade
		if len(sources) != 1 {
			return upgrade()
		}
		// if 1 source is configured and it is not "default" then it's a custom datasource
		// then also prompt an upgrade
		if len(sources) == 1 {
			if sources[0] != "default" {
				return upgrade()
			}
		}
	}
	return nil
}

func ListDatasources(client hasura.CommonMetadataOperations) ([]string, error) {
	metadata, err := client.ExportMetadata()
	if err != nil {
		return nil, err
	}
	jsonb, err := ioutil.ReadAll(metadata)
	if err != nil {
		return nil, err
	}
	yamlb, err := yaml.JSONToYAML(jsonb)
	if err != nil {
		return nil, err
	}
	ast, err := parser.ParseBytes(yamlb, 0)
	if err != nil {
		return nil, err
	}
	if len(ast.Docs) <= 0 {
		return nil, fmt.Errorf("failed listing sources from metadata")
	}
	var sources []string
	path, err := yaml.PathString("$.sources[*].name")
	if err != nil {
		return nil, err
	}
	if err := path.Read(ast.Docs[0], &sources); err != nil {
		return nil, err
	}
	return sources, nil
}