package scripts import ( "io/ioutil" "path/filepath" "regexp" "github.com/fatih/color" "github.com/goccy/go-yaml" "github.com/goccy/go-yaml/parser" "github.com/hasura/graphql-engine/cli/internal/hasura" "github.com/hasura/graphql-engine/cli/migrate" "github.com/hasura/graphql-engine/cli/internal/statestore" "github.com/hasura/graphql-engine/cli/internal/statestore/migrations" "github.com/hasura/graphql-engine/cli/internal/statestore/settings" "github.com/hasura/graphql-engine/cli" "fmt" "github.com/hasura/graphql-engine/cli/util" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/afero" ) type UpgradeToMuUpgradeProjectToMultipleSourcesOpts struct { EC *cli.ExecutionContext Fs afero.Fs // Path to project directory ProjectDirectory string // Directory in which migrations are stored MigrationsAbsDirectoryPath string SeedsAbsDirectoryPath string Logger *logrus.Logger } // UpgradeProjectToMultipleSources will help a project directory move from a single // datasource structure to multiple datasource // The project is expected to be in Config V2 func UpgradeProjectToMultipleSources(opts UpgradeToMuUpgradeProjectToMultipleSourcesOpts) error { /* New flow Config V2 -> Config V3 - Warn user about creating a backup - Ask user for the name of datasource to migrate to - copy state from hdb_tables to catalog state - Move current migration directories to a new source directory - Move seeds belonging to the source to a new directory - Update config file and version */ // Validate config version is if opts.EC.Config.Version != cli.V2 { return fmt.Errorf("project should be using config V2 to be able to update to V3") } opts.Logger.Warn("The upgrade process will make some changes to your project directory, It is advised to create a backup project directory before continuing") response, err := util.GetYesNoPrompt("continue?") if err != nil { return err } if response == "n" { return nil } // move migration child directories // get directory names to move targetDatasource, err := util.GetInputPrompt("what datasource does the current migrations / seeds belong to?") if err != nil { return err } opts.EC.Spinner.Start() opts.EC.Spin("updating project... ") // copy state // if a default datasource is setup copy state from it sources, err := ListDatasources(opts.EC.APIClient.V1Metadata) if err != nil { return err } if len(sources) >= 1 { if err := copyState(opts.EC, targetDatasource); err != nil { return err } } // move migration child directories // get directory names to move migrationDirectoriesToMove, err := getMigrationDirectoryNames(opts.Fs, opts.MigrationsAbsDirectoryPath) if err != nil { return errors.Wrap(err, "getting list of migrations to move") } // move seed child directories // get directory names to move seedFilesToMove, err := getSeedFiles(opts.Fs, opts.SeedsAbsDirectoryPath) if err != nil { return errors.Wrap(err, "getting list of seed files to move") } // create a new directory for TargetDatasource targetMigrationsDirectoryName := filepath.Join(opts.MigrationsAbsDirectoryPath, targetDatasource) if err = opts.Fs.Mkdir(targetMigrationsDirectoryName, 0755); err != nil { errors.Wrap(err, "creating target datasource name") } // create a new directory for TargetDatasource targetSeedsDirectoryName := filepath.Join(opts.SeedsAbsDirectoryPath, targetDatasource) if err = opts.Fs.Mkdir(targetSeedsDirectoryName, 0755); err != nil { errors.Wrap(err, "creating target datasource name") } // move migration directories to target datasource directory if err := copyMigrations(opts.Fs, migrationDirectoriesToMove, opts.MigrationsAbsDirectoryPath, targetMigrationsDirectoryName); err != nil { return errors.Wrap(err, "moving migrations to target datasource directory") } // move seed directories to target datasource directory if err := copyFiles(opts.Fs, seedFilesToMove, opts.SeedsAbsDirectoryPath, targetSeedsDirectoryName); err != nil { return errors.Wrap(err, "moving seeds to target datasource directory") } // write new config file newConfig := *opts.EC.Config newConfig.Version = cli.V3 newConfig.DatasourcesConfig = []cli.DatasourceConfig{ { Name: targetDatasource, MigrationsDirectory: targetDatasource, SeedsDirectory: targetDatasource, }, } if err := opts.EC.WriteConfig(&newConfig); err != nil { return err } opts.EC.Config = &newConfig // delete original migrations if err := removeDirectories(opts.Fs, opts.MigrationsAbsDirectoryPath, migrationDirectoriesToMove); err != nil { return errors.Wrap(err, "removing up original migrations") } // delete original seeds if err := removeDirectories(opts.Fs, opts.SeedsAbsDirectoryPath, seedFilesToMove); err != nil { return errors.Wrap(err, "removing up original migrations") } // remove functions.yaml and tables.yaml files metadataFiles := []string{"functions.yaml", "tables.yaml"} if err := removeDirectories(opts.Fs, opts.EC.MetadataDir, metadataFiles); err != nil { return err } // do a metadata export m, err := migrate.NewMigrate(opts.EC, true, "") if err != nil { return err } files, err := m.ExportMetadata() if err != nil { return err } if err := m.WriteMetadata(files); err != nil { return err } opts.EC.Spinner.Stop() return nil } func removeDirectories(fs afero.Fs, parentDirectory string, dirNames []string) error { for _, d := range dirNames { if err := fs.RemoveAll(filepath.Join(parentDirectory, d)); err != nil { return err } } return nil } func copyMigrations(fs afero.Fs, dirs []string, parentDir, target string) error { for _, dir := range dirs { f, _ := fs.Stat(filepath.Join(parentDir, dir)) if f != nil { if f.IsDir() { err := util.CopyDirAfero(fs, filepath.Join(parentDir, dir), filepath.Join(target, dir)) if err != nil { return errors.Wrapf(err, "moving %s to %s", dir, target) } } else { err := util.CopyFileAfero(fs, filepath.Join(parentDir, dir), filepath.Join(target, dir)) if err != nil { return errors.Wrapf(err, "moving %s to %s", dir, target) } } } } return nil } func copyFiles(fs afero.Fs, files []string, parentDir, target string) error { for _, dir := range files { err := util.CopyFileAfero(fs, filepath.Join(parentDir, dir), filepath.Join(target, dir)) if err != nil { return errors.Wrapf(err, "moving %s to %s", dir, target) } } return nil } func getMigrationDirectoryNames(fs afero.Fs, rootMigrationsDir string) ([]string, error) { return getMatchingFilesAndDirs(fs, rootMigrationsDir, isHasuraCLIGeneratedMigration) } func getSeedFiles(fs afero.Fs, rootSeedDir string) ([]string, error) { // find migrations which are in the format _name var seedFiles []string dirs, err := afero.ReadDir(fs, rootSeedDir) if err != nil { return nil, err } for _, info := range dirs { if !info.IsDir() { seedFiles = append(seedFiles, filepath.Join(info.Name())) } } return seedFiles, nil } func getMatchingFilesAndDirs(fs afero.Fs, parentDir string, matcher func(string) (bool, error)) ([]string, error) { // find migrations which are in the format _name var migs []string dirs, err := afero.ReadDir(fs, parentDir) if err != nil { return nil, err } for _, info := range dirs { if ok, err := matcher(info.Name()); !ok || err != nil { if err != nil { return nil, err } continue } migs = append(migs, filepath.Join(info.Name())) } return migs, nil } func isHasuraCLIGeneratedMigration(dirPath string) (bool, error) { const regex = `^([0-9]{13})_(.*)$` return regexp.MatchString(regex, filepath.Base(dirPath)) } func copyState(ec *cli.ExecutionContext, destdatasource string) error { // copy migrations state src := cli.GetMigrationsStateStore(ec) if err := src.PrepareMigrationsStateStore(); err != nil { return err } dst := migrations.NewCatalogStateStore(statestore.NewCLICatalogState(ec.APIClient.V1Metadata)) if err := dst.PrepareMigrationsStateStore(); err != nil { return err } err := statestore.CopyMigrationState(src, dst, "", destdatasource) if err != nil { return err } // copy settings state srcSettingsStore := cli.GetSettingsStateStore(ec) if err := srcSettingsStore.PrepareSettingsDriver(); err != nil { return err } dstSettingsStore := settings.NewStateStoreCatalog(statestore.NewCLICatalogState(ec.APIClient.V1Metadata)) if err := dstSettingsStore.PrepareSettingsDriver(); err != nil { return err } err = statestore.CopySettingsState(srcSettingsStore, dstSettingsStore) if err != nil { return err } return nil } func CheckIfUpdateToConfigV3IsRequired(ec *cli.ExecutionContext) error { // see if an update to config V3 is necessary if ec.Config.Version <= cli.V1 && ec.HasMetadataV3 { ec.Logger.Info("config v1 is deprecated from v1.4") return errors.New("please upgrade your project to a newer version.\ntip: use " + color.New(color.FgCyan).SprintFunc()("hasura scripts update-project-v2") + " to upgrade your project to config v2") } if ec.Config.Version < cli.V3 && ec.HasMetadataV3 { sources, err := ListDatasources(ec.APIClient.V1Metadata) if err != nil { return err } upgrade := func() error { // server is configured with a default datasource // and other datasources ec.Logger.Info("Looks like you are trying to use hasura with multiple datasources, which requires some changes on your project directory\n") ec.Logger.Info("please use " + color.New(color.FgCyan).SprintFunc()("hasura scripts update-project-v3") + " to make this change") return errors.New("update to config V3") } // if no sources are configured prompt and upgrade if len(sources) != 1 { return upgrade() } // if 1 source is configured and it is not "default" then it's a custom datasource // then also prompt an upgrade if len(sources) == 1 { if sources[0] != "default" { return upgrade() } } } return nil } func ListDatasources(client hasura.CommonMetadataOperations) ([]string, error) { metadata, err := client.ExportMetadata() if err != nil { return nil, err } jsonb, err := ioutil.ReadAll(metadata) if err != nil { return nil, err } yamlb, err := yaml.JSONToYAML(jsonb) if err != nil { return nil, err } ast, err := parser.ParseBytes(yamlb, 0) if err != nil { return nil, err } if len(ast.Docs) <= 0 { return nil, fmt.Errorf("failed listing sources from metadata") } var sources []string path, err := yaml.PathString("$.sources[*].name") if err != nil { return nil, err } if err := path.Read(ast.Docs[0], &sources); err != nil { return nil, err } return sources, nil }