graphql-engine/cli/migrate/util.go

package migrate

import (
	"fmt"
	nurl "net/url"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/hasura/graphql-engine/cli/v2"
	"github.com/hasura/graphql-engine/cli/v2/internal/errors"
	"github.com/hasura/graphql-engine/cli/v2/internal/hasura"
	"github.com/hasura/graphql-engine/cli/v2/internal/scripts"
	"github.com/hasura/graphql-engine/cli/v2/internal/statestore"
	"github.com/hasura/graphql-engine/cli/v2/internal/statestore/migrations"
	migratedb "github.com/hasura/graphql-engine/cli/v2/migrate/database"
)

// MultiError holds multiple errors.
type MultiError struct {
	Errs []error
}

// NewMultiError returns an error type holding multiple errors.
func NewMultiError(errs ...error) MultiError {
	compactErrs := make([]error, 0)
	for _, e := range errs {
		if e != nil {
			compactErrs = append(compactErrs, e)
		}
	}
	return MultiError{compactErrs}
}

// Error implements error. Mulitple errors are concatenated with 'and's.
func (m MultiError) Error() string {
	var strs = make([]string, 0)
	for _, e := range m.Errs {
		if len(e.Error()) > 0 {
			strs = append(strs, e.Error())
		}
	}
	return strings.Join(strs, " and ")
}

// suint64 safely converts int to uint64
// see https://goo.gl/wEcqof
// see https://goo.gl/pai7Dr
func suint64(n int64) uint64 {
	if n < 0 {
		panic(fmt.Sprintf("suint(%v) expects input >= 0", n))
	}
	return uint64(n)
}

/*
// newSlowReader turns an io.ReadCloser into a slow io.ReadCloser.
// Use this to simulate a slow internet connection.
func newSlowReader(r io.ReadCloser) io.ReadCloser {
	return &slowReader{
		rx:     r,
		reader: bufio.NewReader(r),
	}
}

type slowReader struct {
	rx     io.ReadCloser
	reader *bufio.Reader
}

func (b *slowReader) Read(p []byte) (n int, err error) {
	time.Sleep(10 * time.Millisecond)
	c, err := b.reader.ReadByte()
	if err != nil {
		return 0, err
	} else {
		copy(p, []byte{c})
		return 1, nil
	}
}

func (b *slowReader) Close() error {
	return b.rx.Close()
} */

var errNoScheme = fmt.Errorf("no scheme")

// schemeFromUrl returns the scheme from a URL string
func schemeFromUrl(url string) (string, error) {
	var op errors.Op = "migrate.schemeFromUrl"
	u, err := nurl.Parse(url)
	if err != nil {
		return "", errors.E(op, err)
	}

	if len(u.Scheme) == 0 {
		return "", errors.E(op, errNoScheme)
	}

	return u.Scheme, nil
}

// FilterCustomQuery filters all query values starting with `x-`
func FilterCustomQuery(u *nurl.URL) *nurl.URL {
	ux := *u
	vx := make(nurl.Values)
	for k, v := range ux.Query() {
		if len(k) <= 1 || (len(k) > 1 && k[0:2] != "x-") {
			vx[k] = v
		}
	}
	ux.RawQuery = vx.Encode()
	return &ux
}

func NewMigrate(ec *cli.ExecutionContext, isCmd bool, sourceName string, sourceKind hasura.SourceKind) (*Migrate, error) {
	var op errors.Op = "migrate.NewMigrate"
	// set a default source kind
	if len(sourceKind) < 1 {
		return nil, errors.E(op, fmt.Errorf("invalid source kind"))
	}
	// create a new directory for the database if it doesn't exists
	if f, _ := os.Stat(filepath.Join(ec.MigrationDir, sourceName)); f == nil {
		err := os.MkdirAll(filepath.Join(ec.MigrationDir, sourceName), 0755)
		if err != nil {
			return nil, errors.E(op, err)
		}
	}
	dbURL := GetDataPath(ec)
	fileURL := GetFilePath(filepath.Join(ec.MigrationDir, sourceName))
	opts := NewMigrateOpts{
		fileURL.String(),
		dbURL.String(),
		isCmd, int(ec.Config.Version),
		ec.Logger,
		ec.Stdout,
		ec.Stderr,
		&migratedb.HasuraOpts{
			HasMetadataV3: ec.HasMetadataV3,
			SourceName:    sourceName,
			SourceKind:    sourceKind,
			Client:        ec.APIClient,
			V2MetadataOps: func() hasura.V2CommonMetadataOperations {
				if ec.Config.Version >= cli.V3 {
					return ec.APIClient.V1Metadata
				}
				return nil
			}(),
			MetadataOps:          cli.GetCommonMetadataOps(ec),
			MigrationsStateStore: cli.GetMigrationsStateStore(ec),
			SettingsStateStore:   cli.GetSettingsStateStore(ec, sourceName),
		},
	}
	opts.hasuraOpts.PGDumpClient = ec.APIClient.PGDump
	if ec.HasMetadataV3 {
		opts.hasuraOpts.PGSourceOps = ec.APIClient.V2Query
		opts.hasuraOpts.MSSQLSourceOps = ec.APIClient.V2Query
		opts.hasuraOpts.CitusSourceOps = ec.APIClient.V2Query
		opts.hasuraOpts.BigQuerySourceOps = ec.APIClient.V2Query
		opts.hasuraOpts.GenericQueryRequest = ec.APIClient.V2Query.Send
	} else {
		opts.hasuraOpts.PGSourceOps = ec.APIClient.V1Query
		opts.hasuraOpts.GenericQueryRequest = ec.APIClient.V1Query.Send
	}

	t, err := New(opts)
	if err != nil {
		return nil, errors.E(op, fmt.Errorf("cannot create migrate instance: %w", err))
	}
	if ec.Config.Version >= cli.V2 {
		t.databaseDrv.EnableCheckMetadataConsistency(true)
	}
	if ok, err := copyStateToCatalogStateAPIIfRequired(ec, sourceName); err != nil {
		ec.Logger.Warn(err)
	} else if ok {
		if err := t.ReScan(); err != nil {
			return nil, errors.E(op, err)
		}
	}
	return t, nil
}

func GetDataPath(ec *cli.ExecutionContext) *nurl.URL {
	url := ec.Config.ServerConfig.ParsedEndpoint
	host := &nurl.URL{
		Scheme:   "hasuradb",
		Host:     url.Host,
		Path:     url.Path,
		RawQuery: ec.Config.ServerConfig.APIPaths.GetQueryParams().Encode(),
	}
	q := host.Query()
	// Set sslmode in query
	switch scheme := url.Scheme; scheme {
	case "https":
		q.Set("sslmode", "enable")
	default:
		q.Set("sslmode", "disable")
	}
	for k, v := range ec.HGEHeaders {
		q.Add("headers", fmt.Sprintf("%s:%s", k, v))
	}
	host.RawQuery = q.Encode()
	return host
}

func GetFilePath(dir string) *nurl.URL {
	host := &nurl.URL{
		Scheme: "file",
		Path:   dir,
	}

	// Add Prefix / to path if runtime.GOOS equals to windows
	if runtime.GOOS == "windows" && !strings.HasPrefix(host.Path, "/") {
		host.Path = "/" + host.Path
	}
	return host
}

func IsMigrationsSupported(kind hasura.SourceKind) bool {
	switch kind {
	case hasura.SourceKindMSSQL,
		hasura.SourceKindPG,
		hasura.SourceKindCitus,
		hasura.SourceKindCockroach,
		hasura.SourceKindBigQuery:
		return true
	}
	return false
}

func copyStateToCatalogStateAPIIfRequired(ec *cli.ExecutionContext, sourceName string) (bool, error) {
	var op errors.Op = "migrate.copyStateToCatalogStateAPIIfRequired"
	// if
	//		the project is in config v3
	// 		isStateCopyCompleted is false in catalog state
	//		hdb_catalog.schema_migrations is not empty
	if !ec.DisableAutoStateMigration && ec.Config.Version >= cli.V3 {
		// get cli catalog and check isStateCopyCompleted is false
		cs := statestore.NewCLICatalogState(ec.APIClient.V1Metadata)
		state, err := cs.Get()
		if err != nil {
			return false, errors.E(op, err)
		}
		markStateMigrationCompleted := func() error {
			state.IsStateCopyCompleted = true
			if _, err := cs.Set(*state); err != nil {
				return errors.E(op, fmt.Errorf("error settting state: %w", err))
			}
			return nil
		}
		if !state.IsStateCopyCompleted {
			// if control reaches this block we'll set IsStateCopyCompleted to true
			// this makes sure we only attempt to automatically do the state migration once
			// we'll leave it up to the user to correct the errors and use
			// scripts update-project-v3 --move-state-only to move state
			//
			// this will also make sure new config v3 projects will not repeatedly reach this block
			// for a example a user connecting a custom source named default
			// with no read permissions to other schemas ie we cannot access `hdb_catalog.schema_migrations`
			// in the first run it'll encounter an error but will also mark IsStateCopyCompleted to true
			// thereby not running this block again

			// check if hdb_catalog.schema_migrations exists
			// check if migrations state table exists
			query := hasura.PGRunSQLInput{
				Source: sourceName,
				SQL:    `SELECT COUNT(1) FROM information_schema.tables WHERE table_name = '` + migrations.DefaultMigrationsTable + `' AND table_schema = '` + migrations.DefaultSchema + `' LIMIT 1`,
			}

			runsqlResp, err := ec.APIClient.V2Query.PGRunSQL(query)
			if err != nil {
				ec.Logger.Debug("encountered error when trying to move migrations from hdb_catalog.schema_migrations to catalog state\n", err,
					"\nnote: ignore this if you are not updating your project from config v2 -> config v3")
				ec.Logger.Debug("marking IsStateCopyCompleted as true %w", markStateMigrationCompleted())
				return false, nil
			}

			if runsqlResp.ResultType != hasura.TuplesOK {
				ec.Logger.Debug("encountered error when trying to move migrations from hdb_catalog.schema_migrations to catalog state", fmt.Errorf("invalid result Type %s", runsqlResp.ResultType),
					"\nnote: ignore this if you are not updating your project from config v2 -> config v3")
				ec.Logger.Debug("marking IsStateCopyCompleted as true %w", markStateMigrationCompleted())
				return false, nil
			}
			result := runsqlResp.Result
			if result[1][0] == "0" {
				// hdb_catalog.schema_migrations doesn't exist
				ec.Logger.Debug("hdb_catalog.schema_migrations was not found, skipping state migration")
				ec.Logger.Debug("marking IsStateCopyCompleted as true %w", markStateMigrationCompleted())
				return false, nil
			}
			ec.Logger.Debug("copying cli state from hdb_catalog.schema_migrations to catalog state")
			// COPY STATE
			if err := scripts.CopyState(ec, sourceName, sourceName); err != nil {
				return false, errors.E(op, err)
			}
			ec.Logger.Debug("copying cli state from hdb_catalog.schema_migrations to catalog state success")
			return true, nil
		}
		ec.Logger.Debugf("skipping state migration, found IsStateCopyCompleted: %v Migrations: %v", state.IsStateCopyCompleted, state.Migrations)
		return false, nil
	}
	return false, nil
}