ci: stabilize server-upgrade-downgrade test on buildkite

fixes https://github.com/hasura/graphql-engine-mono/issues/2635 The test is running postgres via docker compose, but wasn't waiting for postgres to be ready, which is likely what caused intermittent failures when trying to clean metadata. Also make indentation consistent. PR-URL: https://github.com/hasura/graphql-engine-mono/pull/2670 Co-authored-by: Vishnu Bharathi <4211715+scriptnull@users.noreply.github.com> GitOrigin-RevId: 5a4c03c3c05322695ee9ff9a49efc834dea37074
2024-12-14 08:02:15 +03:00 · 2021-10-25 14:27:45 +02:00 · 2021-10-25 14:27:45 +02:00 · 44977bdf9d
commit 44977bdf9d
parent ec4ef91273
1 changed files with 78 additions and 69 deletions
--- a/.circleci/server-upgrade-downgrade/run.sh
+++ b/.circleci/server-upgrade-downgrade/run.sh
@ -26,57 +26,66 @@ set -euo pipefail

 cd "${BASH_SOURCE[0]%/*}"
 ROOT="${PWD}"
-cd - > /dev/null
+cd - >/dev/null

 download_with_etag_check() {
 	URL="$1"
 	FILE="$2"
 	ETAG="$(curl -I $URL | grep etag: | awk '{print $2}' | sed 's/\r$//')"
 	set -x
-	if ! ( [ -f "$FILE" ] && [ "$(cat "$FILE.etag" 2>/dev/null)" == "$ETAG" ] ) ; then
+	if ! ([ -f "$FILE" ] && [ "$(cat "$FILE.etag" 2>/dev/null)" == "$ETAG" ]); then
 		curl -Lo "$FILE" "$URL"
 		chmod +x "$FILE"
-		echo -e -n "$ETAG" > "$FILE.etag"
+		echo -e -n "$ETAG" >"$FILE.etag"
 	fi
 	set +x
 }

 fail_if_port_busy() {
-    local PORT=$1
-    if nc -z localhost $PORT ; then
-        echo "Port $PORT is busy. Exiting"
-        exit 1
-    fi
+	local PORT=$1
+	if nc -z localhost $PORT; then
+		echo "Port $PORT is busy. Exiting"
+		exit 1
+	fi
 }

 # wait_for_port PORT [PID] [LOG_FILE]
 wait_for_port() {
-    local PORT=$1
-    local PIDMSG=""
-    local PID=${2:-}
-    if [ -n "$PID" ] ; then
-        PIDMSG=", PID ($PID)"
-    fi
-    echo "waiting for ${PORT}${PIDMSG}"
-    for i in `seq 1 60`;
-    do
-        nc -z localhost $PORT && echo "port $PORT is ready" && return
-        echo -n .
-        sleep 1
-	      if [ -n "$PID" ] && ! ps $PID >/dev/null ; then
-		        echo "Process $PID has exited"
-		        if [ -n "${3:-}" ] ; then
-			          cat $3
-		        fi
-            exit 1
-	      fi
-    done
-    echo "Failed waiting for $PORT" && exit 1
+	local PORT=$1
+	local PIDMSG=""
+	local PID=${2:-}
+	if [ -n "$PID" ]; then
+		PIDMSG=", PID ($PID)"
+	fi
+	echo "waiting for ${PORT}${PIDMSG}"
+	for i in $(seq 1 60); do
+		nc -z localhost $PORT && echo "port $PORT is ready" && return
+		echo -n .
+		sleep 1
+		if [ -n "$PID" ] && ! ps $PID >/dev/null; then
+			echo "Process $PID has exited"
+			if [ -n "${3:-}" ]; then
+				cat $3
+			fi
+			exit 1
+		fi
+	done
+	echo "Failed waiting for $PORT" && exit 1
+}
+
+wait_for_postgres() {
+        for i in $(seq 1 60); do
+                psql "$1" -c '' >/dev/null 2>&1 && \
+                        echo "postgres is ready at $1" && \
+                        return
+                echo -n .
+                sleep 1
+        done
+        echo "failed waiting for postgres at $1" && return 1
 }

 log() { echo $'\e[1;33m'"--> $*"$'\e[0m'; }

-
 : ${HASURA_GRAPHQL_SERVER_PORT:=8080}
 : ${API_SERVER_PORT:=3000}
 : ${HASURA_PROJECT_DIR:=$ROOT/hasura}
@ -98,8 +107,8 @@ PYTEST_DIR="${ROOT}/../../server/tests-py"
 # cryptography 3.4.7 version requires Rust dependencies by default. But we don't need them for our tests, hence disabling them via the following env var => https://stackoverflow.com/a/66334084
 export CRYPTOGRAPHY_DONT_BUILD_RUST=1

-pip3 -q install -r "${PYTEST_DIR}/requirements.txt" ||\
-pip3 -q install -i http://mirrors.digitalocean.com/pypi/web/simple --trusted-host mirrors.digitalocean.com  -r "${PYTEST_DIR}/requirements.txt"
+pip3 -q install -r "${PYTEST_DIR}/requirements.txt" ||
+	pip3 -q install -i http://mirrors.digitalocean.com/pypi/web/simple --trusted-host mirrors.digitalocean.com -r "${PYTEST_DIR}/requirements.txt"

 # export them so that GraphQL Engine can use it
 export HASURA_GRAPHQL_STRINGIFY_NUMERIC_TYPES="$HASURA_GRAPHQL_STRINGIFY_NUMERIC_TYPES"
@ -126,7 +135,7 @@ touch $CURRENT_SERVER_LOG
 log "downloading latest release of graphql engine"
 download_with_etag_check 'https://graphql-engine-cdn.hasura.io/server/latest/linux-amd64' "$LATEST_SERVER_BINARY"

-cur_server_version(){
+cur_server_version() {
 	echo "$(curl http://localhost:${HASURA_GRAPHQL_SERVER_PORT}/v1/version -q 2>/dev/null)"
 }

@ -134,8 +143,8 @@ log "Run pytests with server upgrade"

 WORKTREE_DIR="$(mktemp -d)"
 RELEASE_PYTEST_DIR="${WORKTREE_DIR}/server/tests-py"
-RELEASE_VERSION="$( $LATEST_SERVER_BINARY version | cut -d':' -f2 | awk '{print $1}' )"
-rm_worktree(){
+RELEASE_VERSION="$($LATEST_SERVER_BINARY version | cut -d':' -f2 | awk '{print $1}')"
+rm_worktree() {
 	rm -rf "$WORKTREE_DIR"
 }
 trap rm_worktree ERR
@ -145,10 +154,10 @@ make_latest_release_worktree() {
 }

 cleanup_hasura_metadata_if_present() {
-       set -x
-       psql "$HASURA_GRAPHQL_DATABASE_URL" -c 'drop schema if exists hdb_catalog cascade;
+	set -x
+	psql "$HASURA_GRAPHQL_DATABASE_URL" -c 'drop schema if exists hdb_catalog cascade;
               drop schema if exists hdb_views cascade' >/dev/null 2>/dev/null
-       set +x
+	set +x
 }

 get_tables_of_interest() {
@ -190,17 +199,17 @@ get_server_upgrade_tests() {
 		-m 'allow_server_upgrade_test and not skip_server_upgrade_test' \
 		--deselect test_schema_stitching.py::TestRemoteSchemaBasic::test_introspection \
 		--deselect test_schema_stitching.py::TestAddRemoteSchemaCompareRootQueryFields::test_schema_check_arg_default_values_and_field_and_arg_types \
-                --deselect test_graphql_mutations.py::TestGraphqlInsertPermission::test_user_with_no_backend_privilege \
-                --deselect test_graphql_mutations.py::TestGraphqlInsertPermission::test_backend_user_no_admin_secret_fail \
-                --deselect test_graphql_mutations.py::TestGraphqlMutationCustomSchema::test_update_article \
-                --deselect test_graphql_queries.py::TestGraphQLQueryEnums::test_introspect_user_role \
-                --deselect test_schema_stitching.py::TestRemoteSchemaQueriesOverWebsocket::test_remote_query_error \
-                --deselect test_events.py::TestCreateAndDelete::test_create_reset \
-                --deselect test_events.py::TestUpdateEvtQuery::test_update_basic \
-                --deselect test_schema_stitching.py::TestAddRemoteSchemaTbls::test_add_schema \
-                --deselect test_schema_stitching.py::TestAddRemoteSchemaTbls::test_add_conflicting_table \
-                --deselect test_events.py \
-                --deselect test_graphql_queries.py::TestGraphQLQueryFunctions \
+		--deselect test_graphql_mutations.py::TestGraphqlInsertPermission::test_user_with_no_backend_privilege \
+		--deselect test_graphql_mutations.py::TestGraphqlInsertPermission::test_backend_user_no_admin_secret_fail \
+		--deselect test_graphql_mutations.py::TestGraphqlMutationCustomSchema::test_update_article \
+		--deselect test_graphql_queries.py::TestGraphQLQueryEnums::test_introspect_user_role \
+		--deselect test_schema_stitching.py::TestRemoteSchemaQueriesOverWebsocket::test_remote_query_error \
+		--deselect test_events.py::TestCreateAndDelete::test_create_reset \
+		--deselect test_events.py::TestUpdateEvtQuery::test_update_basic \
+		--deselect test_schema_stitching.py::TestAddRemoteSchemaTbls::test_add_schema \
+		--deselect test_schema_stitching.py::TestAddRemoteSchemaTbls::test_add_conflicting_table \
+		--deselect test_events.py \
+		--deselect test_graphql_queries.py::TestGraphQLQueryFunctions \
 		"${args[@]}" 1>/dev/null 2>/dev/null
 	set +x
 	cat "$tmpfile"
@ -212,7 +221,7 @@ get_server_upgrade_tests() {
 # tests passed as the argument.
 run_server_upgrade_pytest() {
 	HGE_PID=""
-	cleanup_hge(){
+	cleanup_hge() {
 		kill $HGE_PID || true
 		wait $HGE_PID || true
 		# cleanup_hasura_metadata_if_present
@ -222,19 +231,19 @@ run_server_upgrade_pytest() {
 	local HGE_URL="http://localhost:${HASURA_GRAPHQL_SERVER_PORT}"
 	local tests_to_run="$1"

-	[ -n "$tests_to_run" ] || ( echo "Got no test as input" && false )
+	[ -n "$tests_to_run" ] || (echo "Got no test as input" && false)

-	run_pytest(){
+	run_pytest() {
 		cd $RELEASE_PYTEST_DIR
 		set -x

 		# With --avoid-error-message-checks, we are only going to throw warnings if the error message has changed between releases
-		pytest --hge-urls "${HGE_URL}"  --pg-urls "$HASURA_GRAPHQL_DATABASE_URL" \
+		pytest --hge-urls "${HGE_URL}" --pg-urls "$HASURA_GRAPHQL_DATABASE_URL" \
 			--avoid-error-message-checks "$@" \
 			-m 'allow_server_upgrade_test and not skip_server_upgrade_test' \
-                        --deselect test_graphql_mutations.py::TestGraphqlInsertPermission::test_user_with_no_backend_privilege \
-                        --deselect test_graphql_mutations.py::TestGraphqlMutationCustomSchema::test_update_article \
-                        --deselect test_graphql_queries.py::TestGraphQLQueryEnums::test_introspect_user_role \
+			--deselect test_graphql_mutations.py::TestGraphqlInsertPermission::test_user_with_no_backend_privilege \
+			--deselect test_graphql_mutations.py::TestGraphqlMutationCustomSchema::test_update_article \
+			--deselect test_graphql_queries.py::TestGraphQLQueryEnums::test_introspect_user_role \
 			-v $tests_to_run
 		set +x
 		cd -
@ -244,10 +253,9 @@ run_server_upgrade_pytest() {

 	# Start the old (latest release) GraphQL Engine
 	log "starting latest graphql engine release"
-	$LATEST_SERVER_BINARY serve > $LATEST_SERVER_LOG 2>&1 &
+	$LATEST_SERVER_BINARY serve >$LATEST_SERVER_LOG 2>&1 &
 	HGE_PID=$!

-
 	# Wait for server start
 	wait_for_port $HASURA_GRAPHQL_SERVER_PORT $HGE_PID $LATEST_SERVER_LOG

@ -266,12 +274,13 @@ run_server_upgrade_pytest() {

 	############## Tests for the current build GraphQL engine #########################

-	if [[ "$1" =~ "test_schema_stitching" ]] ; then
+	if [[ "$1" =~ "test_schema_stitching" ]]; then
 		# In this case, Hasura metadata will have GraphQL servers defined as remote.
 		# We need to have remote GraphQL server running for the graphql-engine to avoid
 		# inconsistent metadata error
 		cd $RELEASE_PYTEST_DIR
-		python3 graphql_server.py & REMOTE_GQL_PID=$!
+		python3 graphql_server.py &
+		REMOTE_GQL_PID=$!
 		wait_for_port 5000
 		cd -
 	fi
@ -279,7 +288,7 @@ run_server_upgrade_pytest() {
 	log "start the current build"
 	set -x
 	rm -f graphql-engine.tix
-	$SERVER_BINARY serve > $CURRENT_SERVER_LOG 2>&1 &
+	$SERVER_BINARY serve >$CURRENT_SERVER_LOG 2>&1 &
 	HGE_PID=$!
 	set +x

@ -289,7 +298,7 @@ run_server_upgrade_pytest() {
 	log "Catalog version for $(cur_server_version)"
 	get_current_catalog_version

-	if [[ "$1" =~ "test_schema_stitching" ]] ; then
+	if [[ "$1" =~ "test_schema_stitching" ]]; then
 		kill $REMOTE_GQL_PID || true
 		wait $REMOTE_GQL_PID || true
 	fi
@ -301,25 +310,24 @@ run_server_upgrade_pytest() {
 	kill $HGE_PID || true
 	wait $HGE_PID || true

-
 	#################### Downgrade to release version ##########################

 	log "Downgrade graphql-engine to $RELEASE_VERSION"
 	$SERVER_BINARY downgrade "--to-$RELEASE_VERSION"

-
 	############## Tests for latest release GraphQL engine once more after downgrade #########################

-	if [[ "$1" =~ "test_schema_stitching" ]] ; then
+	if [[ "$1" =~ "test_schema_stitching" ]]; then
 		cd $RELEASE_PYTEST_DIR
-		python3 graphql_server.py & REMOTE_GQL_PID=$!
+		python3 graphql_server.py &
+		REMOTE_GQL_PID=$!
 		wait_for_port 5000
 		cd -
 	fi

 	# Start the old (latest release) GraphQL Engine
 	log "starting latest graphql engine release"
-	$LATEST_SERVER_BINARY serve > $LATEST_SERVER_LOG 2>&1 &
+	$LATEST_SERVER_BINARY serve >$LATEST_SERVER_LOG 2>&1 &
 	HGE_PID=$!

 	# Wait for server start
@ -328,7 +336,7 @@ run_server_upgrade_pytest() {
 	log "Catalog version for $(cur_server_version)"
 	get_current_catalog_version

-	if [[ "$1" =~ "test_schema_stitching" ]] ; then
+	if [[ "$1" =~ "test_schema_stitching" ]]; then
 		kill $REMOTE_GQL_PID || true
 		wait $REMOTE_GQL_PID || true
 	fi
@ -344,6 +352,7 @@ run_server_upgrade_pytest() {

 make_latest_release_worktree

+wait_for_postgres "$HASURA_GRAPHQL_DATABASE_URL"
 cleanup_hasura_metadata_if_present

 # We run_server_upgrade_pytest over each test individually to minimize the
@ -358,7 +367,7 @@ cleanup_hasura_metadata_if_present
 #    and make them compatible, or small enough in number we can run them
 #    sequentially
 #  - ???
-for pytest in $(get_server_upgrade_tests) ; do
+for pytest in $(get_server_upgrade_tests); do
 	log "Running pytest $pytest"
 	run_server_upgrade_pytest "$pytest"
 done