server: Use make to simplify tests-py venv generation.

Rather than a homebrewed approach, we can use `make` to figure out when
it's necessary to regenerate our venv.

This Makefile will regenerate _requirements.txt_ from
_requirements-top-level.txt_ when the latter is changed.

It will also regenerate the venv when _requirements.txt_ is changed
(i.e. changes are pulled, or it's regenerated as described above).

`make` uses file/directory timestamps to figure out what to rebuild.
This is probably more reliable than expecting people to update a version
number whenever they change a file.

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/5152
GitOrigin-RevId: 24b27d49bf6c4ba1d57ac38ea38ae278216c6d66

Makefile

@@ -11,4 +11,5 @@ include ./scripts/make/build.mk
 include ./scripts/make/lint.mk
 include ./scripts/make/test-infrastructure.mk
 include ./scripts/make/tests.mk
+include ./scripts/make/legacy-tests.mk
 include ./scripts/make/ghcid.mk

scripts/dev.sh

@@ -81,10 +81,6 @@ try_jq() {
   fi
 }
 
-# Bump this to:
-#  - force a reinstall of python dependencies, etc.
-DEVSH_VERSION=1.8
-
 case "${1-}" in
   graphql-engine)
     case "${2-}" in
@@ -565,46 +561,9 @@ elif [ "$MODE" = "test" ]; then
       echo_pretty "  $ rm -r \"$PROJECT_ROOT/server/tests-py/node_modules\""
     fi
 
-    ### Check for and install dependencies in venv
     PY_VENV=.hasura-dev-python-venv
-    DEVSH_VERSION_FILE=.devsh_version
-    # Do we need to force reinstall?
-    if [ "$DEVSH_VERSION" = "$(cat $DEVSH_VERSION_FILE 2>/dev/null || true)" ]; then
-      true # ok
-    else
-      echo_warn 'dev.sh version was bumped or fresh install. Forcing reinstallation of dependencies.'
-      rm -rf "$PY_VENV"
-    fi
-    # cryptography 3.4.7 version requires Rust dependencies by default. But we don't need them for our tests, hence disabling them via the following env var => https://stackoverflow.com/a/66334084
-    export CRYPTOGRAPHY_DONT_BUILD_RUST=1
-    set +u  # for venv activate
-    if [ ! -d "$PY_VENV" ]; then
-      python3 -m venv "$PY_VENV"
-      source "$PY_VENV/bin/activate"
-      pip3 install wheel
-      # If the maintainer of this script or pytests needs to change dependencies:
-      #  - alter requirements-top-level.txt as needed
-      #  - delete requirements.txt
-      #  - run this script, then check in the new frozen requirements.txt
-      if [ -f requirements.txt ]; then
-        pip3 install -r requirements.txt
-      else
-        pip3 install -r requirements-top-level.txt
-        (
-          # shellcheck disable=SC2016
-          echo '# IF YOU CHANGE THIS PLEASE INCREMENT DEVSH_VERSION in `scripts/dev.sh`'
-          echo '# Please add direct dependencies to requirements-top-level.txt'
-          pip3 freeze
-        ) > requirements.txt
-      fi
-      echo "$DEVSH_VERSION" > "$DEVSH_VERSION_FILE"
-    else
-      echo_pretty "It looks like python dependencies have been installed already. Skipping."
-      echo_pretty "If things fail please run this and try again"
-      echo_pretty "  $ rm -r \"$PROJECT_ROOT/server/tests-py/$PY_VENV\""
-
-      source "$PY_VENV/bin/activate"
-    fi
+    make "$PY_VENV"
+    source "$PY_VENV/bin/activate"
 
     # TODO MAYBE: fix deprecation warnings, make them an error
     if ! pytest \
@@ -618,7 +577,6 @@ elif [ "$MODE" = "test" ]; then
       echo_error "^^^ graphql-engine logs from failed test run can be inspected at: $GRAPHQL_ENGINE_TEST_LOG"
     fi
     deactivate  # python venv
-    set -u
 
     cd "$PROJECT_ROOT/server"
     # Kill the cabal new-run and its children. INT so we get hpc report:

scripts/make/legacy-tests.mk

@@ -0,0 +1,30 @@
+LEGACY_TESTS = server/tests-py
+LEGACY_TESTS_VENV = $(LEGACY_TESTS)/.hasura-dev-python-venv
+
+# cryptography v3.4.8 requires Rust dependencies by default, but we don't need them for our tests.
+# The following environment variable disables this requirement.
+# https://cryptography.io/en/3.4.8/faq.html#installing-cryptography-fails-with-error-can-not-find-rust-compiler
+export CRYPTOGRAPHY_DONT_BUILD_RUST = 1
+
+# Make's automatic variables such as `$<` and `$@` can be confusing.
+# If you are unsure what they do, please consult the documentation.
+# https://www.gnu.org/software/make/manual/html_node/Automatic-Variables.html
+
+$(LEGACY_TESTS_VENV): $(LEGACY_TESTS)/requirements.txt
+	rm -rf "$(LEGACY_TESTS_VENV)"
+	python3 -m venv "$(LEGACY_TESTS_VENV)"
+	source "$@/bin/activate"; \
+		pip3 install -r $<
+	touch $@
+
+$(LEGACY_TESTS)/requirements.txt: $(LEGACY_TESTS)/requirements-top-level.txt
+	rm -rf "$(LEGACY_TESTS_VENV)"
+	python3 -m venv "$(LEGACY_TESTS_VENV)"
+	source "$(LEGACY_TESTS_VENV)/bin/activate"; \
+		pip3 install wheel; \
+		pip3 install -r $<; \
+		( \
+			echo '# Do not modify this file directly. Instead, modify $<.'; \
+			echo '# Then, run `make` in this directory to regenerate this file.'; \
+			pip3 freeze \
+		) > $@

server/tests-py/requirements-top-level.txt

@@ -1,4 +1,6 @@
-# IF YOU CHANGE THIS PLEASE INCREMENT DEVSH_VERSION in `scripts/dev.sh`
+# Direct dependencies go here.
+# Regenerate requirements.txt by running `make` in this directory.
+
 croniter
 cryptography == 3.*  # later versions require Rust support
 graphene == 2.*  # v3 has changed the API

server/tests-py/requirements.txt

@@ -1,5 +1,5 @@
-# IF YOU CHANGE THIS PLEASE INCREMENT DEVSH_VERSION in `scripts/dev.sh`
-# Please add direct dependencies to requirements-top-level.txt
+# Do not modify this file directly. Instead, modify server/tests-py/requirements-top-level.txt.
+# Then, run `make` in this directory to regenerate this file.
 aniso8601==7.0.0
 async-timeout==4.0.2
 attrs==21.4.0