tests: Docker Compose healthchecks

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/6282 Co-authored-by: Gil Mizrahi <8547573+soupi@users.noreply.github.com> Co-authored-by: Samir Talwar <47582+SamirTalwar@users.noreply.github.com> GitOrigin-RevId: c2b1f958364ce5e1ef704fa1275d18ac810ca35b
2024-12-13 19:33:55 +03:00 · 2022-10-17 14:19:12 +01:00 · 2022-10-17 14:19:12 +01:00 · f4f4221d04
commit f4f4221d04
parent 3757731228
5 changed files with 130 additions and 88 deletions
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -32,7 +32,7 @@ version: "3.6"

 services:
  mariadb:
-    image: mariadb:10.6.4-focal@sha256:c014ba1efc5dbd711d0520c7762d57807f35549de3414eb31e942a420c8a2ed2
+    image: mariadb:10.9.3
    ports:
      - "65001:3306"
    environment:
@ -40,8 +40,19 @@ services:
      MARIADB_PASSWORD: "hasura"
      MARIADB_DATABASE: "hasura"
      MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: "true"
+      MARIADB_MYSQL_LOCALHOST_USER: "1"
    volumes:
      - mariadb-data:/var/lib/mysql
+    healthcheck:
+      test:
+        - CMD
+        - /usr/local/bin/healthcheck.sh
+        - --connect
+        - --innodb_initialized
+      start_period: 5s
+      interval: 5s
+      timeout: 10s
+      retries: 10

  postgres:
    image: postgis/postgis:15beta4-3.3-alpine
@ -57,8 +68,20 @@ services:
    volumes:
      - postgres-data:/var/lib/postgresql/data
      - ./docker-compose/postgres/init.sh:/docker-entrypoint-initdb.d/init-hasura.sh:ro
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - psql -U hasura < /dev/null && sleep 5 && psql -U hasura < /dev/null
+      start_period: 5s
+      interval: 5s
+      timeout: 10s
+      retries: 10

  sqlserver:
+    # We cannot use this image on arm64, as it's not available.
+    # Instead, we use `mcr.microsoft.com/azure-sql-edge`.
+    # This uses an environment variable so it can be overridden by scripts to
+    # provide that image instead.
    image: ${MSSQL_IMAGE:-mcr.microsoft.com/mssql/server:2019-latest}
    ports:
      - "65003:1433"
@ -69,8 +92,36 @@ services:
    volumes:
      - mssql-data:/var/opt/mssql

-  # Separated from the `sqlserver` container so we can use the `mssql-tools` image to run `sqlcmd`.
-  # `sqlcmd` is not available in the `azure-sql-edge` image on arm64, which we use for testing on macOS.
+  # We would prefer to attach the healthcheck directly to the `sqlserver` container.
+  # However, while we can do this in the `mcr.microsoft.com/mssql/server` image,
+  # it's not possible when running on macOS, as we don't use that image, but
+  # instead `mcr.microsoft.com/azure-sql-edge`.
+  # The `mcr.microsoft.com/azure-sql-edge` image does not provide `sqlcmd`.
+  # We therefore need to run it in a separate container.
+  # We can reconsider this once the MSSQL Server image can run on arm64.
+  sqlserver-healthcheck:
+    image: mcr.microsoft.com/mssql-tools
+    command:
+      - sleep
+      - inf
+    init: true # handle signals correctly
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - |
+          /opt/mssql-tools/bin/sqlcmd -S 'sqlserver,1433' -U SA -P "DockerComposePassword!"
+      start_period: 5s
+      interval: 5s
+      timeout: 10s
+      retries: 10
+    # This is commented out because this file can be extended, and dependencies don't work for extensions.
+    # Instead, ensure you also run this one.
+    # depends_on:
+    #   sqlserver:
+    #     condition: service_started
+
+    # Separated from the `sqlserver` container so we can use the `mssql-tools` image to run `sqlcmd`.
+    # `sqlcmd` is not available in the `azure-sql-edge` image on arm64, which we use for testing on macOS.
  sqlserver-init:
    image: mcr.microsoft.com/mssql-tools
    command:
@ -80,6 +131,11 @@ services:
    volumes:
      - ./docker-compose/sqlserver-init/init.sh:/init.sh
      - ./docker-compose/sqlserver-init/init.sql:/init.sql
+    # This is commented out because this file can be extended, and dependencies don't work for extensions.
+    # Instead, ensure you also run this one.
+    # depends_on:
+    #   sqlserver:
+    #     condition: service_started

  citus:
    image: citusdata/citus:10.1@sha256:7e497e5ca18d7f2ae2a66c1d5d676b548e9221b7e6294adfb03006adad85502c
@ -92,6 +148,14 @@ services:
    volumes:
      - citus-data:/var/lib/postgresql/data
      - ./docker-compose/postgres/init.sh:/docker-entrypoint-initdb.d/init-hasura.sh:ro
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - psql -U hasura < /dev/null && sleep 5 && psql -U hasura < /dev/null
+      start_period: 5s
+      interval: 5s
+      timeout: 10s
+      retries: 10

  cockroach:
    image: us-docker.pkg.dev/cockroach-cloud-images/cockroachdb/cockroach:v22.2.0-beta.2-218-ga7ecaa82cf
@ -107,6 +171,14 @@ services:
    volumes:
      - cockroach-data:/cockroach/cockroach-data
      - ./docker-compose/cockroach/init.sh:/docker-entrypoint-initdb.d/init-hasura.sh:ro
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - cockroach sql --insecure --database "hasura" --execute "SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'hasura';" | grep -q -F 'hasura'
+      start_period: 5s
+      interval: 5s
+      timeout: 10s
+      retries: 10

  dc-reference-agent:
    build:
@ -114,6 +186,17 @@ services:
      dockerfile: ./Dockerfile-reference
    ports:
      - "65005:8100"
+    healthcheck:
+      test:
+        - CMD
+        - nc
+        - -z
+        - localhost
+        - "8100"
+      start_period: 5s
+      interval: 5s
+      timeout: 10s
+      retries: 10

  dc-sqlite-agent:
    build:
@ -127,6 +210,17 @@ services:
      METRICS: y
      PRETTY_PRINT_LOGS: y
      LOG_LEVEL: debug
+    healthcheck:
+      test:
+        - CMD
+        - nc
+        - -z
+        - localhost
+        - "8100"
+      start_period: 5s
+      interval: 5s
+      timeout: 10s
+      retries: 10

 volumes:
  citus-data:
--- a/docker-compose/sqlserver-init/init.sh
+++ b/docker-compose/sqlserver-init/init.sh
@ -5,8 +5,8 @@ set -u
 set -o pipefail

 SERVER_PORT="${1:-1433}"
-SLEEP=3
-MAX_ATTEMPTS=10
+SLEEP=5
+MAX_ATTEMPTS=50

 if [[ -z "${SERVER_HOST-}" ]]; then
  echo 'The server host must be set.'
--- a/scripts/make/test-infrastructure.mk
+++ b/scripts/make/test-infrastructure.mk
@ -21,12 +21,6 @@ COCKROACH_PORT = 65008
 COCKROACH_DBNAME = hasura
 COCKROACH_DBUSER = root

-# utils.sh contains functions used in CI to wait for DBs to be ready.
-# this can be (ab)used like a script; e.g. `$(DB_UTILS) foo` expands to
-# `source ./.buildkite/scripts/util/util.sh; foo`, which will run the `foo`
-# function from util.sh (or anywhere else).
-DB_UTILS = source ./.buildkite/scripts/util/util.sh;
-
 # Use the Azure SQL Edge image instead of the SQL Server image on arm64.
 # The latter doesn't work yet.
 ifeq ($(shell uname -m),arm64)
@ -54,98 +48,44 @@ endef

 .PHONY: start-cockroach
 ## start-cockroach: start local PostgreSQL DB in Docker and wait for it to be ready
-start-cockroach: spawn-cockroach wait-for-cockroach
-
-.PHONY: spawn-cockroach
-spawn-cockroach:
-	docker compose up -d cockroach
-
-.PHONY: wait-for-cockroach
-wait-for-cockroach:
-	$(DB_UTILS) wait_for_cockroach $(COCKROACH_PORT)
-	$(DB_UTILS) wait_for_cockroach_db $(COCKROACH_PORT) "$(COCKROACH_DBNAME)" "$(COCKROACH_DBUSER)"
+start-cockroach:
+	docker compose up -d --wait cockroach

 .PHONY: start-postgres
 ## start-postgres: start local PostgreSQL DB in Docker and wait for it to be ready
-start-postgres: spawn-postgres wait-for-postgres
-
-.PHONY: spawn-postgres
-spawn-postgres:
-	docker compose up -d postgres
-
-.PHONY: wait-for-postgres
-wait-for-postgres:
-	$(DB_UTILS) wait_for_postgres $(PG_PORT)
-	$(DB_UTILS) wait_for_postgres_db $(PG_PORT) "$(PG_DBNAME)" "$(PG_DBUSER)" "$(PG_DBPASSWORD)"
+start-postgres:
+	docker compose up -d --wait postgres

 .PHONY: start-citus
 ## start-citus: start local Citus DB in Docker and wait for it to be ready
-start-citus: spawn-citus wait-for-citus
-
-.PHONY: spawn-citus
-spawn-citus:
-	docker compose up -d citus
-
-.PHONY: wait-for-citus
-wait-for-citus:
-	$(DB_UTILS) wait_for_postgres $(CITUS_PORT)
-	$(DB_UTILS) wait_for_postgres_db $(CITUS_PORT) "$(PG_DBNAME)" "$(PG_DBUSER)" "$(PG_DBPASSWORD)"
+start-citus:
+	docker compose up -d --wait citus

 .PHONY: start-sqlserver
 ## start-sqlserver: start local MS SQL Server DB in Docker and wait for it to be ready
-start-sqlserver: spawn-sqlserver wait-for-sqlserver
-
-.PHONY: spawn-sqlserver
-spawn-sqlserver:
-	docker compose up -d sqlserver
+start-sqlserver:
+	docker compose up -d --wait sqlserver
 	docker compose run sqlserver-init

-.PHONY: wait-for-sqlserver
-wait-for-sqlserver:
-	$(DB_UTILS) wait_for_mssql $(MSSQL_PORT)
-	$(DB_UTILS) wait_for_mssql_db $(MSSQL_PORT) "$(MSSQL_DBNAME)" "$(MSSQL_DBUSER)" "$(MSSQL_DBPASSWORD)"
-
 .PHONY: start-mysql
 ## start-mysql: start local MariaDB in Docker and wait for it to be ready
-start-mysql: spawn-mysql wait-for-mysql
-
-.PHONY: spawn-mysql
-spawn-mysql:
-	docker compose up -d mariadb
-
-.PHONY: wait-for-mysql
-wait-for-mysql:
-	$(DB_UTILS) wait_for_mysql $(MYSQL_PORT) "$(MYSQL_DBNAME)" "$(MYSQL_DBUSER)" "$(MYSQL_DBPASSWORD)"
+start-mysql:
+	docker compose up -d --wait mariadb

 .PHONY: start-dc-reference-agent
 ## start-dc-reference-agent: start the Data Connectors reference agent in Docker and wait for it to be ready
-start-dc-reference-agent: spawn-dc-reference-agent wait-for-dc-reference-agent
-
-.PHONY: spawn-dc-reference-agent
-spawn-dc-reference-agent:
-	docker compose up -d --build dc-reference-agent
-
-.PHONY: wait-for-dc-reference-agent
-wait-for-dc-reference-agent:
-	$(DB_UTILS) wait_for_http_success $(DC_REFERENCE_AGENT_URL) "dc-reference-agent"
+start-dc-reference-agent:
+	docker compose up -d --wait --build dc-reference-agent

 .PHONY: start-dc-sqlite-agent
 ## start-dc-sqlite-agent: start the Data Connectors SQLite agent in Docker and wait for it to be ready
-start-dc-sqlite-agent: spawn-dc-sqlite-agent wait-for-dc-sqlite-agent
-
-.PHONY: spawn-dc-sqlite-agent
-spawn-dc-sqlite-agent:
-	docker compose up -d --build dc-sqlite-agent
-
-.PHONY: wait-for-dc-sqlite-agent
-wait-for-dc-sqlite-agent:
-	$(DB_UTILS) wait_for_http_success $(DC_SQLITE_AGENT_URL) "dc-sqlite-agent"
+start-dc-sqlite-agent:
+	docker compose up -d --wait --build dc-sqlite-agent

 .PHONY: start-backends
 ## start-backends: start all known backends in Docker and wait for them to be ready
 start-backends: \
-	spawn-postgres spawn-sqlserver spawn-mysql spawn-citus spawn-dc-reference-agent spawn-dc-sqlite-agent spawn-cockroach \
-	wait-for-postgres wait-for-sqlserver wait-for-mysql wait-for-citus wait-for-dc-reference-agent wait-for-dc-sqlite-agent wait-for-cockroach
+	start-postgres start-sqlserver start-mysql start-citus start-dc-reference-agent start-dc-sqlite-agent start-cockroach

 .PHONY: stop-everything
 ## stop-everything: tear down test databases
--- a/scripts/make/tests.mk
+++ b/scripts/make/tests.mk
@ -7,31 +7,31 @@ test-bigquery: start-postgres remove-tix-file

 .PHONY: test-sqlserver
 ## test-sqlserver: run tests for SQL Server backend
-test-sqlserver: spawn-postgres spawn-sqlserver wait-for-postgres wait-for-sqlserver remove-tix-file
+test-sqlserver: start-postgres start-sqlserver remove-tix-file
 	$(call stop_after, \
 		cabal run api-tests -- -m 'SQLServer')

 .PHONY: test-mysql
 ## test-mysql: run tests for MySQL backend
-test-mysql: spawn-postgres spawn-mysql wait-for-postgres wait-for-mysql remove-tix-file
+test-mysql: start-postgres start-mysql remove-tix-file
 	$(call stop_after, \
 		cabal run api-tests -- -m 'MySQL')

 .PHONY: test-citus
 ## test-citus: run tests for Citus backend
-test-citus: spawn-postgres spawn-citus wait-for-postgres wait-for-citus remove-tix-file
+test-citus: start-postgres start-citus remove-tix-file
 	$(call stop_after, \
 		cabal run api-tests -- -m 'Citus')

 .PHONY: test-data-connectors
 ## test-data-connectors: run tests for Data Connectors
-test-data-connectors: start-postgres start-dc-reference-agent remove-tix-file
+test-data-connectors: start-postgres start-dc-reference-agent start-dc-sqlite-agent remove-tix-file
 	$(call stop_after, \
 		cabal run api-tests -- -m 'DataConnector')

 .PHONY: test-cockroach
 ## test-cockroach: run tests for Cockroach backend
-test-cockroach: spawn-postgres spawn-cockroach wait-for-postgres wait-for-cockroach remove-tix-file
+test-cockroach: start-postgres start-cockroach remove-tix-file
 	$(call stop_after, \
 		cabal run api-tests -- -m 'Cockroach')

--- a/server/tests-py/docker-compose.yml
+++ b/server/tests-py/docker-compose.yml
@ -89,7 +89,10 @@ services:
      - /var/lib/postgresql/data

  mssql:
-    # Uses a different image for arm64.
+    # We cannot use this image on arm64, as it's not available.
+    # Instead, we use `mcr.microsoft.com/azure-sql-edge`.
+    # This uses an environment variable so it can be overridden by scripts to
+    # provide that image instead.
    image: ${MSSQL_IMAGE:-mcr.microsoft.com/mssql/server:2019-latest@sha256:a098c9ff6fbb8e1c9608ad7511fa42dba8d22e0d50b48302761717840ccc26af}
    ports:
      - 1433
@ -100,8 +103,13 @@ services:
    volumes:
      - /var/opt/mssql

-  # This is necessary because on macOS on amd64, the `mssql/server` image is not available.
-  # We use the `azure-sql-edge` image instead, which does not ship with `sqlcmd` on amd64.
+  # We would prefer to attach the healthcheck directly to the `sqlserver` container.
+  # However, while we can do this in the `mcr.microsoft.com/mssql/server` image,
+  # it's not possible when running on macOS, as we don't use that image, but
+  # instead `mcr.microsoft.com/azure-sql-edge`.
+  # The `mcr.microsoft.com/azure-sql-edge` image does not provide `sqlcmd`.
+  # We therefore need to run it in a separate container.
+  # We can reconsider this once the MSSQL Server image can run on arm64.
  mssql-healthcheck:
    image: mcr.microsoft.com/mssql-tools
    command: