check db connection in healthz endpoint (close #2645) (#3440)

* check db connection in healthz endpoint
* use Spock.text to automatically set Content-Type
* update docs

docs/graphql/manual/api-reference/index.rst

@@ -67,11 +67,25 @@ in JSON format:
 Health check API
 ^^^^^^^^^^^^^^^^
 
-A ``GET`` request to the public ``/healthz`` endpoint will respond with ``200``
-if the GraphQL engine is ready to serve requests and there are no inconsistencies
-with the metadata. The response will be ``500`` if there are metadata
-inconsistencies and you should use the console or check the server logs to find
-out what the errors are.
+A ``GET`` request to the public ``/healthz`` endpoint will respond with the following:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Server condition
+     - HTTP Status
+     - Message
+   * - All healthy
+     - 200
+     - OK
+   * - Serving requests but some metadata objects are inconsistent/not-available
+     - 200
+     - WARN: inconsistent objects in schema
+   * - Unhealthy
+     - 500
+     - ERROR
+
+If there are metadata inconsistencies, you should use the console or use the `get_inconsistent_metadata <schema-metadata-api/manage-metadata.html#get-inconsistent-metadata>`_ API to find out what the inconsistent objects are.
 
 
 .. _pg_dump_api:

server/src-lib/Hasura/Server/App.hs

@@ -8,6 +8,7 @@ import           Control.Concurrent.MVar
 import           Control.Exception                      (IOException, try)
 import           Control.Monad.Stateless
 import           Data.Aeson                             hiding (json)
+import           Data.Either                            (isRight)
 import           Data.Int                               (Int64)
 import           Data.IORef
 import           Data.Time.Clock                        (UTCTime)
@@ -530,9 +531,12 @@ httpApp corsCfg serverCtx enableConsole consoleAssetsDir enableTelemetry = do
     -- Health check endpoint
     Spock.get "healthz" $ do
       sc <- liftIO $ getSCFromRef $ scCacheRef serverCtx
-      if null $ scInconsistentObjs sc
-        then Spock.setStatus HTTP.status200 >> Spock.lazyBytes "OK"
-        else Spock.setStatus HTTP.status500 >> Spock.lazyBytes "ERROR"
+      dbOk <- checkDbConnection
+      if dbOk
+        then Spock.setStatus HTTP.status200 >> (Spock.text $ if null (scInconsistentObjs sc)
+                                                 then "OK"
+                                                 else "WARN: inconsistent objects in schema")
+        else Spock.setStatus HTTP.status500 >> Spock.text "ERROR"
 
     Spock.get "v1/version" $ do
       uncurry Spock.setHeader jsonHeader
@@ -618,6 +622,14 @@ httpApp corsCfg serverCtx enableConsole consoleAssetsDir enableTelemetry = do
     enablePGDump = isPGDumpEnabled serverCtx
     enableConfig = isConfigEnabled serverCtx
 
+    checkDbConnection = do
+      e <- liftIO $ runExceptT $ runLazyTx' (scPGExecCtx serverCtx) select1Query
+      pure $ isRight e
+      where
+        select1Query :: (MonadTx m) => m Int
+        select1Query =   liftTx $ runIdentity . Q.getRow <$> Q.withQE defaultTxErrorHandler
+                         [Q.sql| SELECT 1 |] () False
+
     serveApiConsole = do
       -- redirect / to /console
       Spock.get Spock.root $ Spock.redirect "console"