Add README.md to tests-hspec and reorganize module namespace

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/3421
GitOrigin-RevId: 8802d7e6a360edee62011ef371cc8930f36b25b1

docker-compose.yml

@@ -59,6 +59,7 @@ services:
     environment:
       ACCEPT_EULA: "Y"
       SA_PASSWORD: "DockerComposePassword!"
+      MSSQL_SA_PASSWORD: "DockerComposePassword!"
 
   citus:
     image: citusdata/citus:10.1@sha256:7e497e5ca18d7f2ae2a66c1d5d676b548e9221b7e6294adfb03006adad85502c

docker-compose/sqlserver/Dockerfile

@@ -5,3 +5,5 @@ COPY run-init.sh /run-init.sh
 COPY entrypoint.sh /entrypoint.sh
 
 CMD /bin/bash ./entrypoint.sh
+
+ENV PATH="/opt/mssql-tools/bin:${PATH}"

docker-compose/sqlserver/run-init.sh

@@ -1,11 +1,13 @@
 #!/bin/bash
 
 echo 'Attempting to run script (10 times max)'
+PORT="${1:-1433}"
+SLEEP="3"
 
-sleep 5
+sleep "${SLEEP}"
 for i in 1 2 3 4 5 6 7 8 9 10; do
-  echo Will wait 5 seconds if this fails ...
-  /opt/mssql-tools/bin/sqlcmd -S localhost,1433 -U SA -P "DockerComposePassword!" -i init.sql && break || sleep 5;
+  echo Attempt "#$i" - will wait "${SLEEP}" seconds if this fails ...
+  sqlcmd -S localhost,"${PORT}" -U SA -P "DockerComposePassword!" -i init.sql && break || sleep "${SLEEP}";
 done
 
 echo Finished attempts.

server/graphql-engine.cabal

@@ -961,27 +961,36 @@ test-suite tests-hspec
   main-is: Spec.hs
   other-modules:
     SpecHook
-    Harness.Mysql
-    Harness.Http
-    Harness.Postgres
-    Harness.Citus
-    Harness.Constants
-    Harness.Feature
-    Harness.Yaml
-    Harness.Sqlserver
-    Harness.Sql
-    Harness.State
-    Harness.Graphql
-    Harness.GraphqlEngine
+ -- Harness
+      Harness.Constants
 
-    ArrayRelationshipsSpec
-    NestedRelationshipsSpec
-    BasicFieldsSpec
-    DirectivesSpec
-    HelloWorldSpec
-    LimitOffsetSpec
-    ObjectRelationshipsSpec
-    OrderingSpec
-    ServiceLivenessSpec
-    ViewsSpec
-    WhereSpec
+   -- Harness.Backend
+        Harness.Backend.Mysql
+        Harness.Backend.Sqlserver
+        Harness.Backend.Postgres
+        Harness.Backend.Citus
+
+      Harness.GraphqlEngine
+      Harness.Http
+      Harness.State
+
+   -- Harness.Test
+        Harness.Test.Feature
+
+   -- Harness.Quoter
+        Harness.Quoter.Yaml
+        Harness.Quoter.Sql
+        Harness.Quoter.Graphql
+
+ -- Test
+      Test.ArrayRelationshipsSpec
+      Test.NestedRelationshipsSpec
+      Test.BasicFieldsSpec
+      Test.DirectivesSpec
+      Test.HelloWorldSpec
+      Test.LimitOffsetSpec
+      Test.ObjectRelationshipsSpec
+      Test.OrderingSpec
+      Test.ServiceLivenessSpec
+      Test.ViewsSpec
+      Test.WhereSpec

server/tests-hspec/Harness/Backend/Citus.hs

@@ -2,7 +2,7 @@
 
 -- | CitusQL helpers. Pretty much the same as postgres. Could refactor
 -- if we add more things here.
-module Harness.Citus
+module Harness.Backend.Citus
   ( livenessCheck,
     run_,
   )

server/tests-hspec/Harness/Backend/Mysql.hs

@@ -1,7 +1,7 @@
 {-# OPTIONS -Wno-redundant-constraints #-}
 
 -- | MySQL helpers.
-module Harness.Mysql
+module Harness.Backend.Mysql
   ( livenessCheck,
     run_,
   )

server/tests-hspec/Harness/Backend/Postgres.hs

@@ -1,7 +1,7 @@
 {-# OPTIONS -Wno-redundant-constraints #-}
 
 -- | PostgreSQL helpers.
-module Harness.Postgres
+module Harness.Backend.Postgres
   ( livenessCheck,
     run_,
   )

server/tests-hspec/Harness/Backend/Sqlserver.hs

@@ -1,7 +1,7 @@
 {-# OPTIONS -Wno-redundant-constraints #-}
 
 -- | SQLServer helpers.
-module Harness.Sqlserver
+module Harness.Backend.Sqlserver
   ( livenessCheck,
     run_,
   )

server/tests-hspec/Harness/Constants.hs

@@ -46,6 +46,8 @@ import Hasura.Server.Init
 import Hasura.Server.Types
 import Network.WebSockets qualified as WS
 
+-- * Postgres
+
 postgresPassword :: String
 postgresPassword = "hasura"
 
@@ -74,6 +76,8 @@ postgresqlConnectionString =
     ++ "/"
     ++ postgresDb
 
+-- * Citus
+
 citusPassword :: String
 citusPassword = "hasura"
 
@@ -102,6 +106,8 @@ citusConnectionString =
     ++ "/"
     ++ citusDb
 
+-- * Liveness
+
 postgresLivenessCheckAttempts :: Int
 postgresLivenessCheckAttempts = 5
 
@@ -134,6 +140,8 @@ mysqlLivenessCheckIntervalSeconds = 1
 mysqlLivenessCheckIntervalMicroseconds :: Int
 mysqlLivenessCheckIntervalMicroseconds = 1000 * 1000 * mysqlLivenessCheckIntervalSeconds
 
+-- * MySQL
+
 mysqlPassword :: String
 mysqlPassword = "hasura"
 
@@ -159,6 +167,8 @@ mysqlConnectInfo =
       Mysql.connectPort = mysqlPort
     }
 
+-- * HTTP health checks
+
 httpHealthCheckAttempts :: Int
 httpHealthCheckAttempts = 5
 
@@ -168,6 +178,8 @@ httpHealthCheckIntervalSeconds = 1
 httpHealthCheckIntervalMicroseconds :: Int
 httpHealthCheckIntervalMicroseconds = 1000 * 1000 * httpHealthCheckIntervalSeconds
 
+-- * Server configuration
+
 serveOptions :: ServeOptions impl
 serveOptions =
   ServeOptions

server/tests-hspec/Harness/Quoter/Graphql.hs

@@ -1,6 +1,6 @@
--- | GRAPHQL quasi quoter with built-in interpolation.
+-- | GraphQL quasi quoter with built-in interpolation.
 -- Interpolation works via the #{expression} syntax.
-module Harness.Graphql (graphql) where
+module Harness.Quoter.Graphql (graphql) where
 
 import Data.Aeson (ToJSON)
 import Data.Aeson.Extended (encode)

server/tests-hspec/Harness/Quoter/Sql.hs

@@ -1,7 +1,7 @@
 -- | Simple SQL quasi quoter. Even if this doesn't do anything, it's
 -- still useful. Some editors (Emacs) display [sql| ...|] with SQL
 -- syntax highlighting.
-module Harness.Sql (sql) where
+module Harness.Quoter.Sql (sql) where
 
 import Language.Haskell.TH
 import Language.Haskell.TH.Quote

server/tests-hspec/Harness/Quoter/Yaml.hs

@@ -4,7 +4,7 @@
 {-# OPTIONS_GHC -fno-warn-orphans #-}
 
 -- | Templating yaml files.
-module Harness.Yaml
+module Harness.Quoter.Yaml
   ( yaml,
     shouldReturnYaml,
   )

server/tests-hspec/Harness/Test/Feature.hs

@@ -1,5 +1,5 @@
 -- | Helper functions for easily testing features.
-module Harness.Feature
+module Harness.Test.Feature
   ( feature,
     Feature (..),
     Backend (..),

server/tests-hspec/README.md

@@ -0,0 +1,172 @@
+# tests-hspec
+
+Graphql-engine integration tests written in Haskell using the [hspec](https://hspec.github.io) testing framework.
+
+For motivation, rationale, and more, see the [test suite rfc](../../rfcs/hspec-test-suite.md).
+
+## Running the test suite
+
+1. To run the Haskell integration test suite, we'll first need to start the backends:
+
+   ```sh
+   docker-compose up
+   ```
+
+   This will start up Postgres, SQL Server, Citus and MariaDB.
+
+
+    > __Note__: on ARM64 architecture we'll need additional steps in order to test mssql properly.
+    >
+    > ### Preparation
+    >
+    > 1. Switch the docker image in `docker-compose/sqlserver/Dockerfile` to `azure-sql-edge`:
+    >
+    > ```diff
+    > - FROM mcr.microsoft.com/mssql/server:2019-latest@sha256:a098c9ff6fbb8e1c9608ad7511fa42dba8d22e0d50b48302761717840ccc26af
+    > + FROM mcr.microsoft.com/azure-sql-edge
+    > ```
+    >
+    > 2. Install `sqlcmd` locally. On MacOS, this can be done with brew: `brew install mssql-tools`.
+    >
+    > ### Start the backends
+    >
+    > 1. Run `docker-compose up`
+    > 2. Initialize the SQL Server database
+    >
+    >    ```sh
+    >    (cd docker-compose/sqlserver && bash run-init.sh 65003)
+    >    ```
+
+2. Once the containers are up, you can run the test suite via
+
+   ```sh
+   cabal run tests-hspec
+   ```
+
+    You can also further refine which tests to run using the `-m` flag:
+
+   ```sh
+   cabal run tests-hspec -- -m "SQLServer"
+   ```
+
+    For additional information, consult the help section:
+
+   ```sh
+   cabal run tests-hspec -- --help
+   ```
+
+## Test suite structure
+
+### Harness
+
+Modules under the [Harness/](Harness/) namespace provide the infrastructure
+and supporting code for writing and running the tests.
+It includes quasiquoters, interacting with backends, interfacing with HTTP,
+constants, and so on.
+
+Supporting code should be added under the `Harness.*` namespace instead of
+added ad-hoc in test specs, to improve readability and reuse.
+
+### Test
+
+Modules under the [Test/](Test/) namespace define integration test specifications for various
+features and backends.
+
+## Adding a new test
+
+The module [Test.HelloWorldSpec](Test/HelloWorldSpec.hs) contains a starting point
+which can be built upon.
+
+To create a new test:
+
+1. Create a new module based on `Test.HelloWorldSpec`
+2. Specify the relevant backends on which the tests will run in `spec`
+3. Specify the tests and expectations in `tests`
+
+When creating a new test, make sure that:
+
+1. The module name is suffixed by the word `Spec`
+2. The module exports the entry point `spec :: SpecWith State`
+3. The module is listed in the cabal file under `other-modules`
+
+(1) and (2) are required for `hspec-discover` to find and run the test.
+
+### Specifying backends
+
+To specify a new backend, create a value of the type `Backend` which is defined in
+[Harness.Test.Feature](Harness/Test/Feature.hs). A `Backend` requires a `name`,
+a `setup action` and a `teardown action`.
+
+#### Setup action
+
+A setup action is a function of type `State -> IO ()` which is responsible with
+creating the environment for the test. It needs to:
+
+1. Clear and reconfigure the metadata
+2. Setup tables and insert values
+3. Track tables, add relationships, permissions
+
+etc.
+
+These actions can be created by running POST requests against graphql-engine
+using `Harness.GraphqlEngine.post_`, or by running SQL requests against the
+backend using `Backend.<backend>.run_`.
+
+#### Teardown action
+
+The teardown action is another function of type `State -> IO ()` which is responsible
+for removing the environment created by the test or setup, so that other tests can have
+a "clean slate" with no artifacts.
+
+For example, this test should drop tables, delete custom functions and sequences, and so on.
+
+These actions can be created by running POST requests against graphql-engine
+using `Harness.GraphqlEngine.post_`, or by running SQL requests against the
+backend using `Backend.<backend>.run_`.
+
+### Writing tests
+
+Test should be written (or reachable from) `tests :: SpecWith State`.
+
+A typical test will look similar to this:
+
+```hs
+  it "Where id=1" \state ->
+    shouldReturnYaml
+      ( GraphqlEngine.postGraphql
+          state
+          [graphql|
+query {
+  hasura_author(where: {id: {_eq: 1}}) {
+    name
+    id
+  }
+}
+|]
+      )
+      [yaml|
+data:
+  hasura_author:
+  - name: Author 1
+    id: 1
+|]
+```
+
+- `it` specifies the name of the test
+- `shouldReturnYaml` creates an [`Expectation`](https://hspec.github.io/expectations.html)
+  which does the following:
+  - Runs a POST request against graphql-engine which can be specified using the `graphql` quasi-quoter.
+  - Compares the response to an expected result which can be specified using the `yaml` quasi-quoter.
+
+__Note__: these quasi-quoter can also perform string interpolation. See the relevant modules
+under the [Harness.Quoter](Harness/Quouter) namespace.
+
+### Style guide
+
+- Test suite should be very easy to read for a junior or intermediate Haskell developer.
+  Be mindful of advanced feature use and abstractions!
+- Prefer self-contained specs, even if there's some query duplication.
+- Avoid functions or types in tests, other than calls to the `Harness.*` API.
+  Any supporting code should be in the `Harness.*` hierarchy and apply broadly to the test suites
+  overall.
+- Consider reorganising tests if modules get much longer than 1/2 pagescrolls (~200-300 LOC?).

server/tests-hspec/Test/ArrayRelationshipsSpec.hs

@@ -2,16 +2,16 @@
 {-# LANGUAGE RecordWildCards #-}
 
 -- | Testing array relationships.
-module ArrayRelationshipsSpec (spec) where
+module Test.ArrayRelationshipsSpec (spec) where
 
+import Harness.Backend.Mysql as Mysql
 import Harness.Constants
-import Harness.Feature qualified as Feature
-import Harness.Graphql
 import Harness.GraphqlEngine qualified as GraphqlEngine
-import Harness.Mysql as Mysql
-import Harness.Sql
+import Harness.Quoter.Graphql
+import Harness.Quoter.Sql
+import Harness.Quoter.Yaml
 import Harness.State (State)
-import Harness.Yaml
+import Harness.Test.Feature qualified as Feature
 import Test.Hspec
 import Prelude
 

server/tests-hspec/Test/BasicFieldsSpec.hs

@@ -2,19 +2,19 @@
 {-# LANGUAGE RecordWildCards #-}
 
 -- | Test querying an entity for a couple fields.
-module BasicFieldsSpec (spec) where
+module Test.BasicFieldsSpec (spec) where
 
-import Harness.Citus as Citus
+import Harness.Backend.Citus as Citus
+import Harness.Backend.Mysql as Mysql
+import Harness.Backend.Postgres as Postgres
+import Harness.Backend.Sqlserver as Sqlserver
 import Harness.Constants
-import Harness.Feature qualified as Feature
-import Harness.Graphql
 import Harness.GraphqlEngine qualified as GraphqlEngine
-import Harness.Mysql as Mysql
-import Harness.Postgres as Postgres
-import Harness.Sql
-import Harness.Sqlserver as Sqlserver
+import Harness.Quoter.Graphql
+import Harness.Quoter.Sql
+import Harness.Quoter.Yaml
 import Harness.State (State)
-import Harness.Yaml
+import Harness.Test.Feature qualified as Feature
 import Test.Hspec
 import Prelude
 

server/tests-hspec/Test/DirectivesSpec.hs

@@ -2,17 +2,17 @@
 {-# LANGUAGE RecordWildCards #-}
 
 -- | Test directives.
-module DirectivesSpec (spec) where
+module Test.DirectivesSpec (spec) where
 
 import Data.Aeson (Value)
+import Harness.Backend.Mysql as Mysql
 import Harness.Constants
-import Harness.Feature qualified as Feature
-import Harness.Graphql (graphql)
 import Harness.GraphqlEngine qualified as GraphqlEngine
-import Harness.Mysql as Mysql
-import Harness.Sql
+import Harness.Quoter.Graphql (graphql)
+import Harness.Quoter.Sql
+import Harness.Quoter.Yaml
 import Harness.State (State)
-import Harness.Yaml
+import Harness.Test.Feature qualified as Feature
 import Test.Hspec
 import Prelude
 

server/tests-hspec/Test/HelloWorldSpec.hs

@@ -1,8 +1,8 @@
 -- | A starting point feature test.
-module HelloWorldSpec (spec) where
+module Test.HelloWorldSpec (spec) where
 
-import Harness.Feature qualified as Feature
 import Harness.State (State)
+import Harness.Test.Feature qualified as Feature
 import Test.Hspec
 import Prelude
 

server/tests-hspec/Test/LimitOffsetSpec.hs

@@ -2,16 +2,16 @@
 {-# LANGUAGE RecordWildCards #-}
 
 -- | Tests for limit/offset.
-module LimitOffsetSpec (spec) where
+module Test.LimitOffsetSpec (spec) where
 
+import Harness.Backend.Mysql as Mysql
 import Harness.Constants
-import Harness.Feature qualified as Feature
-import Harness.Graphql
 import Harness.GraphqlEngine qualified as GraphqlEngine
-import Harness.Mysql as Mysql
-import Harness.Sql
+import Harness.Quoter.Graphql
+import Harness.Quoter.Sql
+import Harness.Quoter.Yaml
 import Harness.State (State)
-import Harness.Yaml
+import Harness.Test.Feature qualified as Feature
 import Test.Hspec
 import Prelude
 

server/tests-hspec/Test/NestedRelationshipsSpec.hs

@@ -3,17 +3,17 @@
 
 -- | Testing nested relationships.
 --
--- Original inspiration for this module is <https://github.com/hasura/graphql-engine-mono/blob/08caf7df10cad0aea0916327736147a0a8f808d1/server/tests-py/queries/graphql_query/mysql/nested_select_query_deep.yaml>
-module NestedRelationshipsSpec (spec) where
+-- Original inspiration for this module Test.is <https://github.com/hasura/graphql-engine-mono/blob/08caf7df10cad0aea0916327736147a0a8f808d1/server/tests-py/queries/graphql_query/mysql/nested_select_query_deep.yaml>
+module Test.NestedRelationshipsSpec (spec) where
 
+import Harness.Backend.Mysql as Mysql
 import Harness.Constants
-import Harness.Feature qualified as Feature
-import Harness.Graphql
 import Harness.GraphqlEngine qualified as GraphqlEngine
-import Harness.Mysql as Mysql
-import Harness.Sql
+import Harness.Quoter.Graphql
+import Harness.Quoter.Sql
+import Harness.Quoter.Yaml
 import Harness.State (State)
-import Harness.Yaml
+import Harness.Test.Feature qualified as Feature
 import Test.Hspec
 import Prelude
 

server/tests-hspec/Test/ObjectRelationshipsSpec.hs

@@ -2,16 +2,16 @@
 {-# LANGUAGE RecordWildCards #-}
 
 -- | Testing object relationships.
-module ObjectRelationshipsSpec (spec) where
+module Test.ObjectRelationshipsSpec (spec) where
 
+import Harness.Backend.Mysql as Mysql
 import Harness.Constants
-import Harness.Feature qualified as Feature
-import Harness.Graphql
 import Harness.GraphqlEngine qualified as GraphqlEngine
-import Harness.Mysql as Mysql
-import Harness.Sql
+import Harness.Quoter.Graphql
+import Harness.Quoter.Sql
+import Harness.Quoter.Yaml
 import Harness.State (State)
-import Harness.Yaml
+import Harness.Test.Feature qualified as Feature
 import Test.Hspec
 import Prelude
 

server/tests-hspec/Test/OrderingSpec.hs

@@ -2,16 +2,16 @@
 {-# LANGUAGE RecordWildCards #-}
 
 -- | Test ordering by fields.
-module OrderingSpec (spec) where
+module Test.OrderingSpec (spec) where
 
+import Harness.Backend.Mysql as Mysql
 import Harness.Constants
-import Harness.Feature qualified as Feature
-import Harness.Graphql
 import Harness.GraphqlEngine qualified as GraphqlEngine
-import Harness.Mysql as Mysql
-import Harness.Sql
+import Harness.Quoter.Graphql
+import Harness.Quoter.Sql
+import Harness.Quoter.Yaml
 import Harness.State (State)
-import Harness.Yaml
+import Harness.Test.Feature qualified as Feature
 import Test.Hspec
 import Prelude
 

server/tests-hspec/Test/ServiceLivenessSpec.hs

@@ -1,13 +1,13 @@
 -- | Service liveness tests: Confirm that the harness is working
 -- properly. If this passes, the rest of the tests will pass.
-module ServiceLivenessSpec (spec) where
+module Test.ServiceLivenessSpec (spec) where
 
-import Harness.Citus qualified as Citus
+import Harness.Backend.Citus qualified as Citus
+import Harness.Backend.Mysql qualified as Mysql
+import Harness.Backend.Postgres qualified as Postgres
+import Harness.Backend.Sqlserver qualified as Sqlserver
 import Harness.GraphqlEngine qualified as GraphqlEngine
 import Harness.Http qualified as Http
-import Harness.Mysql qualified as Mysql
-import Harness.Postgres qualified as Postgres
-import Harness.Sqlserver qualified as Sqlserver
 import Harness.State (State, getServer)
 import Test.Hspec
 import Prelude

server/tests-hspec/Test/ViewsSpec.hs

@@ -2,16 +2,16 @@
 {-# LANGUAGE RecordWildCards #-}
 
 -- | Test views.
-module ViewsSpec (spec) where
+module Test.ViewsSpec (spec) where
 
+import Harness.Backend.Mysql as Mysql
 import Harness.Constants
-import Harness.Feature qualified as Feature
-import Harness.Graphql
 import Harness.GraphqlEngine qualified as GraphqlEngine
-import Harness.Mysql as Mysql
-import Harness.Sql
+import Harness.Quoter.Graphql
+import Harness.Quoter.Sql
+import Harness.Quoter.Yaml
 import Harness.State (State)
-import Harness.Yaml
+import Harness.Test.Feature qualified as Feature
 import Test.Hspec
 import Prelude
 

server/tests-hspec/Test/WhereSpec.hs

@@ -2,16 +2,16 @@
 {-# LANGUAGE RecordWildCards #-}
 
 -- | Tests that `where' works.
-module WhereSpec (spec) where
+module Test.WhereSpec (spec) where
 
+import Harness.Backend.Mysql as Mysql
 import Harness.Constants
-import Harness.Feature qualified as Feature
-import Harness.Graphql
 import Harness.GraphqlEngine qualified as GraphqlEngine
-import Harness.Mysql as Mysql
-import Harness.Sql
+import Harness.Quoter.Graphql
+import Harness.Quoter.Sql
+import Harness.Quoter.Yaml
 import Harness.State (State)
-import Harness.Yaml
+import Harness.Test.Feature qualified as Feature
 import Test.Hspec
 import Prelude