feat(server): native Postgres array support

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/9673
GitOrigin-RevId: 2dfbc07acf9da879910acfc9885bb7138b81e883

docs/docs/mutations/postgres/insert.mdx

@@ -532,9 +532,6 @@ the `tags` table via a bridge table `article_tags`.
 
 ## Insert an object with an ARRAY field
 
-To insert fields of array types, you currently have to pass them as a
-[Postgres array literal](https://www.postgresql.org/docs/current/arrays.html#ARRAYS-INPUT).
-
 **Example:** Insert a new `author` with a text array `emails` field:
 
 <GraphiQLIDE
@@ -543,7 +540,7 @@ To insert fields of array types, you currently have to pass them as a
     objects: [
       {
         name: "Ash",
-        emails: "{ash@ash.com, ash123@ash.com}"
+        emails: ["ash@ash.com", "ash123@ash.com"]
       }
     ]
   ) {
@@ -610,6 +607,9 @@ Using variables:
 }`}
 />
 
+To insert fields of nested array types, you have to pass them as a
+[Postgres array literal](https://www.postgresql.org/docs/current/arrays.html#ARRAYS-INPUT).
+
 ## Set a field to its default value during insert
 
 To set a field to its `default` value, just omit it from the input object, irrespective of the

server/graphql-engine.cabal

@@ -1178,6 +1178,7 @@ test-suite graphql-engine-tests
     Hasura.Backends.Postgres.Connection.VersionCheckSpec
     Hasura.Backends.Postgres.Execute.PrepareSpec
     Hasura.Backends.Postgres.NativeQueries.NativeQueriesSpec
+    Hasura.Backends.Postgres.PGScalarTypeSpec
     Hasura.Backends.Postgres.RQLGenerator
     Hasura.Backends.Postgres.RQLGenerator.GenAnnSelectG
     Hasura.Backends.Postgres.RQLGenerator.GenAssociatedTypes

server/lib/api-tests/api-tests.cabal

@@ -20,6 +20,8 @@ common common-all
     -Wno-monomorphism-restriction
     -Wno-missing-kind-signatures
     -Wno-missing-safe-haskell-mode
+    -- please be quiet, Morpheus
+    -Wno-deprecations
     -- We want these warnings, but the code doesn't satisfy them yet:
     -Wno-missing-deriving-strategies
     -Wno-unused-packages
@@ -135,6 +137,7 @@ library
     Test.Databases.BigQuery.Queries.SpatialTypesSpec
     Test.Databases.BigQuery.Queries.TypeInterpretationSpec
     Test.Databases.BigQuery.Schema.ComputedFields.TableSpec
+    Test.Databases.Postgres.ArraySpec
     Test.Databases.Postgres.BackendOnlyPermissionsSpec
     Test.Databases.Postgres.DataValidation.PermissionSpec
     Test.Databases.Postgres.JsonbSpec

server/lib/api-tests/src/Test/Databases/Postgres/ArraySpec.hs

@@ -0,0 +1,316 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+-- |
+-- Tests that we can decode Arrays values correctly
+module Test.Databases.Postgres.ArraySpec (spec) where
+
+import Data.Aeson (Value)
+import Data.List.NonEmpty qualified as NE
+import Harness.Backend.Citus qualified as Citus
+import Harness.Backend.Cockroach qualified as Cockroach
+import Harness.Backend.Postgres qualified as Postgres
+import Harness.GraphqlEngine (postGraphql)
+import Harness.Quoter.Graphql (graphql)
+import Harness.Quoter.Yaml (interpolateYaml)
+import Harness.Schema qualified as Schema
+import Harness.Test.Fixture qualified as Fixture
+import Harness.TestEnvironment (GlobalTestEnvironment, TestEnvironment)
+import Harness.Yaml (shouldReturnYaml)
+import Hasura.Prelude
+import Test.Hspec (SpecWith, describe, it)
+
+spec :: SpecWith GlobalTestEnvironment
+spec = do
+  Fixture.run
+    ( NE.fromList
+        [ (Fixture.fixture $ Fixture.Backend Postgres.backendTypeMetadata)
+            { Fixture.setupTeardown = \(testEnv, _) ->
+                [ Postgres.setupTablesAction schema testEnv
+                ]
+            },
+          (Fixture.fixture $ Fixture.Backend Citus.backendTypeMetadata)
+            { Fixture.setupTeardown = \(testEnv, _) ->
+                [ Citus.setupTablesAction schema testEnv
+                ]
+            },
+          (Fixture.fixture $ Fixture.Backend Cockroach.backendTypeMetadata)
+            { Fixture.setupTeardown = \(testEnv, _) ->
+                [ Cockroach.setupTablesAction schema testEnv
+                ]
+            }
+        ]
+    )
+    singleArrayTests
+
+  -- CockroachDB does not support nested arrays
+  -- https://www.cockroachlabs.com/docs/stable/array.html
+  Fixture.run
+    ( NE.fromList
+        [ (Fixture.fixture $ Fixture.Backend Postgres.backendTypeMetadata)
+            { Fixture.setupTeardown = \(testEnv, _) ->
+                [ Postgres.setupTablesAction schema testEnv
+                ]
+            },
+          (Fixture.fixture $ Fixture.Backend Citus.backendTypeMetadata)
+            { Fixture.setupTeardown = \(testEnv, _) ->
+                [ Citus.setupTablesAction schema testEnv
+                ]
+            }
+        ]
+    )
+    nestedArrayTests
+
+--------------------------------------------------------------------------------
+-- Schema
+
+textArrayType :: Schema.ScalarType
+textArrayType =
+  Schema.TCustomType
+    $ Schema.defaultBackendScalarType
+      { Schema.bstPostgres = Just "text[]",
+        Schema.bstCitus = Just "text[]",
+        Schema.bstCockroach = Just "text[]"
+      }
+
+nestedTextArrayType :: Schema.ScalarType
+nestedTextArrayType =
+  Schema.TCustomType
+    $ Schema.defaultBackendScalarType
+      { Schema.bstPostgres = Just "text[][]",
+        Schema.bstCitus = Just "text[][]",
+        Schema.bstCockroach = Just "text[]" -- nested arrays aren't supported in Cockroach, so we'll skip this test anyway
+      }
+
+schema :: [Schema.Table]
+schema =
+  [ (Schema.table "author")
+      { Schema.tableColumns =
+          [ Schema.column "id" Schema.defaultSerialType,
+            Schema.column "name" Schema.TStr,
+            Schema.column "emails" textArrayType,
+            Schema.column "grid" nestedTextArrayType
+          ],
+        Schema.tablePrimaryKey = ["id"]
+      }
+  ]
+
+--------------------------------------------------------------------------------
+-- Tests
+
+singleArrayTests :: SpecWith TestEnvironment
+singleArrayTests = do
+  describe "Saves arrays" $ do
+    it "Using native postgres array syntax" \testEnvironment -> do
+      let expected :: Value
+          expected =
+            [interpolateYaml|
+              data:
+                insert_hasura_author:
+                  affected_rows: 1
+                  returning:
+                    - name: "Ash"
+                      emails: ["ash@ash.com", "ash123@ash.com"]
+            |]
+
+          actual :: IO Value
+          actual =
+            postGraphql
+              testEnvironment
+              [graphql|
+                mutation {
+                  insert_hasura_author (
+                    objects: [
+                      {
+                        name: "Ash",
+                        emails: "{ash@ash.com, ash123@ash.com}",
+                        grid: "{}"
+                      }
+                    ]
+                  ) {
+                    affected_rows
+                    returning {
+                      name
+                      emails
+                    }
+                  }
+                }
+              |]
+
+      shouldReturnYaml testEnvironment actual expected
+
+    it "Using native GraphQL array syntax" \testEnvironment -> do
+      let expected :: Value
+          expected =
+            [interpolateYaml|
+              data:
+                insert_hasura_author:
+                  affected_rows: 1
+                  returning:
+                    - name: "Ash"
+                      emails: ["ash@ash.com", "ash123@ash.com"]
+            |]
+
+          actual :: IO Value
+          actual =
+            postGraphql
+              testEnvironment
+              [graphql|
+                mutation {
+                  insert_hasura_author (
+                    objects: [
+                      {
+                        name: "Ash",
+                        emails: ["ash@ash.com", "ash123@ash.com"],
+                        grid: []
+                      }
+                    ]
+                  ) {
+                    affected_rows
+                    returning {
+                      name
+                      emails
+                    }
+                  }
+                }
+              |]
+
+      shouldReturnYaml testEnvironment actual expected
+
+  describe "Filters with contains" $ do
+    it "finds values using _contains" \testEnvironment -> do
+      void
+        $ postGraphql
+          testEnvironment
+          [graphql|
+                mutation {
+                  insert_hasura_author (
+                    objects: [
+                      {
+                        name: "contains",
+                        emails: ["horse@horse.com", "dog@dog.com"],
+                        grid: []
+                      }
+                    ]
+                  ) {
+                    affected_rows
+                    returning {
+                      name
+                      emails
+                    }
+                  }
+                }
+              |]
+
+      let expected :: Value
+          expected =
+            [interpolateYaml|
+              data:
+                hasura_author:
+                  - name: contains
+            |]
+
+          actual :: IO Value
+          actual =
+            postGraphql
+              testEnvironment
+              [graphql|
+                query {
+                  hasura_author (where: { emails: {_contains:["horse@horse.com"]}}) {
+                    name
+                  }
+                }
+              |]
+
+      shouldReturnYaml testEnvironment actual expected
+
+    it "finds values using _contained_in" \testEnvironment -> do
+      void
+        $ postGraphql
+          testEnvironment
+          [graphql|
+                mutation {
+                  insert_hasura_author (
+                    objects: [
+                      {
+                        name: "contained_in",
+                        emails: ["horse@horse2.com", "dog@dog2.com"],
+                        grid: []
+                      }
+                    ]
+                  ) {
+                    affected_rows
+                    returning {
+                      name
+                      emails
+                    }
+                  }
+                }
+              |]
+
+      let expected :: Value
+          expected =
+            [interpolateYaml|
+              data:
+                hasura_author:
+                  - name: contained_in
+            |]
+
+          actual :: IO Value
+          actual =
+            postGraphql
+              testEnvironment
+              [graphql|
+                query {
+                  hasura_author (where: { emails: {_contained_in:["cat@cat2.com","dog@dog2.com", "horse@horse2.com"]}}) {
+                    name
+                  }
+                }
+              |]
+
+      shouldReturnYaml testEnvironment actual expected
+
+nestedArrayTests :: SpecWith TestEnvironment
+nestedArrayTests = do
+  describe "Saves nested arrays" $ do
+    -- Postgres introspection is able to tell us about thing[] but thing[][] always comes
+    -- back as unknown, so the only way to operate on these values continues to
+    -- be with Postgres native array syntax. This test is to ensure we have not
+    -- broken that.
+    it "Using native postgres array syntax" \testEnvironment -> do
+      let expected :: Value
+          expected =
+            [interpolateYaml|
+                data:
+                  insert_hasura_author:
+                    affected_rows: 1
+                    returning:
+                      - name: "Ash"
+                        grid: [["one", "two", "three"],
+                              ["four", "five", "six"]]
+              |]
+
+          actual :: IO Value
+          actual =
+            postGraphql
+              testEnvironment
+              [graphql|
+                  mutation {
+                    insert_hasura_author (
+                      objects: [
+                        {
+                          name: "Ash",
+                          emails: "{}",
+                          grid: "{{one,two,three},{four,five,six}}"
+                        }
+                      ]
+                    ) {
+                      affected_rows
+                      returning {
+                        name
+                        grid
+                      }
+                    }
+                  }
+                |]
+
+      shouldReturnYaml testEnvironment actual expected

server/src-lib/Hasura/Backends/Postgres/Instances/Schema.hs

@@ -435,8 +435,18 @@ columnParser columnType nullability = case columnType of
     -- not accept strings.
     --
     -- TODO: introduce new dedicated scalars for Postgres column types.
-    name <- mkScalarTypeName scalarType
+    (name, schemaType) <- case scalarType of
-    let schemaType = P.TNamed P.NonNullable $ P.Definition name Nothing Nothing [] P.TIScalar
+      PGArray innerScalar -> do
+        name <- mkScalarTypeName innerScalar
+        pure
+          ( name,
+            P.TList
+              P.NonNullable
+              (P.TNamed P.NonNullable $ P.Definition name Nothing Nothing [] P.TIScalar)
+          )
+      _ -> do
+        name <- mkScalarTypeName scalarType
+        pure (name, P.TNamed P.NonNullable $ P.Definition name Nothing Nothing [] P.TIScalar)
     pure
       $ peelWithOrigin
       $ fmap (ColumnValue columnType)
@@ -572,7 +582,12 @@ comparisonExps = memoize 'comparisonExps \columnType -> do
   -- `ltxtquery` represents a full-text-search-like pattern for matching `ltree` values.
   ltxtqueryParser <- columnParser (ColumnScalar PGLtxtquery) (G.Nullability False)
   maybeCastParser <- castExp columnType tCase
-  let name = applyTypeNameCaseCust tCase $ P.getName typedParser <> Name.__comparison_exp
+  -- we need to give comparison exps for `thing[]` a different name to `thing`
+  let nameSuffix = case P.pType typedParser of
+        P.TList {} ->
+          Name.__array <> Name.__comparison_exp
+        _ -> Name.__comparison_exp
+  let name = applyTypeNameCaseCust tCase $ P.getName typedParser <> nameSuffix
       desc =
         G.Description
           $ "Boolean expression to compare columns of type "
@@ -685,6 +700,21 @@ comparisonExps = memoize 'comparisonExps \columnType -> do
                  (Just "does the column NOT match the given POSIX regular expression, case insensitive")
                  (ABackendSpecific . ANIREGEX . IR.mkParameter <$> typedParser)
              ],
+        -- Ops for array types
+        guard (isScalarColumnWhere (\case PGArray _ -> True; _ -> False) columnType)
+          *> [ mkBoolOperator
+                 tCase
+                 collapseIfNull
+                 (C.fromAutogeneratedName Name.__contains)
+                 (Just "does the array contain the given value")
+                 (ABackendSpecific . AContains . IR.mkParameter <$> typedParser),
+               mkBoolOperator
+                 tCase
+                 collapseIfNull
+                 (C.fromAutogeneratedTuple $$(G.litGQLIdentifier ["_contained", "in"]))
+                 (Just "is the array contained in the given array value")
+                 (ABackendSpecific . AContainedIn . IR.mkParameter <$> typedParser)
+             ],
         -- Ops for JSONB type
         guard (isScalarColumnWhere (== PGJSONB) columnType)
           *> [ mkBoolOperator

server/src-lib/Hasura/Backends/Postgres/SQL/Types.hs

@@ -573,7 +573,20 @@ instance ToErrorValue PGScalarType where
   toErrorValue = ErrorValue.squote . pgScalarTypeToText
 
 textToPGScalarType :: Text -> PGScalarType
-textToPGScalarType t = fromMaybe (PGUnknown t) (lookup t pgScalarTranslations)
+textToPGScalarType t =
+  fromMaybe
+    (PGUnknown t)
+    (parse $ T.toLower t)
+  where
+    parse = \case
+      txt
+        | T.takeEnd 2 txt == "[]" ->
+            PGArray <$> parse (T.dropEnd 2 txt)
+      txt
+        | T.take 6 txt == "array " ->
+            PGArray <$> parse (T.drop 6 txt)
+      txt ->
+        lookup txt pgScalarTranslations
 
 -- Inlining this results in pretty terrible Core being generated by GHC.
 

server/src-lib/Hasura/Name.hs

@@ -628,3 +628,8 @@ _predicate = [G.name|predicate|]
 
 _filter :: G.Name
 _filter = [G.name|filter|]
+
+-- * Arrays
+
+__array :: G.Name
+__array = [G.name|_array|]

server/src-rsr/citus_table_metadata.sql

@@ -63,7 +63,10 @@ LEFT JOIN LATERAL
   ( SELECT jsonb_agg(jsonb_build_object(
       'name', "column".attname,
       'position', "column".attnum,
-      'type', json_build_object('name', coalesce(base_type.typname, "type".typname), 'type', "type".typtype),
+      'type', json_build_object('name', (CASE WHEN "array_type".typname IS NULL
+                                              THEN coalesce(base_type.typname, "type".typname)
+                                              ELSE "array_type".typname || '[]' END),
+                                'type', "type".typtype),
       'is_nullable', NOT "column".attnotnull,
       'description', pg_catalog.col_description("table".oid, "column".attnum),
       'mutability', jsonb_build_object(
@@ -109,6 +112,8 @@ LEFT JOIN LATERAL
       ON "type".oid = "column".atttypid
     LEFT JOIN pg_catalog.pg_type base_type
       ON "type".typtype = 'd' AND base_type.oid = "type".typbasetype
+    LEFT JOIN pg_catalog.pg_type array_type
+      ON array_type.typarray = "type".oid
     WHERE "column".attrelid = "table".oid
       -- columns where attnum <= 0 are special, system-defined columns
       AND "column".attnum > 0

server/src-rsr/cockroach_table_metadata.sql

@@ -69,7 +69,10 @@ LEFT JOIN LATERAL
   ( SELECT jsonb_agg(jsonb_build_object(
       'name', "column".attname,
       'position', "column".attnum,
-      'type', json_build_object('name', coalesce(base_type.typname, "type".typname), 'type', "type".typtype),
+      'type', json_build_object('name', (CASE WHEN "array_type".typname IS NULL
+                                              THEN coalesce(base_type.typname, "type".typname)
+                                              ELSE "array_type".typname || '[]' END),
+                                'type', "type".typtype),
       'is_nullable', NOT "column".attnotnull,
       'description', '', -- pg_catalog.col_description("table".oid, "column".attnum), -- removing this for now as it takes ~20 seconds per lookup
       'mutability', jsonb_build_object(
@@ -82,6 +85,8 @@ LEFT JOIN LATERAL
       ON "type".oid = "column".atttypid
     LEFT JOIN pg_catalog.pg_type base_type
       ON "type".typtype = 'd' AND base_type.oid = "type".typbasetype
+    LEFT JOIN pg_catalog.pg_type array_type
+      ON array_type.typarray = "type".oid
     WHERE "column".attrelid = "table".oid
       -- columns where attnum <= 0 are special, system-defined columns
       AND "column".attnum > 0

server/src-rsr/pg_table_metadata.sql

@@ -55,7 +55,10 @@ LEFT JOIN LATERAL
   ( SELECT jsonb_agg(jsonb_build_object(
       'name', "column".attname,
       'position', "column".attnum,
-      'type', json_build_object('name', coalesce(base_type.typname, "type".typname), 'type', "type".typtype),
+      'type', json_build_object('name', (CASE WHEN "array_type".typname IS NULL
+                                              THEN coalesce(base_type.typname, "type".typname)
+                                              ELSE "array_type".typname || '[]' END),
+                                'type', "type".typtype),
       'is_nullable', NOT "column".attnotnull,
       'description', pg_catalog.col_description("table".oid, "column".attnum),
       'mutability', jsonb_build_object(
@@ -102,6 +105,8 @@ LEFT JOIN LATERAL
       ON "type".oid = "column".atttypid
     LEFT JOIN pg_catalog.pg_type base_type
       ON "type".typtype = 'd' AND base_type.oid = "type".typbasetype
+    LEFT JOIN pg_catalog.pg_type array_type
+      ON array_type.typarray = "type".oid
     WHERE "column".attrelid = "table".oid
       -- columns where attnum <= 0 are special, system-defined columns
       AND "column".attnum > 0

server/src-test/Hasura/Backends/Postgres/PGScalarTypeSpec.hs

@@ -0,0 +1,31 @@
+module Hasura.Backends.Postgres.PGScalarTypeSpec
+  ( spec,
+  )
+where
+
+import Data.Foldable (traverse_)
+import Data.Text qualified as T
+import Hasura.Backends.Postgres.SQL.Types
+import Test.Hspec
+import Prelude
+
+spec :: Spec
+spec =
+  describe "parse array scalar types" $ do
+    let expectations =
+          [ ("text", PGText),
+            ("text[]", PGArray PGText),
+            ("_text[]", PGUnknown "_text[]"),
+            ("array text", PGArray PGText),
+            ("TEXT", PGText),
+            ("TEXT[]", PGArray PGText),
+            ("_TEXT[]", PGUnknown "_TEXT[]"),
+            ("ARRAY TEXT", PGArray PGText)
+          ]
+
+    traverse_
+      ( \(txt, scalarType) ->
+          it ("successfully parses " <> T.unpack txt) $ do
+            textToPGScalarType txt `shouldBe` scalarType
+      )
+      expectations