Data Connector agent data schema capabilities [GDC-479]

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/6268
GitOrigin-RevId: 4ec29566d3c2ab2144dad8055b4442a4027915ec

dc-agents/README.md

@@ -127,6 +127,11 @@ The `GET /capabilities` endpoint is used by `graphql-engine` to discover the cap
 ```json
 {
   "capabilities": {
+    "data_schema": {
+      "supports_primary_keys": true,
+      "supports_foreign_keys": true,
+      "column_nullability": "nullable_and_non_nullable"
+    },
     "relationships": {},
     "graphql_schema": "scalar DateTime\n\ninput DateTimeComparisons {\n  in_year: Number\n}",
     "scalar_types": {
@@ -158,6 +163,7 @@ The `GET /capabilities` endpoint is used by `graphql-engine` to discover the cap
 ```
 
 The `capabilities` section describes the _capabilities_ of the service. This includes
+- `data_schema`: What sorts of features the agent supports when describing its data schema
 - `relationships`: whether or not the agent supports relationships
 - `scalar_types`: custom scalar types and the operations they support. See [Scalar types capabilities](#scalar-type-capabilities).
 - `graphql_schema`: a GraphQL schema document containing type definitions referenced by the `scalar_types` capabilities.
@@ -166,6 +172,11 @@ The `config_schema` property contains an [OpenAPI 3 Schema](https://swagger.io/s
 
 `graphql-engine` will use the `config_schema` OpenAPI 3 Schema to validate the user's configuration JSON before putting it into the `X-Hasura-DataConnector-Config` header.
 
+#### Data schema capabilities
+The agent can declare whether or not it supports primary keys or foreign keys by setting the `supports_primary_keys` and `supports_foreign_keys` properties under the `data_schema` object on capabilities. If it does not declare support, it is expected that it will not return any such primary/foreign keys in the schema it exposes on the `/schema` endpoint.
+
+If the agent only supports table columns that are always nullable, then it should set `column_nullability` to `"only_nullable"`. However, if it supports both nullable and non-nullable columns, then it should set `"nullable_and_non_nullable"`.
+
 #### Scalar type capabilities
 
 The agent is expected to support a default set of scalar types (`Number`, `String`, `Bool`) and a default set of [comparison operators](#filters) on these types.

dc-agents/dc-api-types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hasura/dc-api-types",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "description": "Hasura GraphQL Engine Data Connector Agent API types",
   "author": "Hasura (https://github.com/hasura/graphql-engine)",
   "license": "Apache-2.0",

dc-agents/dc-api-types/src/agent.openapi.json

@@ -276,6 +276,9 @@
           "comparisons": {
             "$ref": "#/components/schemas/ComparisonCapabilities"
           },
+          "data_schema": {
+            "$ref": "#/components/schemas/DataSchemaCapabilities"
+          },
           "explain": {
             "$ref": "#/components/schemas/ExplainCapabilities"
           },
@@ -306,18 +309,32 @@
         },
         "type": "object"
       },
-      "QueryCapabilities": {
+      "ColumnNullability": {
+        "enum": [
+          "only_nullable",
+          "nullable_and_non_nullable"
+        ],
+        "type": "string"
+      },
+      "DataSchemaCapabilities": {
         "properties": {
+          "column_nullability": {
+            "$ref": "#/components/schemas/ColumnNullability"
+          },
+          "supports_foreign_keys": {
+            "default": false,
+            "description": "Whether tables can have foreign keys",
+            "type": "boolean"
+          },
           "supports_primary_keys": {
-            "description": "Does the agent support querying a table by primary key?",
+            "default": false,
+            "description": "Whether tables can have primary keys",
             "type": "boolean"
           }
         },
-        "required": [
-          "supports_primary_keys"
-        ],
         "type": "object"
       },
+      "QueryCapabilities": {},
       "MutationCapabilities": {},
       "SubscriptionCapabilities": {},
       "GraphQLName": {
@@ -349,13 +366,11 @@
         "nullable": true,
         "properties": {
           "supports_relations": {
+            "default": false,
             "description": "Does the agent support comparisons that involve related tables (ie. joins)?",
             "type": "boolean"
           }
         },
-        "required": [
-          "supports_relations"
-        ],
         "type": "object"
       },
       "ComparisonCapabilities": {

dc-agents/dc-api-types/src/index.ts

@@ -16,11 +16,13 @@ export type { ColumnCountAggregate } from './models/ColumnCountAggregate';
 export type { ColumnField } from './models/ColumnField';
 export type { ColumnFieldValue } from './models/ColumnFieldValue';
 export type { ColumnInfo } from './models/ColumnInfo';
+export type { ColumnNullability } from './models/ColumnNullability';
 export type { ComparisonCapabilities } from './models/ComparisonCapabilities';
 export type { ComparisonColumn } from './models/ComparisonColumn';
 export type { ComparisonValue } from './models/ComparisonValue';
 export type { ConfigSchemaResponse } from './models/ConfigSchemaResponse';
 export type { Constraint } from './models/Constraint';
+export type { DataSchemaCapabilities } from './models/DataSchemaCapabilities';
 export type { ExistsExpression } from './models/ExistsExpression';
 export type { ExistsInTable } from './models/ExistsInTable';
 export type { ExplainCapabilities } from './models/ExplainCapabilities';

dc-agents/dc-api-types/src/models/Capabilities.ts

@@ -3,6 +3,7 @@
 /* eslint-disable */
 
 import type { ComparisonCapabilities } from './ComparisonCapabilities';
+import type { DataSchemaCapabilities } from './DataSchemaCapabilities';
 import type { ExplainCapabilities } from './ExplainCapabilities';
 import type { GraphQLTypeDefinitions } from './GraphQLTypeDefinitions';
 import type { MetricsCapabilities } from './MetricsCapabilities';
@@ -15,6 +16,7 @@ import type { SubscriptionCapabilities } from './SubscriptionCapabilities';
 
 export type Capabilities = {
   comparisons?: ComparisonCapabilities;
+  data_schema?: DataSchemaCapabilities;
   explain?: ExplainCapabilities;
   graphql_schema?: GraphQLTypeDefinitions;
   metrics?: MetricsCapabilities;

dc-agents/dc-api-types/src/models/ColumnNullability.ts

@@ -0,0 +1,5 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+export type ColumnNullability = 'only_nullable' | 'nullable_and_non_nullable';

dc-agents/dc-api-types/src/models/DataSchemaCapabilities.ts

@@ -0,0 +1,18 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { ColumnNullability } from './ColumnNullability';
+
+export type DataSchemaCapabilities = {
+  column_nullability?: ColumnNullability;
+  /**
+   * Whether tables can have foreign keys
+   */
+  supports_foreign_keys?: boolean;
+  /**
+   * Whether tables can have primary keys
+   */
+  supports_primary_keys?: boolean;
+};
+

dc-agents/dc-api-types/src/models/QueryCapabilities.ts

@@ -3,9 +3,5 @@
 /* eslint-disable */
 
 export type QueryCapabilities = {
-  /**
-   * Does the agent support querying a table by primary key?
-   */
-  supports_primary_keys: boolean;
 };
 

dc-agents/dc-api-types/src/models/SubqueryComparisonCapabilities.ts

@@ -6,6 +6,6 @@ export type SubqueryComparisonCapabilities = {
   /**
    * Does the agent support comparisons that involve related tables (ie. joins)?
    */
-  supports_relations: boolean;
+  supports_relations?: boolean;
 };
 

dc-agents/package-lock.json

@@ -24,7 +24,7 @@
     },
     "dc-api-types": {
       "name": "@hasura/dc-api-types",
-      "version": "0.9.0",
+      "version": "0.10.0",
       "license": "Apache-2.0",
       "devDependencies": {
         "@tsconfig/node16": "^1.0.3",
@@ -631,7 +631,7 @@
       "license": "Apache-2.0",
       "dependencies": {
         "@fastify/cors": "^7.0.0",
-        "@hasura/dc-api-types": "0.9.0",
+        "@hasura/dc-api-types": "0.10.0",
         "fastify": "^3.29.0",
         "mathjs": "^11.0.0",
         "pino-pretty": "^8.0.0",
@@ -1389,7 +1389,7 @@
       "license": "Apache-2.0",
       "dependencies": {
         "@fastify/cors": "^8.1.0",
-        "@hasura/dc-api-types": "0.9.0",
+        "@hasura/dc-api-types": "0.10.0",
         "fastify": "^4.4.0",
         "fastify-metrics": "^9.2.1",
         "nanoid": "^3.3.4",
@@ -3122,7 +3122,7 @@
       "version": "file:reference",
       "requires": {
         "@fastify/cors": "^7.0.0",
-        "@hasura/dc-api-types": "0.9.0",
+        "@hasura/dc-api-types": "0.10.0",
         "@tsconfig/node16": "^1.0.3",
         "@types/node": "^16.11.49",
         "@types/xml2js": "^0.4.11",
@@ -3613,7 +3613,7 @@
       "version": "file:sqlite",
       "requires": {
         "@fastify/cors": "^8.1.0",
-        "@hasura/dc-api-types": "0.9.0",
+        "@hasura/dc-api-types": "0.10.0",
         "@tsconfig/node16": "^1.0.3",
         "@types/node": "^16.11.49",
         "@types/sqlite3": "^3.1.8",

dc-agents/reference/package-lock.json

@@ -10,7 +10,7 @@
       "license": "Apache-2.0",
       "dependencies": {
         "@fastify/cors": "^7.0.0",
-        "@hasura/dc-api-types": "0.9.0",
+        "@hasura/dc-api-types": "0.10.0",
         "fastify": "^3.29.0",
         "mathjs": "^11.0.0",
         "pino-pretty": "^8.0.0",
@@ -44,7 +44,7 @@
       }
     },
     "node_modules/@hasura/dc-api-types": {
-      "version": "0.9.0",
+      "version": "0.10.0",
       "license": "Apache-2.0",
       "devDependencies": {
         "@tsconfig/node16": "^1.0.3",

dc-agents/reference/package.json

@@ -22,7 +22,7 @@
   },
   "dependencies": {
     "@fastify/cors": "^7.0.0",
-    "@hasura/dc-api-types": "0.9.0",
+    "@hasura/dc-api-types": "0.10.0",
     "fastify": "^3.29.0",
     "mathjs": "^11.0.0",
     "pino-pretty": "^8.0.0",

dc-agents/reference/src/capabilities.ts

@@ -18,9 +18,12 @@ const scalarTypes: ScalarTypesCapabilities = {
 }
 
 const capabilities: Capabilities = {
-  queries: {
+  data_schema: {
     supports_primary_keys: true,
+    supports_foreign_keys: true,
+    column_nullability: "nullable_and_non_nullable",
   },
+  queries: {},
   relationships: {},
   comparisons: {
     subquery: {

dc-agents/sqlite/package-lock.json

@@ -10,7 +10,7 @@
       "license": "Apache-2.0",
       "dependencies": {
         "@fastify/cors": "^8.1.0",
-        "@hasura/dc-api-types": "0.9.0",
+        "@hasura/dc-api-types": "0.10.0",
         "fastify": "^4.4.0",
         "fastify-metrics": "^9.2.1",
         "nanoid": "^3.3.4",
@@ -54,7 +54,7 @@
       "license": "MIT"
     },
     "node_modules/@hasura/dc-api-types": {
-      "version": "0.9.0",
+      "version": "0.10.0",
       "license": "Apache-2.0",
       "devDependencies": {
         "@tsconfig/node16": "^1.0.3",

dc-agents/sqlite/package.json

@@ -22,7 +22,7 @@
   },
   "dependencies": {
     "@fastify/cors": "^8.1.0",
-    "@hasura/dc-api-types": "0.9.0",
+    "@hasura/dc-api-types": "0.10.0",
     "fastify-metrics": "^9.2.1",
     "fastify": "^4.4.0",
     "nanoid": "^3.3.4",

dc-agents/sqlite/src/capabilities.ts

@@ -5,9 +5,12 @@ import { envToBool } from "./util"
 export const capabilitiesResponse: CapabilitiesResponse = {
   config_schemas: configSchema,
   capabilities: {
-    queries: {
+    data_schema: {
-      supports_primary_keys: true
+      supports_primary_keys: true,
+      supports_foreign_keys: true,
+      column_nullability: "nullable_and_non_nullable",
     },
+    queries: {},
     relationships: {},
     comparisons: {
       subquery: {

server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Capabilities.hs

@@ -1,10 +1,15 @@
 {-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE OverloadedLists #-}
 {-# OPTIONS_GHC -Wno-unrecognised-pragmas #-}
 
 {-# HLINT ignore "Use onNothing" #-}
 
 module Hasura.Backends.DataConnector.API.V0.Capabilities
   ( Capabilities (..),
+    defaultCapabilities,
+    DataSchemaCapabilities (..),
+    defaultDataSchemaCapabilities,
+    ColumnNullability (..),
     QueryCapabilities (..),
     MutationCapabilities (..),
     SubscriptionCapabilities (..),
@@ -18,7 +23,6 @@ module Hasura.Backends.DataConnector.API.V0.Capabilities
     ExplainCapabilities (..),
     RawCapabilities (..),
     CapabilitiesResponse (..),
-    emptyCapabilities,
     lookupComparisonInputObjectDefinition,
     mkGraphQLTypeDefinitions,
   )
@@ -55,7 +59,8 @@ import Prelude
 -- service. Specifically, the service is capable of serving queries
 -- which involve relationships.
 data Capabilities = Capabilities
-  { _cQueries :: Maybe QueryCapabilities,
+  { _cDataSchema :: DataSchemaCapabilities,
+    _cQueries :: Maybe QueryCapabilities,
     _cMutations :: Maybe MutationCapabilities,
     _cSubscriptions :: Maybe SubscriptionCapabilities,
     _cScalarTypes :: Maybe ScalarTypesCapabilities,
@@ -70,14 +75,15 @@ data Capabilities = Capabilities
   deriving anyclass (NFData, Hashable)
   deriving (FromJSON, ToJSON, ToSchema) via Autodocodec Capabilities
 
-emptyCapabilities :: Capabilities
+defaultCapabilities :: Capabilities
-emptyCapabilities = Capabilities Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing
+defaultCapabilities = Capabilities defaultDataSchemaCapabilities Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing
 
 instance HasCodec Capabilities where
   codec =
     object "Capabilities" $
       Capabilities
-        <$> optionalField "queries" "The agent's query capabilities" .= _cQueries
+        <$> optionalFieldWithOmittedDefault "data_schema" defaultDataSchemaCapabilities "The agent's data schema capabilities" .= _cDataSchema
+        <*> optionalField "queries" "The agent's query capabilities" .= _cQueries
         <*> optionalField "mutations" "The agent's mutation capabilities" .= _cMutations
         <*> optionalField "subscriptions" "The agent's subscription capabilities" .= _cSubscriptions
         <*> optionalField "scalar_types" "The agent's scalar types and their capabilities" .= _cScalarTypes
@@ -88,18 +94,50 @@ instance HasCodec Capabilities where
         <*> optionalField "explain" "The agent's explain capabilities" .= _cExplain
         <*> optionalField "raw" "The agent's raw query capabilities" .= _cRaw
 
-data QueryCapabilities = QueryCapabilities
+data DataSchemaCapabilities = DataSchemaCapabilities
-  { _qcSupportsPrimaryKeys :: Bool
+  { _dscSupportsPrimaryKeys :: Bool,
+    _dscSupportsForeignKeys :: Bool,
+    _dscColumnNullability :: ColumnNullability
   }
+  deriving stock (Eq, Ord, Show, Generic, Data)
+  deriving anyclass (NFData, Hashable)
+  deriving (FromJSON, ToJSON, ToSchema) via Autodocodec DataSchemaCapabilities
+
+defaultDataSchemaCapabilities :: DataSchemaCapabilities
+defaultDataSchemaCapabilities =
+  DataSchemaCapabilities False False NullableAndNonNullableColumns
+
+instance HasCodec DataSchemaCapabilities where
+  codec =
+    object "DataSchemaCapabilities" $
+      DataSchemaCapabilities
+        <$> optionalFieldWithOmittedDefault "supports_primary_keys" (_dscSupportsPrimaryKeys defaultDataSchemaCapabilities) "Whether tables can have primary keys" .= _dscSupportsPrimaryKeys
+        <*> optionalFieldWithOmittedDefault "supports_foreign_keys" (_dscSupportsForeignKeys defaultDataSchemaCapabilities) "Whether tables can have foreign keys" .= _dscSupportsForeignKeys
+        <*> optionalFieldWithOmittedDefault "column_nullability" (_dscColumnNullability defaultDataSchemaCapabilities) "The sort of column nullability that is supported" .= _dscColumnNullability
+
+data ColumnNullability
+  = OnlyNullableColumns
+  | NullableAndNonNullableColumns
+  deriving stock (Eq, Ord, Show, Generic, Data)
+  deriving anyclass (NFData, Hashable)
+  deriving (FromJSON, ToJSON, ToSchema) via Autodocodec ColumnNullability
+
+instance HasCodec ColumnNullability where
+  codec =
+    named "ColumnNullability" $
+      stringConstCodec
+        [ (OnlyNullableColumns, "only_nullable"),
+          (NullableAndNonNullableColumns, "nullable_and_non_nullable")
+        ]
+
+data QueryCapabilities = QueryCapabilities {}
   deriving stock (Eq, Ord, Show, Generic, Data)
   deriving anyclass (NFData, Hashable)
   deriving (FromJSON, ToJSON, ToSchema) via Autodocodec QueryCapabilities
 
 instance HasCodec QueryCapabilities where
   codec =
-    object "QueryCapabilities" $
+    object "QueryCapabilities" $ pure QueryCapabilities
-      QueryCapabilities
-        <$> requiredField "supports_primary_keys" "Does the agent support querying a table by primary key?" .= _qcSupportsPrimaryKeys
 
 data MutationCapabilities = MutationCapabilities {}
   deriving stock (Eq, Ord, Show, Generic, Data)
@@ -247,7 +285,7 @@ instance HasCodec SubqueryComparisonCapabilities where
   codec =
     object "SubqueryComparisonCapabilities" $
       SubqueryComparisonCapabilities
-        <$> requiredField "supports_relations" "Does the agent support comparisons that involve related tables (ie. joins)?" .= _ctccSupportsRelations
+        <$> optionalFieldWithOmittedDefault "supports_relations" False "Does the agent support comparisons that involve related tables (ie. joins)?" .= _ctccSupportsRelations
 
 data MetricsCapabilities = MetricsCapabilities {}
   deriving stock (Eq, Ord, Show, Generic, Data)

server/lib/dc-api/test/Command.hs

@@ -1,23 +1,14 @@
-{-# LANGUAGE TemplateHaskell #-}
-
 module Command
   ( Command (..),
     TestConfig (..),
     NameCasing (..),
     TestOptions (..),
-    AgentCapabilities (..),
     parseCommandLine,
   )
 where
 
 import Control.Arrow (left)
-import Control.Lens (contains, modifying, use, (^.), _2)
-import Control.Lens.TH (makeLenses)
-import Control.Monad (when)
-import Control.Monad.State (State, runState)
 import Data.Aeson (FromJSON (..), eitherDecodeStrict')
-import Data.HashSet (HashSet)
-import Data.HashSet qualified as HashSet
 import Data.Text (Text)
 import Data.Text qualified as Text
 import Data.Text.Encoding qualified as Text
@@ -46,7 +37,6 @@ data NameCasing
 data TestOptions = TestOptions
   { _toAgentBaseUrl :: BaseUrl,
     _toAgentConfig :: API.Config,
-    _toAgentCapabilities :: AgentCapabilities,
     _toTestConfig :: TestConfig,
     _toParallelDegree :: Maybe Int,
     _toMatch :: Maybe String,
@@ -55,17 +45,6 @@ data TestOptions = TestOptions
     _toExportMatchStrings :: Bool
   }
 
-data AgentCapabilities
-  = AutoDetect
-  | Explicit API.Capabilities
-
-data CapabilitiesState = CapabilitiesState
-  { _csRemainingCapabilities :: HashSet Text,
-    _csCapabilitiesEnquired :: HashSet Text
-  }
-
-$(makeLenses ''CapabilitiesState)
-
 parseCommandLine :: IO Command
 parseCommandLine =
   execParser $
@@ -150,7 +129,6 @@ testOptionsParser =
           <> metavar "JSON"
           <> help "The configuration JSON to be sent to the agent via the X-Hasura-DataConnector-Config header"
       )
-    <*> agentCapabilitiesParser
     <*> testConfigParser
     <*> optional
       ( option
@@ -202,61 +180,3 @@ configValue = fmap API.Config jsonValue
 
 jsonValue :: FromJSON v => ReadM v
 jsonValue = eitherReader (eitherDecodeStrict' . Text.encodeUtf8 . Text.pack)
-
-agentCapabilitiesParser :: Parser AgentCapabilities
-agentCapabilitiesParser =
-  option
-    agentCapabilities
-    ( long "capabilities"
-        <> short 'c'
-        <> metavar "CAPABILITIES"
-        <> value AutoDetect
-        <> help (Text.unpack helpText)
-    )
-  where
-    helpText =
-      "The capabilities that the agent has, to determine what tests to run. By default, they will be autodetected. The valid capabilities are: " <> allCapabilitiesText
-    allCapabilitiesText =
-      "[autodetect | none | " <> Text.intercalate "," (HashSet.toList allPossibleCapabilities) <> "]"
-
-agentCapabilities :: ReadM AgentCapabilities
-agentCapabilities =
-  str >>= \text -> do
-    let capabilities = HashSet.fromList $ Text.strip <$> Text.split (== ',') text
-    if HashSet.member "autodetect" capabilities
-      then
-        if HashSet.size capabilities == 1
-          then pure AutoDetect
-          else readerError "You can either autodetect capabilities or specify them manually, not both"
-      else
-        if HashSet.member "none" capabilities
-          then
-            if HashSet.size capabilities == 1
-              then pure . Explicit . fst $ readCapabilities mempty
-              else readerError "You cannot specify other capabilities when specifying none"
-          else Explicit <$> readExplicitCapabilities capabilities
-  where
-    readExplicitCapabilities :: HashSet Text -> ReadM API.Capabilities
-    readExplicitCapabilities providedCapabilities =
-      let (capabilities, CapabilitiesState {..}) = readCapabilities providedCapabilities
-       in if _csRemainingCapabilities /= mempty
-            then readerError . Text.unpack $ "Unknown capabilities: " <> Text.intercalate "," (HashSet.toList _csRemainingCapabilities)
-            else pure capabilities
-
-readCapabilities :: HashSet Text -> (API.Capabilities, CapabilitiesState)
-readCapabilities providedCapabilities =
-  flip runState (CapabilitiesState providedCapabilities mempty) $ do
-    supportsRelationships <- readCapability "relationships"
-    pure $ API.emptyCapabilities {API._cRelationships = if supportsRelationships then Just API.RelationshipCapabilities {} else Nothing}
-
-readCapability :: Text -> State CapabilitiesState Bool
-readCapability capability = do
-  modifying csCapabilitiesEnquired $ HashSet.insert capability
-  hasCapability <- use $ csRemainingCapabilities . contains capability
-  when hasCapability $
-    modifying csRemainingCapabilities $ HashSet.delete capability
-  pure hasCapability
-
-allPossibleCapabilities :: HashSet Text
-allPossibleCapabilities =
-  readCapabilities mempty ^. _2 . csCapabilitiesEnquired

server/lib/dc-api/test/Main.hs

@@ -2,7 +2,7 @@ module Main (main) where
 
 --------------------------------------------------------------------------------
 
-import Command (AgentCapabilities (..), Command (..), TestOptions (..), parseCommandLine)
+import Command (Command (..), TestOptions (..), parseCommandLine)
 import Control.Exception (throwIO)
 import Control.Monad (join, (>=>))
 import Data.Aeson.Text (encodeToLazyText)
@@ -50,7 +50,7 @@ main = do
   case command of
     Test testOptions@TestOptions {..} -> do
       api <- mkIOApiClient testOptions
-      agentCapabilities <- getAgentCapabilities api _toAgentCapabilities
+      agentCapabilities <- API._crCapabilities <$> (api // _capabilities)
       let testData = mkTestData _toTestConfig
       let spec = tests testData api testSourceName _toAgentConfig agentCapabilities
       case _toExportMatchStrings of
@@ -72,11 +72,6 @@ mkIOApiClient TestOptions {..} = do
 throwClientError :: Either ClientError a -> IO a
 throwClientError = either throwIO pure
 
-getAgentCapabilities :: Client IO (NamedRoutes Routes) -> AgentCapabilities -> IO API.Capabilities
-getAgentCapabilities api = \case
-  AutoDetect -> API._crCapabilities <$> (api // _capabilities)
-  Explicit capabilities -> pure capabilities
-
 applyTestConfig :: Config -> TestOptions -> Config
 applyTestConfig config TestOptions {..} =
   config

server/lib/dc-api/test/Test/SchemaSpec.hs

@@ -2,11 +2,11 @@ module Test.SchemaSpec (spec) where
 
 --------------------------------------------------------------------------------
 
-import Control.Lens ((%~), (.~))
+import Control.Lens ((%~), (.~), (?~))
 import Control.Lens.At (at)
 import Control.Lens.Lens ((&))
 import Control.Monad (forM_)
-import Data.Aeson (toJSON)
+import Data.Aeson (Value (..), toJSON)
 import Data.Aeson.Lens (_Object)
 import Data.Foldable (find)
 import Data.HashMap.Strict qualified as HashMap
@@ -42,12 +42,17 @@ spec TestData {..} api sourceName config API.Capabilities {..} = describe "schem
                 column
                   & _Object . at "type" .~ Nothing -- Types can vary between agents since underlying datatypes can change
                   & _Object . at "description" .~ Nothing -- Descriptions are not supported by all agents
+                  -- If the agent only supports nullable columns, we make all columns nullable
+    let setExpectedColumnNullability columns =
+          if API._dscColumnNullability _cDataSchema == API.OnlyNullableColumns
+            then columns & traverse %~ (_Object . at "nullable" ?~ Bool True)
+            else columns
     let actualJsonColumns = extractJsonForComparison <$> tables
-    let expectedJsonColumns = Just $ extractJsonForComparison expectedTable
+    let expectedJsonColumns = Just . setExpectedColumnNullability $ extractJsonForComparison expectedTable
 
     actualJsonColumns `jsonShouldBe` expectedJsonColumns
 
-  if (API._qcSupportsPrimaryKeys <$> _cQueries) == Just True
+  if API._dscSupportsPrimaryKeys _cDataSchema
     then testPerTable "returns the correct primary keys for the Chinook tables" $ \API.TableInfo {..} -> do
       tables <- find (\t -> API._tiName t == _tiName) . API._srTables <$> (api // API._schema) sourceName config
       let actualPrimaryKey = API._tiPrimaryKey <$> tables
@@ -57,7 +62,7 @@ spec TestData {..} api sourceName config API.Capabilities {..} = describe "schem
       let actualPrimaryKey = API._tiPrimaryKey <$> tables
       actualPrimaryKey `jsonShouldBe` Just []
 
-  if (API._qcSupportsPrimaryKeys <$> _cQueries) == Just True
+  if API._dscSupportsForeignKeys _cDataSchema
     then testPerTable "returns the correct foreign keys for the Chinook tables" $ \expectedTable@API.TableInfo {..} -> do
       tables <- find (\t -> API._tiName t == _tiName) . API._srTables <$> (api // API._schema) sourceName config
 

server/lib/test-harness/src/Harness/Backend/DataConnector/MockAgent.hs

@@ -33,7 +33,8 @@ capabilities =
   API.CapabilitiesResponse
     { _crCapabilities =
         API.Capabilities
-          { API._cQueries = Just API.QueryCapabilities {API._qcSupportsPrimaryKeys = True},
+          { API._cDataSchema = API.defaultDataSchemaCapabilities,
+            API._cQueries = Just API.QueryCapabilities,
             API._cMutations = Nothing,
             API._cSubscriptions = Nothing,
             API._cScalarTypes = Nothing,

server/lib/test-harness/src/Harness/Test/BackendType.hs

@@ -51,6 +51,10 @@ defaultBackendCapabilities = \case
   DataConnectorSqlite ->
     Just
       [yaml|
+        data_schema:
+          supports_primary_keys: true
+          supports_foreign_keys: true
+        queries: {}
         relationships: {}
         comparisons:
           subquery:
@@ -58,14 +62,14 @@ defaultBackendCapabilities = \case
         explain: {}
         metrics: {}
         raw: {}
-      queries:
-        supports_primary_keys: true
     |]
   DataConnectorReference ->
     Just
       [yaml|
-      queries:
+        data_schema:
           supports_primary_keys: true
+          supports_foreign_keys: true
+        queries: {}
         graphql_schema: |-
           scalar DateTime
 

server/src-test/Hasura/Backends/DataConnector/API/V0/CapabilitiesSpec.hs

@@ -21,11 +21,11 @@ import Test.Hspec
 spec :: Spec
 spec = do
   describe "Capabilities" $ do
-    testToFromJSONToSchema emptyCapabilities [aesonQQ|{}|]
+    testToFromJSONToSchema defaultCapabilities [aesonQQ|{}|]
     jsonOpenApiProperties genCapabilities
   describe "CapabilitiesResponse" $ do
     testToFromJSON
-      (CapabilitiesResponse (emptyCapabilities {_cRelationships = Just RelationshipCapabilities {}}) emptyConfigSchemaResponse)
+      (CapabilitiesResponse (defaultCapabilities {_cRelationships = Just RelationshipCapabilities {}}) emptyConfigSchemaResponse)
       [aesonQQ|{"capabilities": {"relationships": {}}, "config_schemas": {"config_schema": {}, "other_schemas": {}}}|]
   describe "ScalarTypeCapabilities" $ do
     testToFromJSONToSchema (ScalarTypeCapabilities $ Just [G.name|DateTimeComparisons|]) [aesonQQ|{"comparison_type": "DateTimeComparisons"}|]
@@ -56,8 +56,19 @@ input DateTimeComparisons {after: DateTime
   in_year: Int
 }|]
 
+genDataSchemaCapabilities :: MonadGen m => m DataSchemaCapabilities
+genDataSchemaCapabilities =
+  DataSchemaCapabilities
+    <$> Gen.bool
+    <*> Gen.bool
+    <*> genColumnNullability
+
+genColumnNullability :: MonadGen m => m ColumnNullability
+genColumnNullability =
+  Gen.element [NullableAndNonNullableColumns, OnlyNullableColumns]
+
 genQueryCapabilities :: MonadGen m => m QueryCapabilities
-genQueryCapabilities = QueryCapabilities <$> Gen.bool
+genQueryCapabilities = pure QueryCapabilities
 
 genMutationCapabilities :: MonadGen m => m MutationCapabilities
 genMutationCapabilities = pure MutationCapabilities {}
@@ -139,7 +150,8 @@ genRawCapabilities = pure RawCapabilities {}
 genCapabilities :: Gen Capabilities
 genCapabilities =
   Capabilities
-    <$> Gen.maybe genQueryCapabilities
+    <$> genDataSchemaCapabilities
+    <*> Gen.maybe genQueryCapabilities
     <*> Gen.maybe genMutationCapabilities
     <*> Gen.maybe genSubscriptionCapabilities
     <*> Gen.maybe genScalarTypesCapabilities