From 8369cac3bd644aae21db732dcd14f391b5bf95af Mon Sep 17 00:00:00 2001 From: Daniel Chambers Date: Mon, 10 Oct 2022 17:58:12 +1100 Subject: [PATCH] Data Connector agent data schema capabilities [GDC-479] PR-URL: https://github.com/hasura/graphql-engine-mono/pull/6268 GitOrigin-RevId: 4ec29566d3c2ab2144dad8055b4442a4027915ec --- dc-agents/README.md | 11 +++ dc-agents/dc-api-types/package.json | 2 +- dc-agents/dc-api-types/src/agent.openapi.json | 31 +++++-- dc-agents/dc-api-types/src/index.ts | 2 + .../dc-api-types/src/models/Capabilities.ts | 2 + .../src/models/ColumnNullability.ts | 5 ++ .../src/models/DataSchemaCapabilities.ts | 18 +++++ .../src/models/QueryCapabilities.ts | 4 - .../models/SubqueryComparisonCapabilities.ts | 2 +- dc-agents/package-lock.json | 10 +-- dc-agents/reference/package-lock.json | 4 +- dc-agents/reference/package.json | 2 +- dc-agents/reference/src/capabilities.ts | 5 +- dc-agents/sqlite/package-lock.json | 4 +- dc-agents/sqlite/package.json | 2 +- dc-agents/sqlite/src/capabilities.ts | 7 +- .../DataConnector/API/V0/Capabilities.hs | 60 +++++++++++--- server/lib/dc-api/test/Command.hs | 80 ------------------- server/lib/dc-api/test/Main.hs | 9 +-- server/lib/dc-api/test/Test/SchemaSpec.hs | 15 ++-- .../Backend/DataConnector/MockAgent.hs | 3 +- .../src/Harness/Test/BackendType.hs | 50 ++++++------ .../DataConnector/API/V0/CapabilitiesSpec.hs | 20 ++++- 23 files changed, 189 insertions(+), 159 deletions(-) create mode 100644 dc-agents/dc-api-types/src/models/ColumnNullability.ts create mode 100644 dc-agents/dc-api-types/src/models/DataSchemaCapabilities.ts diff --git a/dc-agents/README.md b/dc-agents/README.md index e5a246573c3..afe81ce66f3 100644 --- a/dc-agents/README.md +++ b/dc-agents/README.md @@ -127,6 +127,11 @@ The `GET /capabilities` endpoint is used by `graphql-engine` to discover the cap ```json { "capabilities": { + "data_schema": { + "supports_primary_keys": true, + "supports_foreign_keys": true, + "column_nullability": "nullable_and_non_nullable" + }, "relationships": {}, "graphql_schema": "scalar DateTime\n\ninput DateTimeComparisons {\n in_year: Number\n}", "scalar_types": { @@ -158,6 +163,7 @@ The `GET /capabilities` endpoint is used by `graphql-engine` to discover the cap ``` The `capabilities` section describes the _capabilities_ of the service. This includes +- `data_schema`: What sorts of features the agent supports when describing its data schema - `relationships`: whether or not the agent supports relationships - `scalar_types`: custom scalar types and the operations they support. See [Scalar types capabilities](#scalar-type-capabilities). - `graphql_schema`: a GraphQL schema document containing type definitions referenced by the `scalar_types` capabilities. @@ -166,6 +172,11 @@ The `config_schema` property contains an [OpenAPI 3 Schema](https://swagger.io/s `graphql-engine` will use the `config_schema` OpenAPI 3 Schema to validate the user's configuration JSON before putting it into the `X-Hasura-DataConnector-Config` header. +#### Data schema capabilities +The agent can declare whether or not it supports primary keys or foreign keys by setting the `supports_primary_keys` and `supports_foreign_keys` properties under the `data_schema` object on capabilities. If it does not declare support, it is expected that it will not return any such primary/foreign keys in the schema it exposes on the `/schema` endpoint. + +If the agent only supports table columns that are always nullable, then it should set `column_nullability` to `"only_nullable"`. However, if it supports both nullable and non-nullable columns, then it should set `"nullable_and_non_nullable"`. + #### Scalar type capabilities The agent is expected to support a default set of scalar types (`Number`, `String`, `Bool`) and a default set of [comparison operators](#filters) on these types. diff --git a/dc-agents/dc-api-types/package.json b/dc-agents/dc-api-types/package.json index 99ae47f61b5..1fe94dcb634 100644 --- a/dc-agents/dc-api-types/package.json +++ b/dc-agents/dc-api-types/package.json @@ -1,6 +1,6 @@ { "name": "@hasura/dc-api-types", - "version": "0.9.0", + "version": "0.10.0", "description": "Hasura GraphQL Engine Data Connector Agent API types", "author": "Hasura (https://github.com/hasura/graphql-engine)", "license": "Apache-2.0", diff --git a/dc-agents/dc-api-types/src/agent.openapi.json b/dc-agents/dc-api-types/src/agent.openapi.json index c28e45e6d27..1934fa8d113 100644 --- a/dc-agents/dc-api-types/src/agent.openapi.json +++ b/dc-agents/dc-api-types/src/agent.openapi.json @@ -276,6 +276,9 @@ "comparisons": { "$ref": "#/components/schemas/ComparisonCapabilities" }, + "data_schema": { + "$ref": "#/components/schemas/DataSchemaCapabilities" + }, "explain": { "$ref": "#/components/schemas/ExplainCapabilities" }, @@ -306,18 +309,32 @@ }, "type": "object" }, - "QueryCapabilities": { + "ColumnNullability": { + "enum": [ + "only_nullable", + "nullable_and_non_nullable" + ], + "type": "string" + }, + "DataSchemaCapabilities": { "properties": { + "column_nullability": { + "$ref": "#/components/schemas/ColumnNullability" + }, + "supports_foreign_keys": { + "default": false, + "description": "Whether tables can have foreign keys", + "type": "boolean" + }, "supports_primary_keys": { - "description": "Does the agent support querying a table by primary key?", + "default": false, + "description": "Whether tables can have primary keys", "type": "boolean" } }, - "required": [ - "supports_primary_keys" - ], "type": "object" }, + "QueryCapabilities": {}, "MutationCapabilities": {}, "SubscriptionCapabilities": {}, "GraphQLName": { @@ -349,13 +366,11 @@ "nullable": true, "properties": { "supports_relations": { + "default": false, "description": "Does the agent support comparisons that involve related tables (ie. joins)?", "type": "boolean" } }, - "required": [ - "supports_relations" - ], "type": "object" }, "ComparisonCapabilities": { diff --git a/dc-agents/dc-api-types/src/index.ts b/dc-agents/dc-api-types/src/index.ts index 48f64a7076f..65b25b1bb75 100644 --- a/dc-agents/dc-api-types/src/index.ts +++ b/dc-agents/dc-api-types/src/index.ts @@ -16,11 +16,13 @@ export type { ColumnCountAggregate } from './models/ColumnCountAggregate'; export type { ColumnField } from './models/ColumnField'; export type { ColumnFieldValue } from './models/ColumnFieldValue'; export type { ColumnInfo } from './models/ColumnInfo'; +export type { ColumnNullability } from './models/ColumnNullability'; export type { ComparisonCapabilities } from './models/ComparisonCapabilities'; export type { ComparisonColumn } from './models/ComparisonColumn'; export type { ComparisonValue } from './models/ComparisonValue'; export type { ConfigSchemaResponse } from './models/ConfigSchemaResponse'; export type { Constraint } from './models/Constraint'; +export type { DataSchemaCapabilities } from './models/DataSchemaCapabilities'; export type { ExistsExpression } from './models/ExistsExpression'; export type { ExistsInTable } from './models/ExistsInTable'; export type { ExplainCapabilities } from './models/ExplainCapabilities'; diff --git a/dc-agents/dc-api-types/src/models/Capabilities.ts b/dc-agents/dc-api-types/src/models/Capabilities.ts index 8e819b2189b..7863496407a 100644 --- a/dc-agents/dc-api-types/src/models/Capabilities.ts +++ b/dc-agents/dc-api-types/src/models/Capabilities.ts @@ -3,6 +3,7 @@ /* eslint-disable */ import type { ComparisonCapabilities } from './ComparisonCapabilities'; +import type { DataSchemaCapabilities } from './DataSchemaCapabilities'; import type { ExplainCapabilities } from './ExplainCapabilities'; import type { GraphQLTypeDefinitions } from './GraphQLTypeDefinitions'; import type { MetricsCapabilities } from './MetricsCapabilities'; @@ -15,6 +16,7 @@ import type { SubscriptionCapabilities } from './SubscriptionCapabilities'; export type Capabilities = { comparisons?: ComparisonCapabilities; + data_schema?: DataSchemaCapabilities; explain?: ExplainCapabilities; graphql_schema?: GraphQLTypeDefinitions; metrics?: MetricsCapabilities; diff --git a/dc-agents/dc-api-types/src/models/ColumnNullability.ts b/dc-agents/dc-api-types/src/models/ColumnNullability.ts new file mode 100644 index 00000000000..4790ca01bda --- /dev/null +++ b/dc-agents/dc-api-types/src/models/ColumnNullability.ts @@ -0,0 +1,5 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type ColumnNullability = 'only_nullable' | 'nullable_and_non_nullable'; diff --git a/dc-agents/dc-api-types/src/models/DataSchemaCapabilities.ts b/dc-agents/dc-api-types/src/models/DataSchemaCapabilities.ts new file mode 100644 index 00000000000..cf508c3b4ca --- /dev/null +++ b/dc-agents/dc-api-types/src/models/DataSchemaCapabilities.ts @@ -0,0 +1,18 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ColumnNullability } from './ColumnNullability'; + +export type DataSchemaCapabilities = { + column_nullability?: ColumnNullability; + /** + * Whether tables can have foreign keys + */ + supports_foreign_keys?: boolean; + /** + * Whether tables can have primary keys + */ + supports_primary_keys?: boolean; +}; + diff --git a/dc-agents/dc-api-types/src/models/QueryCapabilities.ts b/dc-agents/dc-api-types/src/models/QueryCapabilities.ts index 2c4c53797f8..48e4f0576bc 100644 --- a/dc-agents/dc-api-types/src/models/QueryCapabilities.ts +++ b/dc-agents/dc-api-types/src/models/QueryCapabilities.ts @@ -3,9 +3,5 @@ /* eslint-disable */ export type QueryCapabilities = { - /** - * Does the agent support querying a table by primary key? - */ - supports_primary_keys: boolean; }; diff --git a/dc-agents/dc-api-types/src/models/SubqueryComparisonCapabilities.ts b/dc-agents/dc-api-types/src/models/SubqueryComparisonCapabilities.ts index 9deb9b39285..f63269455be 100644 --- a/dc-agents/dc-api-types/src/models/SubqueryComparisonCapabilities.ts +++ b/dc-agents/dc-api-types/src/models/SubqueryComparisonCapabilities.ts @@ -6,6 +6,6 @@ export type SubqueryComparisonCapabilities = { /** * Does the agent support comparisons that involve related tables (ie. joins)? */ - supports_relations: boolean; + supports_relations?: boolean; }; diff --git a/dc-agents/package-lock.json b/dc-agents/package-lock.json index aa8e5a4c651..15fcd0176a6 100644 --- a/dc-agents/package-lock.json +++ b/dc-agents/package-lock.json @@ -24,7 +24,7 @@ }, "dc-api-types": { "name": "@hasura/dc-api-types", - "version": "0.9.0", + "version": "0.10.0", "license": "Apache-2.0", "devDependencies": { "@tsconfig/node16": "^1.0.3", @@ -631,7 +631,7 @@ "license": "Apache-2.0", "dependencies": { "@fastify/cors": "^7.0.0", - "@hasura/dc-api-types": "0.9.0", + "@hasura/dc-api-types": "0.10.0", "fastify": "^3.29.0", "mathjs": "^11.0.0", "pino-pretty": "^8.0.0", @@ -1389,7 +1389,7 @@ "license": "Apache-2.0", "dependencies": { "@fastify/cors": "^8.1.0", - "@hasura/dc-api-types": "0.9.0", + "@hasura/dc-api-types": "0.10.0", "fastify": "^4.4.0", "fastify-metrics": "^9.2.1", "nanoid": "^3.3.4", @@ -3122,7 +3122,7 @@ "version": "file:reference", "requires": { "@fastify/cors": "^7.0.0", - "@hasura/dc-api-types": "0.9.0", + "@hasura/dc-api-types": "0.10.0", "@tsconfig/node16": "^1.0.3", "@types/node": "^16.11.49", "@types/xml2js": "^0.4.11", @@ -3613,7 +3613,7 @@ "version": "file:sqlite", "requires": { "@fastify/cors": "^8.1.0", - "@hasura/dc-api-types": "0.9.0", + "@hasura/dc-api-types": "0.10.0", "@tsconfig/node16": "^1.0.3", "@types/node": "^16.11.49", "@types/sqlite3": "^3.1.8", diff --git a/dc-agents/reference/package-lock.json b/dc-agents/reference/package-lock.json index ffea820135e..b750e81489b 100644 --- a/dc-agents/reference/package-lock.json +++ b/dc-agents/reference/package-lock.json @@ -10,7 +10,7 @@ "license": "Apache-2.0", "dependencies": { "@fastify/cors": "^7.0.0", - "@hasura/dc-api-types": "0.9.0", + "@hasura/dc-api-types": "0.10.0", "fastify": "^3.29.0", "mathjs": "^11.0.0", "pino-pretty": "^8.0.0", @@ -44,7 +44,7 @@ } }, "node_modules/@hasura/dc-api-types": { - "version": "0.9.0", + "version": "0.10.0", "license": "Apache-2.0", "devDependencies": { "@tsconfig/node16": "^1.0.3", diff --git a/dc-agents/reference/package.json b/dc-agents/reference/package.json index 99c7c3e11c8..a4ac4451010 100644 --- a/dc-agents/reference/package.json +++ b/dc-agents/reference/package.json @@ -22,7 +22,7 @@ }, "dependencies": { "@fastify/cors": "^7.0.0", - "@hasura/dc-api-types": "0.9.0", + "@hasura/dc-api-types": "0.10.0", "fastify": "^3.29.0", "mathjs": "^11.0.0", "pino-pretty": "^8.0.0", diff --git a/dc-agents/reference/src/capabilities.ts b/dc-agents/reference/src/capabilities.ts index 7aa5e591142..a16d1165dbd 100644 --- a/dc-agents/reference/src/capabilities.ts +++ b/dc-agents/reference/src/capabilities.ts @@ -18,9 +18,12 @@ const scalarTypes: ScalarTypesCapabilities = { } const capabilities: Capabilities = { - queries: { + data_schema: { supports_primary_keys: true, + supports_foreign_keys: true, + column_nullability: "nullable_and_non_nullable", }, + queries: {}, relationships: {}, comparisons: { subquery: { diff --git a/dc-agents/sqlite/package-lock.json b/dc-agents/sqlite/package-lock.json index aaa699ab598..548ce7a636d 100644 --- a/dc-agents/sqlite/package-lock.json +++ b/dc-agents/sqlite/package-lock.json @@ -10,7 +10,7 @@ "license": "Apache-2.0", "dependencies": { "@fastify/cors": "^8.1.0", - "@hasura/dc-api-types": "0.9.0", + "@hasura/dc-api-types": "0.10.0", "fastify": "^4.4.0", "fastify-metrics": "^9.2.1", "nanoid": "^3.3.4", @@ -54,7 +54,7 @@ "license": "MIT" }, "node_modules/@hasura/dc-api-types": { - "version": "0.9.0", + "version": "0.10.0", "license": "Apache-2.0", "devDependencies": { "@tsconfig/node16": "^1.0.3", diff --git a/dc-agents/sqlite/package.json b/dc-agents/sqlite/package.json index b0ab80918e4..d5c81eae4c2 100644 --- a/dc-agents/sqlite/package.json +++ b/dc-agents/sqlite/package.json @@ -22,7 +22,7 @@ }, "dependencies": { "@fastify/cors": "^8.1.0", - "@hasura/dc-api-types": "0.9.0", + "@hasura/dc-api-types": "0.10.0", "fastify-metrics": "^9.2.1", "fastify": "^4.4.0", "nanoid": "^3.3.4", diff --git a/dc-agents/sqlite/src/capabilities.ts b/dc-agents/sqlite/src/capabilities.ts index 2eca47e5eae..88cd6cb71db 100644 --- a/dc-agents/sqlite/src/capabilities.ts +++ b/dc-agents/sqlite/src/capabilities.ts @@ -5,9 +5,12 @@ import { envToBool } from "./util" export const capabilitiesResponse: CapabilitiesResponse = { config_schemas: configSchema, capabilities: { - queries: { - supports_primary_keys: true + data_schema: { + supports_primary_keys: true, + supports_foreign_keys: true, + column_nullability: "nullable_and_non_nullable", }, + queries: {}, relationships: {}, comparisons: { subquery: { diff --git a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Capabilities.hs b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Capabilities.hs index f3c86f1e586..1e8d1583360 100644 --- a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Capabilities.hs +++ b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Capabilities.hs @@ -1,10 +1,15 @@ {-# LANGUAGE DeriveAnyClass #-} +{-# LANGUAGE OverloadedLists #-} {-# OPTIONS_GHC -Wno-unrecognised-pragmas #-} {-# HLINT ignore "Use onNothing" #-} module Hasura.Backends.DataConnector.API.V0.Capabilities ( Capabilities (..), + defaultCapabilities, + DataSchemaCapabilities (..), + defaultDataSchemaCapabilities, + ColumnNullability (..), QueryCapabilities (..), MutationCapabilities (..), SubscriptionCapabilities (..), @@ -18,7 +23,6 @@ module Hasura.Backends.DataConnector.API.V0.Capabilities ExplainCapabilities (..), RawCapabilities (..), CapabilitiesResponse (..), - emptyCapabilities, lookupComparisonInputObjectDefinition, mkGraphQLTypeDefinitions, ) @@ -55,7 +59,8 @@ import Prelude -- service. Specifically, the service is capable of serving queries -- which involve relationships. data Capabilities = Capabilities - { _cQueries :: Maybe QueryCapabilities, + { _cDataSchema :: DataSchemaCapabilities, + _cQueries :: Maybe QueryCapabilities, _cMutations :: Maybe MutationCapabilities, _cSubscriptions :: Maybe SubscriptionCapabilities, _cScalarTypes :: Maybe ScalarTypesCapabilities, @@ -70,14 +75,15 @@ data Capabilities = Capabilities deriving anyclass (NFData, Hashable) deriving (FromJSON, ToJSON, ToSchema) via Autodocodec Capabilities -emptyCapabilities :: Capabilities -emptyCapabilities = Capabilities Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing +defaultCapabilities :: Capabilities +defaultCapabilities = Capabilities defaultDataSchemaCapabilities Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing instance HasCodec Capabilities where codec = object "Capabilities" $ Capabilities - <$> optionalField "queries" "The agent's query capabilities" .= _cQueries + <$> optionalFieldWithOmittedDefault "data_schema" defaultDataSchemaCapabilities "The agent's data schema capabilities" .= _cDataSchema + <*> optionalField "queries" "The agent's query capabilities" .= _cQueries <*> optionalField "mutations" "The agent's mutation capabilities" .= _cMutations <*> optionalField "subscriptions" "The agent's subscription capabilities" .= _cSubscriptions <*> optionalField "scalar_types" "The agent's scalar types and their capabilities" .= _cScalarTypes @@ -88,18 +94,50 @@ instance HasCodec Capabilities where <*> optionalField "explain" "The agent's explain capabilities" .= _cExplain <*> optionalField "raw" "The agent's raw query capabilities" .= _cRaw -data QueryCapabilities = QueryCapabilities - { _qcSupportsPrimaryKeys :: Bool +data DataSchemaCapabilities = DataSchemaCapabilities + { _dscSupportsPrimaryKeys :: Bool, + _dscSupportsForeignKeys :: Bool, + _dscColumnNullability :: ColumnNullability } + deriving stock (Eq, Ord, Show, Generic, Data) + deriving anyclass (NFData, Hashable) + deriving (FromJSON, ToJSON, ToSchema) via Autodocodec DataSchemaCapabilities + +defaultDataSchemaCapabilities :: DataSchemaCapabilities +defaultDataSchemaCapabilities = + DataSchemaCapabilities False False NullableAndNonNullableColumns + +instance HasCodec DataSchemaCapabilities where + codec = + object "DataSchemaCapabilities" $ + DataSchemaCapabilities + <$> optionalFieldWithOmittedDefault "supports_primary_keys" (_dscSupportsPrimaryKeys defaultDataSchemaCapabilities) "Whether tables can have primary keys" .= _dscSupportsPrimaryKeys + <*> optionalFieldWithOmittedDefault "supports_foreign_keys" (_dscSupportsForeignKeys defaultDataSchemaCapabilities) "Whether tables can have foreign keys" .= _dscSupportsForeignKeys + <*> optionalFieldWithOmittedDefault "column_nullability" (_dscColumnNullability defaultDataSchemaCapabilities) "The sort of column nullability that is supported" .= _dscColumnNullability + +data ColumnNullability + = OnlyNullableColumns + | NullableAndNonNullableColumns + deriving stock (Eq, Ord, Show, Generic, Data) + deriving anyclass (NFData, Hashable) + deriving (FromJSON, ToJSON, ToSchema) via Autodocodec ColumnNullability + +instance HasCodec ColumnNullability where + codec = + named "ColumnNullability" $ + stringConstCodec + [ (OnlyNullableColumns, "only_nullable"), + (NullableAndNonNullableColumns, "nullable_and_non_nullable") + ] + +data QueryCapabilities = QueryCapabilities {} deriving stock (Eq, Ord, Show, Generic, Data) deriving anyclass (NFData, Hashable) deriving (FromJSON, ToJSON, ToSchema) via Autodocodec QueryCapabilities instance HasCodec QueryCapabilities where codec = - object "QueryCapabilities" $ - QueryCapabilities - <$> requiredField "supports_primary_keys" "Does the agent support querying a table by primary key?" .= _qcSupportsPrimaryKeys + object "QueryCapabilities" $ pure QueryCapabilities data MutationCapabilities = MutationCapabilities {} deriving stock (Eq, Ord, Show, Generic, Data) @@ -247,7 +285,7 @@ instance HasCodec SubqueryComparisonCapabilities where codec = object "SubqueryComparisonCapabilities" $ SubqueryComparisonCapabilities - <$> requiredField "supports_relations" "Does the agent support comparisons that involve related tables (ie. joins)?" .= _ctccSupportsRelations + <$> optionalFieldWithOmittedDefault "supports_relations" False "Does the agent support comparisons that involve related tables (ie. joins)?" .= _ctccSupportsRelations data MetricsCapabilities = MetricsCapabilities {} deriving stock (Eq, Ord, Show, Generic, Data) diff --git a/server/lib/dc-api/test/Command.hs b/server/lib/dc-api/test/Command.hs index 0bbebf4ec28..dcd23aa6362 100644 --- a/server/lib/dc-api/test/Command.hs +++ b/server/lib/dc-api/test/Command.hs @@ -1,23 +1,14 @@ -{-# LANGUAGE TemplateHaskell #-} - module Command ( Command (..), TestConfig (..), NameCasing (..), TestOptions (..), - AgentCapabilities (..), parseCommandLine, ) where import Control.Arrow (left) -import Control.Lens (contains, modifying, use, (^.), _2) -import Control.Lens.TH (makeLenses) -import Control.Monad (when) -import Control.Monad.State (State, runState) import Data.Aeson (FromJSON (..), eitherDecodeStrict') -import Data.HashSet (HashSet) -import Data.HashSet qualified as HashSet import Data.Text (Text) import Data.Text qualified as Text import Data.Text.Encoding qualified as Text @@ -46,7 +37,6 @@ data NameCasing data TestOptions = TestOptions { _toAgentBaseUrl :: BaseUrl, _toAgentConfig :: API.Config, - _toAgentCapabilities :: AgentCapabilities, _toTestConfig :: TestConfig, _toParallelDegree :: Maybe Int, _toMatch :: Maybe String, @@ -55,17 +45,6 @@ data TestOptions = TestOptions _toExportMatchStrings :: Bool } -data AgentCapabilities - = AutoDetect - | Explicit API.Capabilities - -data CapabilitiesState = CapabilitiesState - { _csRemainingCapabilities :: HashSet Text, - _csCapabilitiesEnquired :: HashSet Text - } - -$(makeLenses ''CapabilitiesState) - parseCommandLine :: IO Command parseCommandLine = execParser $ @@ -150,7 +129,6 @@ testOptionsParser = <> metavar "JSON" <> help "The configuration JSON to be sent to the agent via the X-Hasura-DataConnector-Config header" ) - <*> agentCapabilitiesParser <*> testConfigParser <*> optional ( option @@ -202,61 +180,3 @@ configValue = fmap API.Config jsonValue jsonValue :: FromJSON v => ReadM v jsonValue = eitherReader (eitherDecodeStrict' . Text.encodeUtf8 . Text.pack) - -agentCapabilitiesParser :: Parser AgentCapabilities -agentCapabilitiesParser = - option - agentCapabilities - ( long "capabilities" - <> short 'c' - <> metavar "CAPABILITIES" - <> value AutoDetect - <> help (Text.unpack helpText) - ) - where - helpText = - "The capabilities that the agent has, to determine what tests to run. By default, they will be autodetected. The valid capabilities are: " <> allCapabilitiesText - allCapabilitiesText = - "[autodetect | none | " <> Text.intercalate "," (HashSet.toList allPossibleCapabilities) <> "]" - -agentCapabilities :: ReadM AgentCapabilities -agentCapabilities = - str >>= \text -> do - let capabilities = HashSet.fromList $ Text.strip <$> Text.split (== ',') text - if HashSet.member "autodetect" capabilities - then - if HashSet.size capabilities == 1 - then pure AutoDetect - else readerError "You can either autodetect capabilities or specify them manually, not both" - else - if HashSet.member "none" capabilities - then - if HashSet.size capabilities == 1 - then pure . Explicit . fst $ readCapabilities mempty - else readerError "You cannot specify other capabilities when specifying none" - else Explicit <$> readExplicitCapabilities capabilities - where - readExplicitCapabilities :: HashSet Text -> ReadM API.Capabilities - readExplicitCapabilities providedCapabilities = - let (capabilities, CapabilitiesState {..}) = readCapabilities providedCapabilities - in if _csRemainingCapabilities /= mempty - then readerError . Text.unpack $ "Unknown capabilities: " <> Text.intercalate "," (HashSet.toList _csRemainingCapabilities) - else pure capabilities - -readCapabilities :: HashSet Text -> (API.Capabilities, CapabilitiesState) -readCapabilities providedCapabilities = - flip runState (CapabilitiesState providedCapabilities mempty) $ do - supportsRelationships <- readCapability "relationships" - pure $ API.emptyCapabilities {API._cRelationships = if supportsRelationships then Just API.RelationshipCapabilities {} else Nothing} - -readCapability :: Text -> State CapabilitiesState Bool -readCapability capability = do - modifying csCapabilitiesEnquired $ HashSet.insert capability - hasCapability <- use $ csRemainingCapabilities . contains capability - when hasCapability $ - modifying csRemainingCapabilities $ HashSet.delete capability - pure hasCapability - -allPossibleCapabilities :: HashSet Text -allPossibleCapabilities = - readCapabilities mempty ^. _2 . csCapabilitiesEnquired diff --git a/server/lib/dc-api/test/Main.hs b/server/lib/dc-api/test/Main.hs index 6035ead7b6b..2f405921137 100644 --- a/server/lib/dc-api/test/Main.hs +++ b/server/lib/dc-api/test/Main.hs @@ -2,7 +2,7 @@ module Main (main) where -------------------------------------------------------------------------------- -import Command (AgentCapabilities (..), Command (..), TestOptions (..), parseCommandLine) +import Command (Command (..), TestOptions (..), parseCommandLine) import Control.Exception (throwIO) import Control.Monad (join, (>=>)) import Data.Aeson.Text (encodeToLazyText) @@ -50,7 +50,7 @@ main = do case command of Test testOptions@TestOptions {..} -> do api <- mkIOApiClient testOptions - agentCapabilities <- getAgentCapabilities api _toAgentCapabilities + agentCapabilities <- API._crCapabilities <$> (api // _capabilities) let testData = mkTestData _toTestConfig let spec = tests testData api testSourceName _toAgentConfig agentCapabilities case _toExportMatchStrings of @@ -72,11 +72,6 @@ mkIOApiClient TestOptions {..} = do throwClientError :: Either ClientError a -> IO a throwClientError = either throwIO pure -getAgentCapabilities :: Client IO (NamedRoutes Routes) -> AgentCapabilities -> IO API.Capabilities -getAgentCapabilities api = \case - AutoDetect -> API._crCapabilities <$> (api // _capabilities) - Explicit capabilities -> pure capabilities - applyTestConfig :: Config -> TestOptions -> Config applyTestConfig config TestOptions {..} = config diff --git a/server/lib/dc-api/test/Test/SchemaSpec.hs b/server/lib/dc-api/test/Test/SchemaSpec.hs index 7ac92787881..c76a830c8fa 100644 --- a/server/lib/dc-api/test/Test/SchemaSpec.hs +++ b/server/lib/dc-api/test/Test/SchemaSpec.hs @@ -2,11 +2,11 @@ module Test.SchemaSpec (spec) where -------------------------------------------------------------------------------- -import Control.Lens ((%~), (.~)) +import Control.Lens ((%~), (.~), (?~)) import Control.Lens.At (at) import Control.Lens.Lens ((&)) import Control.Monad (forM_) -import Data.Aeson (toJSON) +import Data.Aeson (Value (..), toJSON) import Data.Aeson.Lens (_Object) import Data.Foldable (find) import Data.HashMap.Strict qualified as HashMap @@ -42,12 +42,17 @@ spec TestData {..} api sourceName config API.Capabilities {..} = describe "schem column & _Object . at "type" .~ Nothing -- Types can vary between agents since underlying datatypes can change & _Object . at "description" .~ Nothing -- Descriptions are not supported by all agents + -- If the agent only supports nullable columns, we make all columns nullable + let setExpectedColumnNullability columns = + if API._dscColumnNullability _cDataSchema == API.OnlyNullableColumns + then columns & traverse %~ (_Object . at "nullable" ?~ Bool True) + else columns let actualJsonColumns = extractJsonForComparison <$> tables - let expectedJsonColumns = Just $ extractJsonForComparison expectedTable + let expectedJsonColumns = Just . setExpectedColumnNullability $ extractJsonForComparison expectedTable actualJsonColumns `jsonShouldBe` expectedJsonColumns - if (API._qcSupportsPrimaryKeys <$> _cQueries) == Just True + if API._dscSupportsPrimaryKeys _cDataSchema then testPerTable "returns the correct primary keys for the Chinook tables" $ \API.TableInfo {..} -> do tables <- find (\t -> API._tiName t == _tiName) . API._srTables <$> (api // API._schema) sourceName config let actualPrimaryKey = API._tiPrimaryKey <$> tables @@ -57,7 +62,7 @@ spec TestData {..} api sourceName config API.Capabilities {..} = describe "schem let actualPrimaryKey = API._tiPrimaryKey <$> tables actualPrimaryKey `jsonShouldBe` Just [] - if (API._qcSupportsPrimaryKeys <$> _cQueries) == Just True + if API._dscSupportsForeignKeys _cDataSchema then testPerTable "returns the correct foreign keys for the Chinook tables" $ \expectedTable@API.TableInfo {..} -> do tables <- find (\t -> API._tiName t == _tiName) . API._srTables <$> (api // API._schema) sourceName config diff --git a/server/lib/test-harness/src/Harness/Backend/DataConnector/MockAgent.hs b/server/lib/test-harness/src/Harness/Backend/DataConnector/MockAgent.hs index 584e775788e..54576f43582 100644 --- a/server/lib/test-harness/src/Harness/Backend/DataConnector/MockAgent.hs +++ b/server/lib/test-harness/src/Harness/Backend/DataConnector/MockAgent.hs @@ -33,7 +33,8 @@ capabilities = API.CapabilitiesResponse { _crCapabilities = API.Capabilities - { API._cQueries = Just API.QueryCapabilities {API._qcSupportsPrimaryKeys = True}, + { API._cDataSchema = API.defaultDataSchemaCapabilities, + API._cQueries = Just API.QueryCapabilities, API._cMutations = Nothing, API._cSubscriptions = Nothing, API._cScalarTypes = Nothing, diff --git a/server/lib/test-harness/src/Harness/Test/BackendType.hs b/server/lib/test-harness/src/Harness/Test/BackendType.hs index 5c33ee57819..eac439d9b31 100644 --- a/server/lib/test-harness/src/Harness/Test/BackendType.hs +++ b/server/lib/test-harness/src/Harness/Test/BackendType.hs @@ -51,34 +51,38 @@ defaultBackendCapabilities = \case DataConnectorSqlite -> Just [yaml| - relationships: {} - comparisons: - subquery: - supports_relations: true - explain: {} - metrics: {} - raw: {} - queries: - supports_primary_keys: true + data_schema: + supports_primary_keys: true + supports_foreign_keys: true + queries: {} + relationships: {} + comparisons: + subquery: + supports_relations: true + explain: {} + metrics: {} + raw: {} |] DataConnectorReference -> Just [yaml| - queries: - supports_primary_keys: true - graphql_schema: |- - scalar DateTime + data_schema: + supports_primary_keys: true + supports_foreign_keys: true + queries: {} + graphql_schema: |- + scalar DateTime - input DateTimeComparisons {in_year: Int - same_day_as: DateTime - } - relationships: {} - comparisons: - subquery: - supports_relations: true - scalar_types: - DateTime: - comparison_type: DateTimeComparisons + input DateTimeComparisons {in_year: Int + same_day_as: DateTime + } + relationships: {} + comparisons: + subquery: + supports_relations: true + scalar_types: + DateTime: + comparison_type: DateTimeComparisons |] _ -> Nothing diff --git a/server/src-test/Hasura/Backends/DataConnector/API/V0/CapabilitiesSpec.hs b/server/src-test/Hasura/Backends/DataConnector/API/V0/CapabilitiesSpec.hs index e582d5e0cae..a4a95c645d2 100644 --- a/server/src-test/Hasura/Backends/DataConnector/API/V0/CapabilitiesSpec.hs +++ b/server/src-test/Hasura/Backends/DataConnector/API/V0/CapabilitiesSpec.hs @@ -21,11 +21,11 @@ import Test.Hspec spec :: Spec spec = do describe "Capabilities" $ do - testToFromJSONToSchema emptyCapabilities [aesonQQ|{}|] + testToFromJSONToSchema defaultCapabilities [aesonQQ|{}|] jsonOpenApiProperties genCapabilities describe "CapabilitiesResponse" $ do testToFromJSON - (CapabilitiesResponse (emptyCapabilities {_cRelationships = Just RelationshipCapabilities {}}) emptyConfigSchemaResponse) + (CapabilitiesResponse (defaultCapabilities {_cRelationships = Just RelationshipCapabilities {}}) emptyConfigSchemaResponse) [aesonQQ|{"capabilities": {"relationships": {}}, "config_schemas": {"config_schema": {}, "other_schemas": {}}}|] describe "ScalarTypeCapabilities" $ do testToFromJSONToSchema (ScalarTypeCapabilities $ Just [G.name|DateTimeComparisons|]) [aesonQQ|{"comparison_type": "DateTimeComparisons"}|] @@ -56,8 +56,19 @@ input DateTimeComparisons {after: DateTime in_year: Int }|] +genDataSchemaCapabilities :: MonadGen m => m DataSchemaCapabilities +genDataSchemaCapabilities = + DataSchemaCapabilities + <$> Gen.bool + <*> Gen.bool + <*> genColumnNullability + +genColumnNullability :: MonadGen m => m ColumnNullability +genColumnNullability = + Gen.element [NullableAndNonNullableColumns, OnlyNullableColumns] + genQueryCapabilities :: MonadGen m => m QueryCapabilities -genQueryCapabilities = QueryCapabilities <$> Gen.bool +genQueryCapabilities = pure QueryCapabilities genMutationCapabilities :: MonadGen m => m MutationCapabilities genMutationCapabilities = pure MutationCapabilities {} @@ -139,7 +150,8 @@ genRawCapabilities = pure RawCapabilities {} genCapabilities :: Gen Capabilities genCapabilities = Capabilities - <$> Gen.maybe genQueryCapabilities + <$> genDataSchemaCapabilities + <*> Gen.maybe genQueryCapabilities <*> Gen.maybe genMutationCapabilities <*> Gen.maybe genSubscriptionCapabilities <*> Gen.maybe genScalarTypesCapabilities