diff --git a/scripts/make/tests.mk b/scripts/make/tests.mk index 028ad51b5ef..455d5cb10ff 100644 --- a/scripts/make/tests.mk +++ b/scripts/make/tests.mk @@ -145,7 +145,7 @@ test-logical-models: GRAPHQL_ENGINE=$(GRAPHQL_ENGINE_PATH) \ cabal run api-tests:exe:api-tests HASURA_TEST_BACKEND_TYPE=BigQuery \ - HSPEC_MATCH=Metadata.LogicalModel \ + HSPEC_MATCH=LogicalModel \ GRAPHQL_ENGINE=$(GRAPHQL_ENGINE_PATH) \ cabal run api-tests:exe:api-tests HASURA_TEST_BACKEND_TYPE=SQLServer \ diff --git a/server/lib/api-tests/api-tests.cabal b/server/lib/api-tests/api-tests.cabal index ac7486deea4..1ab9a74834c 100644 --- a/server/lib/api-tests/api-tests.cabal +++ b/server/lib/api-tests/api-tests.cabal @@ -126,6 +126,7 @@ library Test.DataConnector.MockAgent.UpdateMutationsSpec Test.DataConnector.QuerySpec Test.DataConnector.SelectPermissionsSpec + Test.Databases.BigQuery.LogicalModelsSpec Test.Databases.BigQuery.Queries.SpatialTypesSpec Test.Databases.BigQuery.Queries.TypeInterpretationSpec Test.Databases.BigQuery.Schema.ComputedFields.TableSpec diff --git a/server/lib/api-tests/src/Test/Databases/BigQuery/LogicalModelsSpec.hs b/server/lib/api-tests/src/Test/Databases/BigQuery/LogicalModelsSpec.hs new file mode 100644 index 00000000000..935f164bddc --- /dev/null +++ b/server/lib/api-tests/src/Test/Databases/BigQuery/LogicalModelsSpec.hs @@ -0,0 +1,799 @@ +{-# LANGUAGE QuasiQuotes #-} + +-- | Access to the SQL +module Test.Databases.BigQuery.LogicalModelsSpec (spec) where + +import Data.Aeson (Value) +import Data.List.NonEmpty qualified as NE +import Data.String.Interpolate (i) +import Data.Time.Calendar.OrdinalDate +import Data.Time.Clock +import Harness.Backend.BigQuery qualified as BigQuery +import Harness.GraphqlEngine qualified as GraphqlEngine +import Harness.Quoter.Graphql +import Harness.Quoter.Yaml (interpolateYaml, yaml) +import Harness.Test.BackendType qualified as BackendType +import Harness.Test.Fixture qualified as Fixture +import Harness.Test.Schema (Table (..), table) +import Harness.Test.Schema qualified as Schema +import Harness.Test.SchemaName (unSchemaName) +import Harness.TestEnvironment (GlobalTestEnvironment, TestEnvironment (options), getBackendTypeConfig) +import Harness.Yaml (shouldBeYaml, shouldReturnYaml) +import Hasura.Prelude +import Test.Hspec (SpecWith, describe, it) + +-- ** Preamble + +featureFlagForLogicalModels :: String +featureFlagForLogicalModels = "HASURA_FF_LOGICAL_MODEL_INTERFACE" + +spec :: SpecWith GlobalTestEnvironment +spec = + Fixture.hgeWithEnv [(featureFlagForLogicalModels, "True")] $ + Fixture.runClean -- re-run fixture setup on every test + ( NE.fromList + [ (Fixture.fixture $ Fixture.Backend BigQuery.backendTypeMetadata) + { Fixture.setupTeardown = \(testEnvironment, _) -> + [ BigQuery.setupTablesAction schema testEnvironment + ] + } + ] + ) + tests + +-- ** Setup and teardown + +-- we add and track a table here as it's the only way we can currently define a +-- return type +schema :: [Schema.Table] +schema = + [ (table "article") + { tableColumns = + [ Schema.column "id" Schema.TInt, + Schema.column "title" Schema.TStr, + Schema.column "content" Schema.TStr, + Schema.column "date" Schema.TUTCTime + ], + tableData = + [ [ Schema.VInt 1, + Schema.VStr "Dogs", + Schema.VStr "I like to eat dog food I am a dogs I like to eat dog food I am a dogs I like to eat dog food I am a dogs", + Schema.VUTCTime (UTCTime (fromOrdinalDate 2000 1) 0) + ] + ] + }, + (table "stuff") + { tableColumns = + [ Schema.column "thing" Schema.TInt, + Schema.column "date" Schema.TUTCTime + ] + } + ] + +tests :: SpecWith TestEnvironment +tests = do + let query :: Text + query = "SELECT * FROM UNNEST([STRUCT('hello' as one, 'world' as two), ('welcome', 'friend')])" + + helloWorldLogicalModel :: Schema.LogicalModel + helloWorldLogicalModel = + (Schema.logicalModel "hello_world_function" query) + { Schema.logicalModelColumns = + [ Schema.logicalModelColumn "one" Schema.TStr, + Schema.logicalModelColumn "two" Schema.TStr + ] + } + + describe "Testing Logical Models" $ do + it "Descriptions and nullability appear in the schema" $ \testEnvironment -> do + let backendTypeMetadata = fromMaybe (error "Unknown backend") $ getBackendTypeConfig testEnvironment + sourceName = BackendType.backendSourceName backendTypeMetadata + + nullableQuery = "SELECT thing / 2 AS divided, null as something_nullable FROM stuff" + + descriptionsAndNullableLogicalModel :: Schema.LogicalModel + descriptionsAndNullableLogicalModel = + (Schema.logicalModel "divided_stuff" nullableQuery) + { Schema.logicalModelColumns = + [ (Schema.logicalModelColumn "divided" Schema.TInt) + { Schema.logicalModelColumnDescription = Just "A divided thing" + }, + (Schema.logicalModelColumn "something_nullable" Schema.TInt) + { Schema.logicalModelColumnDescription = Just "Something nullable", + Schema.logicalModelColumnNullable = True + } + ], + Schema.logicalModelArguments = + [ Schema.logicalModelColumn "unused" Schema.TInt + ], + Schema.logicalModelReturnTypeDescription = Just "Return type description" + } + + Schema.trackLogicalModel sourceName descriptionsAndNullableLogicalModel testEnvironment + + let queryTypesIntrospection :: Value + queryTypesIntrospection = + [graphql| + query { + __type(name: "divided_stuff") { + name + description + fields { + name + description + type { + name + kind + ofType { + name + } + } + } + } + } + |] + + expected = + [interpolateYaml| + { + "data": { + "__type": { + "description": "Return type description", + "fields": [ + { + "description": "A divided thing", + "name": "divided", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "name": "Int" + } + } + }, + { + "description": "Something nullable", + "name": "something_nullable", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + } + ], + "name": "divided_stuff" + } + } + } + |] + + actual <- GraphqlEngine.postGraphql testEnvironment queryTypesIntrospection + + actual `shouldBeYaml` expected + + it "Runs the absolute simplest query that takes no parameters" $ \testEnvironment -> do + let backendTypeMetadata = fromMaybe (error "Unknown backend") $ getBackendTypeConfig testEnvironment + source = BackendType.backendSourceName backendTypeMetadata + + Schema.trackLogicalModel source helloWorldLogicalModel testEnvironment + + let expected = + [yaml| + data: + hello_world_function: + - one: "hello" + two: "world" + - one: "welcome" + two: "friend" + |] + + actual :: IO Value + actual = + GraphqlEngine.postGraphql + testEnvironment + [graphql| + query { + hello_world_function { + one + two + } + } + |] + + shouldReturnYaml (options testEnvironment) actual expected + + it "Runs simple query with a basic where clause" $ \testEnvironment -> do + let backendTypeMetadata = fromMaybe (error "Unknown backend") $ getBackendTypeConfig testEnvironment + source = BackendType.backendSourceName backendTypeMetadata + + Schema.trackLogicalModel source helloWorldLogicalModel testEnvironment + + let expected = + [yaml| + data: + hello_world_function: + - one: "hello" + two: "world" + |] + + actual :: IO Value + actual = + GraphqlEngine.postGraphql + testEnvironment + [graphql| + query { + hello_world_function (where: { two: { _eq: "world" } }){ + one + two + } + } + |] + + shouldReturnYaml (options testEnvironment) actual expected + + it "Runs a simple query using distinct_on and order_by" $ \testEnvironment -> do + let backendTypeMetadata = fromMaybe (error "Unknown backend") $ getBackendTypeConfig testEnvironment + source = BackendType.backendSourceName backendTypeMetadata + + queryWithDuplicates :: Text + queryWithDuplicates = "SELECT * FROM UNNEST([STRUCT('hello' as one, 'world' as two), ('hello', 'friend')])" + + helloWorldLogicalModelWithDuplicates :: Schema.LogicalModel + helloWorldLogicalModelWithDuplicates = + (Schema.logicalModel "hello_world_function" queryWithDuplicates) + { Schema.logicalModelColumns = + [ Schema.logicalModelColumn "one" Schema.TStr, + Schema.logicalModelColumn "two" Schema.TStr + ] + } + + Schema.trackLogicalModel source helloWorldLogicalModelWithDuplicates testEnvironment + + let expected = + [yaml| + data: + hello_world_function: + - one: "hello" + two: "world" + |] + + actual :: IO Value + actual = + GraphqlEngine.postGraphql + testEnvironment + [graphql| + query { + hello_world_function ( + distinct_on: [one] + order_by: [{one:asc}] + ){ + one + two + } + } + |] + + shouldReturnYaml (options testEnvironment) actual expected + + it "Runs a simple query that takes no parameters" $ \testEnvironment -> do + let backendTypeMetadata = fromMaybe (error "Unknown backend") $ getBackendTypeConfig testEnvironment + sourceName = BackendType.backendSourceName backendTypeMetadata + + Schema.trackLogicalModel sourceName helloWorldLogicalModel testEnvironment + + let expected = + [yaml| + data: + hello_world_function: + - one: "hello" + two: "world" + |] + + actual :: IO Value + actual = + GraphqlEngine.postGraphql + testEnvironment + [graphql| + query { + hello_world_function(where: {one: {_eq: "hello"}}) { + one + two + } + } + |] + + shouldReturnYaml (options testEnvironment) actual expected + + it "Runs a simple query that takes one dummy parameter and order_by" $ \testEnvironment -> do + let backendTypeMetadata = fromMaybe (error "Unknown backend") $ getBackendTypeConfig testEnvironment + source = BackendType.backendSourceName backendTypeMetadata + + helloWorldLogicalModelWithDummyArgument :: Schema.LogicalModel + helloWorldLogicalModelWithDummyArgument = + (Schema.logicalModel "hello_world_function_with_dummy" query) + { Schema.logicalModelColumns = + [ Schema.logicalModelColumn "one" Schema.TStr, + Schema.logicalModelColumn "two" Schema.TStr + ], + Schema.logicalModelArguments = + [ Schema.logicalModelColumn "dummy" Schema.TStr + ] + } + + Schema.trackLogicalModel source helloWorldLogicalModelWithDummyArgument testEnvironment + + let expected = + [yaml| + data: + hello_world_function_with_dummy: + - two: "world" + - two: "friend" + |] + + actual :: IO Value + actual = + GraphqlEngine.postGraphql + testEnvironment + [graphql| + query { + hello_world_function_with_dummy(args: {dummy: "ignored"}, order_by: {one: asc}) { + two + } + } + |] + + shouldReturnYaml (options testEnvironment) actual expected + + it "Runs a simple query that takes no parameters but ends with a comment" $ \testEnvironment -> do + let spicyQuery :: Text + spicyQuery = "SELECT * FROM UNNEST([STRUCT('hello' as one, 'world' as two), ('welcome', 'friend')]) -- my query" + + let backendTypeMetadata = fromMaybe (error "Unknown backend") $ getBackendTypeConfig testEnvironment + source = BackendType.backendSourceName backendTypeMetadata + + helloCommentLogicalModel :: Schema.LogicalModel + helloCommentLogicalModel = + (Schema.logicalModel "hello_comment_function" spicyQuery) + { Schema.logicalModelColumns = + [ Schema.logicalModelColumn "one" Schema.TStr, + Schema.logicalModelColumn "two" Schema.TStr + ] + } + + Schema.trackLogicalModel source helloCommentLogicalModel testEnvironment + + let expected = + [yaml| + data: + hello_comment_function: + - one: "hello" + two: "world" + - one: "welcome" + two: "friend" + |] + + actual :: IO Value + actual = + GraphqlEngine.postGraphql + testEnvironment + [graphql| + query { + hello_comment_function { + one + two + } + } + |] + + shouldReturnYaml (options testEnvironment) actual expected + + it "Uses a column permission that we are allowed to access" $ \testEnvironment -> do + let backendTypeMetadata = fromMaybe (error "Unknown backend") $ getBackendTypeConfig testEnvironment + source = BackendType.backendSourceName backendTypeMetadata + backendType = BackendType.backendTypeString backendTypeMetadata + createPermRequestType = backendType <> "_create_logical_model_select_permission" + + helloWorldPermLogicalModel :: Schema.LogicalModel + helloWorldPermLogicalModel = + (Schema.logicalModel "hello_world_perms" query) + { Schema.logicalModelColumns = + [ Schema.logicalModelColumn "one" Schema.TStr, + Schema.logicalModelColumn "two" Schema.TStr + ] + } + + Schema.trackLogicalModel source helloWorldPermLogicalModel testEnvironment + + shouldReturnYaml + (options testEnvironment) + ( GraphqlEngine.postMetadata + testEnvironment + [yaml| + type: bulk + args: + - type: *createPermRequestType + args: + source: *source + root_field_name: hello_world_perms + role: "test" + permission: + columns: + - one + filter: {} + |] + ) + [yaml| + - message: success + |] + + let expected = + [yaml| + data: + hello_world_perms: + - one: "hello" + - one: "welcome" + |] + + actual :: IO Value + actual = + GraphqlEngine.postGraphqlWithHeaders + testEnvironment + [("X-Hasura-Role", "test")] + [graphql| + query { + hello_world_perms { + one + } + } + |] + + shouldReturnYaml (options testEnvironment) actual expected + + it "Fails because we access a column we do not have permissions for" $ \testEnvironment -> do + let backendTypeMetadata = fromMaybe (error "Unknown backend") $ getBackendTypeConfig testEnvironment + source = BackendType.backendSourceName backendTypeMetadata + backendType = BackendType.backendTypeString backendTypeMetadata + createPermRequestType = backendType <> "_create_logical_model_select_permission" + + helloWorldPermLogicalModel :: Schema.LogicalModel + helloWorldPermLogicalModel = + (Schema.logicalModel "hello_world_perms" query) + { Schema.logicalModelColumns = + [ Schema.logicalModelColumn "one" Schema.TStr, + Schema.logicalModelColumn "two" Schema.TStr + ] + } + + Schema.trackLogicalModel source helloWorldPermLogicalModel testEnvironment + + shouldReturnYaml + (options testEnvironment) + ( GraphqlEngine.postMetadata + testEnvironment + [yaml| + type: bulk + args: + - type: *createPermRequestType + args: + source: *source + root_field_name: hello_world_perms + role: "test" + permission: + columns: + - two + filter: {} + |] + ) + [yaml| + - message: success + |] + + let expected = + [yaml| + errors: + - extensions: + code: validation-failed + path: $.selectionSet.hello_world_perms.selectionSet.one + message: "field 'one' not found in type: 'hello_world_perms'" + |] + + actual :: IO Value + actual = + GraphqlEngine.postGraphqlWithHeaders + testEnvironment + [("X-Hasura-Role", "test")] + [graphql| + query { + hello_world_perms { + one + } + } + |] + + shouldReturnYaml (options testEnvironment) actual expected + + it "Using row permissions filters out some results" $ \testEnvironment -> do + let backendTypeMetadata = fromMaybe (error "Unknown backend") $ getBackendTypeConfig testEnvironment + source = BackendType.backendSourceName backendTypeMetadata + backendType = BackendType.backendTypeString backendTypeMetadata + createPermRequestType = backendType <> "_create_logical_model_select_permission" + + helloWorldPermLogicalModel :: Schema.LogicalModel + helloWorldPermLogicalModel = + (Schema.logicalModel "hello_world_perms" query) + { Schema.logicalModelColumns = + [ Schema.logicalModelColumn "one" Schema.TStr, + Schema.logicalModelColumn "two" Schema.TStr + ] + } + + Schema.trackLogicalModel source helloWorldPermLogicalModel testEnvironment + + shouldReturnYaml + (options testEnvironment) + ( GraphqlEngine.postMetadata + testEnvironment + [yaml| + type: bulk + args: + - type: *createPermRequestType + args: + source: *source + root_field_name: hello_world_perms + role: "test" + permission: + columns: "*" + filter: + one: + _eq: "welcome" + |] + ) + [yaml| + - message: success + |] + + let expected = + [yaml| + data: + hello_world_perms: + - one: "welcome" + two: "friend" + |] + + actual :: IO Value + actual = + GraphqlEngine.postGraphqlWithHeaders + testEnvironment + [("X-Hasura-Role", "test")] + [graphql| + query { + hello_world_perms { + one + two + } + } + |] + + shouldReturnYaml (options testEnvironment) actual expected + + let articleQuery :: Schema.SchemaName -> Text + articleQuery schemaName = + [i| + SELECT + id, + title, + ( substring(content, 1, {{length}}) || ( + case when length(content) < {{length}} + then '' + else '...' end + )) as excerpt, + date + from #{unSchemaName schemaName}.article + |] + + describe "Testing Logical Models" $ do + it "Runs a simple query that takes one parameter and uses it multiple times" $ \testEnvironment -> do + let backendTypeMetadata = fromMaybe (error "Unknown backend") $ getBackendTypeConfig testEnvironment + source = BackendType.backendSourceName backendTypeMetadata + + schemaName :: Schema.SchemaName + schemaName = Schema.getSchemaName testEnvironment + + articleWithExcerptLogicalModel :: Schema.LogicalModel + articleWithExcerptLogicalModel = + (Schema.logicalModel "article_with_excerpt" (articleQuery schemaName)) + { Schema.logicalModelColumns = + [ Schema.logicalModelColumn "id" Schema.TInt, + Schema.logicalModelColumn "title" Schema.TStr, + Schema.logicalModelColumn "excerpt" Schema.TStr, + Schema.logicalModelColumn "date" Schema.TUTCTime + ], + Schema.logicalModelArguments = + [ Schema.logicalModelColumn "length" Schema.TInt + ] + } + + Schema.trackLogicalModel source articleWithExcerptLogicalModel testEnvironment + + let actual :: IO Value + actual = + GraphqlEngine.postGraphql + testEnvironment + [graphql| + query { + article_with_excerpt(args: { length: 34 }) { + id + title + date + excerpt + } + } + |] + + expected = + [yaml| + data: + article_with_excerpt: + - id: '1' + title: "Dogs" + date: "2000-01-01T00:00:00" + excerpt: "I like to eat dog food I am a dogs..." + |] + + shouldReturnYaml (options testEnvironment) actual expected + + it "Uses two queries with the same argument names and ensure they don't mess with one another" $ \testEnvironment -> do + let backendTypeMetadata = fromMaybe (error "Unknown backend") $ getBackendTypeConfig testEnvironment + source = BackendType.backendSourceName backendTypeMetadata + + schemaName :: Schema.SchemaName + schemaName = Schema.getSchemaName testEnvironment + + mkArticleWithExcerptLogicalModel :: Text -> Schema.LogicalModel + mkArticleWithExcerptLogicalModel name = + (Schema.logicalModel name (articleQuery schemaName)) + { Schema.logicalModelColumns = + [ Schema.logicalModelColumn "id" Schema.TInt, + Schema.logicalModelColumn "title" Schema.TStr, + Schema.logicalModelColumn "excerpt" Schema.TStr, + Schema.logicalModelColumn "date" Schema.TUTCTime + ], + Schema.logicalModelArguments = + [ Schema.logicalModelColumn "length" Schema.TInt + ] + } + + Schema.trackLogicalModel + source + (mkArticleWithExcerptLogicalModel "article_with_excerpt_1") + testEnvironment + + Schema.trackLogicalModel + source + (mkArticleWithExcerptLogicalModel "article_with_excerpt_2") + testEnvironment + + let actual :: IO Value + actual = + GraphqlEngine.postGraphql + testEnvironment + [graphql| + query { + article_with_excerpt_1(args: { length: 34 }) { + excerpt + } + article_with_excerpt_2(args: { length: 13 }) { + excerpt + } + } + |] + + expected = + [yaml| + data: + article_with_excerpt_1: + - excerpt: "I like to eat dog food I am a dogs..." + article_with_excerpt_2: + - excerpt: "I like to eat..." + |] + + shouldReturnYaml (options testEnvironment) actual expected + + it "Uses a one parameter query and uses it multiple times" $ \testEnvironment -> do + let backendTypeMetadata = fromMaybe (error "Unknown backend") $ getBackendTypeConfig testEnvironment + source = BackendType.backendSourceName backendTypeMetadata + + schemaName :: Schema.SchemaName + schemaName = Schema.getSchemaName testEnvironment + + articleWithExcerptLogicalModel :: Schema.LogicalModel + articleWithExcerptLogicalModel = + (Schema.logicalModel "article_with_excerpt" (articleQuery schemaName)) + { Schema.logicalModelColumns = + [ Schema.logicalModelColumn "id" Schema.TInt, + Schema.logicalModelColumn "title" Schema.TStr, + Schema.logicalModelColumn "excerpt" Schema.TStr, + Schema.logicalModelColumn "date" Schema.TUTCTime + ], + Schema.logicalModelArguments = + [ Schema.logicalModelColumn "length" Schema.TInt + ] + } + + Schema.trackLogicalModel source articleWithExcerptLogicalModel testEnvironment + + let actual :: IO Value + actual = + GraphqlEngine.postGraphql + testEnvironment + [graphql| + query { + first: article_with_excerpt(args: { length: 34 }) { + excerpt + } + second: article_with_excerpt(args: { length: 13 }) { + excerpt + } + } + |] + + expected = + [yaml| + data: + first: + - excerpt: "I like to eat dog food I am a dogs..." + second: + - excerpt: "I like to eat..." + |] + + shouldReturnYaml (options testEnvironment) actual expected + + it "Uses a one parameter query, passing it a GraphQL variable" $ \testEnvironment -> do + let backendTypeMetadata = fromMaybe (error "Unknown backend") $ getBackendTypeConfig testEnvironment + source = BackendType.backendSourceName backendTypeMetadata + + schemaName :: Schema.SchemaName + schemaName = Schema.getSchemaName testEnvironment + + articleWithExcerptLogicalModel :: Schema.LogicalModel + articleWithExcerptLogicalModel = + (Schema.logicalModel "article_with_excerpt" (articleQuery schemaName)) + { Schema.logicalModelColumns = + [ Schema.logicalModelColumn "id" Schema.TInt, + Schema.logicalModelColumn "title" Schema.TStr, + Schema.logicalModelColumn "excerpt" Schema.TStr, + Schema.logicalModelColumn "date" Schema.TUTCTime + ], + Schema.logicalModelArguments = + [ Schema.logicalModelColumn "length" Schema.TInt + ] + } + + Schema.trackLogicalModel source articleWithExcerptLogicalModel testEnvironment + + let variables = + [yaml| + length: 34 + |] + + actual :: IO Value + actual = + GraphqlEngine.postGraphqlWithVariables + testEnvironment + [graphql| + query MyQuery($length: Int!) { + article_with_excerpt(args: { length: $length }) { + excerpt + } + } + |] + variables + + expected = + [yaml| + data: + article_with_excerpt: + - excerpt: "I like to eat dog food I am a dogs..." + |] + + shouldReturnYaml (options testEnvironment) actual expected diff --git a/server/lib/graphql-parser/src/Language/GraphQL/Draft/Syntax/Name.hs b/server/lib/graphql-parser/src/Language/GraphQL/Draft/Syntax/Name.hs index 304e1cb8116..6bbfda97674 100644 --- a/server/lib/graphql-parser/src/Language/GraphQL/Draft/Syntax/Name.hs +++ b/server/lib/graphql-parser/src/Language/GraphQL/Draft/Syntax/Name.hs @@ -1,4 +1,5 @@ {-# HLINT ignore "Use onNothing" #-} +{-# LANGUAGE DeriveDataTypeable #-} {-# LANGUAGE TemplateHaskellQuotes #-} {-# OPTIONS_GHC -Wno-unrecognised-pragmas #-} @@ -29,6 +30,7 @@ import Control.DeepSeq (NFData) import Data.Aeson qualified as J import Data.Char qualified as C import Data.Coerce (coerce) +import Data.Data (Data) import Data.Hashable (Hashable) import Data.Text (Text) import Data.Text qualified as T @@ -42,7 +44,7 @@ import Prelude -- Defined here and re-exported in the public module to avoid exporting `unName`.` newtype Name = Name {unName :: Text} - deriving stock (Eq, Lift, Ord, Show) + deriving stock (Data, Eq, Lift, Ord, Show) deriving newtype (Semigroup, Hashable, NFData, Pretty, J.ToJSONKey, J.ToJSON) instance HasCodec Name where diff --git a/server/src-lib/Hasura/Backends/BigQuery/FromIr.hs b/server/src-lib/Hasura/Backends/BigQuery/FromIr.hs index f275274adbf..4103b324bcc 100644 --- a/server/src-lib/Hasura/Backends/BigQuery/FromIr.hs +++ b/server/src-lib/Hasura/Backends/BigQuery/FromIr.hs @@ -6,6 +6,7 @@ module Hasura.Backends.BigQuery.FromIr Error (..), runFromIr, FromIr, + FromIrWriter (..), FromIrConfig (..), defaultFromIrConfig, bigQuerySourceConfigToFromIrConfig, @@ -21,9 +22,13 @@ import Data.List.NonEmpty qualified as NE import Data.Map.Strict (Map) import Data.Map.Strict qualified as M import Data.Text qualified as T +import Data.Text.Extended qualified as T (toTxt) import Hasura.Backends.BigQuery.Instances.Types () import Hasura.Backends.BigQuery.Source (BigQuerySourceConfig (..)) import Hasura.Backends.BigQuery.Types as BigQuery +import Hasura.LogicalModel.IR (LogicalModel (..)) +import Hasura.LogicalModel.Metadata (InterpolatedQuery) +import Hasura.LogicalModel.Types (LogicalModelName (..)) import Hasura.Prelude import Hasura.RQL.IR qualified as Ir import Hasura.RQL.Types.Column qualified as Rql @@ -100,10 +105,28 @@ instance Show Error where -- setting the current entity that a given field name refers to. See -- @fromColumn@. newtype FromIr a = FromIr - { unFromIr :: ReaderT FromIrReader (StateT FromIrState (Validate (NonEmpty Error))) a + { unFromIr :: + ReaderT + FromIrReader + ( StateT + FromIrState + ( WriterT + FromIrWriter + ( Validate (NonEmpty Error) + ) + ) + ) + a } deriving (Functor, Applicative, Monad, MonadValidate (NonEmpty Error)) +-- | Collected from using a logical model in a query. +-- Each entry here because a CTE to be prepended to the query. +newtype FromIrWriter = FromIrWriter + { fromIrWriterLogicalModels :: Map LogicalModelName (InterpolatedQuery Expression) + } + deriving newtype (Semigroup, Monoid) + data FromIrState = FromIrState { indices :: Map Text Int } @@ -160,11 +183,12 @@ data ParentSelectFromEntity -------------------------------------------------------------------------------- -- Runners -runFromIr :: FromIrConfig -> FromIr a -> Validate (NonEmpty Error) a +runFromIr :: FromIrConfig -> FromIr a -> Validate (NonEmpty Error) (a, FromIrWriter) runFromIr config fromIr = - evalStateT - (runReaderT (unFromIr fromIr) (FromIrReader {config})) - (FromIrState {indices = mempty}) + runWriterT $ + evalStateT + (runReaderT (unFromIr fromIr) (FromIrReader {config})) + (FromIrState {indices = mempty}) bigQuerySourceConfigToFromIrConfig :: BigQuerySourceConfig -> FromIrConfig bigQuerySourceConfigToFromIrConfig BigQuerySourceConfig {_scGlobalSelectLimit} = @@ -237,6 +261,7 @@ fromSelectRows parentSelectFromEntity annSelectG = do | functionName nm == "unnest" -> fromUnnestedJSON json columns (map fst fields) Ir.FromFunction functionName (Rql.FunctionArgsExp positionalArgs namedArgs) Nothing -> fromFunction parentSelectFromEntity functionName positionalArgs namedArgs + Ir.FromLogicalModel logicalModel -> fromLogicalModel logicalModel _ -> refute (pure (FromTypeUnsupported from)) Args { argsOrderBy, @@ -258,7 +283,8 @@ fromSelectRows parentSelectFromEntity annSelectG = do globalTop <- getGlobalTop let select = Select - { selectCardinality = Many, + { selectWith = Nothing, + selectCardinality = Many, selectAsStruct = NoAsStruct, selectFinalWantedFields = pure (fieldTextNames fields), selectGroupBy = mempty, @@ -441,7 +467,8 @@ fromSelectAggregate minnerJoinFields annSelectG = do } pure Select - { selectCardinality = One, + { selectWith = Nothing, + selectCardinality = One, selectAsStruct = NoAsStruct, selectFinalWantedFields = Nothing, selectGroupBy = mempty, @@ -452,7 +479,8 @@ fromSelectAggregate minnerJoinFields annSelectG = do ( Aliased { aliasedThing = Select - { selectProjections = innerProjections, + { selectWith = Nothing, + selectProjections = innerProjections, selectAsStruct = NoAsStruct, selectFrom, selectJoins = argsJoins, @@ -611,7 +639,8 @@ unfurlAnnotatedOrderByElement = { joinSource = JoinSelect Select - { selectCardinality = One, + { selectWith = Nothing, + selectCardinality = One, selectAsStruct = NoAsStruct, selectFinalWantedFields = Nothing, selectGroupBy = mempty, @@ -663,7 +692,8 @@ unfurlAnnotatedOrderByElement = { joinSource = JoinSelect Select - { selectCardinality = One, + { selectWith = Nothing, + selectCardinality = One, selectAsStruct = NoAsStruct, selectFinalWantedFields = Nothing, selectTop = NoTop, @@ -771,6 +801,11 @@ fromAnnBoolExp :: ReaderT EntityAlias FromIr Expression fromAnnBoolExp = traverse fromAnnBoolExpFld >=> fromGBoolExp +fromLogicalModel :: LogicalModel 'BigQuery Expression -> FromIr From +fromLogicalModel LogicalModel {..} = FromIr do + tell (FromIrWriter (M.singleton lmRootFieldName lmInterpolatedQuery)) + pure (FromLogicalModel lmRootFieldName) + fromAnnBoolExpFld :: Ir.AnnBoolExpFld 'BigQuery Expression -> ReaderT EntityAlias FromIr Expression fromAnnBoolExpFld = @@ -787,7 +822,8 @@ fromAnnBoolExpFld = pure ( ExistsExpression Select - { selectCardinality = One, + { selectWith = Nothing, + selectCardinality = One, selectAsStruct = NoAsStruct, selectFinalWantedFields = Nothing, selectGroupBy = mempty, @@ -826,7 +862,8 @@ fromGExists Ir.GExists {_geTable, _geWhere} = do local (const (fromAlias selectFrom)) (fromGBoolExp _geWhere) pure Select - { selectCardinality = One, + { selectWith = Nothing, + selectCardinality = One, selectAsStruct = NoAsStruct, selectFinalWantedFields = Nothing, selectGroupBy = mempty, @@ -1219,7 +1256,8 @@ fromObjectRelationSelectG _existingJoins annRelationSelectG = do joinSource = JoinSelect Select - { selectCardinality = One, + { selectWith = Nothing, + selectCardinality = One, selectAsStruct = NoAsStruct, selectFinalWantedFields, selectGroupBy = mempty, @@ -1324,7 +1362,8 @@ fromComputedFieldSelect = \case wrapUnnest from = let starSelect = Select - { selectTop = NoTop, + { selectWith = Nothing, + selectTop = NoTop, selectAsStruct = AsStruct, selectProjections = pure StarProjection, selectFrom = from, @@ -1468,7 +1507,8 @@ fromArrayRelationSelectG annRelationSelectG = do let joinSelect = Select - { selectCardinality = One, + { selectWith = Nothing, + selectCardinality = One, selectAsStruct = NoAsStruct, selectFinalWantedFields = selectFinalWantedFields select, selectTop = NoTop, @@ -1498,7 +1538,8 @@ fromArrayRelationSelectG annRelationSelectG = do { aliasedAlias = coerce (fromAlias (selectFrom select)), aliasedThing = Select - { selectProjections = + { selectWith = Nothing, + selectProjections = selectProjections select <> joinFieldProjections `appendToNonEmpty` foldMap @Maybe @@ -1845,6 +1886,7 @@ fromAlias (FromQualifiedTable Aliased {aliasedAlias}) = EntityAlias aliasedAlias fromAlias (FromSelect Aliased {aliasedAlias}) = EntityAlias aliasedAlias fromAlias (FromSelectJson Aliased {aliasedAlias}) = EntityAlias aliasedAlias fromAlias (FromFunction Aliased {aliasedAlias}) = EntityAlias aliasedAlias +fromAlias (FromLogicalModel (LogicalModelName logicalModelName)) = EntityAlias (T.toTxt logicalModelName) fieldTextNames :: Ir.AnnFieldsG 'BigQuery Void Expression -> [Text] fieldTextNames = fmap (\(Rql.FieldName name, _) -> name) diff --git a/server/src-lib/Hasura/Backends/BigQuery/Instances/Schema.hs b/server/src-lib/Hasura/Backends/BigQuery/Instances/Schema.hs index 34efcd4bcb0..c61a8c8259b 100644 --- a/server/src-lib/Hasura/Backends/BigQuery/Instances/Schema.hs +++ b/server/src-lib/Hasura/Backends/BigQuery/Instances/Schema.hs @@ -33,6 +33,7 @@ import Hasura.GraphQL.Schema.Parser qualified as P import Hasura.GraphQL.Schema.Select import Hasura.GraphQL.Schema.Table import Hasura.GraphQL.Schema.Typename +import Hasura.LogicalModel.Schema (defaultBuildLogicalModelRootFields) import Hasura.Name qualified as Name import Hasura.Prelude import Hasura.RQL.IR.BoolExp @@ -63,6 +64,7 @@ instance BackendSchema 'BigQuery where buildFunctionQueryFields _ _ _ _ = pure [] buildFunctionRelayQueryFields _ _ _ _ _ = pure [] buildFunctionMutationFields _ _ _ _ = pure [] + buildLogicalModelRootFields = defaultBuildLogicalModelRootFields -- backend extensions relayExtension = Nothing @@ -86,6 +88,10 @@ instance BackendTableSelectSchema 'BigQuery where selectTableAggregate = defaultSelectTableAggregate tableSelectionSet = defaultTableSelectionSet +instance BackendCustomTypeSelectSchema 'BigQuery where + logicalModelArguments = defaultLogicalModelArgs + logicalModelSelectionSet = defaultLogicalModelSelectionSet + ---------------------------------------------------------------- -- Individual components diff --git a/server/src-lib/Hasura/Backends/BigQuery/Plan.hs b/server/src-lib/Hasura/Backends/BigQuery/Plan.hs index 1ab05465811..9fdcd414b20 100644 --- a/server/src-lib/Hasura/Backends/BigQuery/Plan.hs +++ b/server/src-lib/Hasura/Backends/BigQuery/Plan.hs @@ -1,3 +1,5 @@ +{-# LANGUAGE MonadComprehensions #-} + -- | Planning T-SQL queries and subscriptions. module Hasura.Backends.BigQuery.Plan ( planNoPlan, @@ -6,6 +8,8 @@ where import Control.Monad.Validate import Data.Aeson.Text +import Data.List.NonEmpty qualified as NE +import Data.Map.Strict qualified as Map import Data.Text.Extended import Data.Text.Lazy qualified as LT import Hasura.Backends.BigQuery.FromIr as BigQuery @@ -29,8 +33,20 @@ planNoPlan :: m Select planNoPlan fromIrConfig userInfo queryDB = do rootField <- traverse (prepareValueNoPlan (_uiSession userInfo)) queryDB - runValidate (BigQuery.runFromIr fromIrConfig (BigQuery.fromRootField rootField)) - `onLeft` (E.throw400 E.NotSupported . (tshow :: NonEmpty Error -> Text)) + + (select, FromIrWriter {fromIrWriterLogicalModels}) <- + runValidate (BigQuery.runFromIr fromIrConfig (BigQuery.fromRootField rootField)) + `onLeft` (E.throw400 E.NotSupported . (tshow :: NonEmpty Error -> Text)) + + -- Logical models used within this query need to be converted into CTEs. + -- These need to come before any other CTEs in case those CTEs also depend on + -- the logical models. + let logicalModels :: Maybe With + logicalModels = do + ctes <- NE.nonEmpty (Map.toList fromIrWriterLogicalModels) + pure (With [Aliased query (toTxt name) | (name, query) <- ctes]) + + pure select {selectWith = logicalModels <> selectWith select} -------------------------------------------------------------------------------- -- Resolving values diff --git a/server/src-lib/Hasura/Backends/BigQuery/ToQuery.hs b/server/src-lib/Hasura/Backends/BigQuery/ToQuery.hs index eb07a7c41b1..e0ac41eaec3 100644 --- a/server/src-lib/Hasura/Backends/BigQuery/ToQuery.hs +++ b/server/src-lib/Hasura/Backends/BigQuery/ToQuery.hs @@ -25,12 +25,15 @@ import Data.List (intersperse) import Data.List.NonEmpty qualified as NE import Data.String import Data.Text qualified as T +import Data.Text.Extended qualified as T (toTxt) import Data.Text.Lazy qualified as LT import Data.Text.Lazy.Builder (Builder) import Data.Text.Lazy.Builder qualified as LT import Data.Tuple import Data.Vector qualified as V import Hasura.Backends.BigQuery.Types +import Hasura.LogicalModel.Metadata (InterpolatedItem (..), InterpolatedQuery (..)) +import Hasura.LogicalModel.Types (LogicalModelName (..)) import Hasura.Prelude hiding (second) -------------------------------------------------------------------------------- @@ -184,10 +187,26 @@ fromSelect Select {..} = finalExpression fromAsStruct = \case AsStruct -> "AS STRUCT" NoAsStruct -> "" + interpolatedQuery = \case + IIText t -> UnsafeTextPrinter t <+> NewlinePrinter + IIVariable v -> fromExpression v + fromWith = \case + Just (With expressions) -> do + let go :: InterpolatedQuery Expression -> Printer + go = foldr ((<+>) . interpolatedQuery) "" . getInterpolatedQuery + + "WITH " + <+> SepByPrinter + ("," <+> NewlinePrinter) + [ fromNameText alias <+> " AS " <+> parens (go thing) + | Aliased thing alias <- toList expressions + ] + Nothing -> "" inner = SepByPrinter NewlinePrinter - [ "SELECT ", + [ fromWith selectWith, + "SELECT ", fromAsStruct selectAsStruct, IndentPrinter 7 projections, "FROM " <+> IndentPrinter 5 (fromFrom selectFrom), @@ -539,6 +558,8 @@ fromFrom = ) selectFromFunction ) + FromLogicalModel (LogicalModelName logicalModelName) -> + fromNameText (T.toTxt logicalModelName) fromTableName :: TableName -> Printer fromTableName TableName {tableName, tableNameSchema} = diff --git a/server/src-lib/Hasura/Backends/BigQuery/Types.hs b/server/src-lib/Hasura/Backends/BigQuery/Types.hs index e7939ffca93..b901ff62fa7 100644 --- a/server/src-lib/Hasura/Backends/BigQuery/Types.hs +++ b/server/src-lib/Hasura/Backends/BigQuery/Types.hs @@ -50,6 +50,7 @@ module Hasura.Backends.BigQuery.Types Top (..), Value (..), Where (..), + With (..), WindowFunction (..), aggregateProjectionsFieldOrigin, doubleToBigDecimal, @@ -96,6 +97,7 @@ import Data.Vector.Instances () import Hasura.Base.Error import Hasura.Base.ErrorValue qualified as ErrorValue import Hasura.Base.ToErrorValue +import Hasura.LogicalModel.Metadata (InterpolatedQuery, LogicalModelName) import Hasura.Metadata.DTO.Utils (boundedEnumCodec) import Hasura.Prelude hiding (state) import Hasura.RQL.IR.BoolExp @@ -105,7 +107,8 @@ import Language.Haskell.TH.Syntax hiding (location) import Text.ParserCombinators.ReadP (eof, readP_to_S) data Select = Select - { selectTop :: Top, + { selectWith :: Maybe With, + selectTop :: Top, selectAsStruct :: AsStruct, selectProjections :: NonEmpty Projection, selectFrom :: From, @@ -118,7 +121,7 @@ data Select = Select selectCardinality :: Cardinality } deriving stock (Eq, Ord, Show, Generic, Data, Lift) - deriving anyclass (FromJSON, Hashable, NFData) + deriving anyclass (Hashable, NFData) -- | Helper type allowing addition of extra fields used -- in PARTITION BY. @@ -149,14 +152,14 @@ data ArrayAgg = ArrayAgg arrayAggTop :: Top } deriving stock (Eq, Ord, Show, Generic, Data, Lift) - deriving anyclass (FromJSON, Hashable, NFData) + deriving anyclass (Hashable, NFData) data Reselect = Reselect { reselectProjections :: NonEmpty Projection, reselectWhere :: Where } deriving stock (Eq, Ord, Show, Generic, Data, Lift) - deriving anyclass (FromJSON, Hashable, NFData) + deriving anyclass (Hashable, NFData) data OrderBy = OrderBy { orderByFieldName :: FieldName, @@ -183,7 +186,7 @@ data FieldOrigin = NoOrigin | AggregateOrigin [Aliased Aggregate] deriving stock (Eq, Ord, Show, Generic, Data, Lift) - deriving anyclass (FromJSON, Hashable, NFData) + deriving anyclass (Hashable, NFData) aggregateProjectionsFieldOrigin :: Projection -> FieldOrigin aggregateProjectionsFieldOrigin = \case @@ -202,7 +205,7 @@ data Projection | ArrayEntityProjection EntityAlias (Aliased [FieldName]) | WindowProjection (Aliased WindowFunction) deriving stock (Eq, Show, Generic, Data, Lift, Ord) - deriving anyclass (FromJSON, Hashable, NFData) + deriving anyclass (Hashable, NFData) data WindowFunction = -- | ROW_NUMBER() OVER(PARTITION BY field) @@ -220,7 +223,7 @@ data Join = Join joinRightTable :: EntityAlias } deriving stock (Eq, Ord, Show, Generic, Data, Lift) - deriving anyclass (FromJSON, Hashable, NFData) + deriving anyclass (Hashable, NFData) data JoinProvenance = OrderByJoinProvenance @@ -229,19 +232,19 @@ data JoinProvenance | ArrayJoinProvenance [Text] | MultiplexProvenance deriving stock (Eq, Ord, Show, Generic, Data, Lift) - deriving anyclass (FromJSON, Hashable, NFData) + deriving anyclass (Hashable, NFData) data JoinSource = JoinSelect Select -- We're not using existingJoins at the moment, which was used to -- avoid re-joining on the same table twice. deriving stock (Eq, Ord, Show, Generic, Data, Lift) - deriving anyclass (FromJSON, Hashable, NFData) + deriving anyclass (Hashable, NFData) newtype Where = Where [Expression] deriving stock (Eq, Ord, Show, Generic, Data, Lift) - deriving newtype (FromJSON, Hashable, Monoid, NFData, Semigroup) + deriving newtype (Hashable, Monoid, NFData, Semigroup) data Cardinality = Many @@ -255,6 +258,11 @@ data AsStruct deriving stock (Eq, Ord, Show, Generic, Data, Lift) deriving anyclass (FromJSON, Hashable, NFData, ToJSON) +-- | A Common Table Expression clause. +newtype With = With (NonEmpty (Aliased (InterpolatedQuery Expression))) + deriving stock (Data, Generic, Lift) + deriving newtype (Eq, Hashable, NFData, Ord, Semigroup, Show) + data Top = NoTop | Top Int.Int64 @@ -300,7 +308,7 @@ data Expression -- `argument_name` => 'argument_value' FunctionNamedArgument Text Expression deriving stock (Eq, Ord, Show, Generic, Data, Lift) - deriving anyclass (FromJSON, Hashable, NFData) + deriving anyclass (Hashable, NFData) data JsonPath = RootPath @@ -315,7 +323,7 @@ data Aggregate | OpAggregate Text Expression | TextAggregate Text deriving stock (Eq, Ord, Show, Generic, Data, Lift) - deriving anyclass (FromJSON, Hashable, NFData) + deriving anyclass (Hashable, NFData) data Countable fieldname = StarCountable @@ -336,29 +344,30 @@ data From | FromSelect (Aliased Select) | FromSelectJson (Aliased SelectJson) | FromFunction (Aliased SelectFromFunction) + | FromLogicalModel LogicalModelName deriving stock (Eq, Show, Generic, Data, Lift, Ord) - deriving anyclass (FromJSON, Hashable, NFData) + deriving anyclass (Hashable, NFData) data SelectJson = SelectJson { selectJsonBody :: Expression, selectJsonFields :: [(ColumnName, ScalarType)] } deriving stock (Eq, Ord, Show, Generic, Data, Lift) - deriving anyclass (FromJSON, Hashable, NFData) + deriving anyclass (Hashable, NFData) data SelectFromFunction = SelectFromFunction { sffFunctionName :: FunctionName, sffArguments :: [Expression] } deriving stock (Eq, Show, Generic, Data, Lift, Ord) - deriving anyclass (FromJSON, Hashable, NFData) + deriving anyclass (Hashable, NFData) data OpenJson = OpenJson { openJsonExpression :: Expression, openJsonWith :: NonEmpty JsonFieldSpec } deriving stock (Eq, Ord, Show, Generic, Data, Lift) - deriving anyclass (FromJSON, Hashable, NFData) + deriving anyclass (Hashable, NFData) data JsonFieldSpec = IntField Text diff --git a/server/src-lib/Hasura/LogicalModel/Metadata.hs b/server/src-lib/Hasura/LogicalModel/Metadata.hs index 00249b5261a..9e479322dcb 100644 --- a/server/src-lib/Hasura/LogicalModel/Metadata.hs +++ b/server/src-lib/Hasura/LogicalModel/Metadata.hs @@ -39,6 +39,7 @@ import Hasura.RQL.Types.Common (SourceName, ToAesonPairs (toAesonPairs), default import Hasura.RQL.Types.Permission (SelPermDef, _pdRole) import Hasura.SQL.Backend import Hasura.Session (RoleName) +import Language.Haskell.TH.Syntax (Lift) newtype RawQuery = RawQuery {getRawQuery :: Text} deriving newtype (Eq, Ord, Show, FromJSON, ToJSON) @@ -54,7 +55,7 @@ data InterpolatedItem variable IIText Text | -- | a captured variable IIVariable variable - deriving stock (Eq, Ord, Show, Functor, Foldable, Data, Generic, Traversable) + deriving stock (Eq, Ord, Show, Functor, Foldable, Data, Generic, Lift, Traversable) -- | Converting an interpolated query back to text. -- Should roundtrip with the 'parseInterpolatedQuery'. @@ -74,7 +75,7 @@ newtype InterpolatedQuery variable = InterpolatedQuery { getInterpolatedQuery :: [InterpolatedItem variable] } deriving newtype (Eq, Ord, Show, Generic) - deriving stock (Data, Functor, Foldable, Traversable) + deriving stock (Data, Functor, Foldable, Lift, Traversable) deriving newtype instance (Hashable variable) => Hashable (InterpolatedQuery variable) diff --git a/server/src-lib/Hasura/LogicalModel/Types.hs b/server/src-lib/Hasura/LogicalModel/Types.hs index 02c1b6dc780..fcf701b13f8 100644 --- a/server/src-lib/Hasura/LogicalModel/Types.hs +++ b/server/src-lib/Hasura/LogicalModel/Types.hs @@ -15,11 +15,12 @@ import Hasura.Metadata.DTO.Utils (codecNamePrefix) import Hasura.Prelude hiding (first) import Hasura.RQL.Types.Backend (Backend (..)) import Language.GraphQL.Draft.Syntax qualified as G +import Language.Haskell.TH.Syntax (Lift) -- The name of a logical model. This appears as a root field name in the graphql schema. newtype LogicalModelName = LogicalModelName {getLogicalModelName :: G.Name} deriving newtype (Eq, Ord, Show, Hashable, NFData, ToJSON, FromJSON, ToTxt) - deriving stock (Generic) + deriving stock (Data, Generic, Lift) instance HasCodec LogicalModelName where codec = dimapCodec LogicalModelName getLogicalModelName codec