mirror of
https://github.com/hasura/graphql-engine.git
synced 2024-09-20 15:09:02 +03:00
BigQuery Feature Branch
This will implement BigQuery support. Co-authored-by: Antoine Leblanc <1618949+nicuveo@users.noreply.github.com> Co-authored-by: Sibi Prabakaran <737477+psibi@users.noreply.github.com> Co-authored-by: Aniket Deshpande <922486+aniketd@users.noreply.github.com> Co-authored-by: Vamshi Surabhi <6562944+0x777@users.noreply.github.com> GitOrigin-RevId: 1a6ffaf34233e13e8125a5c908eaa7e32d65007b
This commit is contained in:
parent
0f7c273873
commit
f7a202a363
@ -36,6 +36,7 @@ In the future, we will probably offer a way to explicitly choose which behaviour
|
|||||||
|
|
||||||
(Add entries here in the order of: server, console, cli, docs, others)
|
(Add entries here in the order of: server, console, cli, docs, others)
|
||||||
|
|
||||||
|
- server: support for bigquery datasets
|
||||||
- console: add custom_column_names to track_table request with replaced invalid characters (#992)
|
- console: add custom_column_names to track_table request with replaced invalid characters (#992)
|
||||||
- console: add details button to the success notification to see inserted row
|
- console: add details button to the success notification to see inserted row
|
||||||
|
|
||||||
|
@ -173,7 +173,8 @@ constraints: any.Cabal ==3.2.0.0,
|
|||||||
any.http-client ==0.7.5,
|
any.http-client ==0.7.5,
|
||||||
http-client +network-uri,
|
http-client +network-uri,
|
||||||
any.http-client-tls ==0.3.5.3,
|
any.http-client-tls ==0.3.5.3,
|
||||||
any.http-conduit ==2.3.7.3,
|
any.http-conduit ==2.3.7.4,
|
||||||
|
http-conduit +aeson,
|
||||||
any.http-date ==0.0.10,
|
any.http-date ==0.0.10,
|
||||||
any.http-types ==0.12.3,
|
any.http-types ==0.12.3,
|
||||||
any.http2 ==2.0.5,
|
any.http2 ==2.0.5,
|
||||||
@ -193,6 +194,7 @@ constraints: any.Cabal ==3.2.0.0,
|
|||||||
jose -demos,
|
jose -demos,
|
||||||
any.js-chart ==2.9.4.1,
|
any.js-chart ==2.9.4.1,
|
||||||
any.kan-extensions ==5.2.1,
|
any.kan-extensions ==5.2.1,
|
||||||
|
any.keys ==3.12.3,
|
||||||
any.lens ==4.19.2,
|
any.lens ==4.19.2,
|
||||||
lens -benchmark-uniplate -dump-splices +inlining -j -old-inline-pragmas -safe +test-doctests +test-hunit +test-properties +test-templates +trustworthy,
|
lens -benchmark-uniplate -dump-splices +inlining -j -old-inline-pragmas -safe +test-doctests +test-hunit +test-properties +test-templates +trustworthy,
|
||||||
any.lens-aeson ==1.1,
|
any.lens-aeson ==1.1,
|
||||||
@ -242,6 +244,8 @@ constraints: any.Cabal ==3.2.0.0,
|
|||||||
parsers +attoparsec +binary +parsec,
|
parsers +attoparsec +binary +parsec,
|
||||||
any.pem ==0.2.4,
|
any.pem ==0.2.4,
|
||||||
any.placeholders ==0.1,
|
any.placeholders ==0.1,
|
||||||
|
any.pointed ==5.0.2,
|
||||||
|
pointed +comonad +containers +kan-extensions +semigroupoids +semigroups +stm +tagged +transformers +unordered-containers,
|
||||||
any.postgresql-binary ==0.12.3.3,
|
any.postgresql-binary ==0.12.3.3,
|
||||||
any.postgresql-libpq ==0.9.4.3,
|
any.postgresql-libpq ==0.9.4.3,
|
||||||
postgresql-libpq -use-pkg-config,
|
postgresql-libpq -use-pkg-config,
|
||||||
@ -274,6 +278,7 @@ constraints: any.Cabal ==3.2.0.0,
|
|||||||
retry -lib-werror,
|
retry -lib-werror,
|
||||||
any.rts ==1.0,
|
any.rts ==1.0,
|
||||||
any.safe ==0.3.19,
|
any.safe ==0.3.19,
|
||||||
|
any.safe-exceptions ==0.1.7.1,
|
||||||
any.scientific ==0.3.6.2,
|
any.scientific ==0.3.6.2,
|
||||||
scientific -bytestring-builder -integer-simple,
|
scientific -bytestring-builder -integer-simple,
|
||||||
any.semialign ==1.1.0.1,
|
any.semialign ==1.1.0.1,
|
||||||
@ -361,6 +366,8 @@ constraints: any.Cabal ==3.2.0.0,
|
|||||||
vector-algorithms +bench +boundschecks -internalchecks -llvm +properties -unsafechecks,
|
vector-algorithms +bench +boundschecks -internalchecks -llvm +properties -unsafechecks,
|
||||||
any.vector-binary-instances ==0.2.5.1,
|
any.vector-binary-instances ==0.2.5.1,
|
||||||
any.vector-builder ==0.3.8,
|
any.vector-builder ==0.3.8,
|
||||||
|
any.vector-instances ==3.4,
|
||||||
|
vector-instances +hashable,
|
||||||
any.vector-th-unbox ==0.2.1.7,
|
any.vector-th-unbox ==0.2.1.7,
|
||||||
any.void ==0.7.3,
|
any.void ==0.7.3,
|
||||||
void -safe,
|
void -safe,
|
||||||
|
@ -112,10 +112,12 @@ library
|
|||||||
, deepseq
|
, deepseq
|
||||||
, dependent-map >=0.4 && <0.5
|
, dependent-map >=0.4 && <0.5
|
||||||
, exceptions
|
, exceptions
|
||||||
|
, safe-exceptions
|
||||||
, fast-logger
|
, fast-logger
|
||||||
, hashable
|
, hashable
|
||||||
, http-client-tls
|
, http-client-tls
|
||||||
, http-types
|
, http-types
|
||||||
|
, http-conduit
|
||||||
, kan-extensions
|
, kan-extensions
|
||||||
, lifted-base
|
, lifted-base
|
||||||
, monad-control
|
, monad-control
|
||||||
@ -142,6 +144,7 @@ library
|
|||||||
, unordered-containers >= 0.2.12
|
, unordered-containers >= 0.2.12
|
||||||
, validation
|
, validation
|
||||||
, vector
|
, vector
|
||||||
|
, vector-instances
|
||||||
, wai
|
, wai
|
||||||
|
|
||||||
-- Encoder related
|
-- Encoder related
|
||||||
@ -260,6 +263,10 @@ library
|
|||||||
, odbc
|
, odbc
|
||||||
, resource-pool
|
, resource-pool
|
||||||
|
|
||||||
|
-- bigquery support
|
||||||
|
, memory
|
||||||
|
, x509-store
|
||||||
|
|
||||||
if !flag(profiling)
|
if !flag(profiling)
|
||||||
build-depends:
|
build-depends:
|
||||||
-- 0.6.1 is supposedly not okay for ghc 8.6:
|
-- 0.6.1 is supposedly not okay for ghc 8.6:
|
||||||
@ -291,6 +298,25 @@ library
|
|||||||
, Hasura.App
|
, Hasura.App
|
||||||
, Hasura.Metadata.Class
|
, Hasura.Metadata.Class
|
||||||
|
|
||||||
|
, Hasura.Backends.BigQuery.Connection
|
||||||
|
, Hasura.Backends.BigQuery.DataLoader.Execute
|
||||||
|
, Hasura.Backends.BigQuery.DataLoader.Plan
|
||||||
|
, Hasura.Backends.BigQuery.DDL
|
||||||
|
, Hasura.Backends.BigQuery.DDL.BoolExp
|
||||||
|
, Hasura.Backends.BigQuery.DDL.RunSQL
|
||||||
|
, Hasura.Backends.BigQuery.DDL.Source
|
||||||
|
, Hasura.Backends.BigQuery.FromIr
|
||||||
|
, Hasura.Backends.BigQuery.Instances.Execute
|
||||||
|
, Hasura.Backends.BigQuery.Instances.Schema
|
||||||
|
, Hasura.Backends.BigQuery.Instances.Transport
|
||||||
|
, Hasura.Backends.BigQuery.Instances.Types
|
||||||
|
, Hasura.Backends.BigQuery.Instances.Metadata
|
||||||
|
, Hasura.Backends.BigQuery.Meta
|
||||||
|
, Hasura.Backends.BigQuery.Plan
|
||||||
|
, Hasura.Backends.BigQuery.Source
|
||||||
|
, Hasura.Backends.BigQuery.ToQuery
|
||||||
|
, Hasura.Backends.BigQuery.Types
|
||||||
|
|
||||||
, Hasura.Backends.Postgres.Connection
|
, Hasura.Backends.Postgres.Connection
|
||||||
, Hasura.Backends.Postgres.DDL
|
, Hasura.Backends.Postgres.DDL
|
||||||
, Hasura.Backends.Postgres.DDL.BoolExp
|
, Hasura.Backends.Postgres.DDL.BoolExp
|
||||||
|
204
server/src-lib/Hasura/Backends/BigQuery/Connection.hs
Normal file
204
server/src-lib/Hasura/Backends/BigQuery/Connection.hs
Normal file
@ -0,0 +1,204 @@
|
|||||||
|
{-# LANGUAGE NumericUnderscores #-}
|
||||||
|
|
||||||
|
module Hasura.Backends.BigQuery.Connection where
|
||||||
|
|
||||||
|
|
||||||
|
import Control.Concurrent.MVar
|
||||||
|
import Control.Exception
|
||||||
|
import Crypto.Hash.Algorithms (SHA256(..))
|
||||||
|
import Crypto.PubKey.RSA.PKCS15 (signSafer)
|
||||||
|
import Crypto.PubKey.RSA.Types as Cry (Error)
|
||||||
|
import Data.Bifunctor (bimap)
|
||||||
|
import qualified Data.ByteArray.Encoding as BAE
|
||||||
|
import qualified Data.ByteString as BS
|
||||||
|
import qualified Data.ByteString.Lazy as BL
|
||||||
|
import qualified Data.ByteString.Char8 as B8
|
||||||
|
import qualified Data.Aeson as J
|
||||||
|
import qualified Data.Aeson.Casing as J
|
||||||
|
import qualified Data.Aeson.TH as J
|
||||||
|
import qualified Data.Environment as Env
|
||||||
|
import qualified Data.Text as T
|
||||||
|
import qualified Data.Text.Encoding as TE
|
||||||
|
import qualified Data.Text.Encoding.Error as TE
|
||||||
|
import Data.Time.Clock.POSIX (getPOSIXTime)
|
||||||
|
import Data.Time.Clock
|
||||||
|
import Hasura.Prelude
|
||||||
|
import Hasura.Backends.BigQuery.Source
|
||||||
|
import qualified Hasura.Backends.MSSQL.Connection as MSSQLConn (getEnv)
|
||||||
|
import Hasura.RQL.Types.Error
|
||||||
|
import Network.HTTP.Simple
|
||||||
|
import Network.HTTP.Types
|
||||||
|
|
||||||
|
|
||||||
|
newtype Scope
|
||||||
|
= Scope { unScope :: T.Text }
|
||||||
|
deriving (Show, Eq, IsString)
|
||||||
|
|
||||||
|
|
||||||
|
data GoogleAccessTokenRequest = GoogleAccessTokenRequest
|
||||||
|
{ _gatrGrantType :: !Text
|
||||||
|
, _gatrAssertion :: !Text
|
||||||
|
} deriving (Show, Eq)
|
||||||
|
$(J.deriveJSON (J.aesonDrop 5 J.snakeCase){J.omitNothingFields=False} ''GoogleAccessTokenRequest)
|
||||||
|
|
||||||
|
mkTokenRequest :: Text -> GoogleAccessTokenRequest
|
||||||
|
mkTokenRequest = GoogleAccessTokenRequest "urn:ietf:params:oauth:grant-type:jwt-bearer"
|
||||||
|
|
||||||
|
|
||||||
|
data TokenProblem
|
||||||
|
= BearerTokenDecodeProblem TE.UnicodeException
|
||||||
|
| BearerTokenSignsaferProblem Cry.Error
|
||||||
|
| TokenFetchProblem JSONException
|
||||||
|
| TokenRequestNonOK Status
|
||||||
|
deriving (Show)
|
||||||
|
instance Exception TokenProblem
|
||||||
|
|
||||||
|
data ServiceAccountProblem
|
||||||
|
= ServiceAccountFileDecodeProblem String
|
||||||
|
deriving (Show)
|
||||||
|
instance Exception ServiceAccountProblem
|
||||||
|
|
||||||
|
|
||||||
|
resolveConfigurationJson ::
|
||||||
|
(QErrM m, J.FromJSON a) =>
|
||||||
|
Env.Environment ->
|
||||||
|
ConfigurationJSON a -> -- REVIEW: Can this be made polymorphic?
|
||||||
|
m (Either String a)
|
||||||
|
resolveConfigurationJson env = \case
|
||||||
|
FromYamlJSON s -> pure . Right $ s
|
||||||
|
FromEnvJSON v -> do
|
||||||
|
fileContents <- MSSQLConn.getEnv env v
|
||||||
|
case J.eitherDecode . BL.fromStrict . TE.encodeUtf8 $ fileContents of
|
||||||
|
Left e -> pure . Left $ e
|
||||||
|
Right sa -> pure . Right $ sa
|
||||||
|
|
||||||
|
|
||||||
|
resolveConfigurationInput ::
|
||||||
|
QErrM m =>
|
||||||
|
Env.Environment ->
|
||||||
|
ConfigurationInput ->
|
||||||
|
m Text
|
||||||
|
resolveConfigurationInput env = \case
|
||||||
|
FromYaml s -> pure s
|
||||||
|
FromEnv v -> MSSQLConn.getEnv env v
|
||||||
|
|
||||||
|
|
||||||
|
resolveConfigurationInputs ::
|
||||||
|
QErrM m =>
|
||||||
|
Env.Environment ->
|
||||||
|
ConfigurationInputs ->
|
||||||
|
m [Text]
|
||||||
|
resolveConfigurationInputs env = \case
|
||||||
|
FromYamls a -> pure a
|
||||||
|
FromEnvs v -> filter (not . T.null) . T.splitOn "," <$> MSSQLConn.getEnv env v
|
||||||
|
|
||||||
|
|
||||||
|
getAccessToken :: MonadIO m => ServiceAccount -> m (Either TokenProblem TokenResp)
|
||||||
|
getAccessToken sa = do
|
||||||
|
eJwt <- encodeBearerJWT sa ["https://www.googleapis.com/auth/cloud-platform"]
|
||||||
|
case eJwt of
|
||||||
|
Left tokenProblem -> pure . Left $ tokenProblem
|
||||||
|
Right jwt ->
|
||||||
|
case TE.decodeUtf8' jwt of
|
||||||
|
Left unicodeEx -> pure . Left . BearerTokenDecodeProblem $ unicodeEx
|
||||||
|
Right assertion -> do
|
||||||
|
tokenFetchResponse :: Response (Either JSONException TokenResp) <-
|
||||||
|
httpJSONEither $
|
||||||
|
setRequestBodyJSON (mkTokenRequest assertion) $
|
||||||
|
parseRequest_ ("POST " <> tokenURL)
|
||||||
|
if getResponseStatusCode tokenFetchResponse /= 200
|
||||||
|
then
|
||||||
|
pure . Left . TokenRequestNonOK . getResponseStatus $ tokenFetchResponse
|
||||||
|
else
|
||||||
|
case getResponseBody tokenFetchResponse of
|
||||||
|
Left jsonEx -> pure . Left . TokenFetchProblem $ jsonEx
|
||||||
|
Right tr@TokenResp{_trExpiresAt} -> do
|
||||||
|
-- We add the current POSIXTime and store the POSIX "moment" at
|
||||||
|
-- which this token will expire, so that at the site where
|
||||||
|
-- we need to check if a token is nearing expiry, we only
|
||||||
|
-- need to compare it with the _then_ "current" POSIXTime.
|
||||||
|
expiresAt <- (fromIntegral _trExpiresAt +) <$> liftIO getPOSIXTime
|
||||||
|
pure . Right $ tr { _trExpiresAt = truncate expiresAt }
|
||||||
|
where
|
||||||
|
-- TODO: use jose for jwt encoding
|
||||||
|
b64EncodeJ :: (J.ToJSON a) => a -> BS.ByteString
|
||||||
|
b64EncodeJ = base64 . BL.toStrict . J.encode
|
||||||
|
base64 :: BS.ByteString -> BS.ByteString
|
||||||
|
base64 = BAE.convertToBase BAE.Base64URLUnpadded
|
||||||
|
tokenURL :: String
|
||||||
|
tokenURL = "https://www.googleapis.com/oauth2/v4/token"
|
||||||
|
maxTokenLifetime :: Int
|
||||||
|
maxTokenLifetime = 3600
|
||||||
|
truncateEquals :: B8.ByteString -> B8.ByteString
|
||||||
|
truncateEquals bs =
|
||||||
|
case B8.unsnoc bs of
|
||||||
|
Nothing -> mempty
|
||||||
|
Just (bs', x)
|
||||||
|
| x == '=' -> bs'
|
||||||
|
| otherwise -> bs
|
||||||
|
encodeBearerJWT :: ( MonadIO m ) => ServiceAccount -> [Scope] -> m (Either TokenProblem BS.ByteString)
|
||||||
|
encodeBearerJWT ServiceAccount{..} scopes = do
|
||||||
|
inp <- mkSigInput . truncate <$> liftIO getPOSIXTime
|
||||||
|
signRes <- liftIO $ signSafer (Just SHA256) (unPKey _saPrivateKey) inp
|
||||||
|
case signRes of
|
||||||
|
Left e -> pure . Left . BearerTokenSignsaferProblem $ e
|
||||||
|
Right sig -> pure . Right $ inp <> "." <> truncateEquals (base64 sig)
|
||||||
|
where
|
||||||
|
mkSigInput :: Int -> BS.ByteString
|
||||||
|
mkSigInput n = header <> "." <> payload
|
||||||
|
where
|
||||||
|
header = b64EncodeJ $ J.object
|
||||||
|
[ "alg" J..= ("RS256" :: T.Text)
|
||||||
|
, "typ" J..= ("JWT" :: T.Text)
|
||||||
|
]
|
||||||
|
payload = b64EncodeJ $ J.object [ "aud" J..= tokenURL , "scope" J..= T.intercalate " " (map unScope scopes)
|
||||||
|
, "iat" J..= n
|
||||||
|
, "exp" J..= (n + maxTokenLifetime)
|
||||||
|
, "iss" J..= _saClientEmail
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
getServiceAccount :: MonadIO m => FilePath -> m (Either ServiceAccountProblem ServiceAccount)
|
||||||
|
getServiceAccount serviceAccountFilePath =
|
||||||
|
bimap ServiceAccountFileDecodeProblem id . J.eitherDecode' <$> liftIO (BL.readFile serviceAccountFilePath)
|
||||||
|
|
||||||
|
|
||||||
|
-- | Get a usable token. If the token has expired refresh it.
|
||||||
|
getUsableToken :: MonadIO m => BigQuerySourceConfig -> m (Either TokenProblem TokenResp)
|
||||||
|
getUsableToken BigQuerySourceConfig{_scServiceAccount, _scAccessTokenMVar} =
|
||||||
|
liftIO $ modifyMVar _scAccessTokenMVar $ \mTokenResp -> do
|
||||||
|
case mTokenResp of
|
||||||
|
Nothing -> do
|
||||||
|
refreshedToken <- getAccessToken _scServiceAccount
|
||||||
|
case refreshedToken of
|
||||||
|
Left e -> pure (Nothing, Left e)
|
||||||
|
Right t -> pure (Just t, Right t)
|
||||||
|
Just t@TokenResp{_trAccessToken, _trExpiresAt} -> do
|
||||||
|
pt <- liftIO $ getPOSIXTime
|
||||||
|
if (pt >= fromIntegral _trExpiresAt - (10 :: NominalDiffTime)) -- when posix-time is greater than expires-at-minus-threshold
|
||||||
|
then do
|
||||||
|
refreshedToken' <- getAccessToken _scServiceAccount
|
||||||
|
case refreshedToken' of
|
||||||
|
Left e -> pure (Just t, Left e)
|
||||||
|
Right t' -> pure (Just t', Right t')
|
||||||
|
else pure (Just t, Right t)
|
||||||
|
|
||||||
|
|
||||||
|
data BigQueryProblem
|
||||||
|
= TokenProblem TokenProblem
|
||||||
|
deriving (Show)
|
||||||
|
|
||||||
|
|
||||||
|
runBigQuery ::
|
||||||
|
(MonadIO m) =>
|
||||||
|
BigQuerySourceConfig ->
|
||||||
|
Request ->
|
||||||
|
m (Either BigQueryProblem (Response BL.ByteString))
|
||||||
|
runBigQuery sc req = do
|
||||||
|
eToken <- getUsableToken sc
|
||||||
|
case eToken of
|
||||||
|
Left e -> pure . Left . TokenProblem $ e
|
||||||
|
Right TokenResp{_trAccessToken, _trExpiresAt} -> do
|
||||||
|
let req' = setRequestHeader "Authorization" ["Bearer " <> (TE.encodeUtf8 . coerce) _trAccessToken] req
|
||||||
|
-- TODO: Make this catch the HTTP exceptions
|
||||||
|
Right <$> httpLBS req'
|
141
server/src-lib/Hasura/Backends/BigQuery/DDL.hs
Normal file
141
server/src-lib/Hasura/Backends/BigQuery/DDL.hs
Normal file
@ -0,0 +1,141 @@
|
|||||||
|
module Hasura.Backends.BigQuery.DDL
|
||||||
|
( buildComputedFieldInfo
|
||||||
|
, buildRemoteFieldInfo
|
||||||
|
, fetchAndValidateEnumValues
|
||||||
|
, createTableEventTrigger
|
||||||
|
, buildEventTriggerInfo
|
||||||
|
, buildFunctionInfo
|
||||||
|
, updateColumnInEventTrigger
|
||||||
|
, parseBoolExpOperations
|
||||||
|
, parseCollectableType
|
||||||
|
, module M
|
||||||
|
)
|
||||||
|
where
|
||||||
|
|
||||||
|
import Hasura.Backends.BigQuery.DDL.BoolExp
|
||||||
|
import Hasura.Prelude
|
||||||
|
|
||||||
|
import Data.Aeson
|
||||||
|
|
||||||
|
import qualified Data.Environment as Env
|
||||||
|
|
||||||
|
import Hasura.RQL.IR.BoolExp
|
||||||
|
import Hasura.RQL.Types.Backend
|
||||||
|
import Hasura.RQL.Types.Column
|
||||||
|
import Hasura.RQL.Types.Common
|
||||||
|
import Hasura.RQL.Types.ComputedField
|
||||||
|
import Hasura.RQL.Types.Error
|
||||||
|
import Hasura.RQL.Types.EventTrigger
|
||||||
|
import Hasura.RQL.Types.Function
|
||||||
|
import Hasura.RQL.Types.RemoteRelationship
|
||||||
|
import Hasura.RQL.Types.SchemaCache
|
||||||
|
import Hasura.RQL.Types.Table
|
||||||
|
import Hasura.SQL.Backend
|
||||||
|
import Hasura.SQL.Types
|
||||||
|
import Hasura.Server.Types
|
||||||
|
import Hasura.Backends.BigQuery.Instances.Types ()
|
||||||
|
import Hasura.Server.Utils
|
||||||
|
import Hasura.Session
|
||||||
|
|
||||||
|
import qualified Hasura.Backends.BigQuery.Types as BigQuery
|
||||||
|
import Hasura.Backends.BigQuery.DDL.Source as M
|
||||||
|
|
||||||
|
buildComputedFieldInfo
|
||||||
|
:: (MonadError QErr m)
|
||||||
|
=> HashSet (TableName 'BigQuery)
|
||||||
|
-> TableName 'BigQuery
|
||||||
|
-> ComputedFieldName
|
||||||
|
-> ComputedFieldDefinition 'BigQuery
|
||||||
|
-> RawFunctionInfo
|
||||||
|
-> Maybe Text
|
||||||
|
-> m (ComputedFieldInfo 'BigQuery)
|
||||||
|
buildComputedFieldInfo _ _ _ _ _ _ =
|
||||||
|
throw400 NotSupported "Computed fields aren't supported for BigQuery sources"
|
||||||
|
|
||||||
|
buildRemoteFieldInfo
|
||||||
|
:: (MonadError QErr m)
|
||||||
|
=> RemoteRelationship 'BigQuery
|
||||||
|
-> [ColumnInfo 'BigQuery]
|
||||||
|
-> RemoteSchemaMap
|
||||||
|
-> m (RemoteFieldInfo 'BigQuery, [SchemaDependency])
|
||||||
|
buildRemoteFieldInfo _ _ _ =
|
||||||
|
throw400 NotSupported "Remote joins aren't supported for BigQuery sources"
|
||||||
|
|
||||||
|
fetchAndValidateEnumValues
|
||||||
|
:: (Monad m)
|
||||||
|
=> SourceConfig 'BigQuery
|
||||||
|
-> TableName 'BigQuery
|
||||||
|
-> Maybe (PrimaryKey 'BigQuery (RawColumnInfo 'BigQuery))
|
||||||
|
-> [RawColumnInfo 'BigQuery]
|
||||||
|
-> m (Either QErr EnumValues)
|
||||||
|
fetchAndValidateEnumValues _ _ _ _ = runExceptT $
|
||||||
|
throw400 NotSupported "Enum tables are not supported for BigQuery sources"
|
||||||
|
|
||||||
|
createTableEventTrigger
|
||||||
|
:: (Monad m)
|
||||||
|
=> ServerConfigCtx
|
||||||
|
-> SourceConfig 'BigQuery
|
||||||
|
-> TableName 'BigQuery
|
||||||
|
-> [ColumnInfo 'BigQuery]
|
||||||
|
-> TriggerName
|
||||||
|
-> TriggerOpsDef
|
||||||
|
-> m (Either QErr ())
|
||||||
|
createTableEventTrigger _ _ _ _ _ _ = runExceptT $
|
||||||
|
throw400 NotSupported "Cannot create table event triggers in BigQuery sources"
|
||||||
|
|
||||||
|
buildEventTriggerInfo
|
||||||
|
:: MonadError QErr m
|
||||||
|
=> Env.Environment
|
||||||
|
-> SourceName
|
||||||
|
-> TableName 'BigQuery
|
||||||
|
-> EventTriggerConf
|
||||||
|
-> m (EventTriggerInfo, [SchemaDependency])
|
||||||
|
buildEventTriggerInfo _ _ _ _ =
|
||||||
|
throw400 NotSupported "Table event triggers are not supported for BigQuery sources"
|
||||||
|
|
||||||
|
buildFunctionInfo
|
||||||
|
:: (MonadError QErr m)
|
||||||
|
=> SourceName
|
||||||
|
-> FunctionName 'BigQuery
|
||||||
|
-> SystemDefined
|
||||||
|
-> FunctionConfig
|
||||||
|
-> [FunctionPermissionMetadata]
|
||||||
|
-> RawFunctionInfo
|
||||||
|
-> m (FunctionInfo 'BigQuery, SchemaDependency)
|
||||||
|
buildFunctionInfo _ _ _ _ _ _ =
|
||||||
|
throw400 NotSupported "SQL Functions are not supported for BigQuery source"
|
||||||
|
|
||||||
|
updateColumnInEventTrigger
|
||||||
|
:: TableName 'BigQuery
|
||||||
|
-> Column 'BigQuery
|
||||||
|
-> Column 'BigQuery
|
||||||
|
-> TableName 'BigQuery
|
||||||
|
-> EventTriggerConf
|
||||||
|
-> EventTriggerConf
|
||||||
|
updateColumnInEventTrigger _ _ _ _ = id
|
||||||
|
|
||||||
|
parseCollectableType
|
||||||
|
:: (MonadError QErr m)
|
||||||
|
=> CollectableType (ColumnType 'BigQuery)
|
||||||
|
-> Value
|
||||||
|
-> m (PartialSQLExp 'BigQuery)
|
||||||
|
parseCollectableType collectableType = \case
|
||||||
|
String t
|
||||||
|
| isSessionVariable t -> pure $ mkTypedSessionVar collectableType $ mkSessionVariable t
|
||||||
|
| isReqUserId t -> pure $ mkTypedSessionVar collectableType userIdHeader
|
||||||
|
val -> case collectableType of
|
||||||
|
CollectableTypeScalar scalarType ->
|
||||||
|
PSESQLExp . BigQuery.ValueExpression <$> parseScalarValueColumnType scalarType val
|
||||||
|
CollectableTypeArray _ ->
|
||||||
|
throw400 NotSupported "Array types are not supported in BigQuery backend"
|
||||||
|
|
||||||
|
mkTypedSessionVar
|
||||||
|
:: CollectableType (ColumnType 'BigQuery)
|
||||||
|
-> SessionVariable -> PartialSQLExp 'BigQuery
|
||||||
|
mkTypedSessionVar columnType =
|
||||||
|
PSESessVar (msColumnTypeToScalarType <$> columnType)
|
||||||
|
|
||||||
|
msColumnTypeToScalarType :: ColumnType 'BigQuery -> ScalarType 'BigQuery
|
||||||
|
msColumnTypeToScalarType = \case
|
||||||
|
ColumnScalar scalarType -> scalarType
|
||||||
|
ColumnEnumReference _ -> BigQuery.StringScalarType
|
69
server/src-lib/Hasura/Backends/BigQuery/DDL/BoolExp.hs
Normal file
69
server/src-lib/Hasura/Backends/BigQuery/DDL/BoolExp.hs
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
module Hasura.Backends.BigQuery.DDL.BoolExp where
|
||||||
|
|
||||||
|
import Hasura.Prelude
|
||||||
|
|
||||||
|
import qualified Data.Aeson as J
|
||||||
|
import qualified Data.HashMap.Strict as Map
|
||||||
|
|
||||||
|
import Hasura.Backends.BigQuery.Instances.Types ()
|
||||||
|
import Hasura.Backends.BigQuery.Types
|
||||||
|
import Hasura.RQL.IR.BoolExp
|
||||||
|
import Hasura.RQL.Types.Column
|
||||||
|
import Hasura.RQL.Types.Error
|
||||||
|
import Hasura.RQL.Types.SchemaCache
|
||||||
|
import Hasura.SQL.Backend
|
||||||
|
import Hasura.SQL.Types
|
||||||
|
|
||||||
|
parseBoolExpOperations
|
||||||
|
:: forall m v
|
||||||
|
. (MonadError QErr m)
|
||||||
|
=> ValueParser 'BigQuery m v
|
||||||
|
-> FieldInfoMap (FieldInfo 'BigQuery)
|
||||||
|
-> ColumnInfo 'BigQuery
|
||||||
|
-> J.Value
|
||||||
|
-> m [OpExpG 'BigQuery v]
|
||||||
|
parseBoolExpOperations rhsParser _fields columnInfo value =
|
||||||
|
withPathK (columnName $ pgiColumn columnInfo) $
|
||||||
|
parseOperations (pgiType columnInfo) value
|
||||||
|
where
|
||||||
|
parseWithTy ty = rhsParser (CollectableTypeScalar ty)
|
||||||
|
|
||||||
|
parseOperations :: ColumnType 'BigQuery -> J.Value -> m [OpExpG 'BigQuery v]
|
||||||
|
parseOperations columnType = \case
|
||||||
|
J.Object o -> mapM (parseOperation columnType) $ Map.toList o
|
||||||
|
v -> pure . AEQ False <$> parseWithTy columnType v
|
||||||
|
|
||||||
|
parseOperation :: ColumnType 'BigQuery -> (Text, J.Value) -> m (OpExpG 'BigQuery v)
|
||||||
|
parseOperation columnType (opStr, val) = withPathK opStr $
|
||||||
|
case opStr of
|
||||||
|
"_eq" -> parseEq
|
||||||
|
"$eq" -> parseEq
|
||||||
|
|
||||||
|
"_neq" -> parseNeq
|
||||||
|
"$neq" -> parseNeq
|
||||||
|
|
||||||
|
"_gt" -> parseGt
|
||||||
|
"$gt" -> parseGt
|
||||||
|
|
||||||
|
"_lt" -> parseLt
|
||||||
|
"$lt" -> parseLt
|
||||||
|
|
||||||
|
"_gte" -> parseGte
|
||||||
|
"$gte" -> parseGte
|
||||||
|
|
||||||
|
"_lte" -> parseLte
|
||||||
|
"$lte" -> parseLte
|
||||||
|
|
||||||
|
-- TODO: support column operators
|
||||||
|
|
||||||
|
x -> throw400 UnexpectedPayload $ "Unknown operator : " <> x
|
||||||
|
|
||||||
|
where
|
||||||
|
parseOne = parseWithTy columnType val
|
||||||
|
|
||||||
|
parseEq = AEQ False <$> parseOne
|
||||||
|
parseNeq = ANE False <$> parseOne
|
||||||
|
parseGt = AGT <$> parseOne
|
||||||
|
parseLt = ALT <$> parseOne
|
||||||
|
parseGte = AGTE <$> parseOne
|
||||||
|
parseLte = ALTE <$> parseOne
|
94
server/src-lib/Hasura/Backends/BigQuery/DDL/RunSQL.hs
Normal file
94
server/src-lib/Hasura/Backends/BigQuery/DDL/RunSQL.hs
Normal file
@ -0,0 +1,94 @@
|
|||||||
|
-- Working example:
|
||||||
|
--
|
||||||
|
-- $ echo '{"type":"bigquery_run_sql","args":{"sql":"select 3 * 4 as foo, \"Hello, World!\" as bar", "source":"chinook"}}' | curl -XPOST -d @- http://localhost:8080/v2/query
|
||||||
|
-- {"result_type":"TuplesOk","result":[["foo","bar"],["12","Hello, World!"]]}
|
||||||
|
|
||||||
|
module Hasura.Backends.BigQuery.DDL.RunSQL
|
||||||
|
( runSQL
|
||||||
|
, runDatabaseInspection
|
||||||
|
, BigQueryRunSQL
|
||||||
|
)
|
||||||
|
where
|
||||||
|
|
||||||
|
import Control.Monad.IO.Class
|
||||||
|
import qualified Data.Aeson as J
|
||||||
|
import Data.Aeson.TH (deriveJSON)
|
||||||
|
import Data.Aeson.Text (encodeToLazyText)
|
||||||
|
import qualified Data.HashMap.Strict.InsOrd as OMap
|
||||||
|
import qualified Data.Text as T
|
||||||
|
import qualified Data.Text.Lazy as LT
|
||||||
|
import qualified Data.Vector as V
|
||||||
|
|
||||||
|
import Hasura.Backends.BigQuery.DataLoader.Execute as Execute (BigQuery (..),
|
||||||
|
OutputValue (..),
|
||||||
|
RecordSet (..),
|
||||||
|
streamBigQuery)
|
||||||
|
import qualified Hasura.Backends.BigQuery.DataLoader.Plan as Plan
|
||||||
|
import Hasura.Backends.BigQuery.Source (BigQuerySourceConfig (..))
|
||||||
|
import Hasura.EncJSON
|
||||||
|
import Hasura.Prelude
|
||||||
|
import Hasura.RQL.DDL.Schema (RunSQLRes (..))
|
||||||
|
import Hasura.RQL.Types (CacheRWM, Code (..), MetadataM, QErr,
|
||||||
|
SourceName, askSourceConfig, throw400)
|
||||||
|
|
||||||
|
data BigQueryRunSQL
|
||||||
|
= BigQueryRunSQL
|
||||||
|
{ _mrsSql :: Text
|
||||||
|
, _mrsSource :: !SourceName
|
||||||
|
} deriving (Show, Eq)
|
||||||
|
$(deriveJSON hasuraJSON ''BigQueryRunSQL)
|
||||||
|
|
||||||
|
runSQL ::
|
||||||
|
(MonadIO m, CacheRWM m, MonadError QErr m, MetadataM m) =>
|
||||||
|
BigQueryRunSQL ->
|
||||||
|
m EncJSON
|
||||||
|
runSQL = runSQL_ recordSetAsHeaderAndRows
|
||||||
|
|
||||||
|
-- | The SQL query in the request is ignored
|
||||||
|
runDatabaseInspection ::
|
||||||
|
(MonadIO m, CacheRWM m, MonadError QErr m, MetadataM m) =>
|
||||||
|
BigQueryRunSQL ->
|
||||||
|
m EncJSON
|
||||||
|
runDatabaseInspection (BigQueryRunSQL _query source) = do
|
||||||
|
BigQuerySourceConfig{_scDatasets = dataSets} <- askSourceConfig source
|
||||||
|
let queries = ["SELECT *, ARRAY(SELECT as STRUCT * from " <>
|
||||||
|
dataSet <> ".INFORMATION_SCHEMA.COLUMNS WHERE table_name = t.table_name) as columns from " <>
|
||||||
|
dataSet <> ".INFORMATION_SCHEMA.TABLES as t" | dataSet <- dataSets]
|
||||||
|
query' = T.intercalate " UNION ALL " queries
|
||||||
|
runSQL_ recordSetAsSchema (BigQueryRunSQL query' source)
|
||||||
|
|
||||||
|
runSQL_ ::
|
||||||
|
(MonadIO m, CacheRWM m, MonadError QErr m, MetadataM m) =>
|
||||||
|
(RecordSet -> J.Value) ->
|
||||||
|
BigQueryRunSQL ->
|
||||||
|
m EncJSON
|
||||||
|
runSQL_ f (BigQueryRunSQL query source) = do
|
||||||
|
sourceConfig <- askSourceConfig source
|
||||||
|
result <-
|
||||||
|
streamBigQuery
|
||||||
|
sourceConfig
|
||||||
|
Execute.BigQuery {query = LT.fromStrict query, parameters = mempty}
|
||||||
|
case result of
|
||||||
|
Left queryError -> throw400 BigQueryError (T.pack (show queryError)) -- TODO: Pretty print the error type.
|
||||||
|
Right recordSet ->
|
||||||
|
pure
|
||||||
|
(encJFromJValue
|
||||||
|
(RunSQLRes "TuplesOk" (f recordSet)))
|
||||||
|
|
||||||
|
recordSetAsHeaderAndRows :: RecordSet -> J.Value
|
||||||
|
recordSetAsHeaderAndRows RecordSet {rows} = J.toJSON (thead : tbody)
|
||||||
|
where
|
||||||
|
thead =
|
||||||
|
case rows V.!? 0 of
|
||||||
|
Nothing -> []
|
||||||
|
Just row ->
|
||||||
|
map (J.toJSON . (coerce :: Plan.FieldName -> Text)) (OMap.keys row)
|
||||||
|
tbody :: [[J.Value]]
|
||||||
|
tbody = map (\row -> map J.toJSON (OMap.elems row)) (toList rows)
|
||||||
|
|
||||||
|
recordSetAsSchema :: RecordSet -> J.Value
|
||||||
|
recordSetAsSchema rs@(RecordSet {rows}) =
|
||||||
|
recordSetAsHeaderAndRows $
|
||||||
|
rs { rows = OMap.adjust
|
||||||
|
(TextOutputValue . LT.toStrict . encodeToLazyText . J.toJSON)
|
||||||
|
(Plan.FieldName "columns") <$> rows }
|
131
server/src-lib/Hasura/Backends/BigQuery/DDL/Source.hs
Normal file
131
server/src-lib/Hasura/Backends/BigQuery/DDL/Source.hs
Normal file
@ -0,0 +1,131 @@
|
|||||||
|
{-# LANGUAGE DisambiguateRecordFields #-}
|
||||||
|
{-# LANGUAGE DuplicateRecordFields #-}
|
||||||
|
|
||||||
|
module Hasura.Backends.BigQuery.DDL.Source
|
||||||
|
( resolveSource
|
||||||
|
, postDropSourceHook
|
||||||
|
, resolveSourceConfig
|
||||||
|
)
|
||||||
|
where
|
||||||
|
|
||||||
|
|
||||||
|
import Control.Concurrent.MVar (newMVar)
|
||||||
|
import qualified Data.Environment as Env
|
||||||
|
import qualified Data.HashMap.Strict as HM
|
||||||
|
import qualified Data.Text as T
|
||||||
|
import Data.Time.Clock.System
|
||||||
|
|
||||||
|
import Hasura.Backends.BigQuery.Connection
|
||||||
|
import Hasura.Backends.BigQuery.Instances.Types ()
|
||||||
|
import Hasura.Backends.BigQuery.Meta
|
||||||
|
import Hasura.Backends.BigQuery.Source
|
||||||
|
import Hasura.Backends.BigQuery.Types
|
||||||
|
import Hasura.Prelude
|
||||||
|
import Hasura.RQL.Types.Column
|
||||||
|
import Hasura.RQL.Types.Common
|
||||||
|
import Hasura.RQL.Types.Error
|
||||||
|
import Hasura.RQL.Types.Source
|
||||||
|
import Hasura.RQL.Types.Table
|
||||||
|
import Hasura.SQL.Backend
|
||||||
|
|
||||||
|
|
||||||
|
resolveSourceConfig ::
|
||||||
|
MonadIO m =>
|
||||||
|
SourceName ->
|
||||||
|
BigQueryConnSourceConfig ->
|
||||||
|
m (Either QErr BigQuerySourceConfig)
|
||||||
|
resolveSourceConfig _name BigQueryConnSourceConfig{..} = runExceptT $ do
|
||||||
|
env <- liftIO Env.getEnvironment
|
||||||
|
eSA <- resolveConfigurationJson env _cscServiceAccount
|
||||||
|
case eSA of
|
||||||
|
Left e -> throw400 Unexpected $ T.pack e
|
||||||
|
Right _scServiceAccount -> do
|
||||||
|
_scDatasets <- resolveConfigurationInputs env _cscDatasets
|
||||||
|
_scProjectId <- resolveConfigurationInput env _cscProjectId
|
||||||
|
trMVar <- liftIO $ newMVar Nothing -- `runBigQuery` initializes the token
|
||||||
|
pure BigQuerySourceConfig
|
||||||
|
{ _scAccessTokenMVar = trMVar
|
||||||
|
, ..
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
resolveSource
|
||||||
|
:: (MonadIO m)
|
||||||
|
=> BigQuerySourceConfig
|
||||||
|
-> m (Either QErr (ResolvedSource 'BigQuery))
|
||||||
|
resolveSource sourceConfig =
|
||||||
|
runExceptT $ do
|
||||||
|
result <- getTables sourceConfig
|
||||||
|
case result of
|
||||||
|
Left err ->
|
||||||
|
throw400 Unexpected $
|
||||||
|
"unexpected exception while connecting to database: " <> tshow err
|
||||||
|
Right restTables -> do
|
||||||
|
seconds <- liftIO $ fmap systemSeconds getSystemTime
|
||||||
|
pure
|
||||||
|
(ResolvedSource
|
||||||
|
{ _rsConfig = sourceConfig
|
||||||
|
, _rsTables =
|
||||||
|
HM.fromList
|
||||||
|
[ ( restTableReferenceToTableName tableReference
|
||||||
|
, DBTableMetadata
|
||||||
|
{ _ptmiOid = OID (fromIntegral seconds + index :: Int) -- TODO: The seconds are used for uniqueness. BigQuery doesn't support a "stable" ID for a table.
|
||||||
|
, _ptmiColumns =
|
||||||
|
[ RawColumnInfo
|
||||||
|
{ prciName = ColumnName name
|
||||||
|
, prciPosition = position
|
||||||
|
, prciType = restTypeToScalarType type'
|
||||||
|
, prciIsNullable =
|
||||||
|
case mode of
|
||||||
|
Nullable -> True
|
||||||
|
_ -> False
|
||||||
|
, prciDescription = Nothing
|
||||||
|
}
|
||||||
|
| (position, RestFieldSchema {name, type', mode}) <-
|
||||||
|
zip [1 ..] fields -- TODO: Same trouble as Oid above.
|
||||||
|
]
|
||||||
|
, _ptmiPrimaryKey = Nothing
|
||||||
|
, _ptmiUniqueConstraints = mempty
|
||||||
|
, _ptmiForeignKeys = mempty
|
||||||
|
, _ptmiViewInfo = Just $ ViewInfo False False False
|
||||||
|
, _ptmiDescription = Nothing
|
||||||
|
})
|
||||||
|
| (index, RestTable {tableReference, schema}) <-
|
||||||
|
zip [0 ..] restTables
|
||||||
|
, let RestTableSchema fields = schema
|
||||||
|
]
|
||||||
|
, _rsFunctions = mempty
|
||||||
|
, _rsPgScalars = mempty
|
||||||
|
})
|
||||||
|
|
||||||
|
restTypeToScalarType :: RestType -> ScalarType
|
||||||
|
restTypeToScalarType =
|
||||||
|
\case
|
||||||
|
STRING -> StringScalarType
|
||||||
|
BYTES -> BytesScalarType
|
||||||
|
INTEGER -> IntegerScalarType
|
||||||
|
FLOAT -> FloatScalarType
|
||||||
|
BOOL -> BoolScalarType
|
||||||
|
TIMESTAMP -> TimestampScalarType
|
||||||
|
DATE -> DateScalarType
|
||||||
|
TIME -> TimeScalarType
|
||||||
|
DATETIME -> DatetimeScalarType
|
||||||
|
GEOGRAPHY -> GeographyScalarType
|
||||||
|
STRUCT -> StructScalarType
|
||||||
|
BIGDECIMAL -> BigDecimalScalarType
|
||||||
|
DECIMAL -> DecimalScalarType
|
||||||
|
|
||||||
|
-- Hierarchy: Project / Dataset / Table
|
||||||
|
-- see <https://cloud.google.com/bigquery/docs/datasets-intro>
|
||||||
|
restTableReferenceToTableName :: RestTableReference -> TableName
|
||||||
|
restTableReferenceToTableName RestTableReference {..} =
|
||||||
|
TableName {tableName = tableId, tableNameSchema = datasetId}
|
||||||
|
-- We ignore project id and push that requirement up to the top to
|
||||||
|
-- the data source level.
|
||||||
|
|
||||||
|
postDropSourceHook
|
||||||
|
:: (MonadIO m)
|
||||||
|
=> BigQuerySourceConfig -> m ()
|
||||||
|
postDropSourceHook _ =
|
||||||
|
-- On BigQuery we don't keep connections open.
|
||||||
|
pure ()
|
1045
server/src-lib/Hasura/Backends/BigQuery/DataLoader/Execute.hs
Normal file
1045
server/src-lib/Hasura/Backends/BigQuery/DataLoader/Execute.hs
Normal file
File diff suppressed because it is too large
Load Diff
427
server/src-lib/Hasura/Backends/BigQuery/DataLoader/Plan.hs
Normal file
427
server/src-lib/Hasura/Backends/BigQuery/DataLoader/Plan.hs
Normal file
@ -0,0 +1,427 @@
|
|||||||
|
{-# LANGUAGE DuplicateRecordFields #-}
|
||||||
|
|
||||||
|
-- | Make a plan for the data loader to execute (.Execute).
|
||||||
|
|
||||||
|
module Hasura.Backends.BigQuery.DataLoader.Plan
|
||||||
|
( prettyPlanned
|
||||||
|
, runPlan
|
||||||
|
, planSelectHeadAndTail
|
||||||
|
, actionsForest
|
||||||
|
, drawActionsForest
|
||||||
|
, drawActionsForestSQL
|
||||||
|
, Ref
|
||||||
|
, PlannedAction(..)
|
||||||
|
, Action(..)
|
||||||
|
, Select(..)
|
||||||
|
, Join(..)
|
||||||
|
, Relationship(..)
|
||||||
|
, FieldName(..)
|
||||||
|
, HeadAndTail(..)
|
||||||
|
, selectQuery
|
||||||
|
, printDrawPlannedActions
|
||||||
|
) where
|
||||||
|
|
||||||
|
import Hasura.Prelude hiding (head, second, tail, tell)
|
||||||
|
|
||||||
|
import qualified Data.ByteString.Char8 as S8
|
||||||
|
import qualified Data.ByteString.Lazy.Char8 as L8
|
||||||
|
import qualified Data.HashMap.Strict.InsOrd as OMap
|
||||||
|
import qualified Data.List as List
|
||||||
|
import qualified Data.Text as T
|
||||||
|
import qualified Data.Text.Encoding as T
|
||||||
|
import qualified Data.Text.Lazy as LT
|
||||||
|
import qualified Data.Text.Lazy.Builder as LT
|
||||||
|
import qualified Data.Text.Lazy.Encoding as LT
|
||||||
|
|
||||||
|
import Data.Aeson
|
||||||
|
import Data.Bifunctor
|
||||||
|
import Data.Graph
|
||||||
|
import Data.Sequence (Seq (..))
|
||||||
|
import Data.String
|
||||||
|
import Data.Tree
|
||||||
|
|
||||||
|
import qualified Hasura.Backends.BigQuery.ToQuery as ToQuery
|
||||||
|
import qualified Hasura.Backends.BigQuery.Types as BigQuery
|
||||||
|
|
||||||
|
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
-- Types
|
||||||
|
|
||||||
|
data Ref = Ref
|
||||||
|
{ idx :: !Int
|
||||||
|
, text :: !Text
|
||||||
|
} deriving (Show, Eq, Generic, Ord)
|
||||||
|
instance Hashable Ref
|
||||||
|
|
||||||
|
data PlanState = PlanState
|
||||||
|
{ actions :: !(Seq PlannedAction)
|
||||||
|
, counter :: !Int
|
||||||
|
}
|
||||||
|
|
||||||
|
data PlannedAction = PlannedAction
|
||||||
|
{ ref :: Ref
|
||||||
|
, action :: Action
|
||||||
|
} deriving (Show)
|
||||||
|
|
||||||
|
newtype Plan a = Plan
|
||||||
|
{ unPlan :: State PlanState a
|
||||||
|
} deriving (Functor, Applicative, Monad, MonadState PlanState)
|
||||||
|
|
||||||
|
data Action
|
||||||
|
= SelectAction Select
|
||||||
|
| JoinAction Join
|
||||||
|
deriving (Show)
|
||||||
|
|
||||||
|
data Select = Select
|
||||||
|
{ selectTop :: !BigQuery.Top
|
||||||
|
, selectProjections :: !(NonEmpty BigQuery.Projection)
|
||||||
|
, selectFrom :: !BigQuery.From
|
||||||
|
, selectGroupBy :: ![BigQuery.FieldName]
|
||||||
|
, selectWhere :: !BigQuery.Where
|
||||||
|
, selectOrderBy :: !(Maybe (NonEmpty BigQuery.OrderBy))
|
||||||
|
, selectOffset :: !(Maybe BigQuery.Expression)
|
||||||
|
, selectRelationship :: !(Maybe Relationship)
|
||||||
|
, selectSqlJoins :: ![BigQuery.Join]
|
||||||
|
, selectHaskellJoins :: ![BigQuery.Join]
|
||||||
|
, selectAggUnwrap :: !(Maybe Text)
|
||||||
|
, wantedFields :: !(Maybe [Text])
|
||||||
|
} deriving (Show)
|
||||||
|
|
||||||
|
data Relationship = Relationship
|
||||||
|
{ leftRecordSet :: Ref
|
||||||
|
, onFields :: [(FieldName, FieldName)]
|
||||||
|
, rightTable :: BigQuery.EntityAlias
|
||||||
|
} deriving (Show)
|
||||||
|
|
||||||
|
newtype FieldName =
|
||||||
|
FieldName Text
|
||||||
|
deriving (Show, Ord, Eq, Hashable, FromJSON, ToJSONKey, IsString)
|
||||||
|
|
||||||
|
data Join = Join
|
||||||
|
{ joinOn :: [(FieldName,FieldName)]
|
||||||
|
, leftRecordSet :: Ref
|
||||||
|
, rightRecordSet :: Ref
|
||||||
|
, joinProvenance :: BigQuery.JoinProvenance
|
||||||
|
, joinFieldName :: !Text
|
||||||
|
, joinExtractPath :: !(Maybe Text)
|
||||||
|
, wantedFields :: !(Maybe [Text])
|
||||||
|
} deriving (Show)
|
||||||
|
|
||||||
|
data HeadAndTail = HeadAndTail
|
||||||
|
{ head :: Ref
|
||||||
|
, tail :: Ref
|
||||||
|
}
|
||||||
|
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
-- Run planner
|
||||||
|
|
||||||
|
runPlan :: Plan r -> (r, [PlannedAction])
|
||||||
|
runPlan =
|
||||||
|
second (toList . actions) .
|
||||||
|
flip runState (PlanState {actions = mempty, counter = 0}) . unPlan
|
||||||
|
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
-- Planners
|
||||||
|
|
||||||
|
planSelectHeadAndTail :: Maybe Relationship -> Maybe Text -> BigQuery.Select -> Plan HeadAndTail
|
||||||
|
planSelectHeadAndTail relationship joinExtractPath select0 = do
|
||||||
|
ref <- generate (selectFromName (BigQuery.selectFrom select0))
|
||||||
|
let select = fromSelect relationship joinExtractPath select0
|
||||||
|
action = SelectAction select
|
||||||
|
tell PlannedAction {ref, action}
|
||||||
|
joinsFinalRef <- foldM planJoin ref (selectHaskellJoins select)
|
||||||
|
pure
|
||||||
|
(let head = ref
|
||||||
|
tail = case selectHaskellJoins select of
|
||||||
|
[] -> ref
|
||||||
|
_ -> joinsFinalRef
|
||||||
|
in HeadAndTail {head,tail})
|
||||||
|
|
||||||
|
planJoin :: Ref -> BigQuery.Join -> Plan Ref
|
||||||
|
planJoin leftRecordSet BigQuery.Join {..} = do
|
||||||
|
ref <- generate (joinAliasName joinAlias)
|
||||||
|
let joinFields = fmap (bimap toFieldName toFieldName) joinOn
|
||||||
|
rightRecordSet <-
|
||||||
|
case joinSource of
|
||||||
|
BigQuery.JoinSelect select ->
|
||||||
|
fmap
|
||||||
|
(\HeadAndTail {..} -> tail)
|
||||||
|
(planSelectHeadAndTail
|
||||||
|
(Just
|
||||||
|
(Relationship
|
||||||
|
{ leftRecordSet
|
||||||
|
, onFields = joinFields
|
||||||
|
, rightTable = joinRightTable
|
||||||
|
}))
|
||||||
|
joinExtractPath
|
||||||
|
select)
|
||||||
|
let action =
|
||||||
|
JoinAction
|
||||||
|
Join
|
||||||
|
{ leftRecordSet
|
||||||
|
, rightRecordSet
|
||||||
|
, joinOn = joinFields
|
||||||
|
, wantedFields
|
||||||
|
, ..
|
||||||
|
}
|
||||||
|
tell PlannedAction {ref, action}
|
||||||
|
pure ref
|
||||||
|
where
|
||||||
|
BigQuery.JoinSelect BigQuery.Select {selectFinalWantedFields = wantedFields} =
|
||||||
|
joinSource
|
||||||
|
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
-- Conversions
|
||||||
|
|
||||||
|
-- TODO: Check this. We're intentionally discarding the table
|
||||||
|
-- qualification.
|
||||||
|
toFieldName :: BigQuery.FieldName -> FieldName
|
||||||
|
toFieldName (BigQuery.FieldName {fieldName = t}) = FieldName t
|
||||||
|
|
||||||
|
joinAliasName :: BigQuery.EntityAlias -> Text
|
||||||
|
joinAliasName (BigQuery.EntityAlias {entityAliasText}) = entityAliasText
|
||||||
|
|
||||||
|
selectFromName :: BigQuery.From -> Text
|
||||||
|
selectFromName (BigQuery.FromQualifiedTable (BigQuery.Aliased {aliasedThing = BigQuery.TableName {tableName}})) =
|
||||||
|
tableName
|
||||||
|
|
||||||
|
fromSelect :: Maybe Relationship -> Maybe Text -> BigQuery.Select -> Select
|
||||||
|
fromSelect selectRelationship selectAggUnwrap BigQuery.Select {..} =
|
||||||
|
Select
|
||||||
|
{ selectSqlJoins =
|
||||||
|
mapMaybe
|
||||||
|
(\case
|
||||||
|
j@BigQuery.Join {joinProvenance = BigQuery.OrderByJoinProvenance} ->
|
||||||
|
pure j
|
||||||
|
j@BigQuery.Join {joinProvenance = BigQuery.ArrayAggregateJoinProvenance} ->
|
||||||
|
pure j
|
||||||
|
_ -> Nothing)
|
||||||
|
selectJoins
|
||||||
|
, selectHaskellJoins =
|
||||||
|
mapMaybe
|
||||||
|
(\case
|
||||||
|
BigQuery.Join {joinProvenance = BigQuery.OrderByJoinProvenance} ->
|
||||||
|
Nothing
|
||||||
|
BigQuery.Join {joinProvenance = BigQuery.ArrayAggregateJoinProvenance} ->
|
||||||
|
Nothing
|
||||||
|
j -> pure j)
|
||||||
|
selectJoins
|
||||||
|
, wantedFields = selectFinalWantedFields
|
||||||
|
, ..
|
||||||
|
}
|
||||||
|
|
||||||
|
tell :: PlannedAction -> Plan ()
|
||||||
|
tell action = modify' (\s -> s {actions = actions s :|> action})
|
||||||
|
|
||||||
|
generate :: Text -> Plan Ref
|
||||||
|
generate text = do
|
||||||
|
idx <- gets counter
|
||||||
|
modify' (\s -> s {counter = counter s + 1})
|
||||||
|
pure (Ref {idx, text})
|
||||||
|
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
-- Plan pretty printer
|
||||||
|
|
||||||
|
prettyPlanned :: [PlannedAction] -> IO ()
|
||||||
|
prettyPlanned =
|
||||||
|
L8.putStrLn .
|
||||||
|
LT.encodeUtf8 . LT.toLazyText . mconcat . List.intersperse "\n\n" . map prettyPlannedAction
|
||||||
|
|
||||||
|
prettyPlannedActionsSQL :: PlannedAction -> Maybe LT.Builder
|
||||||
|
prettyPlannedActionsSQL PlannedAction {action} =
|
||||||
|
case action of
|
||||||
|
JoinAction {} -> Nothing
|
||||||
|
SelectAction select -> pure query
|
||||||
|
where (query, _params) =
|
||||||
|
ToQuery.renderBuilderPretty
|
||||||
|
(ToQuery.fromSelect (selectQuery select))
|
||||||
|
|
||||||
|
prettyPlannedAction :: PlannedAction -> LT.Builder
|
||||||
|
prettyPlannedAction PlannedAction {ref, action} =
|
||||||
|
case action of
|
||||||
|
SelectAction select ->
|
||||||
|
mconcat
|
||||||
|
(List.intersperse
|
||||||
|
"\n"
|
||||||
|
(mconcat
|
||||||
|
(filter
|
||||||
|
(/= mempty)
|
||||||
|
[ [ "Load " <> prettyFrom (selectFrom select) <> " producing " <>
|
||||||
|
prettyRef ref
|
||||||
|
]
|
||||||
|
, [ "Fields: " <>
|
||||||
|
mconcat
|
||||||
|
(List.intersperse
|
||||||
|
", "
|
||||||
|
(map
|
||||||
|
prettyProjection
|
||||||
|
(toList (selectProjections select))))
|
||||||
|
]
|
||||||
|
, case selectRelationship select of
|
||||||
|
Nothing -> []
|
||||||
|
Just relationship -> [prettyRelationship relationship]
|
||||||
|
, (map prettyJoin (selectSqlJoins select))
|
||||||
|
, case selectTop select of
|
||||||
|
BigQuery.NoTop -> []
|
||||||
|
BigQuery.Top top ->
|
||||||
|
["Limit " <> LT.fromText (tshow top)]
|
||||||
|
, ["SQL:"]
|
||||||
|
, [query]
|
||||||
|
, ["Params: " | not (null params)]
|
||||||
|
, map
|
||||||
|
(\(idx :: Int, value) ->
|
||||||
|
fromString (show idx) <> "=" <> fromString (show value))
|
||||||
|
(OMap.toList params)
|
||||||
|
])))
|
||||||
|
where (query, params) =
|
||||||
|
ToQuery.renderBuilderPretty
|
||||||
|
(ToQuery.fromSelect (selectQuery select))
|
||||||
|
JoinAction Join {leftRecordSet, rightRecordSet, joinOn} ->
|
||||||
|
mconcat
|
||||||
|
(List.intersperse
|
||||||
|
"\n"
|
||||||
|
[ "Join " <> prettyRef leftRecordSet <> " with " <>
|
||||||
|
prettyRef rightRecordSet <>
|
||||||
|
" producing " <>
|
||||||
|
prettyRef ref
|
||||||
|
, "On " <> prettyJoinFields joinOn
|
||||||
|
])
|
||||||
|
|
||||||
|
prettyRef :: Ref -> LT.Builder
|
||||||
|
prettyRef Ref {..} = "#" <> LT.fromText (text <> tshow idx)
|
||||||
|
|
||||||
|
prettyFrom :: BigQuery.From -> LT.Builder
|
||||||
|
prettyFrom =
|
||||||
|
\case
|
||||||
|
BigQuery.FromQualifiedTable aliased ->
|
||||||
|
prettyAliased
|
||||||
|
(fmap
|
||||||
|
(\BigQuery.TableName {tableName = t} -> (LT.fromText t))
|
||||||
|
aliased)
|
||||||
|
|
||||||
|
prettyJoin :: BigQuery.Join -> LT.Builder
|
||||||
|
prettyJoin BigQuery.Join {..} =
|
||||||
|
"SQL join with " <> src <> " on " <> prettyJoinFields' joinOn <> " for " <>
|
||||||
|
reason
|
||||||
|
where
|
||||||
|
reason =
|
||||||
|
case joinProvenance of
|
||||||
|
BigQuery.OrderByJoinProvenance -> "order by"
|
||||||
|
BigQuery.ObjectJoinProvenance -> "object relation"
|
||||||
|
BigQuery.ArrayAggregateJoinProvenance -> "array aggregate relation"
|
||||||
|
BigQuery.ArrayJoinProvenance -> "array relation"
|
||||||
|
BigQuery.MultiplexProvenance -> "multiplex"
|
||||||
|
src =
|
||||||
|
case joinSource of
|
||||||
|
BigQuery.JoinSelect select -> prettyFrom (BigQuery.selectFrom select)
|
||||||
|
|
||||||
|
prettyJoinFields' :: [(BigQuery.FieldName, BigQuery.FieldName)] -> LT.Builder
|
||||||
|
prettyJoinFields' onFields =
|
||||||
|
mconcat
|
||||||
|
(List.intersperse
|
||||||
|
", "
|
||||||
|
(map
|
||||||
|
(\(left, right) ->
|
||||||
|
"(" <> prettyFieldName' left <> " = " <> prettyFieldName' right <>
|
||||||
|
")")
|
||||||
|
onFields))
|
||||||
|
|
||||||
|
prettyRelationship :: Relationship -> LT.Builder
|
||||||
|
prettyRelationship Relationship {leftRecordSet, onFields} =
|
||||||
|
"Relationship: " <> prettyRef leftRecordSet <> " on " <>
|
||||||
|
prettyJoinFields onFields
|
||||||
|
|
||||||
|
prettyJoinFields :: [(FieldName, FieldName)] -> LT.Builder
|
||||||
|
prettyJoinFields onFields =
|
||||||
|
mconcat
|
||||||
|
(List.intersperse
|
||||||
|
", "
|
||||||
|
(map
|
||||||
|
(\(left, right) ->
|
||||||
|
"(" <> prettyFieldName left <> " = " <> prettyFieldName right <>
|
||||||
|
")")
|
||||||
|
onFields))
|
||||||
|
|
||||||
|
prettyFieldName :: FieldName -> LT.Builder
|
||||||
|
prettyFieldName (FieldName t) = LT.fromText t
|
||||||
|
|
||||||
|
prettyProjection :: BigQuery.Projection -> LT.Builder
|
||||||
|
prettyProjection =
|
||||||
|
\case
|
||||||
|
BigQuery.ExpressionProjection aliased ->
|
||||||
|
prettyAliased (fmap (\_e -> "<Expression" <> ">") aliased)
|
||||||
|
BigQuery.FieldNameProjection aliased ->
|
||||||
|
prettyAliased (fmap prettyFieldName' aliased)
|
||||||
|
BigQuery.AggregateProjection aliased ->
|
||||||
|
prettyAliased (fmap (const "<Aggregate>") aliased)
|
||||||
|
BigQuery.StarProjection -> "*"
|
||||||
|
BigQuery.ArrayAggProjection {} -> "<ArrayAgg>"
|
||||||
|
BigQuery.EntityProjection aliased ->
|
||||||
|
prettyAliased (fmap (LT.fromText . joinAliasName) aliased)
|
||||||
|
|
||||||
|
prettyAliased :: BigQuery.Aliased LT.Builder -> LT.Builder
|
||||||
|
prettyAliased BigQuery.Aliased {aliasedThing, aliasedAlias} =
|
||||||
|
aliasedThing <> " as " <> LT.fromText aliasedAlias
|
||||||
|
|
||||||
|
prettyFieldName' :: BigQuery.FieldName -> LT.Builder
|
||||||
|
prettyFieldName' (BigQuery.FieldName {fieldName, fieldNameEntity}) =
|
||||||
|
LT.fromText (fieldNameEntity <> "." <> fieldName)
|
||||||
|
|
||||||
|
printDrawPlannedActions :: [PlannedAction] -> IO ()
|
||||||
|
printDrawPlannedActions = S8.putStrLn . T.encodeUtf8 . drawPlannedActions
|
||||||
|
|
||||||
|
drawActionsForest :: Forest PlannedAction -> Text
|
||||||
|
drawActionsForest =
|
||||||
|
T.pack . drawForest .
|
||||||
|
fmap (fmap (LT.unpack . LT.toLazyText . prettyPlannedAction))
|
||||||
|
|
||||||
|
drawActionsForestSQL :: Forest PlannedAction -> Text
|
||||||
|
drawActionsForestSQL =
|
||||||
|
T.intercalate ";\n\n" .
|
||||||
|
mapMaybe (fmap (LT.toStrict . LT.toLazyText) . prettyPlannedActionsSQL) .
|
||||||
|
foldMap toList
|
||||||
|
|
||||||
|
drawPlannedActions :: [PlannedAction] -> Text
|
||||||
|
drawPlannedActions =
|
||||||
|
drawActionsForest . actionsForest id
|
||||||
|
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
-- Graphing the plan to a forest
|
||||||
|
|
||||||
|
actionsForest :: (Graph -> Graph) -> [PlannedAction] -> Forest PlannedAction
|
||||||
|
actionsForest transform actions =
|
||||||
|
let (graph, vertex2Node, _key2Vertex) =
|
||||||
|
graphFromEdges
|
||||||
|
(map
|
||||||
|
(\PlannedAction {ref, action} ->
|
||||||
|
( action
|
||||||
|
, ref
|
||||||
|
, map
|
||||||
|
(\PlannedAction {ref = r} -> r)
|
||||||
|
(filter (elem ref . plannedActionRefs) actions)))
|
||||||
|
actions)
|
||||||
|
in fmap
|
||||||
|
(fmap
|
||||||
|
((\(action, ref, _refs) -> PlannedAction {ref, action}) . vertex2Node))
|
||||||
|
(dff (transform graph))
|
||||||
|
where
|
||||||
|
plannedActionRefs PlannedAction {action} =
|
||||||
|
case action of
|
||||||
|
SelectAction Select {selectRelationship} ->
|
||||||
|
case selectRelationship of
|
||||||
|
Just Relationship {leftRecordSet} -> [leftRecordSet]
|
||||||
|
Nothing -> mempty
|
||||||
|
JoinAction Join {leftRecordSet, rightRecordSet} ->
|
||||||
|
[leftRecordSet, rightRecordSet]
|
||||||
|
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
-- Build a query
|
||||||
|
|
||||||
|
selectQuery :: Select -> BigQuery.Select
|
||||||
|
selectQuery Select {..} =
|
||||||
|
BigQuery.Select
|
||||||
|
{ selectFor = BigQuery.NoFor
|
||||||
|
, selectJoins = selectSqlJoins
|
||||||
|
, selectFinalWantedFields = wantedFields
|
||||||
|
, ..
|
||||||
|
}
|
1063
server/src-lib/Hasura/Backends/BigQuery/FromIr.hs
Normal file
1063
server/src-lib/Hasura/Backends/BigQuery/FromIr.hs
Normal file
File diff suppressed because it is too large
Load Diff
121
server/src-lib/Hasura/Backends/BigQuery/Instances/Execute.hs
Normal file
121
server/src-lib/Hasura/Backends/BigQuery/Instances/Execute.hs
Normal file
@ -0,0 +1,121 @@
|
|||||||
|
{-# OPTIONS_GHC -fno-warn-orphans #-}
|
||||||
|
|
||||||
|
module Hasura.Backends.BigQuery.Instances.Execute () where
|
||||||
|
|
||||||
|
import qualified Data.Aeson as Aeson
|
||||||
|
import qualified Data.HashMap.Strict.InsOrd as OMap
|
||||||
|
import qualified Hasura.Backends.BigQuery.DataLoader.Execute as DataLoader
|
||||||
|
import qualified Hasura.Backends.BigQuery.DataLoader.Plan as DataLoader
|
||||||
|
import Hasura.EncJSON
|
||||||
|
import Hasura.Prelude
|
||||||
|
import qualified Hasura.RQL.Types.Error as RQL
|
||||||
|
import qualified Hasura.SQL.AnyBackend as AB
|
||||||
|
|
||||||
|
import qualified Data.Environment as Env
|
||||||
|
import qualified Language.GraphQL.Draft.Syntax as G
|
||||||
|
import qualified Network.HTTP.Client as HTTP
|
||||||
|
import qualified Network.HTTP.Types as HTTP
|
||||||
|
|
||||||
|
import qualified Hasura.Tracing as Tracing
|
||||||
|
|
||||||
|
import Hasura.Backends.BigQuery.Plan
|
||||||
|
import Hasura.GraphQL.Context
|
||||||
|
import Hasura.GraphQL.Execute.Backend
|
||||||
|
import Hasura.GraphQL.Parser
|
||||||
|
import Hasura.RQL.Types
|
||||||
|
import Hasura.Session
|
||||||
|
|
||||||
|
-- MultiplexedQuery
|
||||||
|
instance BackendExecute 'BigQuery where
|
||||||
|
type PreparedQuery 'BigQuery = Text
|
||||||
|
type ExecutionMonad 'BigQuery = Tracing.TraceT (ExceptT QErr IO)
|
||||||
|
type MultiplexedQuery 'BigQuery = Void
|
||||||
|
getRemoteJoins = const []
|
||||||
|
|
||||||
|
mkDBQueryPlan = msDBQueryPlan
|
||||||
|
mkDBMutationPlan = msDBMutationPlan
|
||||||
|
mkDBSubscriptionPlan _ _ _ _ =
|
||||||
|
throwError $ RQL.internalError "Cannot currently perform subscriptions on BigQuery sources."
|
||||||
|
|
||||||
|
|
||||||
|
-- query
|
||||||
|
|
||||||
|
msDBQueryPlan
|
||||||
|
:: forall m.
|
||||||
|
( MonadError QErr m
|
||||||
|
)
|
||||||
|
=> Env.Environment
|
||||||
|
-> HTTP.Manager
|
||||||
|
-> [HTTP.Header]
|
||||||
|
-> UserInfo
|
||||||
|
-> [G.Directive G.Name]
|
||||||
|
-> SourceName
|
||||||
|
-> SourceConfig 'BigQuery
|
||||||
|
-> QueryDB 'BigQuery (UnpreparedValue 'BigQuery)
|
||||||
|
-> m ExecutionStep
|
||||||
|
msDBQueryPlan _env _manager _reqHeaders userInfo _directives sourceName sourceConfig qrf = do
|
||||||
|
select <- planNoPlan userInfo qrf
|
||||||
|
let (!headAndTail, !plannedActionsList) =
|
||||||
|
DataLoader.runPlan
|
||||||
|
(DataLoader.planSelectHeadAndTail Nothing Nothing select)
|
||||||
|
!actionsForest = DataLoader.actionsForest id plannedActionsList
|
||||||
|
let action = do
|
||||||
|
result <-
|
||||||
|
DataLoader.runExecute
|
||||||
|
sourceConfig
|
||||||
|
headAndTail
|
||||||
|
(DataLoader.execute actionsForest)
|
||||||
|
case result of
|
||||||
|
Left err -> throw500WithDetail "dataLoader error" $ Aeson.toJSON $ show err
|
||||||
|
Right recordSet -> pure $! recordSetToEncJSON recordSet
|
||||||
|
pure
|
||||||
|
$ ExecStepDB []
|
||||||
|
. AB.mkAnyBackend
|
||||||
|
$ DBStepInfo sourceName sourceConfig (Just (DataLoader.drawActionsForest actionsForest)) action
|
||||||
|
|
||||||
|
-- | Convert the dataloader's 'RecordSet' type to JSON.
|
||||||
|
recordSetToEncJSON :: DataLoader.RecordSet -> EncJSON
|
||||||
|
recordSetToEncJSON DataLoader.RecordSet {rows} =
|
||||||
|
encJFromList (toList (fmap encJFromRecord rows))
|
||||||
|
where
|
||||||
|
encJFromRecord =
|
||||||
|
encJFromInsOrdHashMap . fmap encJFromOutputValue . OMap.mapKeys coerce
|
||||||
|
encJFromOutputValue outputValue =
|
||||||
|
case outputValue of
|
||||||
|
DataLoader.NullOutputValue -> encJFromJValue Aeson.Null
|
||||||
|
DataLoader.DecimalOutputValue !i -> encJFromJValue i
|
||||||
|
DataLoader.BigDecimalOutputValue !i -> encJFromJValue i
|
||||||
|
DataLoader.FloatOutputValue !i -> encJFromJValue i
|
||||||
|
DataLoader.TextOutputValue !i -> encJFromJValue i
|
||||||
|
DataLoader.BytesOutputValue !i -> encJFromJValue i
|
||||||
|
DataLoader.DateOutputValue !i -> encJFromJValue i
|
||||||
|
DataLoader.TimestampOutputValue !i -> encJFromJValue i
|
||||||
|
DataLoader.TimeOutputValue !i -> encJFromJValue i
|
||||||
|
DataLoader.DatetimeOutputValue !i -> encJFromJValue i
|
||||||
|
DataLoader.GeographyOutputValue !i -> encJFromJValue i
|
||||||
|
DataLoader.BoolOutputValue !i -> encJFromJValue i
|
||||||
|
DataLoader.IntegerOutputValue !i -> encJFromJValue i
|
||||||
|
DataLoader.ArrayOutputValue !vector ->
|
||||||
|
encJFromList (toList (fmap encJFromOutputValue vector))
|
||||||
|
-- Really, the case below shouldn't be happening. But if it
|
||||||
|
-- does, it's not a problem either. The output will just have
|
||||||
|
-- a record in it.
|
||||||
|
DataLoader.RecordOutputValue !record -> encJFromRecord record
|
||||||
|
|
||||||
|
-- mutation
|
||||||
|
|
||||||
|
msDBMutationPlan
|
||||||
|
:: forall m.
|
||||||
|
( MonadError QErr m
|
||||||
|
)
|
||||||
|
=> Env.Environment
|
||||||
|
-> HTTP.Manager
|
||||||
|
-> [HTTP.Header]
|
||||||
|
-> UserInfo
|
||||||
|
-> Bool
|
||||||
|
-> SourceName
|
||||||
|
-> SourceConfig 'BigQuery
|
||||||
|
-> MutationDB 'BigQuery (UnpreparedValue 'BigQuery)
|
||||||
|
-> m ExecutionStep
|
||||||
|
msDBMutationPlan _env _manager _reqHeaders _userInfo _stringifyNum _sourceName _sourceConfig _mrf =
|
||||||
|
throw500 "mutations are not supported in BigQuery; this should be unreachable"
|
@ -0,0 +1,25 @@
|
|||||||
|
{-# OPTIONS_GHC -fno-warn-orphans #-}
|
||||||
|
|
||||||
|
-- |
|
||||||
|
|
||||||
|
module Hasura.Backends.BigQuery.Instances.Metadata where
|
||||||
|
|
||||||
|
import qualified Hasura.Backends.BigQuery.DDL as BigQuery
|
||||||
|
|
||||||
|
import Hasura.RQL.Types.Metadata.Backend
|
||||||
|
import Hasura.SQL.Backend
|
||||||
|
|
||||||
|
|
||||||
|
instance BackendMetadata 'BigQuery where
|
||||||
|
buildComputedFieldInfo = BigQuery.buildComputedFieldInfo
|
||||||
|
buildRemoteFieldInfo = BigQuery.buildRemoteFieldInfo
|
||||||
|
fetchAndValidateEnumValues = BigQuery.fetchAndValidateEnumValues
|
||||||
|
resolveSourceConfig = BigQuery.resolveSourceConfig
|
||||||
|
resolveDatabaseMetadata = BigQuery.resolveSource
|
||||||
|
createTableEventTrigger = BigQuery.createTableEventTrigger
|
||||||
|
buildEventTriggerInfo = BigQuery.buildEventTriggerInfo
|
||||||
|
parseBoolExpOperations = BigQuery.parseBoolExpOperations
|
||||||
|
buildFunctionInfo = BigQuery.buildFunctionInfo
|
||||||
|
updateColumnInEventTrigger = BigQuery.updateColumnInEventTrigger
|
||||||
|
parseCollectableType = BigQuery.parseCollectableType
|
||||||
|
postDropSourceHook = BigQuery.postDropSourceHook
|
362
server/src-lib/Hasura/Backends/BigQuery/Instances/Schema.hs
Normal file
362
server/src-lib/Hasura/Backends/BigQuery/Instances/Schema.hs
Normal file
@ -0,0 +1,362 @@
|
|||||||
|
{-# OPTIONS_GHC -fno-warn-orphans #-}
|
||||||
|
|
||||||
|
module Hasura.Backends.BigQuery.Instances.Schema () where
|
||||||
|
|
||||||
|
import qualified Data.Aeson as J
|
||||||
|
import qualified Data.HashMap.Strict as Map
|
||||||
|
import qualified Data.List.NonEmpty as NE
|
||||||
|
import qualified Data.Text as T
|
||||||
|
import Data.Text.Extended
|
||||||
|
import qualified Hasura.Backends.BigQuery.Types as BigQuery
|
||||||
|
import Hasura.GraphQL.Context
|
||||||
|
import qualified Hasura.GraphQL.Parser as P
|
||||||
|
import Hasura.GraphQL.Parser hiding (EnumValueInfo, field)
|
||||||
|
import Hasura.GraphQL.Parser.Internal.Parser hiding (field)
|
||||||
|
import Hasura.GraphQL.Schema.Backend
|
||||||
|
import qualified Hasura.GraphQL.Schema.Build as GSB
|
||||||
|
import Hasura.GraphQL.Schema.Common
|
||||||
|
import Hasura.Prelude
|
||||||
|
import qualified Hasura.RQL.IR.Select as IR
|
||||||
|
import qualified Hasura.RQL.IR.Update as IR
|
||||||
|
import Hasura.RQL.Types
|
||||||
|
import qualified Hasura.RQL.Types.Error as RQL
|
||||||
|
import qualified Language.GraphQL.Draft.Syntax as G
|
||||||
|
|
||||||
|
----------------------------------------------------------------
|
||||||
|
-- BackendSchema instance
|
||||||
|
|
||||||
|
instance BackendSchema 'BigQuery where
|
||||||
|
-- top level parsers
|
||||||
|
buildTableQueryFields = GSB.buildTableQueryFields
|
||||||
|
buildTableRelayQueryFields = msBuildTableRelayQueryFields
|
||||||
|
buildTableInsertMutationFields = msBuildTableInsertMutationFields
|
||||||
|
buildTableUpdateMutationFields = msBuildTableUpdateMutationFields
|
||||||
|
buildTableDeleteMutationFields = msBuildTableDeleteMutationFields
|
||||||
|
buildFunctionQueryFields = msBuildFunctionQueryFields
|
||||||
|
buildFunctionRelayQueryFields = msBuildFunctionRelayQueryFields
|
||||||
|
buildFunctionMutationFields = msBuildFunctionMutationFields
|
||||||
|
-- backend extensions
|
||||||
|
relayExtension = const Nothing
|
||||||
|
nodesAggExtension = const $ Just ()
|
||||||
|
-- indivdual components
|
||||||
|
columnParser = msColumnParser
|
||||||
|
jsonPathArg = msJsonPathArg
|
||||||
|
orderByOperators = msOrderByOperators
|
||||||
|
comparisonExps = msComparisonExps
|
||||||
|
updateOperators = msUpdateOperators
|
||||||
|
offsetParser = msOffsetParser
|
||||||
|
mkCountType = msMkCountType
|
||||||
|
aggregateOrderByCountType = BigQuery.IntegerScalarType
|
||||||
|
computedField = msComputedField
|
||||||
|
node = msNode
|
||||||
|
tableDistinctOn = msTableDistinctOn
|
||||||
|
remoteRelationshipField = msRemoteRelationshipField
|
||||||
|
-- SQL literals
|
||||||
|
columnDefaultValue = error "TODO: Make impossible by the type system. BigQuery doesn't support insertions."
|
||||||
|
|
||||||
|
----------------------------------------------------------------
|
||||||
|
-- Top level parsers
|
||||||
|
|
||||||
|
msBuildTableRelayQueryFields
|
||||||
|
:: MonadBuildSchema 'BigQuery r m n
|
||||||
|
=> SourceName
|
||||||
|
-> SourceConfig 'BigQuery
|
||||||
|
-> TableName 'BigQuery
|
||||||
|
-> TableInfo 'BigQuery
|
||||||
|
-> G.Name
|
||||||
|
-> NESeq (ColumnInfo 'BigQuery)
|
||||||
|
-> SelPermInfo 'BigQuery
|
||||||
|
-> m (Maybe (FieldParser n (QueryRootField UnpreparedValue)))
|
||||||
|
msBuildTableRelayQueryFields _sourceName _sourceInfo _tableName _tableInfo _gqlName _pkeyColumns _selPerms =
|
||||||
|
pure Nothing
|
||||||
|
|
||||||
|
msBuildTableInsertMutationFields
|
||||||
|
:: MonadBuildSchema 'BigQuery r m n
|
||||||
|
=> SourceName
|
||||||
|
-> SourceConfig 'BigQuery
|
||||||
|
-> TableName 'BigQuery
|
||||||
|
-> TableInfo 'BigQuery
|
||||||
|
-> G.Name
|
||||||
|
-> InsPermInfo 'BigQuery
|
||||||
|
-> Maybe (SelPermInfo 'BigQuery)
|
||||||
|
-> Maybe (UpdPermInfo 'BigQuery)
|
||||||
|
-> m [FieldParser n (MutationRootField UnpreparedValue)]
|
||||||
|
msBuildTableInsertMutationFields _sourceName _sourceInfo _tableName _tableInfo _gqlName _insPerms _selPerms _updPerms =
|
||||||
|
pure []
|
||||||
|
|
||||||
|
msBuildTableUpdateMutationFields
|
||||||
|
:: MonadBuildSchema 'BigQuery r m n
|
||||||
|
=> SourceName
|
||||||
|
-> SourceConfig 'BigQuery
|
||||||
|
-> TableName 'BigQuery
|
||||||
|
-> TableInfo 'BigQuery
|
||||||
|
-> G.Name
|
||||||
|
-> UpdPermInfo 'BigQuery
|
||||||
|
-> Maybe (SelPermInfo 'BigQuery)
|
||||||
|
-> m [FieldParser n (MutationRootField UnpreparedValue)]
|
||||||
|
msBuildTableUpdateMutationFields _sourceName _sourceInfo _tableName _tableInfo _gqlName _updPerns _selPerms =
|
||||||
|
pure []
|
||||||
|
|
||||||
|
msBuildTableDeleteMutationFields
|
||||||
|
:: MonadBuildSchema 'BigQuery r m n
|
||||||
|
=> SourceName
|
||||||
|
-> SourceConfig 'BigQuery
|
||||||
|
-> TableName 'BigQuery
|
||||||
|
-> TableInfo 'BigQuery
|
||||||
|
-> G.Name
|
||||||
|
-> DelPermInfo 'BigQuery
|
||||||
|
-> Maybe (SelPermInfo 'BigQuery)
|
||||||
|
-> m [FieldParser n (MutationRootField UnpreparedValue)]
|
||||||
|
msBuildTableDeleteMutationFields _sourceName _sourceInfo _tableName _tableInfo _gqlName _delPerns _selPerms =
|
||||||
|
pure []
|
||||||
|
|
||||||
|
msBuildFunctionQueryFields
|
||||||
|
:: MonadBuildSchema 'BigQuery r m n
|
||||||
|
=> SourceName
|
||||||
|
-> SourceConfig 'BigQuery
|
||||||
|
-> FunctionName 'BigQuery
|
||||||
|
-> FunctionInfo 'BigQuery
|
||||||
|
-> TableName 'BigQuery
|
||||||
|
-> SelPermInfo 'BigQuery
|
||||||
|
-> m [FieldParser n (QueryRootField UnpreparedValue)]
|
||||||
|
msBuildFunctionQueryFields _ _ _ _ _ _ =
|
||||||
|
pure []
|
||||||
|
|
||||||
|
msBuildFunctionRelayQueryFields
|
||||||
|
:: MonadBuildSchema 'BigQuery r m n
|
||||||
|
=> SourceName
|
||||||
|
-> SourceConfig 'BigQuery
|
||||||
|
-> FunctionName 'BigQuery
|
||||||
|
-> FunctionInfo 'BigQuery
|
||||||
|
-> TableName 'BigQuery
|
||||||
|
-> NESeq (ColumnInfo 'BigQuery)
|
||||||
|
-> SelPermInfo 'BigQuery
|
||||||
|
-> m (Maybe (FieldParser n (QueryRootField UnpreparedValue)))
|
||||||
|
msBuildFunctionRelayQueryFields _sourceName _sourceInfo _functionName _functionInfo _tableName _pkeyColumns _selPerms =
|
||||||
|
pure Nothing
|
||||||
|
|
||||||
|
msBuildFunctionMutationFields
|
||||||
|
:: MonadBuildSchema 'BigQuery r m n
|
||||||
|
=> SourceName
|
||||||
|
-> SourceConfig 'BigQuery
|
||||||
|
-> FunctionName 'BigQuery
|
||||||
|
-> FunctionInfo 'BigQuery
|
||||||
|
-> TableName 'BigQuery
|
||||||
|
-> SelPermInfo 'BigQuery
|
||||||
|
-> m [FieldParser n (MutationRootField UnpreparedValue)]
|
||||||
|
msBuildFunctionMutationFields _ _ _ _ _ _ =
|
||||||
|
pure []
|
||||||
|
|
||||||
|
|
||||||
|
----------------------------------------------------------------
|
||||||
|
-- Individual components
|
||||||
|
|
||||||
|
msColumnParser
|
||||||
|
:: (MonadSchema n m, MonadError QErr m)
|
||||||
|
=> ColumnType 'BigQuery
|
||||||
|
-> G.Nullability
|
||||||
|
-> m (Parser 'Both n (Opaque (ColumnValue 'BigQuery)))
|
||||||
|
msColumnParser columnType (G.Nullability isNullable) =
|
||||||
|
opaque . fmap (ColumnValue columnType) <$> case columnType of
|
||||||
|
ColumnScalar scalarType -> case scalarType of
|
||||||
|
-- bytestrings
|
||||||
|
-- we only accept string literals
|
||||||
|
BigQuery.BytesScalarType -> pure $ possiblyNullable scalarType $ BigQuery.StringValue <$> P.string
|
||||||
|
-- text
|
||||||
|
BigQuery.StringScalarType -> pure $ possiblyNullable scalarType $ BigQuery.StringValue <$> P.string
|
||||||
|
-- floating point values
|
||||||
|
-- TODO: we do not perform size checks here, meaning we would accept an
|
||||||
|
-- out-of-bounds value as long as it can be represented by a GraphQL float; this
|
||||||
|
-- will in all likelihood error on the BigQuery side. Do we want to handle those
|
||||||
|
-- properly here?
|
||||||
|
BigQuery.FloatScalarType -> pure $ possiblyNullable scalarType $ BigQuery.FloatValue . BigQuery.doubleToFloat64 <$> P.float
|
||||||
|
-- Int types; we cram everything into Double at the moment
|
||||||
|
-- TODO: Distinguish between ints and doubles
|
||||||
|
BigQuery.IntegerScalarType -> pure $ possiblyNullable scalarType $ BigQuery.IntegerValue . BigQuery.intToInt64 . round <$> P.float
|
||||||
|
BigQuery.DecimalScalarType -> pure $ possiblyNullable scalarType $ BigQuery.DecimalValue . BigQuery.doubleToDecimal <$> P.float
|
||||||
|
BigQuery.BigDecimalScalarType -> pure $ possiblyNullable scalarType $ BigQuery.BigDecimalValue . BigQuery.doubleToBigDecimal <$> P.float
|
||||||
|
-- boolean type
|
||||||
|
BigQuery.BoolScalarType -> pure $ possiblyNullable scalarType $ BigQuery.BoolValue <$> P.boolean
|
||||||
|
BigQuery.DateScalarType -> pure $ possiblyNullable scalarType $ BigQuery.DateValue . BigQuery.Date <$> P.string
|
||||||
|
BigQuery.DatetimeScalarType -> pure $ possiblyNullable scalarType $ BigQuery.DatetimeValue . BigQuery.Datetime <$> P.string
|
||||||
|
BigQuery.GeographyScalarType -> pure $ possiblyNullable scalarType $ BigQuery.GeographyValue . BigQuery.Geography <$> P.string
|
||||||
|
BigQuery.TimestampScalarType -> pure $ possiblyNullable scalarType $ BigQuery.TimestampValue . BigQuery.Timestamp <$> P.string
|
||||||
|
ty -> throwError $ RQL.internalError $ T.pack $ "Type currently unsupported for BigQuery: " ++ show ty
|
||||||
|
ColumnEnumReference (EnumReference tableName enumValues) ->
|
||||||
|
case nonEmpty (Map.toList enumValues) of
|
||||||
|
Just enumValuesList -> do
|
||||||
|
tableGQLName <- tableGraphQLName tableName `onLeft` throwError
|
||||||
|
let enumName = tableGQLName <> $$(G.litName "_enum")
|
||||||
|
pure $ possiblyNullable BigQuery.StringScalarType $ P.enum enumName Nothing (mkEnumValue <$> enumValuesList)
|
||||||
|
Nothing -> throw400 ValidationFailed "empty enum values"
|
||||||
|
where
|
||||||
|
-- Sadly, this combinator is not sound in general, so we can’t export it
|
||||||
|
-- for general-purpose use. If we did, someone could write this:
|
||||||
|
--
|
||||||
|
-- mkParameter <$> opaque do
|
||||||
|
-- n <- int
|
||||||
|
-- pure (mkIntColumnValue (n + 1))
|
||||||
|
--
|
||||||
|
-- Now we’d end up with a UVParameter that has a variable in it, so we’d
|
||||||
|
-- parameterize over it. But when we’d reuse the plan, we wouldn’t know to
|
||||||
|
-- increment the value by 1, so we’d use the wrong value!
|
||||||
|
--
|
||||||
|
-- We could theoretically solve this by retaining a reference to the parser
|
||||||
|
-- itself and re-parsing each new value, using the saved parser, which
|
||||||
|
-- would admittedly be neat. But it’s more complicated, and it isn’t clear
|
||||||
|
-- that it would actually be useful, so for now we don’t support it.
|
||||||
|
opaque :: MonadParse m => Parser 'Both m a -> Parser 'Both m (Opaque a)
|
||||||
|
opaque parser = parser
|
||||||
|
{ pParser = \case
|
||||||
|
P.GraphQLValue (G.VVariable var@Variable{ vInfo, vValue }) -> do
|
||||||
|
typeCheck False (P.toGraphQLType $ pType parser) var
|
||||||
|
P.mkOpaque (Just vInfo) <$> pParser parser (absurd <$> vValue)
|
||||||
|
value -> P.mkOpaque Nothing <$> pParser parser value
|
||||||
|
}
|
||||||
|
possiblyNullable _scalarType
|
||||||
|
| isNullable = fmap (fromMaybe BigQuery.NullValue) . P.nullable
|
||||||
|
| otherwise = id
|
||||||
|
mkEnumValue :: (EnumValue, EnumValueInfo) -> (P.Definition P.EnumValueInfo, ScalarValue 'BigQuery)
|
||||||
|
mkEnumValue (EnumValue value, EnumValueInfo description) =
|
||||||
|
( P.mkDefinition value (G.Description <$> description) P.EnumValueInfo
|
||||||
|
, BigQuery.StringValue $ G.unName value
|
||||||
|
)
|
||||||
|
throughJSON scalarName =
|
||||||
|
let schemaType = P.NonNullable $ P.TNamed $ P.mkDefinition scalarName Nothing P.TIScalar
|
||||||
|
in Parser
|
||||||
|
{ pType = schemaType
|
||||||
|
, pParser =
|
||||||
|
valueToJSON (P.toGraphQLType schemaType) >=>
|
||||||
|
either (parseErrorWith ParseFailed . qeError) pure . runAesonParser J.parseJSON
|
||||||
|
}
|
||||||
|
|
||||||
|
msJsonPathArg
|
||||||
|
:: MonadParse n
|
||||||
|
=> ColumnType 'BigQuery
|
||||||
|
-> InputFieldsParser n (Maybe (IR.ColumnOp 'BigQuery))
|
||||||
|
msJsonPathArg _columnType = pure Nothing
|
||||||
|
|
||||||
|
msOrderByOperators
|
||||||
|
:: NonEmpty
|
||||||
|
( Definition P.EnumValueInfo
|
||||||
|
, (BasicOrderType 'BigQuery, NullsOrderType 'BigQuery)
|
||||||
|
)
|
||||||
|
msOrderByOperators = NE.fromList
|
||||||
|
[ ( define $$(G.litName "asc") "in ascending order, nulls first"
|
||||||
|
, (BigQuery.AscOrder, BigQuery.NullsFirst)
|
||||||
|
)
|
||||||
|
, ( define $$(G.litName "asc_nulls_first") "in ascending order, nulls first"
|
||||||
|
, (BigQuery.AscOrder, BigQuery.NullsFirst)
|
||||||
|
)
|
||||||
|
, ( define $$(G.litName "asc_nulls_last") "in ascending order, nulls last"
|
||||||
|
, (BigQuery.AscOrder, BigQuery.NullsLast)
|
||||||
|
)
|
||||||
|
, ( define $$(G.litName "desc") "in descending order, nulls last"
|
||||||
|
, (BigQuery.DescOrder, BigQuery.NullsLast)
|
||||||
|
)
|
||||||
|
, ( define $$(G.litName "desc_nulls_first") "in descending order, nulls first"
|
||||||
|
, (BigQuery.DescOrder, BigQuery.NullsFirst)
|
||||||
|
)
|
||||||
|
, ( define $$(G.litName "desc_nulls_last") "in descending order, nulls last"
|
||||||
|
, (BigQuery.DescOrder, BigQuery.NullsLast)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
where
|
||||||
|
define name desc = P.mkDefinition name (Just desc) P.EnumValueInfo
|
||||||
|
|
||||||
|
msComparisonExps
|
||||||
|
:: forall m n
|
||||||
|
. (BackendSchema 'BigQuery, MonadSchema n m, MonadError QErr m)
|
||||||
|
=> ColumnType 'BigQuery
|
||||||
|
-> m (Parser 'Input n [ComparisonExp 'BigQuery])
|
||||||
|
msComparisonExps = P.memoize 'comparisonExps $ \columnType -> do
|
||||||
|
-- see Note [Columns in comparison expression are never nullable]
|
||||||
|
typedParser <- columnParser columnType (G.Nullability False)
|
||||||
|
nullableTextParser <- columnParser (ColumnScalar BigQuery.StringScalarType) (G.Nullability True)
|
||||||
|
textParser <- columnParser (ColumnScalar BigQuery.StringScalarType) (G.Nullability False)
|
||||||
|
let name = P.getName typedParser <> $$(G.litName "_BigQuery_comparison_exp")
|
||||||
|
desc = G.Description $ "Boolean expression to compare columns of type "
|
||||||
|
<> P.getName typedParser
|
||||||
|
<<> ". All fields are combined with logical 'AND'."
|
||||||
|
textListParser = P.list textParser `P.bind` traverse P.openOpaque
|
||||||
|
columnListParser = P.list typedParser `P.bind` traverse P.openOpaque
|
||||||
|
pure $ P.object name (Just desc) $ catMaybes <$> sequenceA
|
||||||
|
[ P.fieldOptional $$(G.litName "_is_null") Nothing (bool ANISNOTNULL ANISNULL <$> P.boolean)
|
||||||
|
, P.fieldOptional $$(G.litName "_eq") Nothing (AEQ True . mkParameter <$> typedParser)
|
||||||
|
, P.fieldOptional $$(G.litName "_neq") Nothing (ANE True . mkParameter <$> typedParser)
|
||||||
|
, P.fieldOptional $$(G.litName "_gt") Nothing (AGT . mkParameter <$> typedParser)
|
||||||
|
, P.fieldOptional $$(G.litName "_lt") Nothing (ALT . mkParameter <$> typedParser)
|
||||||
|
, P.fieldOptional $$(G.litName "_gte") Nothing (AGTE . mkParameter <$> typedParser)
|
||||||
|
, P.fieldOptional $$(G.litName "_lte") Nothing (ALTE . mkParameter <$> typedParser)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
msOffsetParser :: MonadParse n => Parser 'Both n (SQLExpression 'BigQuery)
|
||||||
|
msOffsetParser =
|
||||||
|
BigQuery.ValueExpression . BigQuery.IntegerValue . BigQuery.intToInt64 . fromIntegral <$>
|
||||||
|
P.int
|
||||||
|
|
||||||
|
msMkCountType
|
||||||
|
:: Maybe Bool
|
||||||
|
-- ^ distinct values
|
||||||
|
-> Maybe [Column 'BigQuery]
|
||||||
|
-> CountType 'BigQuery
|
||||||
|
msMkCountType _ Nothing = BigQuery.StarCountable
|
||||||
|
msMkCountType (Just True) (Just cols) =
|
||||||
|
maybe BigQuery.StarCountable BigQuery.DistinctCountable $ nonEmpty cols
|
||||||
|
msMkCountType _ (Just cols) =
|
||||||
|
maybe BigQuery.StarCountable BigQuery.NonNullFieldCountable $ nonEmpty cols
|
||||||
|
|
||||||
|
-- | Argument to distinct select on columns returned from table selection
|
||||||
|
-- > distinct_on: [table_select_column!]
|
||||||
|
msTableDistinctOn
|
||||||
|
-- :: forall m n. (BackendSchema 'BigQuery, MonadSchema n m, MonadTableInfo r m, MonadRole r m)
|
||||||
|
:: Applicative m
|
||||||
|
=> Applicative n
|
||||||
|
=> TableName 'BigQuery
|
||||||
|
-> SelPermInfo 'BigQuery
|
||||||
|
-> m (InputFieldsParser n (Maybe (XDistinct 'BigQuery, NonEmpty (Column 'BigQuery))))
|
||||||
|
msTableDistinctOn _table _selectPermissions = pure (pure Nothing)
|
||||||
|
|
||||||
|
-- | Various update operators
|
||||||
|
msUpdateOperators
|
||||||
|
-- :: forall m n r. (MonadSchema n m, MonadTableInfo r m)
|
||||||
|
:: Applicative m
|
||||||
|
=> TableName 'BigQuery -- ^ qualified name of the table
|
||||||
|
-> UpdPermInfo 'BigQuery -- ^ update permissions of the table
|
||||||
|
-> m (Maybe (InputFieldsParser n [(Column 'BigQuery, IR.UpdOpExpG (UnpreparedValue 'BigQuery))]))
|
||||||
|
msUpdateOperators _table _updatePermissions = pure Nothing
|
||||||
|
|
||||||
|
-- | Computed field parser.
|
||||||
|
-- Currently unsupported: returns Nothing for now.
|
||||||
|
msComputedField
|
||||||
|
:: MonadBuildSchema 'BigQuery r m n
|
||||||
|
=> ComputedFieldInfo 'BigQuery
|
||||||
|
-> SelPermInfo 'BigQuery
|
||||||
|
-> m (Maybe (FieldParser n (AnnotatedField 'BigQuery)))
|
||||||
|
msComputedField _fieldInfo _selectPemissions = pure Nothing
|
||||||
|
|
||||||
|
-- | Remote join field parser.
|
||||||
|
-- Currently unsupported: returns Nothing for now.
|
||||||
|
msRemoteRelationshipField
|
||||||
|
:: MonadBuildSchema 'BigQuery r m n
|
||||||
|
=> RemoteFieldInfo 'BigQuery
|
||||||
|
-> m (Maybe [FieldParser n (AnnotatedField 'BigQuery)])
|
||||||
|
msRemoteRelationshipField _remoteFieldInfo = pure Nothing
|
||||||
|
|
||||||
|
-- | The 'node' root field of a Relay request. Relay is currently unsupported on BigQuery,
|
||||||
|
-- meaning this parser will never be called: any attempt to create this parser should
|
||||||
|
-- therefore fail.
|
||||||
|
msNode
|
||||||
|
:: MonadBuildSchema 'BigQuery r m n
|
||||||
|
=> m ( Parser 'Output n
|
||||||
|
( HashMap
|
||||||
|
( TableName 'BigQuery)
|
||||||
|
( SourceName, SourceConfig 'BigQuery
|
||||||
|
, SelPermInfo 'BigQuery
|
||||||
|
, PrimaryKeyColumns 'BigQuery
|
||||||
|
, AnnotatedFields 'BigQuery
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
msNode = throw500 "BigQuery does not support relay; `node` should never be exposed in the schema."
|
@ -0,0 +1,81 @@
|
|||||||
|
{-# OPTIONS_GHC -fno-warn-orphans #-}
|
||||||
|
|
||||||
|
module Hasura.Backends.BigQuery.Instances.Transport () where
|
||||||
|
|
||||||
|
import qualified Data.Aeson as J
|
||||||
|
import Hasura.Backends.BigQuery.Instances.Execute ()
|
||||||
|
import Hasura.Backends.MSSQL.Instances.Execute ()
|
||||||
|
import Hasura.EncJSON
|
||||||
|
import Hasura.GraphQL.Logging (MonadQueryLog (..), QueryLog(..), GeneratedQuery(..))
|
||||||
|
import Hasura.GraphQL.Transport.Backend
|
||||||
|
import Hasura.GraphQL.Transport.HTTP.Protocol
|
||||||
|
import qualified Hasura.Logging as L
|
||||||
|
import Hasura.Prelude
|
||||||
|
import Hasura.RQL.Types
|
||||||
|
import Hasura.Server.Types (RequestId)
|
||||||
|
import Hasura.Session
|
||||||
|
import Hasura.Tracing
|
||||||
|
import qualified Hasura.Tracing as Tracing
|
||||||
|
import qualified Language.GraphQL.Draft.Syntax as G
|
||||||
|
|
||||||
|
instance BackendTransport 'BigQuery where
|
||||||
|
runDBQuery = runQuery
|
||||||
|
runDBMutation = runMutation
|
||||||
|
runDBSubscription = error "Not supported."
|
||||||
|
|
||||||
|
runQuery
|
||||||
|
:: ( MonadIO m
|
||||||
|
, MonadQueryLog m
|
||||||
|
, MonadTrace m
|
||||||
|
, MonadError QErr m
|
||||||
|
)
|
||||||
|
=> RequestId
|
||||||
|
-> GQLReqUnparsed
|
||||||
|
-> G.Name
|
||||||
|
-> UserInfo
|
||||||
|
-> L.Logger L.Hasura
|
||||||
|
-> SourceConfig 'BigQuery
|
||||||
|
-> Tracing.TraceT (ExceptT QErr IO) EncJSON
|
||||||
|
-> Maybe Text
|
||||||
|
-> m (DiffTime, EncJSON)
|
||||||
|
-- ^ Also return the time spent in the PG query; for telemetry.
|
||||||
|
runQuery reqId query fieldName _userInfo logger _sourceConfig tx genSql = do
|
||||||
|
-- log the generated SQL and the graphql query
|
||||||
|
-- FIXME: fix logging by making logQueryLog expect something backend agnostic!
|
||||||
|
logQueryLog logger $ mkQueryLog query fieldName genSql reqId
|
||||||
|
withElapsedTime $
|
||||||
|
flip Tracing.interpTraceT tx $ \m -> run m
|
||||||
|
|
||||||
|
run :: (MonadIO m, MonadError QErr m) => ExceptT QErr IO a -> m a
|
||||||
|
run action = do
|
||||||
|
result <- liftIO $ runExceptT action
|
||||||
|
result `onLeft` throwError
|
||||||
|
|
||||||
|
runMutation
|
||||||
|
:: ( MonadError QErr m
|
||||||
|
)
|
||||||
|
=> RequestId
|
||||||
|
-> GQLReqUnparsed
|
||||||
|
-> G.Name
|
||||||
|
-> UserInfo
|
||||||
|
-> L.Logger L.Hasura
|
||||||
|
-> SourceConfig 'BigQuery
|
||||||
|
-> Tracing.TraceT (ExceptT QErr IO) EncJSON
|
||||||
|
-> Maybe Text
|
||||||
|
-> m (DiffTime, EncJSON)
|
||||||
|
-- ^ Also return 'Mutation' when the operation was a mutation, and the time
|
||||||
|
-- spent in the PG query; for telemetry.
|
||||||
|
runMutation _reqId _query _fieldName _userInfo _logger _sourceConfig _tx _genSql = -- do
|
||||||
|
throw500 "BigQuery does not support mutations!"
|
||||||
|
|
||||||
|
|
||||||
|
mkQueryLog
|
||||||
|
:: GQLReqUnparsed
|
||||||
|
-> G.Name
|
||||||
|
-> Maybe Text
|
||||||
|
-> RequestId
|
||||||
|
-> QueryLog
|
||||||
|
mkQueryLog gqlQuery fieldName preparedSql requestId =
|
||||||
|
QueryLog gqlQuery ((fieldName,) <$> generatedQuery) requestId
|
||||||
|
where
|
||||||
|
generatedQuery = preparedSql <&> \qs -> GeneratedQuery qs J.Null
|
86
server/src-lib/Hasura/Backends/BigQuery/Instances/Types.hs
Normal file
86
server/src-lib/Hasura/Backends/BigQuery/Instances/Types.hs
Normal file
@ -0,0 +1,86 @@
|
|||||||
|
{-# OPTIONS_GHC -fno-warn-orphans #-}
|
||||||
|
|
||||||
|
module Hasura.Backends.BigQuery.Instances.Types where
|
||||||
|
|
||||||
|
import Data.Coerce
|
||||||
|
import Data.Functor.Const
|
||||||
|
import Hasura.Prelude
|
||||||
|
import Hasura.SQL.Types
|
||||||
|
import qualified Text.Builder as TB
|
||||||
|
|
||||||
|
import qualified Language.GraphQL.Draft.Syntax as G
|
||||||
|
|
||||||
|
import Data.Aeson
|
||||||
|
|
||||||
|
import qualified Hasura.Backends.BigQuery.Types as BigQuery
|
||||||
|
import qualified Hasura.Backends.BigQuery.Source as BigQuery
|
||||||
|
|
||||||
|
import qualified Hasura.Backends.BigQuery.ToQuery as BigQuery (toTextPretty, fromExpression)
|
||||||
|
import Hasura.RQL.DDL.Headers ()
|
||||||
|
import Hasura.RQL.Types.Backend
|
||||||
|
import Hasura.RQL.Types.Error
|
||||||
|
import Hasura.SQL.Backend
|
||||||
|
|
||||||
|
instance ToSQL BigQuery.Expression where
|
||||||
|
toSQL = TB.text . BigQuery.toTextPretty . BigQuery.fromExpression
|
||||||
|
|
||||||
|
instance Backend 'BigQuery where
|
||||||
|
type SourceConfig 'BigQuery = BigQuery.BigQuerySourceConfig
|
||||||
|
type SourceConnConfiguration 'BigQuery = BigQuery.BigQueryConnSourceConfig
|
||||||
|
type Identifier 'BigQuery = Void
|
||||||
|
type Alias 'BigQuery = BigQuery.EntityAlias
|
||||||
|
type TableName 'BigQuery = BigQuery.TableName
|
||||||
|
type FunctionName 'BigQuery = BigQuery.FunctionName
|
||||||
|
type FunctionArgType 'BigQuery = Void
|
||||||
|
type ConstraintName 'BigQuery = Void
|
||||||
|
type BasicOrderType 'BigQuery = BigQuery.Order
|
||||||
|
type NullsOrderType 'BigQuery = BigQuery.NullsOrder
|
||||||
|
type CountType 'BigQuery = BigQuery.Countable BigQuery.ColumnName
|
||||||
|
type Column 'BigQuery = BigQuery.ColumnName
|
||||||
|
type ScalarValue 'BigQuery = BigQuery.Value
|
||||||
|
type ScalarType 'BigQuery = BigQuery.ScalarType
|
||||||
|
type SQLExpression 'BigQuery = BigQuery.Expression
|
||||||
|
type SQLOperator 'BigQuery = BigQuery.Op
|
||||||
|
type BooleanOperators 'BigQuery = Const Void
|
||||||
|
type XComputedField 'BigQuery = Void
|
||||||
|
type XRemoteField 'BigQuery = Void
|
||||||
|
|
||||||
|
type XRelay 'BigQuery = Void
|
||||||
|
type XNodesAgg 'BigQuery = XEnable
|
||||||
|
type XDistinct 'BigQuery = Void
|
||||||
|
|
||||||
|
functionArgScalarType :: FunctionArgType 'BigQuery -> ScalarType 'BigQuery
|
||||||
|
functionArgScalarType = absurd
|
||||||
|
|
||||||
|
isComparableType :: ScalarType 'BigQuery -> Bool
|
||||||
|
isComparableType = BigQuery.isComparableType
|
||||||
|
|
||||||
|
isNumType :: ScalarType 'BigQuery -> Bool
|
||||||
|
isNumType = BigQuery.isNumType
|
||||||
|
|
||||||
|
textToScalarValue :: Maybe Text -> ScalarValue 'BigQuery
|
||||||
|
textToScalarValue = maybe BigQuery.NullValue BigQuery.StringValue
|
||||||
|
|
||||||
|
parseScalarValue :: ScalarType 'BigQuery -> Value -> Either QErr (ScalarValue 'BigQuery)
|
||||||
|
parseScalarValue = BigQuery.parseScalarValue
|
||||||
|
|
||||||
|
scalarValueToJSON :: ScalarValue 'BigQuery -> Value
|
||||||
|
scalarValueToJSON = error "scalarValueToJSON"
|
||||||
|
|
||||||
|
functionToTable :: FunctionName 'BigQuery -> TableName 'BigQuery
|
||||||
|
functionToTable = error "functionToTable"
|
||||||
|
|
||||||
|
tableToFunction :: TableName 'BigQuery -> FunctionName 'BigQuery
|
||||||
|
tableToFunction = coerce . BigQuery.tableName
|
||||||
|
|
||||||
|
tableGraphQLName :: TableName 'BigQuery -> Either QErr G.Name
|
||||||
|
tableGraphQLName = BigQuery.getGQLTableName
|
||||||
|
|
||||||
|
functionGraphQLName :: FunctionName 'BigQuery -> Either QErr G.Name
|
||||||
|
functionGraphQLName = error "functionGraphQLName"
|
||||||
|
|
||||||
|
scalarTypeGraphQLName :: ScalarType 'BigQuery -> Either QErr G.Name
|
||||||
|
scalarTypeGraphQLName = error "scalarTypeGraphQLName"
|
||||||
|
|
||||||
|
snakeCaseTableName :: TableName 'BigQuery -> Text
|
||||||
|
snakeCaseTableName = error "snakeCaseTableName"
|
265
server/src-lib/Hasura/Backends/BigQuery/Meta.hs
Normal file
265
server/src-lib/Hasura/Backends/BigQuery/Meta.hs
Normal file
@ -0,0 +1,265 @@
|
|||||||
|
{-# LANGUAGE DuplicateRecordFields #-}
|
||||||
|
|
||||||
|
-- |
|
||||||
|
|
||||||
|
module Hasura.Backends.BigQuery.Meta
|
||||||
|
( MetadataError(..)
|
||||||
|
, getTables
|
||||||
|
, RestTableReference(..)
|
||||||
|
, RestTable(..)
|
||||||
|
, RestTableSchema(..)
|
||||||
|
, RestFieldSchema(..)
|
||||||
|
, RestType(..)
|
||||||
|
, Mode(..)
|
||||||
|
) where
|
||||||
|
|
||||||
|
|
||||||
|
import Control.Exception.Safe
|
||||||
|
import Control.Monad.IO.Class
|
||||||
|
import Control.Monad.Trans.Except
|
||||||
|
import Data.Aeson
|
||||||
|
import qualified Data.Aeson as Aeson
|
||||||
|
import Data.Foldable
|
||||||
|
import Data.Maybe
|
||||||
|
import qualified Data.Sequence as Seq
|
||||||
|
import Data.Text (Text)
|
||||||
|
import qualified Data.Text as T
|
||||||
|
import GHC.Generics
|
||||||
|
import Hasura.Backends.BigQuery.Connection
|
||||||
|
import Hasura.Backends.BigQuery.Source
|
||||||
|
import Network.HTTP.Simple
|
||||||
|
import Network.HTTP.Types
|
||||||
|
import Prelude
|
||||||
|
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
-- Types
|
||||||
|
|
||||||
|
data MetadataError
|
||||||
|
= RestProblem RestProblem
|
||||||
|
deriving (Show)
|
||||||
|
|
||||||
|
data RestProblem
|
||||||
|
= GetTablesProblem SomeException
|
||||||
|
| GetTableProblem SomeException
|
||||||
|
| GetMetaDecodeProblem String
|
||||||
|
| GetTablesBigQueryProblem BigQueryProblem
|
||||||
|
| RESTRequestNonOK Status
|
||||||
|
deriving (Show)
|
||||||
|
|
||||||
|
data RestTableList = RestTableList
|
||||||
|
{ nextPageToken :: Maybe Text
|
||||||
|
, tables :: [RestTableBrief]
|
||||||
|
} deriving (Show)
|
||||||
|
|
||||||
|
instance FromJSON RestTableList where
|
||||||
|
parseJSON =
|
||||||
|
withObject
|
||||||
|
"RestTableList"
|
||||||
|
(\o -> do
|
||||||
|
kind <- o .: "kind"
|
||||||
|
case kind of
|
||||||
|
("bigquery#tableList"::Text) -> do
|
||||||
|
nextPageToken <- o .:? "nextPageToken"
|
||||||
|
tables <- o .: "tables"
|
||||||
|
pure RestTableList {..}
|
||||||
|
_ -> fail "Expected kind of bigquery#tableList")
|
||||||
|
|
||||||
|
data RestTableBrief = RestTableBrief
|
||||||
|
{ tableReference :: RestTableReference
|
||||||
|
} deriving (Show, Generic)
|
||||||
|
instance FromJSON RestTableBrief
|
||||||
|
|
||||||
|
data RestTableReference = RestTableReference
|
||||||
|
{ datasetId :: Text
|
||||||
|
, projectId :: Text
|
||||||
|
, tableId :: Text
|
||||||
|
} deriving (Show, Generic)
|
||||||
|
instance FromJSON RestTableReference
|
||||||
|
|
||||||
|
data RestTable = RestTable
|
||||||
|
{ tableReference :: RestTableReference
|
||||||
|
, schema :: RestTableSchema
|
||||||
|
} deriving (Show, Generic)
|
||||||
|
instance FromJSON RestTable
|
||||||
|
|
||||||
|
data RestTableSchema = RestTableSchema
|
||||||
|
{ fields :: [RestFieldSchema]
|
||||||
|
} deriving (Show, Generic)
|
||||||
|
instance FromJSON RestTableSchema
|
||||||
|
|
||||||
|
data RestFieldSchema = RestFieldSchema
|
||||||
|
{ name :: Text
|
||||||
|
, type' :: RestType
|
||||||
|
-- ^ The field data type. Possible values include STRING, BYTES,
|
||||||
|
-- INTEGER, INT64 (same as INTEGER), FLOAT, FLOAT64 (same as
|
||||||
|
-- FLOAT), BOOLEAN, BOOL (same as BOOLEAN), TIMESTAMP, DATE, TIME,
|
||||||
|
-- DATETIME, GEOGRAPHY, NUMERIC, RECORD (where RECORD indicates
|
||||||
|
-- that the field contains a nested schema) or STRUCT (same as
|
||||||
|
-- RECORD).
|
||||||
|
, mode :: Mode
|
||||||
|
-- The field mode. Possible values include NULLABLE, REQUIRED and
|
||||||
|
-- REPEATED. The default value is NULLABLE.
|
||||||
|
} deriving (Show, Generic)
|
||||||
|
instance FromJSON RestFieldSchema where
|
||||||
|
parseJSON =
|
||||||
|
withObject
|
||||||
|
"RestFieldSchema"
|
||||||
|
(\o -> do
|
||||||
|
type' <- o .: "type"
|
||||||
|
name <- o .: "name"
|
||||||
|
mode <- fmap (fromMaybe Nullable) (o .:? "mode")
|
||||||
|
pure RestFieldSchema {..})
|
||||||
|
|
||||||
|
data Mode = Nullable | Required | Repeated deriving (Show)
|
||||||
|
instance FromJSON Mode where
|
||||||
|
parseJSON j = do
|
||||||
|
s <- parseJSON j
|
||||||
|
case s :: Text of
|
||||||
|
"NULLABLE" -> pure Nullable
|
||||||
|
"REQUIRED" -> pure Required
|
||||||
|
"REPEATED" -> pure Repeated
|
||||||
|
_ -> fail ("invalid mode " ++ show s)
|
||||||
|
|
||||||
|
data RestType
|
||||||
|
= STRING
|
||||||
|
| BYTES
|
||||||
|
| INTEGER
|
||||||
|
| FLOAT
|
||||||
|
| BOOL
|
||||||
|
| TIMESTAMP
|
||||||
|
| DATE
|
||||||
|
| TIME
|
||||||
|
| DATETIME
|
||||||
|
| GEOGRAPHY
|
||||||
|
| DECIMAL
|
||||||
|
| BIGDECIMAL
|
||||||
|
| STRUCT -- (same as RECORD).
|
||||||
|
deriving (Show)
|
||||||
|
instance FromJSON RestType where
|
||||||
|
parseJSON j = do
|
||||||
|
s <- parseJSON j
|
||||||
|
case s :: Text of
|
||||||
|
"STRING" -> pure STRING
|
||||||
|
"BYTES" -> pure BYTES
|
||||||
|
"INTEGER" -> pure INTEGER
|
||||||
|
"INT64" -> pure INTEGER
|
||||||
|
"FLOAT" -> pure FLOAT
|
||||||
|
"FLOAT64" -> pure FLOAT
|
||||||
|
"BOOLEAN" -> pure BOOL
|
||||||
|
"BOOL" -> pure BOOL
|
||||||
|
"TIMESTAMP" -> pure TIMESTAMP
|
||||||
|
"DATE" -> pure DATE
|
||||||
|
"TIME" -> pure TIME
|
||||||
|
"DATETIME" -> pure DATETIME
|
||||||
|
"GEOGRAPHY" -> pure GEOGRAPHY
|
||||||
|
"NUMERIC" -> pure DECIMAL
|
||||||
|
"DECIMAL" -> pure DECIMAL
|
||||||
|
"BIGNUMERIC" -> pure BIGDECIMAL
|
||||||
|
"BIGDECIMAL" -> pure BIGDECIMAL
|
||||||
|
"RECORD" -> pure STRUCT
|
||||||
|
"STRUCT" -> pure STRUCT
|
||||||
|
_ -> fail ("invalid type " ++ show s)
|
||||||
|
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
-- REST request
|
||||||
|
|
||||||
|
-- | Get all tables from all specified data sets.
|
||||||
|
getTables ::
|
||||||
|
MonadIO m
|
||||||
|
=> BigQuerySourceConfig
|
||||||
|
-> m (Either RestProblem [RestTable])
|
||||||
|
getTables sc@BigQuerySourceConfig {..} =
|
||||||
|
runExceptT
|
||||||
|
(fmap concat (traverse (ExceptT . getTablesForDataSet sc) _scDatasets))
|
||||||
|
|
||||||
|
-- | Get tables in the dataset.
|
||||||
|
getTablesForDataSet ::
|
||||||
|
MonadIO m
|
||||||
|
=> BigQuerySourceConfig
|
||||||
|
-> Text
|
||||||
|
-> m (Either RestProblem [RestTable])
|
||||||
|
getTablesForDataSet sc@BigQuerySourceConfig{..} dataSet = do
|
||||||
|
result <-
|
||||||
|
liftIO (catchAny (run Nothing mempty) (pure . Left . GetTablesProblem))
|
||||||
|
case result of
|
||||||
|
Left e -> pure (Left e)
|
||||||
|
Right briefs ->
|
||||||
|
fmap
|
||||||
|
sequence
|
||||||
|
(traverse
|
||||||
|
(\RestTableBrief {tableReference = RestTableReference {tableId}} ->
|
||||||
|
getTable sc dataSet tableId)
|
||||||
|
briefs)
|
||||||
|
where
|
||||||
|
run pageToken acc = do
|
||||||
|
let req = setRequestHeader "Content-Type" ["application/json"]
|
||||||
|
$ parseRequest_ url
|
||||||
|
eResp <- runBigQuery sc req
|
||||||
|
case eResp of
|
||||||
|
Left e -> pure (Left (GetTablesBigQueryProblem e))
|
||||||
|
Right resp ->
|
||||||
|
case getResponseStatusCode resp of
|
||||||
|
200 ->
|
||||||
|
case Aeson.eitherDecode (getResponseBody resp) of
|
||||||
|
Left e -> pure (Left (GetMetaDecodeProblem e))
|
||||||
|
Right RestTableList {nextPageToken, tables} ->
|
||||||
|
case nextPageToken of
|
||||||
|
Nothing -> pure (Right (toList (acc <> Seq.fromList tables)))
|
||||||
|
Just token -> run (pure token) (acc <> Seq.fromList tables)
|
||||||
|
_ -> pure (Left (RESTRequestNonOK (getResponseStatus resp)))
|
||||||
|
where
|
||||||
|
url =
|
||||||
|
"GET https://bigquery.googleapis.com/bigquery/v2/projects/" <>
|
||||||
|
T.unpack _scProjectId <>
|
||||||
|
"/datasets/" <>
|
||||||
|
T.unpack dataSet <>
|
||||||
|
"/tables?alt=json&key=" <>
|
||||||
|
-- T.unpack apiToken <>
|
||||||
|
"&" <>
|
||||||
|
T.unpack (encodeParams extraParameters)
|
||||||
|
extraParameters = pageTokenParam
|
||||||
|
where
|
||||||
|
pageTokenParam =
|
||||||
|
case pageToken of
|
||||||
|
Nothing -> []
|
||||||
|
Just token -> [("pageToken", token)]
|
||||||
|
|
||||||
|
-- | Get tables in the schema.
|
||||||
|
getTable ::
|
||||||
|
MonadIO m
|
||||||
|
=> BigQuerySourceConfig
|
||||||
|
-> Text -> Text
|
||||||
|
-> m (Either RestProblem RestTable)
|
||||||
|
getTable sc@BigQuerySourceConfig {..} dataSet tableId = do
|
||||||
|
liftIO (catchAny run (pure . Left . GetTableProblem))
|
||||||
|
where
|
||||||
|
run = do
|
||||||
|
let req =
|
||||||
|
setRequestHeader "Content-Type" ["application/json"] $
|
||||||
|
parseRequest_ url
|
||||||
|
eResp <- runBigQuery sc req
|
||||||
|
case eResp of
|
||||||
|
Left e -> pure (Left (GetTablesBigQueryProblem e))
|
||||||
|
Right resp ->
|
||||||
|
case getResponseStatusCode resp of
|
||||||
|
200 ->
|
||||||
|
case Aeson.eitherDecode (getResponseBody resp) of
|
||||||
|
Left e -> pure (Left (GetMetaDecodeProblem e))
|
||||||
|
Right table -> pure (Right table)
|
||||||
|
_ -> pure (Left (RESTRequestNonOK (getResponseStatus resp)))
|
||||||
|
where
|
||||||
|
url =
|
||||||
|
"GET https://bigquery.googleapis.com/bigquery/v2/projects/" <>
|
||||||
|
T.unpack _scProjectId <>
|
||||||
|
"/datasets/" <>
|
||||||
|
T.unpack dataSet <>
|
||||||
|
"/tables/" <>
|
||||||
|
T.unpack tableId <>
|
||||||
|
"?alt=json&key=" <>
|
||||||
|
-- T.unpack apiToken <>
|
||||||
|
"&" <>
|
||||||
|
T.unpack (encodeParams extraParameters)
|
||||||
|
extraParameters = []
|
||||||
|
|
||||||
|
encodeParams :: [(Text, Text)] -> Text
|
||||||
|
encodeParams = T.intercalate "&" . map (\(k, v) -> k <> "=" <> v)
|
93
server/src-lib/Hasura/Backends/BigQuery/Plan.hs
Normal file
93
server/src-lib/Hasura/Backends/BigQuery/Plan.hs
Normal file
@ -0,0 +1,93 @@
|
|||||||
|
-- | Planning T-SQL queries and subscriptions.
|
||||||
|
|
||||||
|
module Hasura.Backends.BigQuery.Plan
|
||||||
|
( planNoPlan
|
||||||
|
, planToForest
|
||||||
|
) where
|
||||||
|
|
||||||
|
import Data.Aeson.Text
|
||||||
|
import qualified Data.Text.Lazy as LT
|
||||||
|
import Data.Tree
|
||||||
|
import qualified Hasura.Backends.BigQuery.DataLoader.Plan as DataLoader
|
||||||
|
import Hasura.Prelude hiding (first)
|
||||||
|
import Hasura.SQL.Types
|
||||||
|
import Hasura.Session
|
||||||
|
|
||||||
|
|
||||||
|
import Control.Monad.Validate
|
||||||
|
import Data.Text.Extended
|
||||||
|
|
||||||
|
import qualified Hasura.GraphQL.Parser as GraphQL
|
||||||
|
import qualified Hasura.RQL.Types.Column as RQL
|
||||||
|
|
||||||
|
import Hasura.Backends.BigQuery.FromIr as BigQuery
|
||||||
|
import Hasura.Backends.BigQuery.Types as BigQuery
|
||||||
|
import Hasura.GraphQL.Context
|
||||||
|
import qualified Hasura.RQL.Types.Error as RQL
|
||||||
|
import Hasura.SQL.Backend
|
||||||
|
|
||||||
|
-- --------------------------------------------------------------------------------
|
||||||
|
-- -- Top-level planner
|
||||||
|
|
||||||
|
planToForest ::
|
||||||
|
MonadError RQL.QErr m
|
||||||
|
=> UserInfo
|
||||||
|
-> QueryDB 'BigQuery (GraphQL.UnpreparedValue 'BigQuery)
|
||||||
|
-> m (Forest DataLoader.PlannedAction)
|
||||||
|
planToForest userInfo qrf = do
|
||||||
|
select <- planNoPlan userInfo qrf
|
||||||
|
let (!_headAndTail, !plannedActionsList) =
|
||||||
|
DataLoader.runPlan
|
||||||
|
(DataLoader.planSelectHeadAndTail Nothing Nothing select)
|
||||||
|
!actionsForest = DataLoader.actionsForest id plannedActionsList
|
||||||
|
pure actionsForest
|
||||||
|
|
||||||
|
planNoPlan ::
|
||||||
|
MonadError RQL.QErr m
|
||||||
|
=> UserInfo
|
||||||
|
-> QueryDB 'BigQuery (GraphQL.UnpreparedValue 'BigQuery)
|
||||||
|
-> m Select
|
||||||
|
planNoPlan userInfo queryDB = do
|
||||||
|
rootField <- traverseQueryDB (prepareValueNoPlan (_uiSession userInfo)) queryDB
|
||||||
|
select <-
|
||||||
|
runValidate (BigQuery.runFromIr (BigQuery.fromRootField rootField))
|
||||||
|
`onLeft` (RQL.throw400 RQL.NotSupported . (tshow :: NonEmpty Error -> Text))
|
||||||
|
pure
|
||||||
|
select
|
||||||
|
{ selectFor =
|
||||||
|
case selectFor select of
|
||||||
|
NoFor -> NoFor
|
||||||
|
JsonFor forJson -> JsonFor forJson {jsonRoot = Root "root"}
|
||||||
|
}
|
||||||
|
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
-- Resolving values
|
||||||
|
|
||||||
|
-- | Prepare a value without any query planning; we just execute the
|
||||||
|
-- query with the values embedded.
|
||||||
|
prepareValueNoPlan ::
|
||||||
|
(MonadError RQL.QErr m)
|
||||||
|
=> SessionVariables
|
||||||
|
-> GraphQL.UnpreparedValue 'BigQuery
|
||||||
|
-> m BigQuery.Expression
|
||||||
|
prepareValueNoPlan sessionVariables =
|
||||||
|
\case
|
||||||
|
GraphQL.UVLiteral x -> pure x
|
||||||
|
GraphQL.UVSession -> pure globalSessionExpression
|
||||||
|
-- To be honest, I'm not sure if it's indeed the JSON_VALUE operator we need here...
|
||||||
|
GraphQL.UVSessionVar typ text ->
|
||||||
|
case typ of
|
||||||
|
CollectableTypeScalar scalarType ->
|
||||||
|
pure
|
||||||
|
(CastExpression
|
||||||
|
(JsonValueExpression
|
||||||
|
globalSessionExpression
|
||||||
|
(FieldPath RootPath (toTxt text)))
|
||||||
|
scalarType)
|
||||||
|
CollectableTypeArray {} ->
|
||||||
|
throwError $ RQL.internalError "Cannot currently prepare array types in BigQuery."
|
||||||
|
GraphQL.UVParameter _ RQL.ColumnValue {..} -> pure (ValueExpression cvValue)
|
||||||
|
where
|
||||||
|
globalSessionExpression =
|
||||||
|
ValueExpression
|
||||||
|
(StringValue (LT.toStrict (encodeToLazyText sessionVariables)))
|
206
server/src-lib/Hasura/Backends/BigQuery/Source.hs
Normal file
206
server/src-lib/Hasura/Backends/BigQuery/Source.hs
Normal file
@ -0,0 +1,206 @@
|
|||||||
|
{-# LANGUAGE DeriveAnyClass #-}
|
||||||
|
{-# LANGUAGE NoGeneralisedNewtypeDeriving #-}
|
||||||
|
{-# OPTIONS_GHC -fno-warn-orphans #-}
|
||||||
|
|
||||||
|
module Hasura.Backends.BigQuery.Source where
|
||||||
|
|
||||||
|
|
||||||
|
import Control.Concurrent.MVar
|
||||||
|
import Control.DeepSeq
|
||||||
|
import qualified Crypto.PubKey.RSA.Types as Cry
|
||||||
|
import qualified Data.Aeson as J
|
||||||
|
import qualified Data.Aeson.Casing as J
|
||||||
|
import qualified Data.Aeson.TH as J
|
||||||
|
import qualified Data.ByteString.Lazy as BL
|
||||||
|
import Data.Hashable (hashWithSalt)
|
||||||
|
import qualified Data.HashMap.Strict as HM
|
||||||
|
import Data.Text (pack)
|
||||||
|
import qualified Data.Text.Encoding as TE
|
||||||
|
import qualified Data.X509 as X509
|
||||||
|
import qualified Data.X509.Memory as X509
|
||||||
|
import Hasura.Incremental (Cacheable (..))
|
||||||
|
import Hasura.Prelude
|
||||||
|
import qualified System.Environment as SE (getEnv)
|
||||||
|
import System.FilePath (isRelative)
|
||||||
|
import System.IO.Unsafe (unsafePerformIO)
|
||||||
|
|
||||||
|
|
||||||
|
data PKey = PKey
|
||||||
|
{ unPKey :: Cry.PrivateKey
|
||||||
|
, originalBS :: Text
|
||||||
|
}
|
||||||
|
deriving (Show, Eq, Data, Generic, NFData, Hashable)
|
||||||
|
deriving instance Generic Cry.PrivateKey -- orphan
|
||||||
|
deriving instance Generic Cry.PublicKey -- orphan
|
||||||
|
deriving instance J.ToJSON Cry.PrivateKey -- orphan
|
||||||
|
deriving instance J.ToJSON Cry.PublicKey -- orphan
|
||||||
|
deriving instance Hashable Cry.PrivateKey -- orphan
|
||||||
|
deriving instance Hashable Cry.PublicKey -- orphan
|
||||||
|
instance Arbitrary Cry.PrivateKey where -- orphan
|
||||||
|
arbitrary = genericArbitrary
|
||||||
|
instance Arbitrary Cry.PublicKey where -- orphan
|
||||||
|
arbitrary = genericArbitrary
|
||||||
|
instance Arbitrary PKey where
|
||||||
|
arbitrary = genericArbitrary
|
||||||
|
instance J.FromJSON PKey where
|
||||||
|
parseJSON = J.withText "private_key" $ \k ->
|
||||||
|
case X509.readKeyFileFromMemory $ TE.encodeUtf8 k of
|
||||||
|
[X509.PrivKeyRSA k'] -> return $ PKey k' k
|
||||||
|
_ -> fail "unable to parse private key"
|
||||||
|
instance J.ToJSON PKey where
|
||||||
|
toJSON PKey{..} = J.toJSON originalBS
|
||||||
|
|
||||||
|
|
||||||
|
newtype GoogleAccessToken
|
||||||
|
= GoogleAccessToken Text
|
||||||
|
deriving (Show, Eq, J.FromJSON, J.ToJSON, Hashable, Generic, Data, NFData)
|
||||||
|
instance Arbitrary GoogleAccessToken where
|
||||||
|
arbitrary = genericArbitrary
|
||||||
|
|
||||||
|
|
||||||
|
data TokenResp
|
||||||
|
= TokenResp
|
||||||
|
{ _trAccessToken :: !GoogleAccessToken
|
||||||
|
, _trExpiresAt :: !Integer -- Number of seconds until expiry from `now`, but we add `now` seconds to this for easy tracking
|
||||||
|
} deriving (Eq, Show, Data, NFData, Generic, Hashable)
|
||||||
|
instance J.FromJSON TokenResp where
|
||||||
|
parseJSON = J.withObject "TokenResp" $ \o -> TokenResp
|
||||||
|
<$> o J..: "access_token"
|
||||||
|
<*> o J..: "expires_in"
|
||||||
|
instance Arbitrary TokenResp where
|
||||||
|
arbitrary = genericArbitrary
|
||||||
|
|
||||||
|
|
||||||
|
dummyTokenResp :: TokenResp
|
||||||
|
dummyTokenResp = TokenResp (GoogleAccessToken "DUMMY") 0
|
||||||
|
|
||||||
|
|
||||||
|
data ServiceAccount
|
||||||
|
= ServiceAccount
|
||||||
|
{ _saClientEmail :: !Text
|
||||||
|
, _saPrivateKey :: !PKey
|
||||||
|
, _saProjectId :: !Text
|
||||||
|
} deriving (Eq, Show, Data, NFData, Generic, Hashable)
|
||||||
|
$(J.deriveJSON (J.aesonDrop 3 J.snakeCase){J.omitNothingFields=False} ''ServiceAccount)
|
||||||
|
instance Arbitrary ServiceAccount where
|
||||||
|
arbitrary = genericArbitrary
|
||||||
|
|
||||||
|
|
||||||
|
data ConfigurationJSON a
|
||||||
|
= FromEnvJSON Text
|
||||||
|
| FromYamlJSON a
|
||||||
|
deriving stock (Show, Eq, Generic)
|
||||||
|
deriving (NFData, Hashable)
|
||||||
|
instance Arbitrary a => Arbitrary (ConfigurationJSON a) where
|
||||||
|
arbitrary = genericArbitrary
|
||||||
|
instance J.FromJSON a => J.FromJSON (ConfigurationJSON a) where
|
||||||
|
parseJSON = \case
|
||||||
|
J.Object o | Just (J.String text) <- HM.lookup "from_env" o -> pure (FromEnvJSON text)
|
||||||
|
J.String s -> case J.eitherDecode . BL.fromStrict . TE.encodeUtf8 $ s of
|
||||||
|
Left {} -> fail "error parsing configuration json"
|
||||||
|
Right sa -> pure sa
|
||||||
|
j -> fmap FromYamlJSON (J.parseJSON j)
|
||||||
|
instance J.ToJSON a => J.ToJSON (ConfigurationJSON a) where
|
||||||
|
toJSON = \case
|
||||||
|
FromEnvJSON i -> J.object ["from_env" J..= i]
|
||||||
|
FromYamlJSON j -> J.toJSON j
|
||||||
|
|
||||||
|
|
||||||
|
-- | Configuration inputs when they are a YAML array or an Env var whos value is
|
||||||
|
-- a comma-separated string
|
||||||
|
data ConfigurationInputs
|
||||||
|
= FromYamls ![Text]
|
||||||
|
| FromEnvs !Text
|
||||||
|
deriving stock (Show, Eq, Generic)
|
||||||
|
deriving (NFData, Hashable)
|
||||||
|
instance Arbitrary ConfigurationInputs where
|
||||||
|
arbitrary = genericArbitrary
|
||||||
|
instance J.ToJSON ConfigurationInputs where
|
||||||
|
toJSON = \case
|
||||||
|
FromYamls i -> J.toJSON i
|
||||||
|
FromEnvs i -> J.object ["from_env" J..= i]
|
||||||
|
instance J.FromJSON ConfigurationInputs where
|
||||||
|
parseJSON = \case
|
||||||
|
J.Object o -> FromEnvs <$> o J..: "from_env"
|
||||||
|
s@(J.Array _) -> FromYamls <$> J.parseJSON s
|
||||||
|
_ -> fail "one of array or object must be provided"
|
||||||
|
|
||||||
|
|
||||||
|
-- | Configuration input when the YAML value as well as the Env var have
|
||||||
|
-- singlular values
|
||||||
|
data ConfigurationInput
|
||||||
|
= FromYaml !Text
|
||||||
|
| FromEnv !Text
|
||||||
|
deriving stock (Show, Eq, Generic)
|
||||||
|
deriving (NFData, Hashable)
|
||||||
|
instance Arbitrary ConfigurationInput where
|
||||||
|
arbitrary = genericArbitrary
|
||||||
|
instance J.ToJSON ConfigurationInput where
|
||||||
|
toJSON = \case
|
||||||
|
FromYaml i -> J.toJSON i
|
||||||
|
FromEnv i -> J.object ["from_env" J..= i]
|
||||||
|
instance J.FromJSON ConfigurationInput where
|
||||||
|
parseJSON = \case
|
||||||
|
J.Object o -> FromEnv <$> o J..: "from_env"
|
||||||
|
s@(J.String _) -> FromYaml <$> J.parseJSON s
|
||||||
|
_ -> fail "one of string or object must be provided"
|
||||||
|
|
||||||
|
|
||||||
|
data BigQueryConnSourceConfig
|
||||||
|
= BigQueryConnSourceConfig
|
||||||
|
{ _cscServiceAccount :: !(ConfigurationJSON ServiceAccount)
|
||||||
|
, _cscDatasets :: !ConfigurationInputs
|
||||||
|
, _cscProjectId :: !ConfigurationInput -- this is part of service-account.json, but we put it here on purpose
|
||||||
|
} deriving (Eq, Generic, NFData)
|
||||||
|
$(J.deriveJSON (J.aesonDrop 4 J.snakeCase){J.omitNothingFields=True} ''BigQueryConnSourceConfig)
|
||||||
|
deriving instance Show BigQueryConnSourceConfig
|
||||||
|
deriving instance Hashable BigQueryConnSourceConfig
|
||||||
|
instance Arbitrary BigQueryConnSourceConfig where
|
||||||
|
arbitrary = genericArbitrary
|
||||||
|
instance Cacheable BigQueryConnSourceConfig where
|
||||||
|
unchanged _ = (==)
|
||||||
|
|
||||||
|
|
||||||
|
data BigQuerySourceConfig
|
||||||
|
= BigQuerySourceConfig
|
||||||
|
{ _scServiceAccount :: !ServiceAccount
|
||||||
|
, _scDatasets :: ![Text]
|
||||||
|
, _scProjectId :: !Text -- this is part of service-account.json, but we put it here on purpose
|
||||||
|
, _scAccessTokenMVar :: !(MVar (Maybe TokenResp))
|
||||||
|
} deriving (Eq, Generic, NFData)
|
||||||
|
$(J.deriveJSON (J.aesonDrop 3 J.snakeCase){J.omitNothingFields=True} ''BigQuerySourceConfig)
|
||||||
|
deriving instance Show BigQuerySourceConfig
|
||||||
|
deriving instance Hashable BigQuerySourceConfig
|
||||||
|
instance Arbitrary BigQuerySourceConfig where
|
||||||
|
arbitrary = genericArbitrary
|
||||||
|
instance Cacheable BigQuerySourceConfig where
|
||||||
|
unchanged _ = (==)
|
||||||
|
|
||||||
|
instance Show (MVar (Maybe TokenResp)) where
|
||||||
|
-- show = maybe "NOTHING" (const "_REDACTED_") . unsafePerformIO . readIORef
|
||||||
|
show = (const "_REDACTED_")
|
||||||
|
instance J.FromJSON (MVar (Maybe TokenResp)) where
|
||||||
|
parseJSON _ = pure $ unsafePerformIO $ newEmptyMVar
|
||||||
|
instance J.ToJSON (MVar (Maybe TokenResp)) where
|
||||||
|
toJSON _ = J.String "_REDACTED_"
|
||||||
|
instance Hashable (MVar (Maybe TokenResp)) where
|
||||||
|
hashWithSalt i r = hashWithSalt i (unsafePerformIO $ readMVar r)
|
||||||
|
instance Arbitrary (MVar (Maybe TokenResp)) where
|
||||||
|
arbitrary = genericArbitrary @(Maybe TokenResp) <&> (unsafePerformIO . newMVar)
|
||||||
|
|
||||||
|
|
||||||
|
-- | for testing
|
||||||
|
getBigQuerySourceConfigEnv :: IO BigQuerySourceConfig
|
||||||
|
getBigQuerySourceConfigEnv = do
|
||||||
|
_scServiceAccountFilePath <- getEnvUnline _safpEnvKey
|
||||||
|
if isRelative _scServiceAccountFilePath
|
||||||
|
then error $ _safpEnvKey <> " needs to be an absolute file-path"
|
||||||
|
else do
|
||||||
|
_scDatasets <- pure . pack <$> getEnvUnline "HASURA_BIGQUERY_DATASET"
|
||||||
|
_scProjectId <- pack <$> getEnvUnline "HASURA_BIGQUERY_PROJECT_ID"
|
||||||
|
_scServiceAccount :: ServiceAccount <- either error id . J.eitherDecode' <$> BL.readFile _scServiceAccountFilePath
|
||||||
|
_scAccessTokenMVar <- newMVar Nothing
|
||||||
|
pure BigQuerySourceConfig {..}
|
||||||
|
where
|
||||||
|
_safpEnvKey = "HASURA_BIGQUERY_SERVICE_ACCOUNT_FILE_PATH"
|
||||||
|
getEnvUnline key = fmap (concat . take 1 . lines) (SE.getEnv key)
|
457
server/src-lib/Hasura/Backends/BigQuery/ToQuery.hs
Normal file
457
server/src-lib/Hasura/Backends/BigQuery/ToQuery.hs
Normal file
@ -0,0 +1,457 @@
|
|||||||
|
{-# LANGUAGE OverloadedStrings #-}
|
||||||
|
|
||||||
|
-- | Convert the simple BigQuery AST to an SQL query, ready to be passed
|
||||||
|
-- to the odbc package's query/exec functions.
|
||||||
|
|
||||||
|
module Hasura.Backends.BigQuery.ToQuery
|
||||||
|
( fromSelect
|
||||||
|
, fromReselect
|
||||||
|
, fromExpression
|
||||||
|
, toBuilderFlat
|
||||||
|
, toBuilderPretty
|
||||||
|
, toTextFlat
|
||||||
|
, toTextPretty
|
||||||
|
, Printer(..)
|
||||||
|
, renderBuilderFlat
|
||||||
|
, renderBuilderPretty
|
||||||
|
, paramName
|
||||||
|
) where
|
||||||
|
|
||||||
|
import Control.Monad.State.Strict
|
||||||
|
import Data.Bifunctor
|
||||||
|
import Data.Foldable
|
||||||
|
import Data.HashMap.Strict.InsOrd (InsOrdHashMap)
|
||||||
|
import qualified Data.HashMap.Strict.InsOrd as OMap
|
||||||
|
import Data.List (intersperse)
|
||||||
|
import Data.List.NonEmpty (NonEmpty(..))
|
||||||
|
import qualified Data.List.NonEmpty as NE
|
||||||
|
import Data.Maybe
|
||||||
|
import Data.String
|
||||||
|
import Data.Text (Text)
|
||||||
|
import qualified Data.Text as T
|
||||||
|
import qualified Data.Text.Lazy as LT
|
||||||
|
import Data.Text.Lazy.Builder (Builder)
|
||||||
|
import qualified Data.Text.Lazy.Builder as LT
|
||||||
|
import Data.Tuple
|
||||||
|
import qualified Data.Vector as V
|
||||||
|
import Hasura.Backends.BigQuery.Types
|
||||||
|
import Prelude
|
||||||
|
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
-- Types
|
||||||
|
|
||||||
|
data Printer
|
||||||
|
= SeqPrinter [Printer]
|
||||||
|
| SepByPrinter Printer [Printer]
|
||||||
|
| NewlinePrinter
|
||||||
|
| UnsafeTextPrinter Text
|
||||||
|
| IndentPrinter Int Printer
|
||||||
|
| ValuePrinter Value
|
||||||
|
deriving (Show, Eq)
|
||||||
|
|
||||||
|
instance IsString Printer where
|
||||||
|
fromString = UnsafeTextPrinter . fromString
|
||||||
|
|
||||||
|
(<+>) :: Printer -> Printer -> Printer
|
||||||
|
(<+>) x y = SeqPrinter [x,y]
|
||||||
|
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
-- Printer generators
|
||||||
|
|
||||||
|
fromExpression :: Expression -> Printer
|
||||||
|
fromExpression =
|
||||||
|
\case
|
||||||
|
CastExpression e scalarType ->
|
||||||
|
"CAST(" <+> fromExpression e <+> " AS " <+> fromScalarType scalarType <+> ")"
|
||||||
|
InExpression e value ->
|
||||||
|
"(" <+> fromExpression e <+> ") IN UNNEST(" <+> fromValue value <+> ")"
|
||||||
|
JsonQueryExpression e -> "JSON_QUERY(" <+> fromExpression e <+> ")"
|
||||||
|
JsonValueExpression e path ->
|
||||||
|
"JSON_VALUE(" <+> fromExpression e <+> fromPath path <+> ")"
|
||||||
|
ValueExpression value -> fromValue value
|
||||||
|
AndExpression xs ->
|
||||||
|
SepByPrinter
|
||||||
|
(NewlinePrinter <+> "AND ")
|
||||||
|
(toList
|
||||||
|
(fmap
|
||||||
|
(\x -> "(" <+> fromExpression x <+> ")")
|
||||||
|
(fromMaybe (pure trueExpression) (NE.nonEmpty xs))))
|
||||||
|
OrExpression xs ->
|
||||||
|
SepByPrinter
|
||||||
|
(NewlinePrinter <+> " OR ")
|
||||||
|
(toList
|
||||||
|
(fmap
|
||||||
|
(\x -> "(" <+> fromExpression x <+> ")")
|
||||||
|
(fromMaybe (pure falseExpression) (NE.nonEmpty xs))))
|
||||||
|
NotExpression expression -> "NOT " <+> (fromExpression expression)
|
||||||
|
ExistsExpression select ->
|
||||||
|
"EXISTS (" <+> IndentPrinter 9 (fromSelect select) <+> ")"
|
||||||
|
IsNullExpression expression ->
|
||||||
|
"(" <+> fromExpression expression <+> ") IS NULL"
|
||||||
|
IsNotNullExpression expression ->
|
||||||
|
"(" <+> fromExpression expression <+> ") IS NOT NULL"
|
||||||
|
ColumnExpression fieldName -> fromFieldName fieldName
|
||||||
|
EqualExpression x y ->
|
||||||
|
"(" <+> fromExpression x <+> ") = (" <+> fromExpression y <+> ")"
|
||||||
|
NotEqualExpression x y ->
|
||||||
|
"(" <+> fromExpression x <+> ") != (" <+> fromExpression y <+> ")"
|
||||||
|
ToStringExpression e -> "CONCAT(" <+> fromExpression e <+> ", '')"
|
||||||
|
SelectExpression s -> "(" <+> IndentPrinter 1 (fromSelect s) <+> ")"
|
||||||
|
OpExpression op x y ->
|
||||||
|
"(" <+>
|
||||||
|
fromExpression x <+>
|
||||||
|
") " <+> fromOp op <+> " (" <+> fromExpression y <+> ")"
|
||||||
|
ConditionalProjection expression fieldName ->
|
||||||
|
"(CASE WHEN(" <+> fromExpression expression <+>
|
||||||
|
") THEN " <+> fromFieldName fieldName <+>
|
||||||
|
" ELSE NULL END)"
|
||||||
|
|
||||||
|
fromScalarType :: ScalarType -> Printer
|
||||||
|
fromScalarType =
|
||||||
|
\case
|
||||||
|
StringScalarType -> "STRING"
|
||||||
|
BytesScalarType -> "BYTES"
|
||||||
|
IntegerScalarType -> "INT64"
|
||||||
|
FloatScalarType -> "FLOAT64"
|
||||||
|
BoolScalarType -> "BOOL"
|
||||||
|
TimestampScalarType -> "TIMESTAMP"
|
||||||
|
DateScalarType -> "DATE"
|
||||||
|
TimeScalarType -> "TIME"
|
||||||
|
DatetimeScalarType -> "DATETIME"
|
||||||
|
GeographyScalarType -> "GEOGRAPHY"
|
||||||
|
StructScalarType -> "STRUCT"
|
||||||
|
DecimalScalarType -> "DECIMAL"
|
||||||
|
BigDecimalScalarType -> "BIGDECIMAL"
|
||||||
|
|
||||||
|
fromOp :: Op -> Printer
|
||||||
|
fromOp =
|
||||||
|
\case
|
||||||
|
LessOp -> "<"
|
||||||
|
MoreOp -> ">"
|
||||||
|
MoreOrEqualOp -> ">="
|
||||||
|
LessOrEqualOp -> "<="
|
||||||
|
|
||||||
|
fromPath :: JsonPath -> Printer
|
||||||
|
fromPath path =
|
||||||
|
", " <+> string path
|
||||||
|
where
|
||||||
|
string = fromExpression .
|
||||||
|
ValueExpression . StringValue . LT.toStrict . LT.toLazyText . go
|
||||||
|
go =
|
||||||
|
\case
|
||||||
|
RootPath -> "$"
|
||||||
|
IndexPath r i -> go r <> "[" <> LT.fromString (show i) <> "]"
|
||||||
|
FieldPath r f -> go r <> "." <> LT.fromText f
|
||||||
|
|
||||||
|
fromFieldName :: FieldName -> Printer
|
||||||
|
fromFieldName (FieldName {..}) =
|
||||||
|
fromNameText fieldNameEntity <+> "." <+> fromNameText fieldName
|
||||||
|
|
||||||
|
fromSelect :: Select -> Printer
|
||||||
|
fromSelect Select {..} = finalExpression
|
||||||
|
where
|
||||||
|
finalExpression = inner
|
||||||
|
projections =
|
||||||
|
SepByPrinter
|
||||||
|
("," <+> NewlinePrinter)
|
||||||
|
(map fromProjection (toList selectProjections))
|
||||||
|
inner =
|
||||||
|
SepByPrinter
|
||||||
|
NewlinePrinter
|
||||||
|
[ "SELECT " <+> IndentPrinter 7 projections
|
||||||
|
, "FROM " <+> IndentPrinter 5 (fromFrom selectFrom)
|
||||||
|
, SepByPrinter
|
||||||
|
NewlinePrinter
|
||||||
|
(map
|
||||||
|
(\Join {..} ->
|
||||||
|
SeqPrinter
|
||||||
|
[ "LEFT OUTER JOIN " <+>
|
||||||
|
IndentPrinter 16 (fromJoinSource joinSource)
|
||||||
|
, NewlinePrinter
|
||||||
|
, "AS " <+> fromJoinAlias joinAlias
|
||||||
|
, NewlinePrinter
|
||||||
|
, "ON (" <+>
|
||||||
|
IndentPrinter
|
||||||
|
4
|
||||||
|
(SepByPrinter (", " <+> NewlinePrinter) (map fromOn joinOn)) <+>
|
||||||
|
")"
|
||||||
|
])
|
||||||
|
selectJoins)
|
||||||
|
, fromWhere selectWhere
|
||||||
|
, fromOrderBys selectTop selectOffset selectOrderBy
|
||||||
|
, case selectGroupBy of
|
||||||
|
[] -> ""
|
||||||
|
fieldNames -> "GROUP BY " <+> SepByPrinter ", " (map fromFieldName fieldNames)
|
||||||
|
]
|
||||||
|
|
||||||
|
fromOn :: (FieldName, FieldName) -> Printer
|
||||||
|
fromOn (x,y) = fromFieldName x <+> " = " <+> fromFieldName y
|
||||||
|
|
||||||
|
fromJoinSource :: JoinSource -> Printer
|
||||||
|
fromJoinSource =
|
||||||
|
\case
|
||||||
|
JoinSelect select -> "(" <+> fromSelect select <+> ")"
|
||||||
|
-- We're not using existingJoins at the moment, which was used to
|
||||||
|
-- avoid re-joining on the same table twice.
|
||||||
|
-- JoinReselect reselect -> "(" <+> fromReselect reselect <+> ")"
|
||||||
|
|
||||||
|
fromReselect :: Reselect -> Printer
|
||||||
|
fromReselect Reselect {..} =
|
||||||
|
SepByPrinter
|
||||||
|
NewlinePrinter
|
||||||
|
[ "SELECT " <+>
|
||||||
|
IndentPrinter 7 projections
|
||||||
|
, fromWhere reselectWhere
|
||||||
|
]
|
||||||
|
where
|
||||||
|
projections =
|
||||||
|
SepByPrinter
|
||||||
|
("," <+> NewlinePrinter)
|
||||||
|
(map fromProjection (toList reselectProjections))
|
||||||
|
|
||||||
|
fromOrderBys ::
|
||||||
|
Top -> Maybe Expression -> Maybe (NonEmpty OrderBy) -> Printer
|
||||||
|
fromOrderBys NoTop Nothing Nothing = "" -- An ORDER BY is wasteful if not needed.
|
||||||
|
fromOrderBys top moffset morderBys =
|
||||||
|
SepByPrinter
|
||||||
|
NewlinePrinter
|
||||||
|
[ case morderBys of
|
||||||
|
Nothing -> ""
|
||||||
|
Just orderBys ->
|
||||||
|
SeqPrinter
|
||||||
|
[ "ORDER BY "
|
||||||
|
, SepByPrinter
|
||||||
|
("," <+> NewlinePrinter)
|
||||||
|
(map fromOrderBy (toList orderBys))
|
||||||
|
]
|
||||||
|
, case (top, moffset) of
|
||||||
|
(NoTop, Nothing) -> ""
|
||||||
|
(NoTop, Just offset) -> "OFFSET " <+> fromExpression offset
|
||||||
|
(Top n, Nothing) -> "LIMIT " <+> fromValue (IntegerValue (intToInt64 n))
|
||||||
|
(Top n, Just offset) ->
|
||||||
|
"OFFSET " <+>
|
||||||
|
fromExpression offset <+>
|
||||||
|
" LIMIT " <+> fromValue (IntegerValue (intToInt64 n))
|
||||||
|
]
|
||||||
|
|
||||||
|
fromOrderBy :: OrderBy -> Printer
|
||||||
|
fromOrderBy OrderBy {..} =
|
||||||
|
"(" <+>
|
||||||
|
fromFieldName orderByFieldName <+>
|
||||||
|
") " <+>
|
||||||
|
fromOrder orderByOrder <+>
|
||||||
|
fromNullsOrder orderByNullsOrder
|
||||||
|
|
||||||
|
fromOrder :: Order -> Printer
|
||||||
|
fromOrder =
|
||||||
|
\case
|
||||||
|
AscOrder -> "ASC"
|
||||||
|
DescOrder -> "DESC"
|
||||||
|
|
||||||
|
fromNullsOrder :: NullsOrder -> Printer
|
||||||
|
fromNullsOrder =
|
||||||
|
\case
|
||||||
|
NullsAnyOrder -> ""
|
||||||
|
NullsFirst -> " NULLS FIRST"
|
||||||
|
NullsLast -> " NULLS LAST"
|
||||||
|
|
||||||
|
fromJoinAlias :: EntityAlias -> Printer
|
||||||
|
fromJoinAlias EntityAlias {entityAliasText} =
|
||||||
|
fromNameText entityAliasText
|
||||||
|
|
||||||
|
fromProjection :: Projection -> Printer
|
||||||
|
fromProjection =
|
||||||
|
\case
|
||||||
|
ExpressionProjection aliasedExpression ->
|
||||||
|
fromAliased (fmap fromExpression aliasedExpression)
|
||||||
|
FieldNameProjection aliasedFieldName ->
|
||||||
|
fromAliased (fmap fromFieldName aliasedFieldName)
|
||||||
|
AggregateProjection aliasedAggregate ->
|
||||||
|
fromAliased (fmap fromAggregate aliasedAggregate)
|
||||||
|
StarProjection -> "*"
|
||||||
|
ArrayAggProjection aliasedAgg -> fromAliased (fmap fromArrayAgg aliasedAgg)
|
||||||
|
EntityProjection aliasedEntity -> fromAliased (fmap fromJoinAlias aliasedEntity)
|
||||||
|
|
||||||
|
fromArrayAgg :: ArrayAgg -> Printer
|
||||||
|
fromArrayAgg ArrayAgg {..} =
|
||||||
|
SeqPrinter
|
||||||
|
[ "ARRAY_AGG("
|
||||||
|
, SepByPrinter
|
||||||
|
" "
|
||||||
|
[ "STRUCT(" <+> projections <+> ")"
|
||||||
|
, fromOrderBys
|
||||||
|
arrayAggTop
|
||||||
|
arrayAggOffset
|
||||||
|
(fmap
|
||||||
|
(fmap
|
||||||
|
(\orderBy ->
|
||||||
|
orderBy
|
||||||
|
{ orderByNullsOrder = NullsAnyOrder
|
||||||
|
-- Because BigQuery reports:
|
||||||
|
-- > NULLS FIRST not supported with descending sort order in aggregate functions
|
||||||
|
-- And the same error with 'ascending'.
|
||||||
|
}))
|
||||||
|
arrayAggOrderBy)
|
||||||
|
]
|
||||||
|
, ")"
|
||||||
|
]
|
||||||
|
where
|
||||||
|
projections =
|
||||||
|
SepByPrinter
|
||||||
|
("," <+> NewlinePrinter)
|
||||||
|
(map fromProjection (toList arrayAggProjections))
|
||||||
|
|
||||||
|
fromAggregate :: Aggregate -> Printer
|
||||||
|
fromAggregate =
|
||||||
|
\case
|
||||||
|
CountAggregate countable -> "COUNT(" <+> fromCountable countable <+> ")"
|
||||||
|
OpAggregate text arg ->
|
||||||
|
UnsafeTextPrinter text <+> "(" <+> fromExpression arg <+> ")"
|
||||||
|
OpAggregates text args ->
|
||||||
|
"STRUCT(" <+>
|
||||||
|
SepByPrinter
|
||||||
|
", "
|
||||||
|
(map
|
||||||
|
(\(alias, arg) ->
|
||||||
|
UnsafeTextPrinter text <+>
|
||||||
|
"(" <+> fromExpression arg <+> ") AS " <+> fromNameText alias)
|
||||||
|
(toList args)) <+>
|
||||||
|
")"
|
||||||
|
TextAggregate text -> fromExpression (ValueExpression (StringValue text))
|
||||||
|
|
||||||
|
fromCountable :: Countable FieldName -> Printer
|
||||||
|
fromCountable =
|
||||||
|
\case
|
||||||
|
StarCountable -> "*"
|
||||||
|
NonNullFieldCountable fields ->
|
||||||
|
SepByPrinter ", " (map fromFieldName (toList fields))
|
||||||
|
DistinctCountable fields ->
|
||||||
|
"DISTINCT " <+>
|
||||||
|
SepByPrinter ", " (map fromFieldName (toList fields))
|
||||||
|
|
||||||
|
fromWhere :: Where -> Printer
|
||||||
|
fromWhere =
|
||||||
|
\case
|
||||||
|
Where expressions ->
|
||||||
|
case (filter ((/= trueExpression) . collapse)) expressions of
|
||||||
|
[] -> ""
|
||||||
|
collapsedExpressions ->
|
||||||
|
"WHERE " <+>
|
||||||
|
IndentPrinter 6 (fromExpression (AndExpression collapsedExpressions))
|
||||||
|
where collapse (AndExpression [x]) = collapse x
|
||||||
|
collapse (AndExpression []) = trueExpression
|
||||||
|
collapse (OrExpression [x]) = collapse x
|
||||||
|
collapse x = x
|
||||||
|
|
||||||
|
fromFrom :: From -> Printer
|
||||||
|
fromFrom =
|
||||||
|
\case
|
||||||
|
FromQualifiedTable aliasedQualifiedTableName ->
|
||||||
|
fromAliased (fmap fromTableName aliasedQualifiedTableName)
|
||||||
|
|
||||||
|
fromTableName :: TableName -> Printer
|
||||||
|
fromTableName TableName {tableName, tableNameSchema} =
|
||||||
|
fromNameText tableNameSchema <+> "." <+> fromNameText tableName
|
||||||
|
|
||||||
|
fromAliased :: Aliased Printer -> Printer
|
||||||
|
fromAliased Aliased {..} =
|
||||||
|
aliasedThing <+>
|
||||||
|
((" AS " <+>) . fromNameText) aliasedAlias
|
||||||
|
|
||||||
|
fromNameText :: Text -> Printer
|
||||||
|
fromNameText t = UnsafeTextPrinter ("`" <> t <> "`")
|
||||||
|
|
||||||
|
trueExpression :: Expression
|
||||||
|
trueExpression = ValueExpression (BoolValue True)
|
||||||
|
|
||||||
|
falseExpression :: Expression
|
||||||
|
falseExpression = ValueExpression (BoolValue False)
|
||||||
|
|
||||||
|
fromValue :: Value -> Printer
|
||||||
|
fromValue = ValuePrinter
|
||||||
|
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
-- Quick and easy query printer
|
||||||
|
|
||||||
|
toBuilderFlat :: Printer -> Builder
|
||||||
|
toBuilderFlat = flip evalState mempty . runBuilderFlat
|
||||||
|
|
||||||
|
toBuilderPretty :: Printer -> Builder
|
||||||
|
toBuilderPretty = flip evalState mempty . runBuilderPretty
|
||||||
|
|
||||||
|
toTextPretty :: Printer -> Text
|
||||||
|
toTextPretty = LT.toStrict . LT.toLazyText . toBuilderPretty
|
||||||
|
|
||||||
|
toTextFlat :: Printer -> Text
|
||||||
|
toTextFlat = LT.toStrict . LT.toLazyText . toBuilderFlat
|
||||||
|
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
-- Printer ready for consumption
|
||||||
|
|
||||||
|
-- | Produces a query with holes, and a mapping for each
|
||||||
|
renderBuilderFlat :: Printer -> (Builder, InsOrdHashMap Int Value)
|
||||||
|
renderBuilderFlat =
|
||||||
|
second (OMap.fromList . map swap . OMap.toList) . flip runState mempty .
|
||||||
|
runBuilderFlat
|
||||||
|
|
||||||
|
-- | Produces a query with holes, and a mapping for each
|
||||||
|
renderBuilderPretty :: Printer -> (Builder, InsOrdHashMap Int Value)
|
||||||
|
renderBuilderPretty =
|
||||||
|
second (OMap.fromList . map swap . OMap.toList) . flip runState mempty .
|
||||||
|
runBuilderPretty
|
||||||
|
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
-- Real printer engines
|
||||||
|
|
||||||
|
paramName :: Int -> Builder
|
||||||
|
paramName next = "param" <> fromString (show next)
|
||||||
|
|
||||||
|
runBuilderFlat :: Printer -> State (InsOrdHashMap Value Int) Builder
|
||||||
|
runBuilderFlat = go 0
|
||||||
|
where
|
||||||
|
go level =
|
||||||
|
\case
|
||||||
|
UnsafeTextPrinter q -> pure (LT.fromText q)
|
||||||
|
SeqPrinter xs -> fmap (mconcat . filter notEmpty) (mapM (go level) xs)
|
||||||
|
SepByPrinter x xs -> do
|
||||||
|
i <- go level x
|
||||||
|
fmap (mconcat . intersperse i . filter notEmpty) (mapM (go level) xs)
|
||||||
|
NewlinePrinter -> pure " "
|
||||||
|
IndentPrinter n p -> go (level + n) p
|
||||||
|
ValuePrinter (ArrayValue x) | V.null x -> pure "[]"
|
||||||
|
ValuePrinter v -> do
|
||||||
|
themap <- get
|
||||||
|
next <- case OMap.lookup v themap of
|
||||||
|
Just next -> pure next
|
||||||
|
Nothing -> do next <- gets OMap.size
|
||||||
|
modify (OMap.insert v next)
|
||||||
|
pure next
|
||||||
|
pure ("@" <> paramName next)
|
||||||
|
notEmpty = (/= mempty)
|
||||||
|
|
||||||
|
runBuilderPretty :: Printer -> State (InsOrdHashMap Value Int) Builder
|
||||||
|
runBuilderPretty = go 0
|
||||||
|
where
|
||||||
|
go level =
|
||||||
|
\case
|
||||||
|
UnsafeTextPrinter q -> pure (LT.fromText q)
|
||||||
|
SeqPrinter xs -> fmap (mconcat . filter notEmpty) (mapM (go level) xs)
|
||||||
|
SepByPrinter x xs -> do
|
||||||
|
i <- go level x
|
||||||
|
fmap (mconcat . intersperse i . filter notEmpty) (mapM (go level) xs)
|
||||||
|
NewlinePrinter -> pure ("\n" <> indentation level)
|
||||||
|
IndentPrinter n p -> go (level + n) p
|
||||||
|
ValuePrinter (ArrayValue x)
|
||||||
|
| V.null x -> pure "[]"
|
||||||
|
ValuePrinter v -> do
|
||||||
|
themap <- get
|
||||||
|
next <-
|
||||||
|
case OMap.lookup v themap of
|
||||||
|
Just next -> pure next
|
||||||
|
Nothing -> do
|
||||||
|
next <- gets OMap.size
|
||||||
|
modify (OMap.insert v next)
|
||||||
|
pure next
|
||||||
|
pure ("@" <> paramName next)
|
||||||
|
indentation n = LT.fromText (T.replicate n " ")
|
||||||
|
notEmpty = (/= mempty)
|
623
server/src-lib/Hasura/Backends/BigQuery/Types.hs
Normal file
623
server/src-lib/Hasura/Backends/BigQuery/Types.hs
Normal file
@ -0,0 +1,623 @@
|
|||||||
|
{-# LANGUAGE DuplicateRecordFields #-}
|
||||||
|
|
||||||
|
-- | Types for Transact-SQL aka T-SQL; the language of SQL Server.
|
||||||
|
|
||||||
|
module Hasura.Backends.BigQuery.Types where
|
||||||
|
|
||||||
|
import qualified Data.Text.Encoding as T
|
||||||
|
import Control.DeepSeq
|
||||||
|
import Data.Aeson (ToJSONKey,FromJSONKey,ToJSON,FromJSON)
|
||||||
|
import qualified Data.Aeson as J
|
||||||
|
import Data.ByteString (ByteString)
|
||||||
|
import qualified Data.ByteString.Base64 as Base64
|
||||||
|
import qualified Data.ByteString.Lazy as L
|
||||||
|
import Data.Data
|
||||||
|
import Data.Hashable
|
||||||
|
import qualified Data.Text as T
|
||||||
|
import Data.Text.Extended
|
||||||
|
import Data.Vector (Vector)
|
||||||
|
import Data.Vector.Instances ()
|
||||||
|
import GHC.Generics
|
||||||
|
import Hasura.Incremental.Internal.Dependency
|
||||||
|
import Hasura.Prelude
|
||||||
|
import Hasura.RQL.Types.Error
|
||||||
|
import qualified Language.GraphQL.Draft.Syntax as G
|
||||||
|
import Language.Haskell.TH.Syntax
|
||||||
|
|
||||||
|
data Select = Select
|
||||||
|
{ selectTop :: !Top
|
||||||
|
, selectProjections :: !(NonEmpty Projection)
|
||||||
|
, selectFrom :: !From
|
||||||
|
, selectJoins :: ![Join]
|
||||||
|
, selectWhere :: !Where
|
||||||
|
, selectFor :: !For
|
||||||
|
, selectOrderBy :: !(Maybe (NonEmpty OrderBy))
|
||||||
|
, selectOffset :: !(Maybe Expression)
|
||||||
|
, selectGroupBy :: [FieldName]
|
||||||
|
, selectFinalWantedFields :: !(Maybe [Text])
|
||||||
|
} deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON Select
|
||||||
|
instance Hashable Select
|
||||||
|
instance Cacheable Select
|
||||||
|
instance ToJSON Select
|
||||||
|
instance NFData Select
|
||||||
|
|
||||||
|
data ArrayAgg = ArrayAgg
|
||||||
|
{ arrayAggProjections :: !(NonEmpty Projection)
|
||||||
|
, arrayAggOrderBy :: !(Maybe (NonEmpty OrderBy))
|
||||||
|
, arrayAggTop :: !Top
|
||||||
|
, arrayAggOffset :: !(Maybe Expression)
|
||||||
|
} deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON ArrayAgg
|
||||||
|
instance Hashable ArrayAgg
|
||||||
|
instance Cacheable ArrayAgg
|
||||||
|
instance ToJSON ArrayAgg
|
||||||
|
instance NFData ArrayAgg
|
||||||
|
|
||||||
|
data Reselect = Reselect
|
||||||
|
{ reselectProjections :: !(NonEmpty Projection)
|
||||||
|
, reselectFor :: !For
|
||||||
|
, reselectWhere :: !Where
|
||||||
|
} deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON Reselect
|
||||||
|
instance Hashable Reselect
|
||||||
|
instance Cacheable Reselect
|
||||||
|
instance ToJSON Reselect
|
||||||
|
instance NFData Reselect
|
||||||
|
|
||||||
|
data OrderBy = OrderBy
|
||||||
|
{ orderByFieldName :: FieldName
|
||||||
|
, orderByOrder :: Order
|
||||||
|
, orderByNullsOrder :: NullsOrder
|
||||||
|
} deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON OrderBy
|
||||||
|
instance Hashable OrderBy
|
||||||
|
instance Cacheable OrderBy
|
||||||
|
instance ToJSON OrderBy
|
||||||
|
instance NFData OrderBy
|
||||||
|
|
||||||
|
data Order
|
||||||
|
= AscOrder
|
||||||
|
| DescOrder
|
||||||
|
deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON Order
|
||||||
|
instance Hashable Order
|
||||||
|
instance Cacheable Order
|
||||||
|
instance ToJSON Order
|
||||||
|
instance NFData Order
|
||||||
|
|
||||||
|
data NullsOrder
|
||||||
|
= NullsFirst
|
||||||
|
| NullsLast
|
||||||
|
| NullsAnyOrder
|
||||||
|
deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON NullsOrder
|
||||||
|
instance Hashable NullsOrder
|
||||||
|
instance Cacheable NullsOrder
|
||||||
|
instance ToJSON NullsOrder
|
||||||
|
instance NFData NullsOrder
|
||||||
|
|
||||||
|
data For
|
||||||
|
= JsonFor ForJson
|
||||||
|
| NoFor
|
||||||
|
deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON For
|
||||||
|
instance Hashable For
|
||||||
|
instance Cacheable For
|
||||||
|
instance ToJSON For
|
||||||
|
instance NFData For
|
||||||
|
|
||||||
|
data ForJson = ForJson
|
||||||
|
{ jsonCardinality :: JsonCardinality
|
||||||
|
, jsonRoot :: Root
|
||||||
|
} deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON ForJson
|
||||||
|
instance Hashable ForJson
|
||||||
|
instance Cacheable ForJson
|
||||||
|
instance ToJSON ForJson
|
||||||
|
instance NFData ForJson
|
||||||
|
|
||||||
|
data Root
|
||||||
|
= NoRoot
|
||||||
|
| Root Text
|
||||||
|
deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON Root
|
||||||
|
instance Hashable Root
|
||||||
|
instance Cacheable Root
|
||||||
|
instance ToJSON Root
|
||||||
|
instance NFData Root
|
||||||
|
|
||||||
|
data JsonCardinality
|
||||||
|
= JsonArray
|
||||||
|
| JsonSingleton
|
||||||
|
deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON JsonCardinality
|
||||||
|
instance Hashable JsonCardinality
|
||||||
|
instance Cacheable JsonCardinality
|
||||||
|
instance ToJSON JsonCardinality
|
||||||
|
instance NFData JsonCardinality
|
||||||
|
|
||||||
|
data Projection
|
||||||
|
= ExpressionProjection (Aliased Expression)
|
||||||
|
| FieldNameProjection (Aliased FieldName)
|
||||||
|
| AggregateProjection (Aliased Aggregate)
|
||||||
|
| StarProjection
|
||||||
|
| ArrayAggProjection (Aliased ArrayAgg)
|
||||||
|
| EntityProjection (Aliased EntityAlias)
|
||||||
|
deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON Projection
|
||||||
|
instance Hashable Projection
|
||||||
|
instance Cacheable Projection
|
||||||
|
instance ToJSON Projection
|
||||||
|
instance NFData Projection
|
||||||
|
|
||||||
|
data Join = Join
|
||||||
|
{ joinSource :: !JoinSource
|
||||||
|
, joinAlias :: !EntityAlias
|
||||||
|
, joinOn :: [(FieldName,FieldName)]
|
||||||
|
, joinProvenance :: !JoinProvenance
|
||||||
|
, joinFieldName :: !Text
|
||||||
|
, joinExtractPath :: !(Maybe Text)
|
||||||
|
, joinRightTable :: !EntityAlias
|
||||||
|
} deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON Join
|
||||||
|
instance Hashable Join
|
||||||
|
instance Cacheable Join
|
||||||
|
instance ToJSON Join
|
||||||
|
instance NFData Join
|
||||||
|
|
||||||
|
data JoinProvenance
|
||||||
|
= OrderByJoinProvenance
|
||||||
|
| ObjectJoinProvenance
|
||||||
|
| ArrayAggregateJoinProvenance
|
||||||
|
| ArrayJoinProvenance
|
||||||
|
| MultiplexProvenance
|
||||||
|
deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON JoinProvenance
|
||||||
|
instance Hashable JoinProvenance
|
||||||
|
instance Cacheable JoinProvenance
|
||||||
|
instance ToJSON JoinProvenance
|
||||||
|
instance NFData JoinProvenance
|
||||||
|
|
||||||
|
data JoinSource
|
||||||
|
= JoinSelect Select
|
||||||
|
-- We're not using existingJoins at the moment, which was used to
|
||||||
|
-- avoid re-joining on the same table twice.
|
||||||
|
-- | JoinReselect Reselect
|
||||||
|
deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON JoinSource
|
||||||
|
instance Hashable JoinSource
|
||||||
|
instance Cacheable JoinSource
|
||||||
|
instance ToJSON JoinSource
|
||||||
|
instance NFData JoinSource
|
||||||
|
|
||||||
|
newtype Where =
|
||||||
|
Where [Expression]
|
||||||
|
deriving (NFData, Eq, Show, Generic, Data, Lift, FromJSON, ToJSON, Hashable, Cacheable)
|
||||||
|
|
||||||
|
instance Monoid Where where
|
||||||
|
mempty = Where mempty
|
||||||
|
|
||||||
|
instance Semigroup Where where
|
||||||
|
(Where x) <> (Where y) = Where (x <> y)
|
||||||
|
|
||||||
|
data Top
|
||||||
|
= NoTop
|
||||||
|
| Top Int
|
||||||
|
deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON Top
|
||||||
|
instance Hashable Top
|
||||||
|
instance Cacheable Top
|
||||||
|
instance ToJSON Top
|
||||||
|
instance NFData Top
|
||||||
|
|
||||||
|
instance Monoid Top where
|
||||||
|
mempty = NoTop
|
||||||
|
|
||||||
|
instance Semigroup Top where
|
||||||
|
(<>) :: Top -> Top -> Top
|
||||||
|
(<>) NoTop x = x
|
||||||
|
(<>) x NoTop = x
|
||||||
|
(<>) (Top x) (Top y) = Top (min x y)
|
||||||
|
|
||||||
|
data Expression
|
||||||
|
= ValueExpression Value
|
||||||
|
| InExpression Expression Value
|
||||||
|
| AndExpression [Expression]
|
||||||
|
| OrExpression [Expression]
|
||||||
|
| NotExpression Expression
|
||||||
|
| ExistsExpression Select
|
||||||
|
| SelectExpression Select
|
||||||
|
| IsNullExpression Expression
|
||||||
|
| IsNotNullExpression Expression
|
||||||
|
| ColumnExpression FieldName
|
||||||
|
| EqualExpression Expression Expression
|
||||||
|
| NotEqualExpression Expression Expression
|
||||||
|
| JsonQueryExpression Expression
|
||||||
|
-- ^ This one acts like a "cast to JSON" and makes SQL Server
|
||||||
|
-- behave like it knows your field is JSON and not double-encode
|
||||||
|
-- it.
|
||||||
|
| ToStringExpression Expression
|
||||||
|
| JsonValueExpression Expression JsonPath
|
||||||
|
-- ^ This is for getting actual atomic values out of a JSON
|
||||||
|
-- string.
|
||||||
|
| OpExpression Op Expression Expression
|
||||||
|
| CastExpression Expression ScalarType
|
||||||
|
| ConditionalProjection Expression FieldName
|
||||||
|
deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON Expression
|
||||||
|
instance Hashable Expression
|
||||||
|
instance Cacheable Expression
|
||||||
|
instance ToJSON Expression
|
||||||
|
instance NFData Expression
|
||||||
|
|
||||||
|
data JsonPath
|
||||||
|
= RootPath
|
||||||
|
| FieldPath JsonPath Text
|
||||||
|
| IndexPath JsonPath Integer
|
||||||
|
deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON JsonPath
|
||||||
|
instance Hashable JsonPath
|
||||||
|
instance Cacheable JsonPath
|
||||||
|
instance ToJSON JsonPath
|
||||||
|
instance NFData JsonPath
|
||||||
|
|
||||||
|
data Aggregate
|
||||||
|
= CountAggregate (Countable FieldName)
|
||||||
|
| OpAggregates !Text (NonEmpty (Text, Expression))
|
||||||
|
| OpAggregate !Text Expression
|
||||||
|
| TextAggregate !Text
|
||||||
|
deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON Aggregate
|
||||||
|
instance Hashable Aggregate
|
||||||
|
instance Cacheable Aggregate
|
||||||
|
instance ToJSON Aggregate
|
||||||
|
instance NFData Aggregate
|
||||||
|
|
||||||
|
data Countable fieldname
|
||||||
|
= StarCountable
|
||||||
|
| NonNullFieldCountable (NonEmpty fieldname)
|
||||||
|
| DistinctCountable (NonEmpty fieldname)
|
||||||
|
deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON a => FromJSON (Countable a)
|
||||||
|
instance Hashable a => Hashable (Countable a)
|
||||||
|
instance Cacheable a => Cacheable (Countable a)
|
||||||
|
instance ToJSON a => ToJSON (Countable a)
|
||||||
|
instance NFData a => NFData (Countable a)
|
||||||
|
|
||||||
|
data From
|
||||||
|
= FromQualifiedTable (Aliased TableName)
|
||||||
|
deriving (Eq, Show, Generic, Data, Lift, Ord)
|
||||||
|
instance FromJSON From
|
||||||
|
instance Hashable From
|
||||||
|
instance Cacheable From
|
||||||
|
instance ToJSON From
|
||||||
|
instance NFData From
|
||||||
|
|
||||||
|
data OpenJson = OpenJson
|
||||||
|
{ openJsonExpression :: Expression
|
||||||
|
, openJsonWith :: NonEmpty JsonFieldSpec
|
||||||
|
} deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON OpenJson
|
||||||
|
instance Hashable OpenJson
|
||||||
|
instance Cacheable OpenJson
|
||||||
|
instance ToJSON OpenJson
|
||||||
|
instance NFData OpenJson
|
||||||
|
|
||||||
|
data JsonFieldSpec
|
||||||
|
= IntField Text
|
||||||
|
| JsonField Text
|
||||||
|
deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON JsonFieldSpec
|
||||||
|
instance Hashable JsonFieldSpec
|
||||||
|
instance Cacheable JsonFieldSpec
|
||||||
|
instance ToJSON JsonFieldSpec
|
||||||
|
instance NFData JsonFieldSpec
|
||||||
|
|
||||||
|
data Aliased a = Aliased
|
||||||
|
{ aliasedThing :: !a
|
||||||
|
, aliasedAlias :: !Text
|
||||||
|
} deriving (Eq, Show, Generic, Data, Lift, Functor)
|
||||||
|
instance FromJSON a => FromJSON (Aliased a)
|
||||||
|
instance Hashable a => Hashable (Aliased a)
|
||||||
|
instance Cacheable a => Cacheable (Aliased a)
|
||||||
|
instance ToJSON a => ToJSON (Aliased a)
|
||||||
|
instance NFData a => NFData (Aliased a)
|
||||||
|
deriving instance Ord a => Ord (Aliased a)
|
||||||
|
|
||||||
|
newtype SchemaName = SchemaName
|
||||||
|
{ schemaNameParts :: [Text]
|
||||||
|
} deriving (NFData, Eq, Show, Generic, Data, Lift, FromJSON, ToJSON, Hashable, Cacheable)
|
||||||
|
|
||||||
|
data TableName = TableName
|
||||||
|
{ tableName :: Text
|
||||||
|
, tableNameSchema :: Text
|
||||||
|
} deriving (Eq, Show, Generic, Data, Lift, Ord)
|
||||||
|
instance FromJSON TableName where
|
||||||
|
parseJSON =
|
||||||
|
J.withObject
|
||||||
|
"TableName"
|
||||||
|
(\o -> TableName <$> o J..: "name" <*> o J..: "dataset")
|
||||||
|
instance ToJSON TableName where
|
||||||
|
toJSON TableName{..} = J.object [ "name" J..= tableName, "dataset" J..= tableNameSchema ]
|
||||||
|
instance Hashable TableName
|
||||||
|
instance Cacheable TableName
|
||||||
|
instance ToJSONKey TableName
|
||||||
|
instance NFData TableName
|
||||||
|
instance Arbitrary TableName where arbitrary = genericArbitrary
|
||||||
|
|
||||||
|
instance ToTxt TableName where toTxt = T.pack . show
|
||||||
|
|
||||||
|
data FieldName = FieldName
|
||||||
|
{ fieldName :: Text
|
||||||
|
, fieldNameEntity :: !Text
|
||||||
|
} deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON FieldName
|
||||||
|
instance Hashable FieldName
|
||||||
|
instance Cacheable FieldName
|
||||||
|
instance ToJSON FieldName
|
||||||
|
instance NFData FieldName
|
||||||
|
|
||||||
|
newtype ColumnName = ColumnName
|
||||||
|
{ columnName :: Text
|
||||||
|
} deriving (Eq, Ord, Show, Generic, Data, Lift, FromJSON, ToJSON, ToJSONKey, FromJSONKey, Hashable, Cacheable, NFData, ToTxt)
|
||||||
|
|
||||||
|
data Comment = DueToPermission | RequestedSingleObject
|
||||||
|
deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON Comment
|
||||||
|
instance Hashable Comment
|
||||||
|
instance Cacheable Comment
|
||||||
|
instance ToJSON Comment
|
||||||
|
instance NFData Comment
|
||||||
|
instance Arbitrary ColumnName where arbitrary = genericArbitrary
|
||||||
|
|
||||||
|
newtype EntityAlias = EntityAlias
|
||||||
|
{ entityAliasText :: Text
|
||||||
|
} deriving (NFData, Eq, Show, Generic, Data, Lift, FromJSON, ToJSON, Hashable, Cacheable)
|
||||||
|
|
||||||
|
data Op
|
||||||
|
= LessOp
|
||||||
|
| LessOrEqualOp
|
||||||
|
| MoreOp
|
||||||
|
| MoreOrEqualOp
|
||||||
|
-- | SIN
|
||||||
|
-- | SNE
|
||||||
|
-- | SLIKE
|
||||||
|
-- | SNLIKE
|
||||||
|
-- | SILIKE
|
||||||
|
-- | SNILIKE
|
||||||
|
-- | SSIMILAR
|
||||||
|
-- | SNSIMILAR
|
||||||
|
-- | SGTE
|
||||||
|
-- | SLTE
|
||||||
|
-- | SNIN
|
||||||
|
-- | SContains
|
||||||
|
-- | SContainedIn
|
||||||
|
-- | SHasKey
|
||||||
|
-- | SHasKeysAny
|
||||||
|
-- | SHasKeysAll
|
||||||
|
deriving (Eq, Show, Generic, Data, Lift)
|
||||||
|
instance FromJSON Op
|
||||||
|
instance Hashable Op
|
||||||
|
instance Cacheable Op
|
||||||
|
instance ToJSON Op
|
||||||
|
instance NFData Op
|
||||||
|
|
||||||
|
-- | Source for this represenation type:
|
||||||
|
--
|
||||||
|
-- https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#google.protobuf.Value
|
||||||
|
--
|
||||||
|
-- BigQuery results come in via the REST API as one of these simply types.
|
||||||
|
--
|
||||||
|
-- TODO: This omits StructValue -- do we need it?
|
||||||
|
data Value
|
||||||
|
= NullValue
|
||||||
|
| IntegerValue !Int64
|
||||||
|
-- ^ 64-bit <https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#integer_type>
|
||||||
|
| DecimalValue !Decimal
|
||||||
|
-- ^ Fixed precision <https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#decimal_types>
|
||||||
|
| BigDecimalValue !BigDecimal
|
||||||
|
-- ^ Fixed precision <https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#decimal_types>
|
||||||
|
| FloatValue !Float64
|
||||||
|
-- ^ Floating point <https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#floating_point_types>
|
||||||
|
| GeographyValue !Geography
|
||||||
|
| StringValue !Text
|
||||||
|
| BytesValue !Base64
|
||||||
|
| BoolValue !Bool
|
||||||
|
| ArrayValue !(Vector Value)
|
||||||
|
| TimestampValue !Timestamp
|
||||||
|
| DateValue !Date
|
||||||
|
| TimeValue !Time
|
||||||
|
| DatetimeValue !Datetime
|
||||||
|
deriving (Show, Eq, Ord, Generic, Data, Lift)
|
||||||
|
instance FromJSON Value
|
||||||
|
instance Cacheable Value
|
||||||
|
instance ToJSON Value
|
||||||
|
instance NFData Value
|
||||||
|
instance Hashable Value
|
||||||
|
|
||||||
|
-- | BigQuery's conception of a timestamp.
|
||||||
|
newtype Timestamp = Timestamp Text
|
||||||
|
deriving (Show, Eq, Ord, Generic, Data, Lift, ToJSON, FromJSON, Cacheable, NFData, Hashable)
|
||||||
|
|
||||||
|
-- | BigQuery's conception of a date.
|
||||||
|
newtype Date = Date Text
|
||||||
|
deriving (Show, Eq, Ord, Generic, Data, Lift, ToJSON, FromJSON, Cacheable, NFData, Hashable)
|
||||||
|
|
||||||
|
-- | BigQuery's conception of a time.
|
||||||
|
newtype Time = Time Text
|
||||||
|
deriving (Show, Eq, Ord, Generic, Data, Lift, ToJSON, FromJSON, Cacheable, NFData, Hashable)
|
||||||
|
|
||||||
|
-- | BigQuery's conception of a datetime.
|
||||||
|
newtype Datetime = Datetime Text
|
||||||
|
deriving (Show, Eq, Ord, Generic, Data, Lift, ToJSON, FromJSON, Cacheable, NFData, Hashable)
|
||||||
|
|
||||||
|
-- | BigQuery's conception of an INTEGER/INT64 (they are the same).
|
||||||
|
newtype Int64 = Int64 Text
|
||||||
|
deriving (Show, Eq, Ord, Generic, Data, Lift, ToJSON, FromJSON, Cacheable, NFData, Hashable)
|
||||||
|
|
||||||
|
intToInt64 :: Int -> Int64
|
||||||
|
intToInt64 = Int64 . T.pack . show
|
||||||
|
|
||||||
|
-- | BigQuery's conception of a fixed precision decimal.
|
||||||
|
newtype Decimal = Decimal Text
|
||||||
|
deriving (Show, Eq, Ord, Generic, Data, ToJSON, FromJSON, Cacheable, NFData, Hashable, Lift)
|
||||||
|
|
||||||
|
doubleToDecimal :: Double -> Decimal
|
||||||
|
doubleToDecimal = Decimal . T.decodeUtf8 . L.toStrict . J.encode
|
||||||
|
|
||||||
|
-- | BigQuery's conception of a \"big\" fixed precision decimal.
|
||||||
|
newtype BigDecimal = BigDecimal Text
|
||||||
|
deriving (Show, Eq, Ord, Generic, Data, ToJSON, FromJSON, Cacheable, NFData, Hashable, Lift)
|
||||||
|
|
||||||
|
doubleToBigDecimal :: Double -> BigDecimal
|
||||||
|
doubleToBigDecimal = BigDecimal . T.decodeUtf8 . L.toStrict . J.encode
|
||||||
|
|
||||||
|
-- | BigQuery's conception of a fixed precision decimal.
|
||||||
|
newtype Float64 = Float64 Text
|
||||||
|
deriving (Show, Eq, Ord, Generic, Data, ToJSON, FromJSON, Cacheable, NFData, Hashable, Lift)
|
||||||
|
|
||||||
|
doubleToFloat64 :: Double -> Float64
|
||||||
|
doubleToFloat64 = Float64 . T.decodeUtf8 . L.toStrict . J.encode
|
||||||
|
|
||||||
|
-- | A base-64 encoded binary string.
|
||||||
|
newtype Base64 = Base64
|
||||||
|
{ unBase64 :: ByteString
|
||||||
|
} deriving (Show, Eq, Ord, Generic, Data, Lift)
|
||||||
|
instance FromJSON Base64 where parseJSON = fmap (Base64 . L.toStrict . base64Decode) . J.parseJSON
|
||||||
|
instance ToJSON Base64 where toJSON = J.toJSON . T.decodeUtf8 . Base64.encode . unBase64
|
||||||
|
instance Cacheable Base64
|
||||||
|
instance NFData Base64
|
||||||
|
instance Hashable Base64
|
||||||
|
|
||||||
|
newtype Geography = Geography
|
||||||
|
{ unGeography :: Text
|
||||||
|
} deriving (Show, Eq, Ord, Generic, Data, Lift, FromJSON, ToJSON)
|
||||||
|
instance Cacheable Geography
|
||||||
|
instance NFData Geography
|
||||||
|
instance Hashable Geography
|
||||||
|
|
||||||
|
data ScalarType
|
||||||
|
= StringScalarType
|
||||||
|
| BytesScalarType
|
||||||
|
| IntegerScalarType
|
||||||
|
| FloatScalarType
|
||||||
|
| BoolScalarType
|
||||||
|
| TimestampScalarType
|
||||||
|
| DateScalarType
|
||||||
|
| TimeScalarType
|
||||||
|
| DatetimeScalarType
|
||||||
|
| GeographyScalarType
|
||||||
|
| DecimalScalarType
|
||||||
|
| BigDecimalScalarType
|
||||||
|
| StructScalarType
|
||||||
|
deriving (Show, Eq, Ord, Generic, Data, Lift)
|
||||||
|
instance FromJSON ScalarType
|
||||||
|
instance Cacheable ScalarType
|
||||||
|
instance ToJSON ScalarType
|
||||||
|
instance ToJSONKey ScalarType
|
||||||
|
instance NFData ScalarType
|
||||||
|
instance Hashable ScalarType
|
||||||
|
instance ToTxt ScalarType where toTxt = T.pack . show
|
||||||
|
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
-- Unified table metadata
|
||||||
|
|
||||||
|
data UnifiedMetadata = UnifiedMetadata
|
||||||
|
{ tables :: ![UnifiedTableMetadata]
|
||||||
|
}deriving (Eq, Show)
|
||||||
|
|
||||||
|
data UnifiedTableMetadata = UnifiedTableMetadata
|
||||||
|
{ table :: !UnifiedTableName
|
||||||
|
, object_relationships :: ![UnifiedObjectRelationship]
|
||||||
|
, array_relationships :: ![UnifiedArrayRelationship]
|
||||||
|
, columns :: ![UnifiedColumn]
|
||||||
|
}deriving (Eq, Show)
|
||||||
|
|
||||||
|
data UnifiedColumn = UnifiedColumn
|
||||||
|
{ name :: !Text
|
||||||
|
, type' :: !ScalarType
|
||||||
|
}deriving (Eq, Show)
|
||||||
|
|
||||||
|
data UnifiedTableName = UnifiedTableName
|
||||||
|
{ schema :: !Text
|
||||||
|
, name :: !Text
|
||||||
|
}deriving (Eq, Show)
|
||||||
|
|
||||||
|
data UnifiedObjectRelationship = UnifiedObjectRelationship
|
||||||
|
{ using :: !UnifiedUsing
|
||||||
|
, name :: !Text
|
||||||
|
}deriving (Eq, Show)
|
||||||
|
|
||||||
|
data UnifiedArrayRelationship = UnifiedArrayRelationship
|
||||||
|
{ using :: !UnifiedUsing
|
||||||
|
, name :: !Text
|
||||||
|
}deriving (Eq, Show)
|
||||||
|
|
||||||
|
data UnifiedUsing = UnifiedUsing
|
||||||
|
{ foreign_key_constraint_on :: !UnifiedOn
|
||||||
|
}deriving (Eq, Show)
|
||||||
|
|
||||||
|
data UnifiedOn = UnifiedOn
|
||||||
|
{ table :: !UnifiedTableName
|
||||||
|
, column :: !Text
|
||||||
|
}deriving (Eq, Show)
|
||||||
|
|
||||||
|
-- Copied from feature/mssql
|
||||||
|
newtype FunctionName = FunctionName Text -- TODO: Improve this type when SQL function support added
|
||||||
|
deriving (FromJSON, ToJSON, ToJSONKey, ToTxt, Arbitrary, Show, Eq, Ord, Hashable, Cacheable, NFData)
|
||||||
|
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
-- Backend-related stuff
|
||||||
|
|
||||||
|
parseScalarValue :: ScalarType -> J.Value -> Either QErr Value
|
||||||
|
parseScalarValue scalarType jValue = case scalarType of
|
||||||
|
StringScalarType -> StringValue <$> parseJValue jValue
|
||||||
|
BytesScalarType -> StringValue <$> parseJValue jValue
|
||||||
|
IntegerScalarType -> IntegerValue <$> parseJValue jValue
|
||||||
|
FloatScalarType -> FloatValue <$> parseJValue jValue
|
||||||
|
BoolScalarType -> BoolValue <$> parseJValue jValue
|
||||||
|
DecimalScalarType -> DecimalValue <$> parseJValue jValue
|
||||||
|
BigDecimalScalarType -> BigDecimalValue <$> parseJValue jValue
|
||||||
|
_ -> Left (internalError (T.pack ("Unsupported scalar type: " <> show scalarType <> ": " <> show jValue)))
|
||||||
|
-- TODO: These types:
|
||||||
|
-- TimestampScalarType -> TimestampValue <$> parseJValue jValue
|
||||||
|
-- DateScalarType -> DateValue <$> parseJValue jValue
|
||||||
|
-- TimeScalarType -> TimeValue <$> parseJValue jValue
|
||||||
|
-- DatetimeScalarType -> DatetimeValue <$> parseJValue jValue
|
||||||
|
-- GeographyScalarType -> GeographyValue <$> parseJValue jValue
|
||||||
|
-- RecordScalarType -> RecordValue <$> parseJValue jValue
|
||||||
|
-- StructScalarType -> StructValue <$> parseJValue jValue
|
||||||
|
where
|
||||||
|
parseJValue :: (J.FromJSON a) => J.Value -> Either QErr a
|
||||||
|
parseJValue = runAesonParser J.parseJSON
|
||||||
|
|
||||||
|
isComparableType, isNumType :: ScalarType -> Bool
|
||||||
|
|
||||||
|
-- TODO: What does this mean?
|
||||||
|
isComparableType = \case
|
||||||
|
BoolScalarType -> True
|
||||||
|
BytesScalarType -> True
|
||||||
|
_ -> False
|
||||||
|
|
||||||
|
isNumType =
|
||||||
|
\case
|
||||||
|
StringScalarType -> False
|
||||||
|
BytesScalarType -> False
|
||||||
|
IntegerScalarType -> True
|
||||||
|
FloatScalarType -> True
|
||||||
|
BoolScalarType -> False
|
||||||
|
TimestampScalarType -> False
|
||||||
|
DateScalarType -> False
|
||||||
|
TimeScalarType -> False
|
||||||
|
DatetimeScalarType -> False
|
||||||
|
GeographyScalarType -> False
|
||||||
|
DecimalScalarType -> True
|
||||||
|
BigDecimalScalarType -> True
|
||||||
|
StructScalarType -> False
|
||||||
|
|
||||||
|
getGQLTableName :: TableName -> Either QErr G.Name
|
||||||
|
getGQLTableName (TableName table schema) = do
|
||||||
|
let textName = schema <> "_" <> table
|
||||||
|
onNothing (G.mkName textName) $ throw400 ValidationFailed $
|
||||||
|
"cannot include " <> textName <> " in the GraphQL schema because it is not a valid GraphQL identifier"
|
@ -4,6 +4,7 @@ module Hasura.Backends.MSSQL.Instances.Schema () where
|
|||||||
|
|
||||||
import Hasura.Prelude
|
import Hasura.Prelude
|
||||||
|
|
||||||
|
|
||||||
import qualified Data.HashMap.Strict as Map
|
import qualified Data.HashMap.Strict as Map
|
||||||
import qualified Data.List.NonEmpty as NE
|
import qualified Data.List.NonEmpty as NE
|
||||||
import qualified Database.ODBC.SQLServer as ODBC
|
import qualified Database.ODBC.SQLServer as ODBC
|
||||||
|
@ -4,3 +4,4 @@ module Hasura.GraphQL.Execute.Instances (module B) where
|
|||||||
|
|
||||||
import Hasura.Backends.MSSQL.Instances.Execute as B ()
|
import Hasura.Backends.MSSQL.Instances.Execute as B ()
|
||||||
import Hasura.Backends.Postgres.Instances.Execute as B ()
|
import Hasura.Backends.Postgres.Instances.Execute as B ()
|
||||||
|
import Hasura.Backends.BigQuery.Instances.Execute as B ()
|
||||||
|
@ -15,6 +15,7 @@ import qualified Language.GraphQL.Draft.Syntax as G
|
|||||||
|
|
||||||
import Control.Monad.Trans.Control (MonadBaseControl)
|
import Control.Monad.Trans.Control (MonadBaseControl)
|
||||||
import Data.Text.Extended
|
import Data.Text.Extended
|
||||||
|
import qualified Data.Text as T
|
||||||
|
|
||||||
import qualified Hasura.Backends.Postgres.Execute.RemoteJoin as RR
|
import qualified Hasura.Backends.Postgres.Execute.RemoteJoin as RR
|
||||||
import qualified Hasura.Backends.Postgres.SQL.DML as S
|
import qualified Hasura.Backends.Postgres.SQL.DML as S
|
||||||
@ -27,6 +28,8 @@ import qualified Hasura.GraphQL.Execute.Query as E
|
|||||||
import qualified Hasura.GraphQL.Execute.RemoteJoin as RR
|
import qualified Hasura.GraphQL.Execute.RemoteJoin as RR
|
||||||
import qualified Hasura.GraphQL.Transport.HTTP.Protocol as GH
|
import qualified Hasura.GraphQL.Transport.HTTP.Protocol as GH
|
||||||
import qualified Hasura.SQL.AnyBackend as AB
|
import qualified Hasura.SQL.AnyBackend as AB
|
||||||
|
import qualified Hasura.Backends.BigQuery.DataLoader.Plan as BigQuery
|
||||||
|
import qualified Hasura.Backends.BigQuery.Plan as BigQuery
|
||||||
|
|
||||||
import Hasura.Backends.Postgres.SQL.Value
|
import Hasura.Backends.Postgres.SQL.Value
|
||||||
import Hasura.Backends.Postgres.Translate.Column (toTxtValue)
|
import Hasura.Backends.Postgres.Translate.Column (toTxtValue)
|
||||||
@ -95,7 +98,7 @@ explainQueryField userInfo fieldName rootField = do
|
|||||||
RFRemote _ -> throw400 InvalidParams "only hasura queries can be explained"
|
RFRemote _ -> throw400 InvalidParams "only hasura queries can be explained"
|
||||||
RFAction _ -> throw400 InvalidParams "query actions cannot be explained"
|
RFAction _ -> throw400 InvalidParams "query actions cannot be explained"
|
||||||
RFRaw _ -> pure $ Just $ FieldPlan fieldName Nothing Nothing
|
RFRaw _ -> pure $ Just $ FieldPlan fieldName Nothing Nothing
|
||||||
RFDB _ exists -> runMaybeT $ do
|
RFDB _ exists -> dispatch [do
|
||||||
-- TEMPORARY!!!
|
-- TEMPORARY!!!
|
||||||
-- We don't handle non-Postgres backends yet: for now, we filter root fields to only keep those
|
-- We don't handle non-Postgres backends yet: for now, we filter root fields to only keep those
|
||||||
-- that are targeting postgres, and we *silently* discard all the others. This is fine for now, as
|
-- that are targeting postgres, and we *silently* discard all the others. This is fine for now, as
|
||||||
@ -120,6 +123,23 @@ explainQueryField userInfo fieldName rootField = do
|
|||||||
liftTx $ map runIdentity <$>
|
liftTx $ map runIdentity <$>
|
||||||
Q.listQE dmlTxErrorHandler (Q.fromText withExplain) () True
|
Q.listQE dmlTxErrorHandler (Q.fromText withExplain) () True
|
||||||
pure $ FieldPlan fieldName (Just textSQL) $ Just planLines
|
pure $ FieldPlan fieldName (Just textSQL) $ Just planLines
|
||||||
|
,do
|
||||||
|
-- BigQuery case
|
||||||
|
SourceConfigWith _ (QDBR bqQDB) <-
|
||||||
|
hoistMaybe $ AB.unpackAnyBackend exists
|
||||||
|
lift $ do
|
||||||
|
actionsForest <- BigQuery.planToForest userInfo bqQDB
|
||||||
|
pure $
|
||||||
|
FieldPlan
|
||||||
|
fieldName
|
||||||
|
(Just ("--\n" <> BigQuery.drawActionsForestSQL actionsForest))
|
||||||
|
(Just ("": T.lines (BigQuery.drawActionsForest actionsForest)))]
|
||||||
|
where dispatch [] = pure Nothing
|
||||||
|
dispatch (x:xs) = do
|
||||||
|
mv <- runMaybeT x
|
||||||
|
case mv of
|
||||||
|
Nothing -> dispatch xs
|
||||||
|
Just v -> pure (Just v)
|
||||||
|
|
||||||
-- NOTE: This function has a 'MonadTrace' constraint in master, but we don't need it
|
-- NOTE: This function has a 'MonadTrace' constraint in master, but we don't need it
|
||||||
-- here. We should evaluate if we need it here.
|
-- here. We should evaluate if we need it here.
|
||||||
|
@ -4,3 +4,4 @@ module Hasura.GraphQL.Schema.Instances (module B) where
|
|||||||
|
|
||||||
import Hasura.Backends.MSSQL.Instances.Schema as B ()
|
import Hasura.Backends.MSSQL.Instances.Schema as B ()
|
||||||
import Hasura.Backends.Postgres.Instances.Schema as B ()
|
import Hasura.Backends.Postgres.Instances.Schema as B ()
|
||||||
|
import Hasura.Backends.BigQuery.Instances.Schema as B ()
|
||||||
|
@ -4,3 +4,4 @@ module Hasura.GraphQL.Transport.Instances (module B) where
|
|||||||
|
|
||||||
import Hasura.Backends.MSSQL.Instances.Transport as B ()
|
import Hasura.Backends.MSSQL.Instances.Transport as B ()
|
||||||
import Hasura.Backends.Postgres.Instances.Transport as B ()
|
import Hasura.Backends.Postgres.Instances.Transport as B ()
|
||||||
|
import Hasura.Backends.BigQuery.Instances.Transport as B ()
|
||||||
|
@ -73,6 +73,7 @@ import Data.Text.Extended
|
|||||||
import Data.Text.NonEmpty
|
import Data.Text.NonEmpty
|
||||||
import Data.URL.Template
|
import Data.URL.Template
|
||||||
|
|
||||||
|
|
||||||
import qualified Hasura.Backends.Postgres.SQL.Types as PG
|
import qualified Hasura.Backends.Postgres.SQL.Types as PG
|
||||||
|
|
||||||
import Hasura.EncJSON
|
import Hasura.EncJSON
|
||||||
|
@ -71,6 +71,7 @@ data Code
|
|||||||
| PostgresMaxConnectionsError
|
| PostgresMaxConnectionsError
|
||||||
| MSSQLError
|
| MSSQLError
|
||||||
| DatabaseConnectionTimeout
|
| DatabaseConnectionTimeout
|
||||||
|
| BigQueryError
|
||||||
| NotSupported
|
| NotSupported
|
||||||
| DependencyError
|
| DependencyError
|
||||||
| InvalidHeaders
|
| InvalidHeaders
|
||||||
@ -158,6 +159,7 @@ instance Show Code where
|
|||||||
InvalidCustomTypes -> "invalid-custom-types"
|
InvalidCustomTypes -> "invalid-custom-types"
|
||||||
MethodNotAllowed -> "method-not-allowed"
|
MethodNotAllowed -> "method-not-allowed"
|
||||||
Conflict -> "conflict"
|
Conflict -> "conflict"
|
||||||
|
BigQueryError -> "bigquery-error"
|
||||||
ActionWebhookCode t -> T.unpack t
|
ActionWebhookCode t -> T.unpack t
|
||||||
CustomCode t -> T.unpack t
|
CustomCode t -> T.unpack t
|
||||||
|
|
||||||
|
@ -4,3 +4,4 @@ module Hasura.RQL.Types.Instances (module B) where
|
|||||||
|
|
||||||
import Hasura.Backends.MSSQL.Instances.Types as B ()
|
import Hasura.Backends.MSSQL.Instances.Types as B ()
|
||||||
import Hasura.Backends.Postgres.Instances.Types as B ()
|
import Hasura.Backends.Postgres.Instances.Types as B ()
|
||||||
|
import Hasura.Backends.BigQuery.Instances.Types as B ()
|
||||||
|
@ -4,3 +4,4 @@ module Hasura.RQL.Types.Metadata.Instances (module B) where
|
|||||||
|
|
||||||
import Hasura.Backends.MSSQL.Instances.Metadata as B ()
|
import Hasura.Backends.MSSQL.Instances.Metadata as B ()
|
||||||
import Hasura.Backends.Postgres.Instances.Metadata as B ()
|
import Hasura.Backends.Postgres.Instances.Metadata as B ()
|
||||||
|
import Hasura.Backends.BigQuery.Instances.Metadata as B ()
|
||||||
|
@ -16,12 +16,14 @@ import Data.Text.Extended
|
|||||||
data BackendType
|
data BackendType
|
||||||
= Postgres
|
= Postgres
|
||||||
| MSSQL
|
| MSSQL
|
||||||
|
| BigQuery
|
||||||
deriving (Eq, Ord, Bounded, Enum)
|
deriving (Eq, Ord, Bounded, Enum)
|
||||||
|
|
||||||
-- | The name of the backend, as we expect it to appear in our metadata and API.
|
-- | The name of the backend, as we expect it to appear in our metadata and API.
|
||||||
instance ToTxt BackendType where
|
instance ToTxt BackendType where
|
||||||
toTxt Postgres = "postgres"
|
toTxt Postgres = "postgres"
|
||||||
toTxt MSSQL = "mssql"
|
toTxt MSSQL = "mssql"
|
||||||
|
toTxt BigQuery = "bigquery"
|
||||||
|
|
||||||
-- | The FromJSON instance uses this lookup mechanism to avoid having
|
-- | The FromJSON instance uses this lookup mechanism to avoid having
|
||||||
-- to duplicate and hardcode the backend string.
|
-- to duplicate and hardcode the backend string.
|
||||||
|
@ -115,6 +115,28 @@ data RQLMetadataV1
|
|||||||
| RMMssqlDropDeletePermission !(DropPerm 'MSSQL (DelPerm 'MSSQL))
|
| RMMssqlDropDeletePermission !(DropPerm 'MSSQL (DelPerm 'MSSQL))
|
||||||
| RMMssqlSetPermissionComment !(SetPermComment 'MSSQL)
|
| RMMssqlSetPermissionComment !(SetPermComment 'MSSQL)
|
||||||
|
|
||||||
|
-- BigQuery sources
|
||||||
|
| RMBigqueryAddSource !(AddSource 'BigQuery)
|
||||||
|
| RMBigqueryDropSource !DropSource
|
||||||
|
| RMBigqueryTrackTable !(TrackTableV2 'BigQuery)
|
||||||
|
| RMBigqueryUntrackTable !(UntrackTable 'BigQuery)
|
||||||
|
| RMBigqueryCreateObjectRelationship !(CreateObjRel 'BigQuery)
|
||||||
|
| RMBigqueryCreateArrayRelationship !(CreateArrRel 'BigQuery)
|
||||||
|
| RMBigqueryDropRelationship !(DropRel 'BigQuery)
|
||||||
|
| RMBigquerySetRelationshipComment !(SetRelComment 'BigQuery)
|
||||||
|
| RMBigqueryRenameRelationship !(RenameRel 'BigQuery)
|
||||||
|
|
||||||
|
| RMBigqueryCreateInsertPermission !(CreateInsPerm 'BigQuery)
|
||||||
|
| RMBigqueryCreateSelectPermission !(CreateSelPerm 'BigQuery)
|
||||||
|
| RMBigqueryCreateUpdatePermission !(CreateUpdPerm 'BigQuery)
|
||||||
|
| RMBigqueryCreateDeletePermission !(CreateDelPerm 'BigQuery)
|
||||||
|
|
||||||
|
| RMBigqueryDropInsertPermission !(DropPerm 'BigQuery (InsPerm 'BigQuery))
|
||||||
|
| RMBigqueryDropSelectPermission !(DropPerm 'BigQuery (SelPerm 'BigQuery))
|
||||||
|
| RMBigqueryDropUpdatePermission !(DropPerm 'BigQuery (UpdPerm 'BigQuery))
|
||||||
|
| RMBigqueryDropDeletePermission !(DropPerm 'BigQuery (DelPerm 'BigQuery))
|
||||||
|
| RMBigquerySetPermissionComment !(SetPermComment 'BigQuery)
|
||||||
|
|
||||||
-- Inconsistent metadata
|
-- Inconsistent metadata
|
||||||
| RMGetInconsistentMetadata !GetInconsistentMetadata
|
| RMGetInconsistentMetadata !GetInconsistentMetadata
|
||||||
| RMDropInconsistentMetadata !DropInconsistentMetadata
|
| RMDropInconsistentMetadata !DropInconsistentMetadata
|
||||||
@ -390,6 +412,28 @@ runMetadataQueryV1M env currentResourceVersion = \case
|
|||||||
RMPgRedeliverEvent q -> runRedeliverEvent q
|
RMPgRedeliverEvent q -> runRedeliverEvent q
|
||||||
RMPgInvokeEventTrigger q -> runInvokeEventTrigger q
|
RMPgInvokeEventTrigger q -> runInvokeEventTrigger q
|
||||||
|
|
||||||
|
RMBigqueryAddSource q -> runAddSource q
|
||||||
|
RMBigqueryDropSource q -> runDropSource q
|
||||||
|
RMBigqueryTrackTable q -> runTrackTableV2Q q
|
||||||
|
RMBigqueryUntrackTable q -> runUntrackTableQ q
|
||||||
|
|
||||||
|
RMBigqueryCreateObjectRelationship q -> runCreateRelationship ObjRel q
|
||||||
|
RMBigqueryCreateArrayRelationship q -> runCreateRelationship ArrRel q
|
||||||
|
RMBigqueryDropRelationship q -> runDropRel q
|
||||||
|
RMBigquerySetRelationshipComment q -> runSetRelComment q
|
||||||
|
RMBigqueryRenameRelationship q -> runRenameRel q
|
||||||
|
|
||||||
|
RMBigqueryCreateInsertPermission q -> runCreatePerm q
|
||||||
|
RMBigqueryCreateSelectPermission q -> runCreatePerm q
|
||||||
|
RMBigqueryCreateUpdatePermission q -> runCreatePerm q
|
||||||
|
RMBigqueryCreateDeletePermission q -> runCreatePerm q
|
||||||
|
|
||||||
|
RMBigqueryDropInsertPermission q -> runDropPerm q
|
||||||
|
RMBigqueryDropSelectPermission q -> runDropPerm q
|
||||||
|
RMBigqueryDropUpdatePermission q -> runDropPerm q
|
||||||
|
RMBigqueryDropDeletePermission q -> runDropPerm q
|
||||||
|
RMBigquerySetPermissionComment q -> runSetPermComment q
|
||||||
|
|
||||||
RMMssqlAddSource q -> runAddSource q
|
RMMssqlAddSource q -> runAddSource q
|
||||||
RMMssqlDropSource q -> runDropSource q
|
RMMssqlDropSource q -> runDropSource q
|
||||||
RMMssqlTrackTable q -> runTrackTableV2Q q
|
RMMssqlTrackTable q -> runTrackTableV2Q q
|
||||||
|
@ -27,6 +27,7 @@ import Hasura.Server.Version (HasVersion)
|
|||||||
import Hasura.Session
|
import Hasura.Session
|
||||||
|
|
||||||
import qualified Hasura.Backends.MSSQL.DDL.RunSQL as MSSQL
|
import qualified Hasura.Backends.MSSQL.DDL.RunSQL as MSSQL
|
||||||
|
import qualified Hasura.Backends.BigQuery.DDL.RunSQL as BigQuery
|
||||||
import qualified Hasura.Tracing as Tracing
|
import qualified Hasura.Tracing as Tracing
|
||||||
|
|
||||||
data RQLQuery
|
data RQLQuery
|
||||||
@ -37,6 +38,8 @@ data RQLQuery
|
|||||||
| RQCount !CountQuery
|
| RQCount !CountQuery
|
||||||
| RQRunSql !RunSQL
|
| RQRunSql !RunSQL
|
||||||
| RQMssqlRunSql !MSSQL.MSSQLRunSQL
|
| RQMssqlRunSql !MSSQL.MSSQLRunSQL
|
||||||
|
| RQBigqueryRunSql !BigQuery.BigQueryRunSQL
|
||||||
|
| RQBigqueryDatabaseInspection !BigQuery.BigQueryRunSQL
|
||||||
| RQBulk ![RQLQuery]
|
| RQBulk ![RQLQuery]
|
||||||
deriving (Show)
|
deriving (Show)
|
||||||
|
|
||||||
@ -120,4 +123,6 @@ runQueryM env = \case
|
|||||||
RQCount q -> runCount q
|
RQCount q -> runCount q
|
||||||
RQRunSql q -> runRunSQL q
|
RQRunSql q -> runRunSQL q
|
||||||
RQMssqlRunSql q -> MSSQL.runSQL q
|
RQMssqlRunSql q -> MSSQL.runSQL q
|
||||||
|
RQBigqueryRunSql q -> BigQuery.runSQL q
|
||||||
|
RQBigqueryDatabaseInspection q -> BigQuery.runDatabaseInspection q
|
||||||
RQBulk l -> encJFromList <$> indexedMapM (runQueryM env) l
|
RQBulk l -> encJFromList <$> indexedMapM (runQueryM env) l
|
||||||
|
Loading…
Reference in New Issue
Block a user