graphql-engine/server/src-exec/Main.hs

180 lines
8.2 KiB
Haskell
Raw Normal View History

{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE RecordWildCards #-}
2018-06-27 16:11:32 +03:00
module Main where
Clean metadata arguments ## Description Thanks to #1664, the Metadata API types no longer require a `ToJSON` instance. This PR follows up with a cleanup of the types of the arguments to the metadata API: - whenever possible, it moves those argument types to where they're used (RQL.DDL.*) - it removes all unrequired instances (mostly `ToJSON`) This PR does not attempt to do it for _all_ such argument types. For some of the metadata operations, the type used to describe the argument to the API and used to represent the value in the metadata are one and the same (like for `CreateEndpoint`). Sometimes, the two types are intertwined in complex ways (`RemoteRelationship` and `RemoteRelationshipDef`). In the spirit of only doing uncontroversial cleaning work, this PR only moves types that are not used outside of RQL.DDL. Furthermore, this is a small step towards separating the different types all jumbled together in RQL.Types. ## Notes This PR also improves several `FromJSON` instances to make use of `withObject`, and to use a human readable string instead of a type name in error messages whenever possible. For instance: - before: `expected Object for Object, but encountered X` after: `expected Object for add computed field, but encountered X` - before: `Expecting an object for update query` after: `expected Object for update query, but encountered X` This PR also renames `CreateFunctionPermission` to `FunctionPermissionArgument`, to remove the quite surprising `type DropFunctionPermission = CreateFunctionPermission`. This PR also deletes some dead code, mostly in RQL.DML. This PR also moves a PG-specific source resolving function from DDL.Schema.Source to the only place where it is used: App.hs. https://github.com/hasura/graphql-engine-mono/pull/1844 GitOrigin-RevId: a594521194bb7fe6a111b02a9e099896f9fed59c
2021-07-27 13:41:42 +03:00
import Hasura.Prelude
import qualified Control.Concurrent.Extended as C
import qualified Data.ByteString.Char8 as BC
import qualified Data.ByteString.Lazy as BL
import qualified Data.ByteString.Lazy.Char8 as BLC
import qualified Data.Environment as Env
import qualified Database.PG.Query as Q
import qualified System.Exit as Sys
import qualified System.Metrics as EKG
import qualified System.Posix.Signals as Signals
import Control.Exception
import Control.Monad.Trans.Managed (ManagedT (..), lowerManagedT)
import Data.Int (Int64)
import Data.Kind (Type)
import Data.Text.Conversions (convertText)
import Data.Time.Clock (getCurrentTime)
import Data.Time.Clock.POSIX (getPOSIXTime)
import GHC.TypeLits (Symbol)
Clean metadata arguments ## Description Thanks to #1664, the Metadata API types no longer require a `ToJSON` instance. This PR follows up with a cleanup of the types of the arguments to the metadata API: - whenever possible, it moves those argument types to where they're used (RQL.DDL.*) - it removes all unrequired instances (mostly `ToJSON`) This PR does not attempt to do it for _all_ such argument types. For some of the metadata operations, the type used to describe the argument to the API and used to represent the value in the metadata are one and the same (like for `CreateEndpoint`). Sometimes, the two types are intertwined in complex ways (`RemoteRelationship` and `RemoteRelationshipDef`). In the spirit of only doing uncontroversial cleaning work, this PR only moves types that are not used outside of RQL.DDL. Furthermore, this is a small step towards separating the different types all jumbled together in RQL.Types. ## Notes This PR also improves several `FromJSON` instances to make use of `withObject`, and to use a human readable string instead of a type name in error messages whenever possible. For instance: - before: `expected Object for Object, but encountered X` after: `expected Object for add computed field, but encountered X` - before: `Expecting an object for update query` after: `expected Object for update query, but encountered X` This PR also renames `CreateFunctionPermission` to `FunctionPermissionArgument`, to remove the quite surprising `type DropFunctionPermission = CreateFunctionPermission`. This PR also deletes some dead code, mostly in RQL.DML. This PR also moves a PG-specific source resolving function from DDL.Schema.Source to the only place where it is used: App.hs. https://github.com/hasura/graphql-engine-mono/pull/1844 GitOrigin-RevId: a594521194bb7fe6a111b02a9e099896f9fed59c
2021-07-27 13:41:42 +03:00
import qualified Hasura.GC as GC
import qualified Hasura.Tracing as Tracing
import Hasura.App
import Hasura.Logging (Hasura, LogLevel (..),
defaultEnabledEngineLogTypes)
import Hasura.Metadata.Class
import Hasura.RQL.DDL.Schema
import Hasura.RQL.DDL.Schema.Cache.Common
import Hasura.RQL.Types
2018-06-27 16:11:32 +03:00
import Hasura.Server.Init
import Hasura.Server.Metrics (ServerMetricsSpec, createServerMetrics)
import Hasura.Server.Migrate (downgradeCatalog)
import Hasura.Server.Types (MaintenanceMode (..))
import Hasura.Server.Version
import Hasura.Server.Version.TH
2018-06-27 16:11:32 +03:00
main :: IO ()
main = do
tryExit $ do
args <- parseArgs
env <- Env.getEnvironment
runApp env args
where
tryExit io = try io >>= \case
Left (ExitException _code msg) -> BC.putStrLn msg >> Sys.exitFailure
Right r -> return r
runApp :: Env.Environment -> HGEOptions Hasura -> IO ()
runApp env (HGEOptionsG rci metadataDbUrl hgeCmd) = do
initTime <- liftIO getCurrentTime
globalCtx@GlobalCtx{..} <- initGlobalCtx env metadataDbUrl rci
let
(maybeDefaultPgConnInfo, maybeRetries) = _gcDefaultPostgresConnInfo
withVersion $$(getVersionFromEnvironment) $ case hgeCmd of
HCServe serveOptions -> do
(ekgStore, serverMetrics) <- liftIO $ do
store <- EKG.newStore @AppMetricsSpec
EKG.register (EKG.subset GcSubset store) EKG.registerGcMetrics
let getTimeMs :: IO Int64
getTimeMs = (round . (* 1000)) `fmap` getPOSIXTime
EKG.register store $ EKG.registerCounter ServerTimestampMs () getTimeMs
serverMetrics <-
liftIO $ createServerMetrics $ EKG.subset ServerSubset store
pure (EKG.subset EKG.emptyOf store, serverMetrics)
-- It'd be nice if we didn't have to call runManagedT twice here, but
-- there is a data dependency problem since the call to runPGMetadataStorageApp
-- below depends on serveCtx.
runManagedT (initialiseServeCtx env globalCtx serveOptions) $ \serveCtx -> do
-- Catches the SIGTERM signal and initiates a graceful shutdown.
-- Graceful shutdown for regular HTTP requests is already implemented in
-- Warp, and is triggered by invoking the 'closeSocket' callback.
-- We only catch the SIGTERM signal once, that is, if the user hits CTRL-C
-- once again, we terminate the process immediately.
-- The function is written in this style to avoid the shutdown
-- handler retaining a reference to the entire serveCtx (see #344)
-- If you modify this code then you should check the core to see
-- that serveCtx is not retained.
_ <- case serveCtx of
ServeCtx{_scShutdownLatch} ->
liftIO $ do
void $ Signals.installHandler Signals.sigTERM (Signals.CatchOnce (shutdownGracefully _scShutdownLatch)) Nothing
void $ Signals.installHandler Signals.sigINT (Signals.CatchOnce (shutdownGracefully _scShutdownLatch)) Nothing
let Loggers _ logger pgLogger = _scLoggers serveCtx
_idleGCThread <- C.forkImmortal "ourIdleGC" logger $
GC.ourIdleGC logger (seconds 0.3) (seconds 10) (seconds 60)
flip runPGMetadataStorageAppT (_scMetadataDbPool serveCtx, pgLogger) . lowerManagedT $ do
Caching, Rate Limiting, Metrics & Session Variable Improvements (#376) * server: use a leaky bucket algorithm for bytes-per-second cache rate limiting * Use evalsha properly * Adds redis cache limit parameters to PoliciesConfig * Loads Leaky Bucket Script On Server Start * Adds more redis logging and moves cache update into lua script * reverts setex in lua and adds notes * Refactors cacheStore and adds max TTL and cache size limits * Filter session vars in cache key * WIP * parens * cache-clear-hander POC implementation * cache-clear-hander POC implementation * Pro projectId used as cache key * POC working! * prefixing query-response keys in redis * Add cacheClearer to RedisScripts * Partial implementation of cacheClearer from scripts record * updating tests * [automated] stylish-haskell commit * Adds query look with up with metrics script * Adds missing module and lua script from last commit * Changes redis script module structure to match cache clearing branch * minor change to lua script * cleaning up cache clearing * generalising JsonLog * [automated] stylish-haskell commit * Draft Cache Metrics Endpoint * Adds Cache Metrics Handler * Adds hook handler module * Missed HandlerHook module in last commit * glob * Fixes redis mget bug * Removes cache totals and changes dashes to colons in metric cache keys * Adds query param to clear clear endpoint for deleting specific keys * Adds query param to clear clear endpoint for deleting specific keys * Cache Metrics on query families rather then queries * Replace Set with nub * Base16 Redis Hashes * Query Family Redis Keys With Roles * response headers for cache keys * fixing bug in family key by excluding operation name; using hash for response header instead of entire key * Adds query family to redis cache keys and cache clear endpoint * Fixes queryfamily hash bug * Moves cache endpoints to /pro * Moved cache clear to POST * Refactors cache clear function * Fixes query family format bug * Adds query cache tests and optional --redis-url flag to python test suite * Adds session variable cache test * Update pro changelog * adding documentation for additional caching features * more docs * clearing up units of leaky bucket params * Adds comments to leaky bucket script * removes old todo * Fixes session variable filtering to work with new query rootfield * more advanced defaulting behaviour for bucket rate and capacity. * Updates Docs * Moves Role into QueryFamily hash * Use Aeson for Cache Clear endpoint response * Moves trace to bracket the leaky bucket script * Misc review tweaks * Adds sum type for cache clear query params * Hardcodes RegisReplyLog log level * Update docs/graphql/cloud/response-caching.rst Co-authored-by: Phil Freeman <phil@hasura.io> * new prose for rate limiting docs * [automated] stylish-haskell commit * make rootToSessVarPreds total * [automated] stylish-haskell commit * Fixes out of scope error * Renamed _acRedis to _acCacheStore Co-authored-by: Solomon Bothwell <ssbothwell@gmail.com> Co-authored-by: Lyndon Maydwell <lyndon@sordina.net> Co-authored-by: David Overton <david@hasura.io> Co-authored-by: Stylish Haskell Bot <stylish-haskell@users.noreply.github.com> Co-authored-by: Lyndon Maydwell <lyndon@hasura.io> GitOrigin-RevId: dda5c1a3f902967b3d78310f950541a55fabb1b0
2021-02-13 03:05:23 +03:00
runHGEServer (const $ pure ()) env serveOptions serveCtx initTime Nothing serverMetrics ekgStore
HCExport -> do
res <- runTxWithMinimalPool _gcMetadataDbConnInfo fetchMetadataFromCatalog
either (printErrJExit MetadataExportError) printJSON res
HCClean -> do
res <- runTxWithMinimalPool _gcMetadataDbConnInfo dropHdbCatalogSchema
let cleanSuccessMsg = "successfully cleaned graphql-engine related data"
either (printErrJExit MetadataCleanError) (const $ liftIO $ putStrLn cleanSuccessMsg) res
HCExecute -> do
queryBs <- liftIO BL.getContents
let sqlGenCtx = SQLGenCtx False False
remoteSchemaPermsCtx = RemoteSchemaPermsDisabled
pgLogger = print
pgSourceResolver = mkPgSourceResolver pgLogger
functionPermsCtx = FunctionPermissionsInferred
maintenanceMode = MaintenanceModeDisabled
serverConfigCtx =
[Preview] Inherited roles for postgres read queries fixes #3868 docker image - `hasura/graphql-engine:inherited-roles-preview-48b73a2de` Note: To be able to use the inherited roles feature, the graphql-engine should be started with the env variable `HASURA_GRAPHQL_EXPERIMENTAL_FEATURES` set to `inherited_roles`. Introduction ------------ This PR implements the idea of multiple roles as presented in this [paper](https://www.microsoft.com/en-us/research/wp-content/uploads/2016/02/FGALanguageICDE07.pdf). The multiple roles feature in this PR can be used via inherited roles. An inherited role is a role which can be created by combining multiple singular roles. For example, if there are two roles `author` and `editor` configured in the graphql-engine, then we can create a inherited role with the name of `combined_author_editor` role which will combine the select permissions of the `author` and `editor` roles and then make GraphQL queries using the `combined_author_editor`. How are select permissions of different roles are combined? ------------------------------------------------------------ A select permission includes 5 things: 1. Columns accessible to the role 2. Row selection filter 3. Limit 4. Allow aggregation 5. Scalar computed fields accessible to the role Suppose there are two roles, `role1` gives access to the `address` column with row filter `P1` and `role2` gives access to both the `address` and the `phone` column with row filter `P2` and we create a new role `combined_roles` which combines `role1` and `role2`. Let's say the following GraphQL query is queried with the `combined_roles` role. ```graphql query { employees { address phone } } ``` This will translate to the following SQL query: ```sql select (case when (P1 or P2) then address else null end) as address, (case when P2 then phone else null end) as phone from employee where (P1 or P2) ``` The other parameters of the select permission will be combined in the following manner: 1. Limit - Minimum of the limits will be the limit of the inherited role 2. Allow aggregations - If any of the role allows aggregation, then the inherited role will allow aggregation 3. Scalar computed fields - same as table column fields, as in the above example APIs for inherited roles: ---------------------- 1. `add_inherited_role` `add_inherited_role` is the [metadata API](https://hasura.io/docs/1.0/graphql/core/api-reference/index.html#schema-metadata-api) to create a new inherited role. It accepts two arguments `role_name`: the name of the inherited role to be added (String) `role_set`: list of roles that need to be combined (Array of Strings) Example: ```json { "type": "add_inherited_role", "args": { "role_name":"combined_user", "role_set":[ "user", "user1" ] } } ``` After adding the inherited role, the inherited role can be used like single roles like earlier Note: An inherited role can only be created with non-inherited/singular roles. 2. `drop_inherited_role` The `drop_inherited_role` API accepts the name of the inherited role and drops it from the metadata. It accepts a single argument: `role_name`: name of the inherited role to be dropped Example: ```json { "type": "drop_inherited_role", "args": { "role_name":"combined_user" } } ``` Metadata --------- The derived roles metadata will be included under the `experimental_features` key while exporting the metadata. ```json { "experimental_features": { "derived_roles": [ { "role_name": "manager_is_employee_too", "role_set": [ "employee", "manager" ] } ] } } ``` Scope ------ Only postgres queries and subscriptions are supported in this PR. Important points: ----------------- 1. All columns exposed to an inherited role will be marked as `nullable`, this is done so that cell value nullification can be done. TODOs ------- - [ ] Tests - [ ] Test a GraphQL query running with a inherited role without enabling inherited roles in experimental features - [] Tests for aggregate queries, limit, computed fields, functions, subscriptions (?) - [ ] Introspection test with a inherited role (nullability changes in a inherited role) - [ ] Docs - [ ] Changelog Co-authored-by: Vamshi Surabhi <6562944+0x777@users.noreply.github.com> GitOrigin-RevId: 3b8ee1e11f5ceca80fe294f8c074d42fbccfec63
2021-03-08 14:14:13 +03:00
ServerConfigCtx functionPermsCtx remoteSchemaPermsCtx sqlGenCtx maintenanceMode mempty
cacheBuildParams =
CacheBuildParams _gcHttpManager pgSourceResolver serverConfigCtx
runManagedT (mkMinimalPool _gcMetadataDbConnInfo) $ \metadataDbPool -> do
res <- flip runPGMetadataStorageAppT (metadataDbPool, pgLogger) $
runMetadataStorageT $ liftEitherM do
(metadata, _) <- fetchMetadata
runAsAdmin _gcHttpManager serverConfigCtx $ do
schemaCache <- runCacheBuild cacheBuildParams $
buildRebuildableSchemaCache env metadata
execQuery env queryBs
& Tracing.runTraceTWithReporter Tracing.noReporter "execute"
& runMetadataT metadata
& runCacheRWT schemaCache
& fmap (\((res, _), _, _) -> res)
either (printErrJExit ExecuteProcessError) (liftIO . BLC.putStrLn) res
2019-01-28 16:55:28 +03:00
HCDowngrade opts -> do
let defaultSourceConfig = maybeDefaultPgConnInfo <&> \(dbUrlConf, _) ->
let pgSourceConnInfo = PostgresSourceConnInfo dbUrlConf
(Just setPostgresPoolSettings{_ppsRetries = maybeRetries <|> Just 1})
False
Q.ReadCommitted
Nothing
in PostgresConnConfiguration pgSourceConnInfo Nothing
res <- runTxWithMinimalPool _gcMetadataDbConnInfo $ downgradeCatalog defaultSourceConfig opts initTime
either (printErrJExit DowngradeProcessError) (liftIO . print) res
HCVersion -> liftIO $ putStrLn $ "Hasura GraphQL Engine: " ++ convertText currentVersion
where
runTxWithMinimalPool connInfo tx = lowerManagedT $ do
minimalPool <- mkMinimalPool connInfo
liftIO $ runExceptT $ Q.runTx minimalPool (Q.ReadCommitted, Nothing) tx
-- | Generate Postgres pool with single connection.
-- It is useful when graphql-engine executes a transaction on database
-- and exits in commands other than 'serve'.
mkMinimalPool connInfo = do
pgLogger <- _lsPgLogger <$> mkLoggers defaultEnabledEngineLogTypes LevelInfo
let connParams = Q.defaultConnParams { Q.cpConns = 1 }
liftIO $ Q.initPGPool connInfo connParams pgLogger
-- | A specification of all EKG metrics tracked in `runApp`.
data AppMetricsSpec
:: Symbol -- Metric name
-> EKG.MetricType -- Metric type, e.g. Counter, Gauge
-> Type -- Tag structure
-> Type
where
ServerSubset
:: ServerMetricsSpec name metricType tags
-> AppMetricsSpec name metricType tags
GcSubset
:: EKG.GcMetrics name metricType tags
-> AppMetricsSpec name metricType tags
ServerTimestampMs
:: AppMetricsSpec "ekg.server_timestamp_ms" 'EKG.CounterType ()