server: change log level to error for triggers in Cloud

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/9873
GitOrigin-RevId: 703a16da479d35908a9a8c2862884d11a3135731

docs/docs/deployment/graphql-engine-flags/reference.mdx

@@ -352,6 +352,18 @@ Sets the maximum cumulative length of all headers in bytes.
 | **Default**         | `1024*1024` (1MB)                        |
 | **Supported in**    | CE, Enterprise Edition                   |
 
+### Enable Error Log Level for Trigger Errors
+
+Sets the log-level as `error` for Trigger type error logs (Event Triggers, Scheduled Triggers, Cron Triggers).
+
+|                     |                                                  |
+| ------------------- | ------------------------------------------------ |
+| **Flag**            | `--enable-triggers-error-log-level`              |
+| **Env var**         | `HASURA_GRAPHQL_ENABLE_TRIGGERS_ERROR_LOG_LEVEL` |
+| **Accepted values** | Boolean                                          |
+| **Default**         | `false`                                          |
+| **Supported in**    | CE, Enterprise Edition                           |
+
 ### Enable Console
 
 Enable the Hasura Console (served by the server on `/` and `/console`).

server/lib/test-harness/src/Harness/Constants.hs

@@ -311,7 +311,8 @@ serveOptions =
       soMetadataDefaults = emptyMetadataDefaults,
       soApolloFederationStatus = ApolloFederationDisabled,
       soCloseWebsocketsOnMetadataChangeStatus = Init._default Init.closeWebsocketsOnMetadataChangeOption,
-      soMaxTotalHeaderLength = Init._default Init.maxTotalHeaderLengthOption
+      soMaxTotalHeaderLength = Init._default Init.maxTotalHeaderLengthOption,
+      soTriggersErrorLogLevelStatus = Init._default Init.triggersErrorLogLevelStatusOption
     }
 
 -- | What log level should be used by the engine; this is not exported, and

server/src-lib/Hasura/App.hs

@@ -484,7 +484,8 @@ initialiseAppEnv env BasicConnectionInfo {..} serveOptions@ServeOptions {..} liv
           appEnvCheckFeatureFlag = ceCheckFeatureFlag env,
           appEnvSchemaPollInterval = soSchemaPollInterval,
           appEnvLicenseKeyCache = Nothing,
-          appEnvMaxTotalHeaderLength = soMaxTotalHeaderLength
+          appEnvMaxTotalHeaderLength = soMaxTotalHeaderLength,
+          appEnvTriggersErrorLogLevelStatus = soTriggersErrorLogLevelStatus
         }
     )
 
@@ -1275,6 +1276,7 @@ mkHGEServer setupHook appStateRef consoleType ekgStore = do
           appEnvServerMetrics
           (pmEventTriggerMetrics appEnvPrometheusMetrics)
           appEnvEnableMaintenanceMode
+          appEnvTriggersErrorLogLevelStatus
 
     startAsyncActionsPollerThread logger lockedEventsCtx actionSubState = do
       AppEnv {..} <- lift askAppEnv
@@ -1346,6 +1348,7 @@ mkHGEServer setupHook appStateRef consoleType ekgStore = do
           (pmScheduledTriggerMetrics appEnvPrometheusMetrics)
           (getSchemaCache appStateRef)
           lockedEventsCtx
+          appEnvTriggersErrorLogLevelStatus
 
 runInSeparateTx ::
   PG.TxE QErr a ->

server/src-lib/Hasura/App/State.hs

@@ -136,7 +136,8 @@ data AppEnv = AppEnv
     appEnvSchemaPollInterval :: OptionalInterval,
     appEnvCheckFeatureFlag :: CheckFeatureFlag,
     appEnvLicenseKeyCache :: Maybe (CredentialCache AgentLicenseKey),
-    appEnvMaxTotalHeaderLength :: Int
+    appEnvMaxTotalHeaderLength :: Int,
+    appEnvTriggersErrorLogLevelStatus :: TriggersErrorLogLevelStatus
   }
 
 -- | Represents the Dynamic Hasura State, these field are mutable and can be changed

server/src-lib/Hasura/Eventing/EventTrigger.hs

@@ -309,8 +309,9 @@ processEventQueue ::
   ServerMetrics ->
   EventTriggerMetrics ->
   MaintenanceMode () ->
+  TriggersErrorLogLevelStatus ->
   m (Forever m)
-processEventQueue logger statsLogger httpMgr getSchemaCache getEventEngineCtx activeEventProcessingThreads LockedEventsCtx {leEvents} serverMetrics eventTriggerMetrics maintenanceMode = do
+processEventQueue logger statsLogger httpMgr getSchemaCache getEventEngineCtx activeEventProcessingThreads LockedEventsCtx {leEvents} serverMetrics eventTriggerMetrics maintenanceMode triggersErrorLogLevelStatus = do
   events0 <- popEventsBatch
   return $ Forever (events0, 0, False) go
   where
@@ -542,7 +543,7 @@ processEventQueue logger statsLogger httpMgr getSchemaCache getEventEngineCtx ac
                         >>= \reqDetails -> do
                           let request = extractRequest reqDetails
                               logger' res details = do
-                                logHTTPForET res extraLogCtx details (_envVarName webhook) logHeaders
+                                logHTTPForET res extraLogCtx details (_envVarName webhook) logHeaders triggersErrorLogLevelStatus
                                 liftIO $ do
                                   case res of
                                     Left _err -> pure ()

server/src-lib/Hasura/Eventing/HTTP.hs

@@ -68,6 +68,7 @@ import Hasura.RQL.Types.Common (ResolvedWebhook (..))
 import Hasura.RQL.Types.EventTrigger
 import Hasura.RQL.Types.Eventing
 import Hasura.RQL.Types.Headers
+import Hasura.Server.Types (TriggersErrorLogLevelStatus, isTriggersErrorLogLevelEnabled)
 import Hasura.Session (SessionVariables)
 import Hasura.Tracing
 import Network.HTTP.Client.Transformable qualified as HTTP
@@ -207,11 +208,13 @@ instance J.ToJSON (HTTPRespExtra a) where
         Nothing -> updateReqDetail v "original_request"
         Just _ -> updateReqDetail v "transformed_request"
 
-instance ToEngineLog (HTTPRespExtra 'EventType) Hasura where
+data HTTPRespExtraLog a = HTTPRespExtraLog {_hrelLevel :: !LogLevel, _hrelpayload :: HTTPRespExtra a}
-  toEngineLog resp = (LevelInfo, eventTriggerLogType, J.toJSON resp)
 
-instance ToEngineLog (HTTPRespExtra 'ScheduledType) Hasura where
+instance ToEngineLog (HTTPRespExtraLog 'EventType) Hasura where
-  toEngineLog resp = (LevelInfo, scheduledTriggerLogType, J.toJSON resp)
+  toEngineLog (HTTPRespExtraLog level resp) = (level, eventTriggerLogType, J.toJSON resp)
+
+instance ToEngineLog (HTTPRespExtraLog 'ScheduledType) Hasura where
+  toEngineLog (HTTPRespExtraLog level resp) = (level, scheduledTriggerLogType, J.toJSON resp)
 
 isNetworkError :: HTTPErr a -> Bool
 isNetworkError = \case
@@ -251,6 +254,25 @@ instance J.ToJSON HTTPReq where
 instance ToEngineLog HTTPReq Hasura where
   toEngineLog req = (LevelInfo, eventTriggerLogType, J.toJSON req)
 
+logHTTPForTriggers ::
+  ( MonadReader r m,
+    Has (Logger Hasura) r,
+    MonadIO m,
+    ToEngineLog (HTTPRespExtraLog a) Hasura
+  ) =>
+  Either (HTTPErr a) (HTTPResp a) ->
+  ExtraLogContext ->
+  RequestDetails ->
+  Text ->
+  [HeaderConf] ->
+  TriggersErrorLogLevelStatus ->
+  m ()
+logHTTPForTriggers eitherResp extraLogCtx reqDetails webhookVarName logHeaders triggersErrorLogLevelStatus = do
+  logger :: Logger Hasura <- asks getter
+  case (eitherResp, isTriggersErrorLogLevelEnabled triggersErrorLogLevelStatus) of
+    (Left _, True) -> unLogger logger $ HTTPRespExtraLog LevelError $ HTTPRespExtra eitherResp extraLogCtx reqDetails webhookVarName logHeaders
+    (_, _) -> unLogger logger $ HTTPRespExtraLog LevelInfo $ HTTPRespExtra eitherResp extraLogCtx reqDetails webhookVarName logHeaders
+
 logHTTPForET ::
   ( MonadReader r m,
     Has (Logger Hasura) r,
@@ -261,10 +283,9 @@ logHTTPForET ::
   RequestDetails ->
   Text ->
   [HeaderConf] ->
+  TriggersErrorLogLevelStatus ->
   m ()
-logHTTPForET eitherResp extraLogCtx reqDetails webhookVarName logHeaders = do
+logHTTPForET = logHTTPForTriggers
-  logger :: Logger Hasura <- asks getter
-  unLogger logger $ HTTPRespExtra eitherResp extraLogCtx reqDetails webhookVarName logHeaders
 
 logHTTPForST ::
   ( MonadReader r m,
@@ -276,10 +297,9 @@ logHTTPForST ::
   RequestDetails ->
   Text ->
   [HeaderConf] ->
+  TriggersErrorLogLevelStatus ->
   m ()
-logHTTPForST eitherResp extraLogCtx reqDetails webhookVarName logHeaders = do
+logHTTPForST = logHTTPForTriggers
-  logger :: Logger Hasura <- asks getter
-  unLogger logger $ HTTPRespExtra eitherResp extraLogCtx reqDetails webhookVarName logHeaders
 
 runHTTP :: (MonadIO m) => HTTP.Manager -> HTTP.Request -> m (Either (HTTPErr a) (HTTPResp a))
 runHTTP manager req = do

server/src-lib/Hasura/Eventing/ScheduledTrigger.hs

@@ -160,6 +160,7 @@ import Hasura.RQL.Types.ScheduledTrigger
 import Hasura.RQL.Types.SchemaCache
 import Hasura.SQL.Types
 import Hasura.Server.Prometheus (ScheduledTriggerMetrics (..))
+import Hasura.Server.Types (TriggersErrorLogLevelStatus (..))
 import Hasura.Tracing qualified as Tracing
 import Network.HTTP.Client.Transformable qualified as HTTP
 import Refined (unrefine)
@@ -250,8 +251,9 @@ processCronEvents ::
   [CronEvent] ->
   HashMap TriggerName CronTriggerInfo ->
   TVar (Set.Set CronEventId) ->
+  TriggersErrorLogLevelStatus ->
   m ()
-processCronEvents logger httpMgr scheduledTriggerMetrics cronEvents cronTriggersInfo lockedCronEvents = do
+processCronEvents logger httpMgr scheduledTriggerMetrics cronEvents cronTriggersInfo lockedCronEvents triggersErrorLogLevelStatus = do
   -- save the locked cron events that have been fetched from the
   -- database, the events stored here will be unlocked in case a
   -- graceful shutdown is initiated in midst of processing these events
@@ -289,6 +291,7 @@ processCronEvents logger httpMgr scheduledTriggerMetrics cronEvents cronTriggers
                   payload
                   ctiWebhookInfo
                   Cron
+                  triggersErrorLogLevelStatus
         eventProcessedMaybe <-
           timeout (fromInteger (diffTimeToMicroSeconds eventProcessingTimeout)) $ processScheduledEventAction
         case eventProcessedMaybe of
@@ -319,6 +322,7 @@ processOneOffScheduledEvents ::
   ScheduledTriggerMetrics ->
   [OneOffScheduledEvent] ->
   TVar (Set.Set OneOffScheduledEventId) ->
+  TriggersErrorLogLevelStatus ->
   m ()
 processOneOffScheduledEvents
   env
@@ -326,7 +330,8 @@ processOneOffScheduledEvents
   httpMgr
   scheduledTriggerMetrics
   oneOffEvents
-  lockedOneOffScheduledEvents = do
+  lockedOneOffScheduledEvents
+  triggersErrorLogLevelStatus = do
     -- save the locked one-off events that have been fetched from the
     -- database, the events stored here will be unlocked in case a
     -- graceful shutdown is initiated in midst of processing these events
@@ -358,7 +363,7 @@ processOneOffScheduledEvents
           Right (webhookEnvRecord, eventHeaderInfo) -> do
             let processScheduledEventAction =
                   flip runReaderT (logger, httpMgr)
-                    $ processScheduledEvent scheduledTriggerMetrics _ooseId eventHeaderInfo retryCtx payload webhookEnvRecord OneOff
+                    $ processScheduledEvent scheduledTriggerMetrics _ooseId eventHeaderInfo retryCtx payload webhookEnvRecord OneOff triggersErrorLogLevelStatus
 
                 eventTimeout = unrefine $ strcTimeoutSeconds $ _ooseRetryConf
 
@@ -408,8 +413,9 @@ processScheduledTriggers ::
   ScheduledTriggerMetrics ->
   IO SchemaCache ->
   LockedEventsCtx ->
+  TriggersErrorLogLevelStatus ->
   m (Forever m)
-processScheduledTriggers getEnvHook logger statsLogger httpMgr scheduledTriggerMetrics getSC LockedEventsCtx {..} = do
+processScheduledTriggers getEnvHook logger statsLogger httpMgr scheduledTriggerMetrics getSC LockedEventsCtx {..} triggersErrorLogLevelStatus = do
   return
     $ Forever ()
     $ const do
@@ -419,8 +425,8 @@ processScheduledTriggers getEnvHook logger statsLogger httpMgr scheduledTriggerM
         Left e -> logInternalError e
         Right (cronEvents, oneOffEvents) -> do
           logFetchedScheduledEventsStats statsLogger (CronEventsCount $ length cronEvents) (OneOffScheduledEventsCount $ length oneOffEvents)
-          processCronEvents logger httpMgr scheduledTriggerMetrics cronEvents cronTriggersInfo leCronEvents
+          processCronEvents logger httpMgr scheduledTriggerMetrics cronEvents cronTriggersInfo leCronEvents triggersErrorLogLevelStatus
-          processOneOffScheduledEvents env logger httpMgr scheduledTriggerMetrics oneOffEvents leOneOffEvents
+          processOneOffScheduledEvents env logger httpMgr scheduledTriggerMetrics oneOffEvents leOneOffEvents triggersErrorLogLevelStatus
       -- NOTE: cron events are scheduled at times with minute resolution (as on
       -- unix), while one-off events can be set for arbitrary times. The sleep
       -- time here determines how overdue a scheduled event (cron or one-off)
@@ -445,8 +451,9 @@ processScheduledEvent ::
   ScheduledEventWebhookPayload ->
   EnvRecord ResolvedWebhook ->
   ScheduledEventType ->
+  TriggersErrorLogLevelStatus ->
   m ()
-processScheduledEvent scheduledTriggerMetrics eventId eventHeaders retryCtx payload webhookUrl type' =
+processScheduledEvent scheduledTriggerMetrics eventId eventHeaders retryCtx payload webhookUrl type' triggersErrorLogLevelStatus =
   Tracing.newTrace Tracing.sampleAlways traceNote do
     currentTime <- liftIO getCurrentTime
     let retryConf = _rctxConf retryCtx
@@ -470,7 +477,7 @@ processScheduledEvent scheduledTriggerMetrics eventId eventHeaders retryCtx payl
             >>= \reqDetails -> do
               let request = extractRequest reqDetails
                   logger e d = do
-                    logHTTPForST e extraLogCtx d (_envVarName webhookUrl) decodedHeaders
+                    logHTTPForST e extraLogCtx d (_envVarName webhookUrl) decodedHeaders triggersErrorLogLevelStatus
                     liftIO $ do
                       case e of
                         Left _err -> pure ()

server/src-lib/Hasura/Server/Init.hs

@@ -218,6 +218,7 @@ mkServeOptions sor@ServeOptionsRaw {..} = do
   soCloseWebsocketsOnMetadataChangeStatus <- do
     withOptionDefault rsoCloseWebsocketsOnMetadataChangeStatus closeWebsocketsOnMetadataChangeOption
   soMaxTotalHeaderLength <- withOptionDefault rsoMaxTotalHeaderLength maxTotalHeaderLengthOption
+  soTriggersErrorLogLevelStatus <- withOptionDefault rsoTriggersErrorLogLevelStatus triggersErrorLogLevelStatusOption
   pure ServeOptions {..}
 
 -- | Fetch Postgres 'Query.ConnParams' components from the environment

server/src-lib/Hasura/Server/Init/Arg/Command/Serve.hs

@@ -61,6 +61,7 @@ module Hasura.Server.Init.Arg.Command.Serve
     parseMetadataDefaults,
     metadataDefaultsOption,
     apolloFederationStatusOption,
+    triggersErrorLogLevelStatusOption,
     closeWebsocketsOnMetadataChangeOption,
     maxTotalHeaderLengthOption,
 
@@ -152,6 +153,7 @@ serveCommandParser =
     <*> parseApolloFederationStatus
     <*> parseEnableCloseWebsocketsOnMetadataChange
     <*> parseMaxTotalHeaderLength
+    <*> parseTriggersErrorLoglevelStatus
 
 --------------------------------------------------------------------------------
 -- Serve Options
@@ -1215,6 +1217,22 @@ maxTotalHeaderLengthOption =
       Config._helpMessage = "Max cumulative length of all headers in bytes (Default: 1MB)"
     }
 
+triggersErrorLogLevelStatusOption :: Config.Option (Types.TriggersErrorLogLevelStatus)
+triggersErrorLogLevelStatusOption =
+  Config.Option
+    { Config._default = Types.TriggersErrorLogLevelDisabled,
+      Config._envVar = "HASURA_GRAPHQL_ENABLE_TRIGGERS_ERROR_LOG_LEVEL",
+      Config._helpMessage = "Set log-level as error for Trigger error logs (Event Triggers, Scheduled Triggers, Cron Triggers) (default: false)."
+    }
+
+parseTriggersErrorLoglevelStatus :: Opt.Parser (Maybe Types.TriggersErrorLogLevelStatus)
+parseTriggersErrorLoglevelStatus =
+  (bool Nothing (Just Types.TriggersErrorLogLevelEnabled))
+    <$> Opt.switch
+      ( Opt.long "enable-triggers-error-log-level"
+          <> Opt.help (Config._helpMessage triggersErrorLogLevelStatusOption)
+      )
+
 --------------------------------------------------------------------------------
 -- Pretty Printer
 
@@ -1316,6 +1334,7 @@ serveCmdFooter =
         Config.optionPP apolloFederationStatusOption,
         Config.optionPP closeWebsocketsOnMetadataChangeOption,
         Config.optionPP maxTotalHeaderLengthOption,
-        Config.optionPP remoteNullForwardingPolicyOption
+        Config.optionPP remoteNullForwardingPolicyOption,
+        Config.optionPP triggersErrorLogLevelStatusOption
       ]
     eventEnvs = [Config.optionPP graphqlEventsHttpPoolSizeOption, Config.optionPP graphqlEventsFetchIntervalOption]

server/src-lib/Hasura/Server/Init/Config.hs

@@ -323,7 +323,8 @@ data ServeOptionsRaw impl = ServeOptionsRaw
     rsoMetadataDefaults :: Maybe MetadataDefaults,
     rsoApolloFederationStatus :: Maybe Server.Types.ApolloFederationStatus,
     rsoCloseWebsocketsOnMetadataChangeStatus :: Maybe Server.Types.CloseWebsocketsOnMetadataChangeStatus,
-    rsoMaxTotalHeaderLength :: Maybe Int
+    rsoMaxTotalHeaderLength :: Maybe Int,
+    rsoTriggersErrorLogLevelStatus :: Maybe Server.Types.TriggersErrorLogLevelStatus
   }
 
 -- | Whether or not to serve Console assets.
@@ -624,7 +625,8 @@ data ServeOptions impl = ServeOptions
     soMetadataDefaults :: MetadataDefaults,
     soApolloFederationStatus :: Server.Types.ApolloFederationStatus,
     soCloseWebsocketsOnMetadataChangeStatus :: Server.Types.CloseWebsocketsOnMetadataChangeStatus,
-    soMaxTotalHeaderLength :: Int
+    soMaxTotalHeaderLength :: Int,
+    soTriggersErrorLogLevelStatus :: Server.Types.TriggersErrorLogLevelStatus
   }
 
 -- | 'ResponseInternalErrorsConfig' represents the encoding of the

server/src-lib/Hasura/Server/Init/Env.hs

@@ -378,3 +378,6 @@ instance FromEnv GranularPrometheusMetricsState where
 
 instance FromEnv Server.Types.CloseWebsocketsOnMetadataChangeStatus where
   fromEnv = fmap (bool Server.Types.CWMCDisabled Server.Types.CWMCEnabled) . fromEnv @Bool
+
+instance FromEnv Server.Types.TriggersErrorLogLevelStatus where
+  fromEnv = fmap (bool Server.Types.TriggersErrorLogLevelDisabled Server.Types.TriggersErrorLogLevelEnabled) . fromEnv @Bool

server/src-lib/Hasura/Server/Types.hs

@@ -16,7 +16,9 @@ module Hasura.Server.Types
     CheckFeatureFlag (..),
     getRequestId,
     ApolloFederationStatus (..),
+    TriggersErrorLogLevelStatus (..),
     isApolloFederationEnabled,
+    isTriggersErrorLogLevelEnabled,
     GranularPrometheusMetricsState (..),
     OpenTelemetryExporterState (..),
     CloseWebsocketsOnMetadataChangeStatus (..),
@@ -165,6 +167,20 @@ isApolloFederationEnabled = \case
 instance ToJSON ApolloFederationStatus where
   toJSON = toJSON . isApolloFederationEnabled
 
+data TriggersErrorLogLevelStatus = TriggersErrorLogLevelEnabled | TriggersErrorLogLevelDisabled
+  deriving stock (Show, Eq, Ord, Generic)
+
+instance FromJSON TriggersErrorLogLevelStatus where
+  parseJSON = fmap (bool TriggersErrorLogLevelDisabled TriggersErrorLogLevelEnabled) . parseJSON
+
+isTriggersErrorLogLevelEnabled :: TriggersErrorLogLevelStatus -> Bool
+isTriggersErrorLogLevelEnabled = \case
+  TriggersErrorLogLevelEnabled -> True
+  TriggersErrorLogLevelDisabled -> False
+
+instance ToJSON TriggersErrorLogLevelStatus where
+  toJSON = toJSON . isTriggersErrorLogLevelEnabled
+
 -- | Whether or not to enable granular metrics for Prometheus.
 --
 -- `GranularMetricsOn` will enable the dynamic labels for the metrics.

server/src-test/Hasura/Server/InitSpec.hs

@@ -94,7 +94,8 @@ emptyServeOptionsRaw =
       rsoMetadataDefaults = Nothing,
       rsoApolloFederationStatus = Nothing,
       rsoCloseWebsocketsOnMetadataChangeStatus = Nothing,
-      rsoMaxTotalHeaderLength = Nothing
+      rsoMaxTotalHeaderLength = Nothing,
+      rsoTriggersErrorLogLevelStatus = Nothing
     }
 
 mkServeOptionsSpec :: Hspec.Spec

server/test-postgres/Constants.hs

@@ -93,7 +93,8 @@ serveOptions =
       soMetadataDefaults = emptyMetadataDefaults,
       soApolloFederationStatus = ApolloFederationDisabled,
       soCloseWebsocketsOnMetadataChangeStatus = Init._default Init.closeWebsocketsOnMetadataChangeOption,
-      soMaxTotalHeaderLength = Init._default Init.maxTotalHeaderLengthOption
+      soMaxTotalHeaderLength = Init._default Init.maxTotalHeaderLengthOption,
+      soTriggersErrorLogLevelStatus = Init._default Init.triggersErrorLogLevelStatusOption
     }
 
 -- | What log level should be used by the engine; this is not exported, and