From f0430ef2c76d5e22742b70fa4b5739bc22dd4e20 Mon Sep 17 00:00:00 2001 From: Krushan Bauva Date: Wed, 27 Sep 2023 12:00:27 +0530 Subject: [PATCH] skip `/healthz` and `/v1/version` data transfers from Prometheus metrics PR-URL: https://github.com/hasura/graphql-engine-mono/pull/10332 GitOrigin-RevId: a44001272edc394a42542f2cac5c3692e771d165 --- .../enterprise-edition/prometheus/metrics.mdx | 22 +++++++++++++++++++ server/src-lib/Hasura/App.hs | 4 ++-- server/src-lib/Hasura/Server/App.hs | 16 +++++++++----- server/src-lib/Hasura/Server/Logging.hs | 16 +++++++++----- 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/docs/docs/observability/enterprise-edition/prometheus/metrics.mdx b/docs/docs/observability/enterprise-edition/prometheus/metrics.mdx index 4e0f9630b20..1c5f156237e 100644 --- a/docs/docs/observability/enterprise-edition/prometheus/metrics.mdx +++ b/docs/docs/observability/enterprise-edition/prometheus/metrics.mdx @@ -471,6 +471,28 @@ Health check status of a particular data source, corresponding to the output of | Type | Gauge | | Labels | `source_name`: name of the database | +### HTTP Egress + +Total size of HTTP response bodies sent via the HTTP server excluding responses from requests to `/healthz` +and `/v1/version` endpoints or any other undefined resource/endpoint (for example `/foobar`). + +| | | +| ------ | ---------------------------------- | +| Name | `hasura_http_response_bytes_total` | +| Type | Counter | +| Labels | none | + +### HTTP Ingress + +Total size of HTTP request bodies received via the HTTP server excluding requests to `/healthz` and +`/v1/version` endpoints or any other undefined resource/endpoint (for example `/foobar`). + +| | | +| ------ | --------------------------------- | +| Name | `hasura_http_request_bytes_total` | +| Type | Counter | +| Labels | none | + ### OpenTelemetry OTLP Export Metrics These metrics allow for monitoring the reliability and performance of OTLP diff --git a/server/src-lib/Hasura/App.hs b/server/src-lib/Hasura/App.hs index c3a8e17e060..0dd08f268b1 100644 --- a/server/src-lib/Hasura/App.hs +++ b/server/src-lib/Hasura/App.hs @@ -700,12 +700,12 @@ instance HttpLog AppM where buildExtraHttpLogMetadata _ _ = () - logHttpError logger loggingSettings userInfoM reqId waiReq req qErr headers _ = + logHttpError logger loggingSettings userInfoM reqId waiReq req qErr headers _ _ = unLoggerTracing logger $ mkHttpLog $ mkHttpErrorLogContext userInfoM loggingSettings reqId waiReq req qErr Nothing Nothing headers - logHttpSuccess logger loggingSettings userInfoM reqId waiReq reqBody response compressedResponse qTime cType headers (CommonHttpLogMetadata rb batchQueryOpLogs, ()) = + logHttpSuccess logger loggingSettings userInfoM reqId waiReq reqBody response compressedResponse qTime cType headers (CommonHttpLogMetadata rb batchQueryOpLogs, ()) _ = unLoggerTracing logger $ mkHttpLog $ mkHttpAccessLogContext userInfoM loggingSettings reqId waiReq reqBody (BL.length response) compressedResponse qTime cType headers rb batchQueryOpLogs diff --git a/server/src-lib/Hasura/Server/App.hs b/server/src-lib/Hasura/Server/App.hs index deebde06e82..498e646f45b 100644 --- a/server/src-lib/Hasura/Server/App.hs +++ b/server/src-lib/Hasura/Server/App.hs @@ -402,7 +402,7 @@ mkSpockAction appStateRef qErrEncoder qErrModifier apiHandler = do allHeaders = [contentLength, jsonHeader] -- https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/http/#common-attributes lift $ Tracing.attachMetadata [("http.response_content_length", bsToTxt $ snd contentLength)] - lift $ logHttpError (_lsLogger appEnvLoggers) appEnvLoggingSettings userInfo reqId waiReq req qErr headers httpLogMetadata + lift $ logHttpError (_lsLogger appEnvLoggers) appEnvLoggingSettings userInfo reqId waiReq req qErr headers httpLogMetadata True mapM_ setHeader allHeaders Spock.setStatus $ qeStatus qErr Spock.lazyBytes jsonResponse @@ -419,7 +419,7 @@ mkSpockAction appStateRef qErrEncoder qErrModifier apiHandler = do allRespHeaders = [reqIdHeader, contentLength] <> encodingHeader <> respHeaders <> authHdrs -- https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/http/#common-attributes lift $ Tracing.attachMetadata [("http.response_content_length", bsToTxt $ snd contentLength)] - lift $ logHttpSuccess (_lsLogger appEnvLoggers) appEnvLoggingSettings userInfo reqId waiReq req respBytes compressedResp qTime encodingType reqHeaders httpLoggingMetadata + lift $ logHttpSuccess (_lsLogger appEnvLoggers) appEnvLoggingSettings userInfo reqId waiReq req respBytes compressedResp qTime encodingType reqHeaders httpLoggingMetadata True mapM_ setHeader allRespHeaders Spock.lazyBytes compressedResp @@ -1071,14 +1071,18 @@ httpApp setupHook appStateRef AppEnv {..} consoleType ekgStore closeWebsocketsOn let headers = Wai.requestHeaders req blMsg = TL.encodeUtf8 msg (reqId, _newHeaders) <- getRequestId headers - lift $ logHttpSuccess logger appEnvLoggingSettings Nothing reqId req (reqBody, Nothing) blMsg blMsg Nothing Nothing headers (emptyHttpLogMetadata @m) + -- setting the bool flag countDataTransferBytes to False here since we don't want to count the data + -- transfer bytes for requests to `/heatlhz` and `/v1/version` endpoints + lift $ logHttpSuccess logger appEnvLoggingSettings Nothing reqId req (reqBody, Nothing) blMsg blMsg Nothing Nothing headers (emptyHttpLogMetadata @m) False logError err = do req <- Spock.request reqBody <- liftIO $ Wai.strictRequestBody req let headers = Wai.requestHeaders req (reqId, _newHeaders) <- getRequestId headers - lift $ logHttpError logger appEnvLoggingSettings Nothing reqId req (reqBody, Nothing) err headers (emptyHttpLogMetadata @m) + -- setting the bool flag countDataTransferBytes to False here since we don't want to count the data + -- transfer bytes for requests to `/heatlhz` and `/v1/version` endpoints + lift $ logHttpError logger appEnvLoggingSettings Nothing reqId req (reqBody, Nothing) err headers (emptyHttpLogMetadata @m) False spockAction :: forall a. @@ -1148,7 +1152,9 @@ raiseGenericApiError logger loggingSetting headers qErr = do req <- Spock.request reqBody <- liftIO $ Wai.strictRequestBody req (reqId, _newHeaders) <- getRequestId $ Wai.requestHeaders req - lift $ logHttpError logger loggingSetting Nothing reqId req (reqBody, Nothing) qErr headers (emptyHttpLogMetadata @m) + -- setting the bool flag countDataTransferBytes to False here since we don't want to count the data + -- transfer bytes for requests to undefined resources + lift $ logHttpError logger loggingSetting Nothing reqId req (reqBody, Nothing) qErr headers (emptyHttpLogMetadata @m) False setHeader jsonHeader Spock.setStatus $ qeStatus qErr Spock.lazyBytes $ encode qErr diff --git a/server/src-lib/Hasura/Server/Logging.hs b/server/src-lib/Hasura/Server/Logging.hs index bdb1243b7b1..aa243db3e90 100644 --- a/server/src-lib/Hasura/Server/Logging.hs +++ b/server/src-lib/Hasura/Server/Logging.hs @@ -307,6 +307,8 @@ class (Monad m) => HttpLog m where -- | list of request headers [HTTP.Header] -> HttpLogMetadata m -> + -- | flag to indicate if the request/response size should be added to the Prometheus Counter + Bool -> m () logHttpSuccess :: @@ -334,6 +336,8 @@ class (Monad m) => HttpLog m where -- | list of request headers [HTTP.Header] -> HttpLogMetadata m -> + -- | flag to indicate if the request/response size should be added to the Prometheus Counter + Bool -> m () instance (HttpLog m) => HttpLog (TraceT m) where @@ -342,9 +346,9 @@ instance (HttpLog m) => HttpLog (TraceT m) where buildExtraHttpLogMetadata a = buildExtraHttpLogMetadata @m a emptyExtraHttpLogMetadata = emptyExtraHttpLogMetadata @m - logHttpError a b c d e f g h i = lift $ logHttpError a b c d e f g h i + logHttpError a b c d e f g h i j = lift $ logHttpError a b c d e f g h i j - logHttpSuccess a b c d e f g h i j k l = lift $ logHttpSuccess a b c d e f g h i j k l + logHttpSuccess a b c d e f g h i j k l m = lift $ logHttpSuccess a b c d e f g h i j k l m instance (HttpLog m) => HttpLog (ReaderT r m) where type ExtraHttpLogMetadata (ReaderT r m) = ExtraHttpLogMetadata m @@ -352,9 +356,9 @@ instance (HttpLog m) => HttpLog (ReaderT r m) where buildExtraHttpLogMetadata a = buildExtraHttpLogMetadata @m a emptyExtraHttpLogMetadata = emptyExtraHttpLogMetadata @m - logHttpError a b c d e f g h i = lift $ logHttpError a b c d e f g h i + logHttpError a b c d e f g h i j = lift $ logHttpError a b c d e f g h i j - logHttpSuccess a b c d e f g h i j k l = lift $ logHttpSuccess a b c d e f g h i j k l + logHttpSuccess a b c d e f g h i j k l m = lift $ logHttpSuccess a b c d e f g h i j k l m instance (HttpLog m) => HttpLog (ExceptT e m) where type ExtraHttpLogMetadata (ExceptT e m) = ExtraHttpLogMetadata m @@ -362,9 +366,9 @@ instance (HttpLog m) => HttpLog (ExceptT e m) where buildExtraHttpLogMetadata a = buildExtraHttpLogMetadata @m a emptyExtraHttpLogMetadata = emptyExtraHttpLogMetadata @m - logHttpError a b c d e f g h i = lift $ logHttpError a b c d e f g h i + logHttpError a b c d e f g h i j = lift $ logHttpError a b c d e f g h i j - logHttpSuccess a b c d e f g h i j k l = lift $ logHttpSuccess a b c d e f g h i j k l + logHttpSuccess a b c d e f g h i j k l m = lift $ logHttpSuccess a b c d e f g h i j k l m -- | Log information about the HTTP request data HttpInfoLog = HttpInfoLog