server: add Prometheus metric hasura_websocket_message_write_time and fix hasura_websocket_message_queue_time

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/9879
GitOrigin-RevId: 9dd82d01faa367fa67950a7f5f55a61fc396b27f

docs/docs/observability/enterprise-edition/prometheus/metrics.mdx

@@ -303,6 +303,26 @@ consider investigating the subscription query and see if indexes can help improv
 | Type   | Histogram<br /><br />Buckets: 0.000001, 0.0001, 0.01, 0.1, 0.3, 1, 3, 10, 30, 100          |
 | Labels | `subscription_kind`: streaming \| live-query, `operation_name`, `parameterized_query_hash` |
 
+#### Websocket Message Queue Time
+
+The time for which a websocket message remains queued in the GraphQL engine's websocket queue.
+
+|        |                                                                                   |
+| ------ | --------------------------------------------------------------------------------- |
+| Name   | `hasura_websocket_message_queue_time`                                             |
+| Type   | Histogram<br /><br />Buckets: 0.000001, 0.0001, 0.01, 0.1, 0.3, 1, 3, 10, 30, 100 |
+| Labels | none                                                                              |
+
+#### Websocket Message Write Time
+
+The time taken to write a websocket message into the TCP send buffer.
+
+|        |                                                                                   |
+| ------ | --------------------------------------------------------------------------------- |
+| Name   | `hasura_websocket_message_write_time`                                             |
+| Type   | Histogram<br /><br />Buckets: 0.000001, 0.0001, 0.01, 0.1, 0.3, 1, 3, 10, 30, 100 |
+| Labels | none                                                                              |
+
 ### Cache metrics
 
 See more details on caching metrics [here](/caching/caching-metrics.mdx)

server/src-lib/Hasura/GraphQL/Transport/WebSocket/Server.hs

@@ -626,8 +626,8 @@ createServerApp getMetricsConfig wsConnInitTimeout (WSServer logger@(L.Logger wr
 
             let send = forever $ do
                   WSQueueResponse msg wsInfo wsTimer <- liftIO $ STM.atomically $ STM.readTQueue sendQ
-                  liftIO $ WS.sendTextData conn msg
                   messageQueueTime <- liftIO $ realToFrac <$> wsTimer
+                  (messageWriteTime, _) <- liftIO $ withElapsedTime $ WS.sendTextData conn msg
                   let messageLength = BL.length msg
                       messageDetails = MessageDetails (SB.fromLBS msg) messageLength
                   liftIO $ do
@@ -637,6 +637,9 @@ createServerApp getMetricsConfig wsConnInitTimeout (WSServer logger@(L.Logger wr
                     Prometheus.Histogram.observe
                       (pmWebsocketMsgQueueTimeSeconds prometheusMetrics)
                       messageQueueTime
+                    Prometheus.Histogram.observe
+                      (pmWebsocketMsgWriteTimeSeconds prometheusMetrics)
+                      (realToFrac messageWriteTime)
                   logWSLog logger $ WSLog wsId (EMessageSent messageDetails) wsInfo
 
             -- withAsync lets us be very sure that if e.g. an async exception is raised while we're

server/src-lib/Hasura/Server/Prometheus.hs

@@ -76,6 +76,7 @@ data PrometheusMetrics = PrometheusMetrics
     pmScheduledTriggerMetrics :: ScheduledTriggerMetrics,
     pmSubscriptionMetrics :: SubscriptionMetrics,
     pmWebsocketMsgQueueTimeSeconds :: Histogram,
+    pmWebsocketMsgWriteTimeSeconds :: Histogram,
     pmCacheRequestMetrics :: CacheRequestMetrics
   }
 
@@ -146,6 +147,7 @@ makeDummyPrometheusMetrics = do
   pmScheduledTriggerMetrics <- makeDummyScheduledTriggerMetrics
   pmSubscriptionMetrics <- makeDummySubscriptionMetrics
   pmWebsocketMsgQueueTimeSeconds <- Histogram.new []
+  pmWebsocketMsgWriteTimeSeconds <- Histogram.new []
   pmCacheRequestMetrics <- makeDummyCacheRequestMetrics
   pure PrometheusMetrics {..}