Add additional tracing spans to HGE GraphQL queries and the Super Connector

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/9332
GitOrigin-RevId: ecde2383a42acf93fa8c6abb8bbd4c3b074b77fb

server/src-lib/Hasura/Backends/BigQuery/Instances/Types.hs

@@ -120,3 +120,4 @@ instance HasSourceConfiguration 'BigQuery where
   type SourceConnConfiguration 'BigQuery = BigQuery.BigQueryConnSourceConfig
   sourceConfigNumReadReplicas = const 0 -- not supported
   sourceConfigConnectonTemplateEnabled = const False -- not supported
+  sourceConfigBackendSourceKind _sourceConfig = BigQueryKind

server/src-lib/Hasura/Backends/DataConnector/Adapter/Backend.hs

@@ -25,7 +25,7 @@ import Hasura.Base.Error (Code (ValidationFailed), QErr, runAesonParser, throw40
 import Hasura.Prelude
 import Hasura.RQL.IR.BoolExp
 import Hasura.RQL.Types.Backend (Backend (..), ComputedFieldReturnType, HasSourceConfiguration (..), SupportedNamingCase (..), XDisable, XEnable)
-import Hasura.RQL.Types.BackendType (BackendType (DataConnector))
+import Hasura.RQL.Types.BackendType (BackendSourceKind (DataConnectorKind), BackendType (DataConnector))
 import Hasura.RQL.Types.Column (ColumnType (..))
 import Hasura.RQL.Types.ResizePool
 import Language.GraphQL.Draft.Syntax qualified as G
@@ -168,6 +168,7 @@ instance HasSourceConfiguration 'DataConnector where
   type SourceConnConfiguration 'DataConnector = DC.ConnSourceConfig
   sourceConfigNumReadReplicas = const 0 -- not supported
   sourceConfigConnectonTemplateEnabled = const False -- not supported
+  sourceConfigBackendSourceKind DC.SourceConfig {..} = DataConnectorKind _scDataConnectorName
 
 data CustomBooleanOperator a = CustomBooleanOperator
   { _cboName :: Text,

server/src-lib/Hasura/Backends/DataConnector/Adapter/Execute.hs

@@ -30,6 +30,7 @@ import Hasura.RQL.Types.Common qualified as RQL
 import Hasura.SQL.AnyBackend (mkAnyBackend)
 import Hasura.Session
 import Hasura.Tracing (MonadTrace)
+import Hasura.Tracing qualified as Tracing
 
 data DataConnectorPreparedQuery
   = QueryRequest API.QueryRequest
@@ -112,7 +113,7 @@ instance BackendExecute 'DataConnector where
 buildQueryAction :: (MonadIO m, MonadTrace m, MonadError QErr m) => RQL.SourceName -> SourceConfig -> Plan API.QueryRequest API.QueryResponse -> AgentClientT m EncJSON
 buildQueryAction sourceName SourceConfig {..} Plan {..} = do
   queryResponse <- Client.query sourceName _scConfig _pRequest
-  reshapedResponse <- _pResponseReshaper queryResponse
+  reshapedResponse <- Tracing.newSpan "QueryResponse reshaping" $ _pResponseReshaper queryResponse
   pure . encJFromBuilder $ J.fromEncoding reshapedResponse
 
 -- Delegates the generation to the Agent's /explain endpoint if it has that capability,
@@ -136,6 +137,6 @@ toExplainPlan fieldName queryRequest =
 
 buildMutationAction :: (MonadIO m, MonadTrace m, MonadError QErr m) => RQL.SourceName -> SourceConfig -> Plan API.MutationRequest API.MutationResponse -> AgentClientT m EncJSON
 buildMutationAction sourceName SourceConfig {..} Plan {..} = do
-  queryResponse <- Client.mutation sourceName _scConfig _pRequest
-  reshapedResponse <- _pResponseReshaper queryResponse
+  mutationResponse <- Client.mutation sourceName _scConfig _pRequest
+  reshapedResponse <- Tracing.newSpan "MutationResponse reshaping" $ _pResponseReshaper mutationResponse
   pure . encJFromBuilder $ J.fromEncoding reshapedResponse

server/src-lib/Hasura/Backends/DataConnector/Adapter/Transport.hs

@@ -58,7 +58,7 @@ runDBQuery' ::
   Maybe DataConnectorPreparedQuery ->
   ResolvedConnectionTemplate 'DataConnector ->
   m (DiffTime, EncJSON)
-runDBQuery' requestId query fieldName _userInfo logger licenseKeyCacheMaybe SourceConfig {..} action queryRequest _ = do
+runDBQuery' requestId query fieldName _userInfo logger licenseKeyCacheMaybe sourceConfig@SourceConfig {..} action queryRequest _ = do
   agentAuthKey <-
     for licenseKeyCacheMaybe \licenseKeyCache -> do
       (key, _requestKeyRefresh) <- liftIO $ atomically $ getCredential licenseKeyCache
@@ -73,6 +73,7 @@ runDBQuery' requestId query fieldName _userInfo logger licenseKeyCacheMaybe Sour
   void $ HGL.logQueryLog logger $ mkQueryLog query fieldName queryRequest requestId
   withElapsedTime
     . Tracing.newSpan ("Data Connector backend query for root field " <>> fieldName)
+    . (<* Tracing.attachSourceConfigAttributes @'DataConnector sourceConfig)
     . flip runAgentClientT (AgentClientContext logger _scEndpoint _scManager _scTimeoutMicroseconds agentAuthKey)
     . runOnBaseMonad
     . fmap snd
@@ -134,7 +135,7 @@ runDBMutation' ::
   Maybe DataConnectorPreparedQuery ->
   ResolvedConnectionTemplate 'DataConnector ->
   m (DiffTime, a)
-runDBMutation' requestId query fieldName _userInfo logger licenseKeyCacheMaybe SourceConfig {..} action queryRequest _ = do
+runDBMutation' requestId query fieldName _userInfo logger licenseKeyCacheMaybe sourceConfig@SourceConfig {..} action queryRequest _ = do
   agentAuthKey <-
     for licenseKeyCacheMaybe \licenseKeyCache -> do
       (key, _requestKeyRefresh) <- liftIO $ atomically $ getCredential licenseKeyCache
@@ -149,6 +150,7 @@ runDBMutation' requestId query fieldName _userInfo logger licenseKeyCacheMaybe S
   void $ HGL.logQueryLog logger $ mkQueryLog query fieldName queryRequest requestId
   withElapsedTime
     . Tracing.newSpan ("Data Connector backend mutation for root field " <>> fieldName)
+    . (<* Tracing.attachSourceConfigAttributes @'DataConnector sourceConfig)
     . flip runAgentClientT (AgentClientContext logger _scEndpoint _scManager _scTimeoutMicroseconds agentAuthKey)
     . runOnBaseMonad
     $ action

server/src-lib/Hasura/Backends/MSSQL/Instances/Transport.hs

@@ -75,10 +75,11 @@ runQuery ::
   ResolvedConnectionTemplate 'MSSQL ->
   -- | Also return the time spent in the PG query; for telemetry.
   m (DiffTime, EncJSON)
-runQuery reqId query fieldName _userInfo logger _ _sourceConfig tx genSql _ = do
+runQuery reqId query fieldName _userInfo logger _ sourceConfig tx genSql _ = do
   logQueryLog logger $ mkQueryLog query fieldName genSql reqId
   withElapsedTime
     $ newSpan ("MSSQL Query for root field " <>> fieldName)
+    $ (<* attachSourceConfigAttributes @'MSSQL sourceConfig)
     $ fmap snd (run tx)
 
 runQueryExplain ::
@@ -112,10 +113,11 @@ runMutation ::
   -- | Also return 'Mutation' when the operation was a mutation, and the time
   -- spent in the PG query; for telemetry.
   m (DiffTime, EncJSON)
-runMutation reqId query fieldName _userInfo logger _ _sourceConfig tx _genSql _ = do
+runMutation reqId query fieldName _userInfo logger _ sourceConfig tx _genSql _ = do
   logQueryLog logger $ mkQueryLog query fieldName Nothing reqId
   withElapsedTime
     $ newSpan ("MSSQL Mutation for root field " <>> fieldName)
+    $ (<* attachSourceConfigAttributes @'MSSQL sourceConfig)
     $ run tx
 
 runSubscription ::

server/src-lib/Hasura/Backends/MSSQL/Instances/Types.hs

@@ -126,3 +126,4 @@ instance HasSourceConfiguration 'MSSQL where
   type SourceConnConfiguration 'MSSQL = MSSQL.MSSQLConnConfiguration
   sourceConfigNumReadReplicas = MSSQL._mscReadReplicas
   sourceConfigConnectonTemplateEnabled = const False -- not supported
+  sourceConfigBackendSourceKind _sourceConfig = MSSQLKind

server/src-lib/Hasura/Backends/Postgres/Instances/Transport.hs

@@ -41,6 +41,7 @@ import Hasura.Name qualified as Name
 import Hasura.Prelude
 import Hasura.RQL.DDL.ConnectionTemplate (BackendResolvedConnectionTemplate (..), ResolvedConnectionTemplateWrapper (..))
 import Hasura.RQL.Types.Backend
+import Hasura.RQL.Types.BackendTag (HasTag)
 import Hasura.RQL.Types.BackendType
 import Hasura.SQL.AnyBackend qualified as AB
 import Hasura.Server.Types (RequestId)
@@ -53,14 +54,16 @@ instance
   ) =>
   BackendTransport ('Postgres pgKind)
   where
-  runDBQuery = runPGQuery
-  runDBMutation = runPGMutation
+  runDBQuery = runPGQuery @pgKind
+  runDBMutation = runPGMutation @pgKind
   runDBSubscription = runPGSubscription
   runDBStreamingSubscription = runPGStreamingSubscription
   runDBQueryExplain = runPGQueryExplain
 
 runPGQuery ::
-  ( MonadIO m,
+  forall pgKind m.
+  ( HasTag ('Postgres pgKind),
+    MonadIO m,
     MonadBaseControl IO m,
     MonadError QErr m,
     MonadQueryLog m,
@@ -83,11 +86,14 @@ runPGQuery reqId query fieldName _userInfo logger _ sourceConfig tx genSql resol
   logQueryLog logger $ mkQueryLog query fieldName genSql reqId (resolvedConnectionTemplate <$ resolvedConnectionTemplate)
   withElapsedTime
     $ newSpan ("Postgres Query for root field " <>> fieldName)
+    $ (<* attachSourceConfigAttributes @('Postgres pgKind) sourceConfig)
     $ runQueryTx (_pscExecCtx sourceConfig) (GraphQLQuery resolvedConnectionTemplate)
     $ fmap snd (runOnBaseMonad tx)
 
 runPGMutation ::
-  ( MonadIO m,
+  forall pgKind m.
+  ( HasTag ('Postgres pgKind),
+    MonadIO m,
     MonadBaseControl IO m,
     MonadError QErr m,
     MonadQueryLog m,
@@ -109,6 +115,7 @@ runPGMutation reqId query fieldName userInfo logger _ sourceConfig tx _genSql re
   logQueryLog logger $ mkQueryLog query fieldName Nothing reqId (resolvedConnectionTemplate <$ resolvedConnectionTemplate)
   withElapsedTime
     $ newSpan ("Postgres Mutation for root field " <>> fieldName)
+    $ (<* attachSourceConfigAttributes @('Postgres pgKind) sourceConfig)
     $ runTxWithCtxAndUserInfo userInfo (_pscExecCtx sourceConfig) (Tx PG.ReadWrite Nothing) (GraphQLQuery resolvedConnectionTemplate)
     $ runOnBaseMonad tx
 
@@ -176,7 +183,9 @@ mkQueryLog gqlQuery fieldName preparedSql requestId resolvedConnectionTemplate =
 -- see Note [Backwards-compatible transaction optimisation]
 
 runPGMutationTransaction ::
-  ( MonadIO m,
+  forall pgKind m.
+  ( HasTag ('Postgres pgKind),
+    MonadIO m,
     MonadBaseControl IO m,
     MonadError QErr m,
     MonadQueryLog m,
@@ -196,6 +205,7 @@ runPGMutationTransaction reqId query userInfo logger sourceConfig resolvedConnec
     $ runTxWithCtxAndUserInfo userInfo (_pscExecCtx sourceConfig) (Tx PG.ReadWrite Nothing) (GraphQLQuery resolvedConnectionTemplate)
     $ flip InsOrdHashMap.traverseWithKey mutations \fieldName dbsi ->
       newSpan ("Postgres Mutation for root field " <>> fieldName)
+        $ (<* attachSourceConfigAttributes @('Postgres pgKind) sourceConfig)
         $ fmap arResult
         $ runOnBaseMonad
         $ dbsiAction dbsi

server/src-lib/Hasura/Backends/Postgres/Instances/Types.hs

@@ -171,3 +171,8 @@ instance
   type SourceConnConfiguration ('Postgres pgKind) = Postgres.PostgresConnConfiguration
   sourceConfigNumReadReplicas = Postgres.sourceConfigNumReadReplicas
   sourceConfigConnectonTemplateEnabled = Postgres.sourceConfigConnectonTemplateEnabled
+  sourceConfigBackendSourceKind _sourceConfig =
+    case backendTag @('Postgres pgKind) of
+      PostgresVanillaTag -> PostgresVanillaKind
+      PostgresCitusTag -> PostgresCitusKind
+      PostgresCockroachTag -> PostgresCockroachKind

server/src-lib/Hasura/GraphQL/Execute.hs

@@ -352,7 +352,7 @@ getResolvedExecPlan
     -- Construct the full 'ResolvedExecutionPlan' from the 'queryParts :: SingleOperation'.
     (parameterizedQueryHash, resolvedExecPlan) <-
       case queryParts of
-        G.TypedOperationDefinition G.OperationTypeQuery _ varDefs directives inlinedSelSet -> do
+        G.TypedOperationDefinition G.OperationTypeQuery _ varDefs directives inlinedSelSet -> Tracing.newSpan "Resolve query execution plan" $ do
           (executionPlan, queryRootFields, dirMap, parameterizedQueryHash) <-
             EQ.convertQuerySelSet
               env
@@ -368,9 +368,9 @@ getResolvedExecPlan
               (scSetGraphqlIntrospectionOptions sc)
               reqId
               maybeOperationName
-          Tracing.attachMetadata [("parameterized_query_hash", bsToTxt $ unParamQueryHash parameterizedQueryHash)]
+          Tracing.attachMetadata [("graphql.operation.type", "query"), ("parameterized_query_hash", bsToTxt $ unParamQueryHash parameterizedQueryHash)]
           pure (parameterizedQueryHash, QueryExecutionPlan executionPlan queryRootFields dirMap)
-        G.TypedOperationDefinition G.OperationTypeMutation _ varDefs directives inlinedSelSet -> do
+        G.TypedOperationDefinition G.OperationTypeMutation _ varDefs directives inlinedSelSet -> Tracing.newSpan "Resolve mutation execution plan" $ do
           when (readOnlyMode == ReadOnlyModeEnabled)
             $ throw400 NotSupported "Mutations are not allowed when read-only mode is enabled"
           (executionPlan, parameterizedQueryHash) <-
@@ -389,8 +389,9 @@ getResolvedExecPlan
               (scSetGraphqlIntrospectionOptions sc)
               reqId
               maybeOperationName
+          Tracing.attachMetadata [("graphql.operation.type", "mutation")]
           pure (parameterizedQueryHash, MutationExecutionPlan executionPlan)
-        G.TypedOperationDefinition G.OperationTypeSubscription _ varDefs directives inlinedSelSet -> do
+        G.TypedOperationDefinition G.OperationTypeSubscription _ varDefs directives inlinedSelSet -> Tracing.newSpan "Resolve subscription execution plan" $ do
           (normalizedDirectives, normalizedSelectionSet) <-
             ER.resolveVariables
               varDefs
@@ -398,7 +399,7 @@ getResolvedExecPlan
               directives
               inlinedSelSet
           subscriptionParser <- C.gqlSubscriptionParser gCtx `onNothing` throw400 ValidationFailed "no subscriptions exist"
-          unpreparedAST <- liftEither $ subscriptionParser normalizedSelectionSet
+          unpreparedAST <- Tracing.newSpan "Parse subscription IR" $ liftEither $ subscriptionParser normalizedSelectionSet
           let parameterizedQueryHash = calculateParameterizedQueryHash normalizedSelectionSet
           -- Process directives on the subscription
           dirMap <-
@@ -416,6 +417,7 @@ getResolvedExecPlan
               unless (allowMultipleRootFields && isSingleNamespace unpreparedAST)
                 $ throw400 ValidationFailed "subscriptions must select one top level field"
           subscriptionPlan <- buildSubscriptionPlan userInfo unpreparedAST parameterizedQueryHash reqHeaders maybeOperationName
+          Tracing.attachMetadata [("graphql.operation.type", "subscription")]
           pure (parameterizedQueryHash, SubscriptionExecutionPlan subscriptionPlan)
     -- the parameterized query hash is calculated here because it is used in multiple
     -- places and instead of calculating it separately, this is a common place to calculate

server/src-lib/Hasura/GraphQL/Execute/Mutation.hs

@@ -7,6 +7,7 @@ import Data.Environment qualified as Env
 import Data.HashMap.Strict qualified as HashMap
 import Data.HashMap.Strict.InsOrd qualified as InsOrdHashMap
 import Data.Tagged qualified as Tagged
+import Data.Text.Extended ((<>>))
 import Hasura.Base.Error
 import Hasura.GraphQL.Context
 import Hasura.GraphQL.Execute.Action
@@ -112,16 +113,15 @@ convertMutationSelectionSet
 
     (resolvedDirectives, resolvedSelSet) <- resolveVariables varDefs (fromMaybe HashMap.empty (GH._grVariables gqlUnparsed)) directives fields
     -- Parse the GraphQL query into the RQL AST
-    unpreparedQueries ::
-      RootFieldMap (MutationRootField UnpreparedValue) <-
-      liftEither $ mutationParser resolvedSelSet
+    (unpreparedQueries :: RootFieldMap (MutationRootField UnpreparedValue)) <-
+      Tracing.newSpan "Parse mutation IR" $ liftEither $ mutationParser resolvedSelSet
 
     -- Process directives on the mutation
     _dirMap <- toQErr $ runParse (parseDirectives customDirectives (G.DLExecutable G.EDLMUTATION) resolvedDirectives)
 
     let parameterizedQueryHash = calculateParameterizedQueryHash resolvedSelSet
 
-        resolveExecutionSteps rootFieldName rootFieldUnpreparedValue = do
+        resolveExecutionSteps rootFieldName rootFieldUnpreparedValue = Tracing.newSpan ("Resolve execution step for " <>> rootFieldName) do
           case rootFieldUnpreparedValue of
             RFDB sourceName exists ->
               AB.dispatchAnyBackend @BackendExecute

server/src-lib/Hasura/GraphQL/Execute/Query.hs

@@ -9,6 +9,7 @@ import Data.Environment qualified as Env
 import Data.HashMap.Strict qualified as HashMap
 import Data.HashMap.Strict.InsOrd qualified as InsOrdHashMap
 import Data.Tagged qualified as Tagged
+import Data.Text.Extended ((<>>))
 import Hasura.Base.Error
 import Hasura.GraphQL.Context
 import Hasura.GraphQL.Execute.Action
@@ -36,6 +37,8 @@ import Hasura.Server.Prometheus (PrometheusMetrics (..))
 import Hasura.Server.Types (RequestId (..))
 import Hasura.Services.Network
 import Hasura.Session
+import Hasura.Tracing (MonadTrace)
+import Hasura.Tracing qualified as Tracing
 import Language.GraphQL.Draft.Syntax qualified as G
 import Network.HTTP.Types qualified as HTTP
 
@@ -60,6 +63,8 @@ parseGraphQLQuery gqlContext varDefs varValsM directives fields = do
 convertQuerySelSet ::
   forall m.
   ( MonadError QErr m,
+    MonadTrace m,
+    MonadIO m,
     MonadGQLExecutionCheck m,
     MonadQueryTags m,
     ProvidesNetwork m
@@ -95,14 +100,14 @@ convertQuerySelSet
   maybeOperationName = do
     -- 1. Parse the GraphQL query into the 'RootFieldMap' and a 'SelectionSet'
     (unpreparedQueries, normalizedDirectives, normalizedSelectionSet) <-
-      parseGraphQLQuery gqlContext varDefs (GH._grVariables gqlUnparsed) directives fields
+      Tracing.newSpan "Parse query IR" $ parseGraphQLQuery gqlContext varDefs (GH._grVariables gqlUnparsed) directives fields
 
     -- 2. Parse directives on the query
     dirMap <- toQErr $ runParse (parseDirectives customDirectives (G.DLExecutable G.EDLQUERY) normalizedDirectives)
 
     let parameterizedQueryHash = calculateParameterizedQueryHash normalizedSelectionSet
 
-        resolveExecutionSteps rootFieldName rootFieldUnpreparedValue = do
+        resolveExecutionSteps rootFieldName rootFieldUnpreparedValue = Tracing.newSpan ("Resolve execution step for " <>> rootFieldName) do
           case rootFieldUnpreparedValue of
             RFMulti lst -> do
               allSteps <- traverse (resolveExecutionSteps rootFieldName) lst
@@ -111,6 +116,7 @@ convertQuerySelSet
               AB.dispatchAnyBackend @BackendExecute
                 exists
                 \(SourceConfigWith (sourceConfig :: (SourceConfig b)) queryTagsConfig (QDBR db)) -> do
+                  Tracing.attachSourceConfigAttributes @b sourceConfig
                   let mReqId =
                         case _qtcOmitRequestId <$> queryTagsConfig of
                           -- we include the request id only if a user explicitly wishes for it to be included.

server/src-lib/Hasura/GraphQL/Execute/RemoteJoin/Join.hs

@@ -76,7 +76,7 @@ processRemoteJoins ::
   GQLReqUnparsed ->
   m EncJSON
 processRemoteJoins requestId logger agentLicenseKey env requestHeaders userInfo lhs maybeJoinTree gqlreq =
-  forRemoteJoins maybeJoinTree lhs \joinTree -> do
+  Tracing.newSpan "Process remote joins" $ forRemoteJoins maybeJoinTree lhs \joinTree -> do
     lhsParsed <-
       JO.eitherDecode (encJToLBS lhs)
         `onLeft` (throw500 . T.pack)
@@ -133,7 +133,9 @@ processRemoteJoins requestId logger agentLicenseKey env requestHeaders userInfo
 foldJoinTreeWith ::
   ( MonadError QErr m,
     MonadQueryTags m,
-    Traversable f
+    Traversable f,
+    Tracing.MonadTrace m,
+    MonadIO m
   ) =>
   -- | How to process a call to a source.
   (AB.AnyBackend S.SourceJoinCall -> m BL.ByteString) ->
@@ -156,7 +158,7 @@ foldJoinTreeWith callSource callRemoteSchema userInfo lhs joinTree reqHeaders op
       previousStep <- case _jalJoin of
         RemoteJoinRemoteSchema remoteSchemaJoin childJoinTree -> do
           let remoteSchemaInfo = rsDef $ _rsjRemoteSchema remoteSchemaJoin
-          maybeJoinIndex <- RS.makeRemoteSchemaJoinCall (callRemoteSchema remoteSchemaInfo) userInfo remoteSchemaJoin joinArguments
+          maybeJoinIndex <- RS.makeRemoteSchemaJoinCall (callRemoteSchema remoteSchemaInfo) userInfo remoteSchemaJoin _jalFieldName joinArguments
           pure $ fmap (childJoinTree,) maybeJoinIndex
         RemoteJoinSource sourceJoin childJoinTree -> do
           maybeJoinIndex <- S.makeSourceJoinCall callSource userInfo sourceJoin _jalFieldName joinArguments reqHeaders operationName
@@ -173,7 +175,8 @@ foldJoinTreeWith callSource callRemoteSchema userInfo lhs joinTree reqHeaders op
               reqHeaders
               operationName
           pure $ IntMap.fromAscList $ zip (IntMap.keys joinIndex) results
-  joinResults joinIndices compositeValue
+  Tracing.newSpan "Join remote join results"
+    $ joinResults joinIndices compositeValue
 
 -------------------------------------------------------------------------------
 

server/src-lib/Hasura/GraphQL/Execute/RemoteJoin/RemoteSchema.hs

@@ -29,7 +29,7 @@ import Data.HashMap.Strict.Extended qualified as HashMap
 import Data.IntMap.Strict qualified as IntMap
 import Data.List.NonEmpty qualified as NE
 import Data.Text qualified as T
-import Data.Text.Extended (commaSeparated, toTxt, (<<>))
+import Data.Text.Extended (commaSeparated, toTxt, (<<>), (<>>))
 import Data.Validation (Validation (..), toEither)
 import Hasura.Base.Error
 import Hasura.Base.ErrorMessage (fromErrorMessage)
@@ -47,6 +47,8 @@ import Hasura.RQL.Types.Common
 import Hasura.RQL.Types.ResultCustomization
 import Hasura.RemoteSchema.SchemaCache
 import Hasura.Session
+import Hasura.Tracing (MonadTrace)
+import Hasura.Tracing qualified as Tracing
 import Language.GraphQL.Draft.Syntax qualified as G
 
 -------------------------------------------------------------------------------
@@ -54,26 +56,32 @@ import Language.GraphQL.Draft.Syntax qualified as G
 
 -- | Construct and execute a call to a remote schema for a remote join.
 makeRemoteSchemaJoinCall ::
-  (MonadError QErr m) =>
+  (MonadError QErr m, MonadTrace m, MonadIO m) =>
   -- | Function to send a request over the network.
   (GQLReqOutgoing -> m BL.ByteString) ->
   -- | User information.
   UserInfo ->
   -- | Information about that remote join.
   RemoteSchemaJoin ->
+  -- | Name of the field from the join arguments.
+  FieldName ->
   -- | Mapping from 'JoinArgumentId' to its corresponding 'JoinArgument'.
   IntMap.IntMap JoinArgument ->
   -- | The resulting join index (see 'buildJoinIndex') if any.
   m (Maybe (IntMap.IntMap AO.Value))
-makeRemoteSchemaJoinCall networkFunction userInfo remoteSchemaJoin joinArguments = do
-  -- step 1: construct the internal intermediary representation
-  maybeRemoteCall <- buildRemoteSchemaCall remoteSchemaJoin joinArguments userInfo
-  -- if there actually is a remote call:
-  for maybeRemoteCall \remoteCall -> do
-    -- step 2: execute it over the network
-    responseValue <- executeRemoteSchemaCall networkFunction remoteCall
-    -- step 3: build the join index
-    buildJoinIndex remoteCall responseValue
+makeRemoteSchemaJoinCall networkFunction userInfo remoteSchemaJoin jaFieldName joinArguments = do
+  Tracing.newSpan ("Remote join to remote schema for field " <>> jaFieldName) do
+    -- step 1: construct the internal intermediary representation
+    maybeRemoteCall <-
+      Tracing.newSpan "Resolve execution step for remote join field"
+        $ buildRemoteSchemaCall remoteSchemaJoin joinArguments userInfo
+    -- if there actually is a remote call:
+    for maybeRemoteCall \remoteCall -> do
+      -- step 2: execute it over the network
+      responseValue <- executeRemoteSchemaCall networkFunction remoteCall
+      -- step 3: build the join index
+      Tracing.newSpan "Build remote join index"
+        $ buildJoinIndex remoteCall responseValue
 
 -------------------------------------------------------------------------------
 -- Internal representation

server/src-lib/Hasura/GraphQL/Execute/RemoteJoin/Source.hs

@@ -31,6 +31,7 @@ import Data.IntMap.Strict qualified as IntMap
 import Data.List.NonEmpty qualified as NE
 import Data.Scientific qualified as Scientific
 import Data.Text qualified as T
+import Data.Text.Extended ((<<>), (<>>))
 import Data.Text.Read qualified as TR
 import Hasura.Base.Error
 import Hasura.GraphQL.Execute.Backend qualified as EB
@@ -44,6 +45,8 @@ import Hasura.RQL.Types.Backend
 import Hasura.RQL.Types.Common
 import Hasura.SQL.AnyBackend qualified as AB
 import Hasura.Session
+import Hasura.Tracing (MonadTrace)
+import Hasura.Tracing qualified as Tracing
 import Language.GraphQL.Draft.Syntax qualified as G
 import Network.HTTP.Types qualified as HTTP
 
@@ -52,7 +55,7 @@ import Network.HTTP.Types qualified as HTTP
 
 -- | Construct and execute a call to a source for a remote join.
 makeSourceJoinCall ::
-  (MonadQueryTags m, MonadError QErr m) =>
+  (MonadQueryTags m, MonadError QErr m, MonadTrace m, MonadIO m) =>
   -- | Function to dispatch a request to a source.
   (AB.AnyBackend SourceJoinCall -> m BL.ByteString) ->
   -- | User information.
@@ -67,20 +70,25 @@ makeSourceJoinCall ::
   Maybe G.Name ->
   -- | The resulting join index (see 'buildJoinIndex') if any.
   m (Maybe (IntMap.IntMap AO.Value))
-makeSourceJoinCall networkFunction userInfo remoteSourceJoin jaFieldName joinArguments reqHeaders operationName = do
-  -- step 1: create the SourceJoinCall
-  -- maybeSourceCall <-
-  --   AB.dispatchAnyBackend @EB.BackendExecute remoteSourceJoin \(sjc :: SourceJoinCall b) ->
-  --     buildSourceJoinCall @b userInfo jaFieldName joinArguments sjc
-  maybeSourceCall <-
-    AB.dispatchAnyBackend @EB.BackendExecute remoteSourceJoin
-      $ buildSourceJoinCall userInfo jaFieldName joinArguments reqHeaders operationName
-  -- if there actually is a remote call:
-  for maybeSourceCall \sourceCall -> do
-    -- step 2: send this call over the network
-    sourceResponse <- networkFunction sourceCall
-    -- step 3: build the join index
-    buildJoinIndex sourceResponse
+makeSourceJoinCall networkFunction userInfo remoteSourceJoin jaFieldName joinArguments reqHeaders operationName =
+  Tracing.newSpan ("Remote join to data source " <> sourceName <<> " for field " <>> jaFieldName) do
+    -- step 1: create the SourceJoinCall
+    -- maybeSourceCall <-
+    --   AB.dispatchAnyBackend @EB.BackendExecute remoteSourceJoin \(sjc :: SourceJoinCall b) ->
+    --     buildSourceJoinCall @b userInfo jaFieldName joinArguments sjc
+    maybeSourceCall <-
+      AB.dispatchAnyBackend @EB.BackendExecute remoteSourceJoin
+        $ buildSourceJoinCall userInfo jaFieldName joinArguments reqHeaders operationName
+    -- if there actually is a remote call:
+    for maybeSourceCall \sourceCall -> do
+      -- step 2: send this call over the network
+      sourceResponse <- networkFunction sourceCall
+      -- step 3: build the join index
+      Tracing.newSpan "Build remote join index"
+        $ buildJoinIndex sourceResponse
+  where
+    sourceName :: SourceName
+    sourceName = AB.dispatchAnyBackend @Backend remoteSourceJoin _rsjSource
 
 -------------------------------------------------------------------------------
 -- Internal representation
@@ -97,7 +105,8 @@ data SourceJoinCall b = SourceJoinCall
 -- Step 1: building the source call
 
 buildSourceJoinCall ::
-  (EB.BackendExecute b, MonadQueryTags m, MonadError QErr m) =>
+  forall b m.
+  (EB.BackendExecute b, MonadQueryTags m, MonadError QErr m, MonadTrace m, MonadIO m) =>
   UserInfo ->
   FieldName ->
   IntMap.IntMap JoinArgument ->
@@ -106,39 +115,41 @@ buildSourceJoinCall ::
   RemoteSourceJoin b ->
   m (Maybe (AB.AnyBackend SourceJoinCall))
 buildSourceJoinCall userInfo jaFieldName joinArguments reqHeaders operationName remoteSourceJoin = do
-  let rows =
-        IntMap.toList joinArguments <&> \(argumentId, argument) ->
-          KM.insert "__argument_id__" (J.toJSON argumentId)
-            $ KM.fromList
-            $ map (bimap (K.fromText . getFieldNameTxt) JO.fromOrdered)
-            $ HashMap.toList
-            $ unJoinArgument argument
-      rowSchema = fmap snd (_rsjJoinColumns remoteSourceJoin)
-  for (NE.nonEmpty rows) $ \nonEmptyRows -> do
-    let sourceConfig = _rsjSourceConfig remoteSourceJoin
-    stepInfo <-
-      EB.mkDBRemoteRelationshipPlan
-        userInfo
-        (_rsjSource remoteSourceJoin)
-        sourceConfig
-        nonEmptyRows
-        rowSchema
-        (FieldName "__argument_id__")
-        (FieldName "f", _rsjRelationship remoteSourceJoin)
-        reqHeaders
-        operationName
-        (_rsjStringifyNum remoteSourceJoin)
-    -- This should never fail, as field names in remote relationships are
-    -- validated when building the schema cache.
-    fieldName <-
-      G.mkName (getFieldNameTxt jaFieldName)
-        `onNothing` throw500 ("'" <> getFieldNameTxt jaFieldName <> "' is not a valid GraphQL name")
-    -- NOTE: We're making an assumption that the 'FieldName' propagated upwards
-    -- from 'collectJoinArguments' is reasonable to use for logging.
-    let rootFieldAlias = mkUnNamespacedRootFieldAlias fieldName
-    pure
-      $ AB.mkAnyBackend
-      $ SourceJoinCall rootFieldAlias sourceConfig stepInfo
+  Tracing.newSpan "Resolve execution step for remote join field" do
+    let rows =
+          IntMap.toList joinArguments <&> \(argumentId, argument) ->
+            KM.insert "__argument_id__" (J.toJSON argumentId)
+              $ KM.fromList
+              $ map (bimap (K.fromText . getFieldNameTxt) JO.fromOrdered)
+              $ HashMap.toList
+              $ unJoinArgument argument
+        rowSchema = fmap snd (_rsjJoinColumns remoteSourceJoin)
+    for (NE.nonEmpty rows) $ \nonEmptyRows -> do
+      let sourceConfig = _rsjSourceConfig remoteSourceJoin
+      Tracing.attachSourceConfigAttributes @b sourceConfig
+      stepInfo <-
+        EB.mkDBRemoteRelationshipPlan
+          userInfo
+          (_rsjSource remoteSourceJoin)
+          sourceConfig
+          nonEmptyRows
+          rowSchema
+          (FieldName "__argument_id__")
+          (FieldName "f", _rsjRelationship remoteSourceJoin)
+          reqHeaders
+          operationName
+          (_rsjStringifyNum remoteSourceJoin)
+      -- This should never fail, as field names in remote relationships are
+      -- validated when building the schema cache.
+      fieldName <-
+        G.mkName (getFieldNameTxt jaFieldName)
+          `onNothing` throw500 ("'" <> getFieldNameTxt jaFieldName <> "' is not a valid GraphQL name")
+      -- NOTE: We're making an assumption that the 'FieldName' propagated upwards
+      -- from 'collectJoinArguments' is reasonable to use for logging.
+      let rootFieldAlias = mkUnNamespacedRootFieldAlias fieldName
+      pure
+        $ AB.mkAnyBackend
+        $ SourceJoinCall rootFieldAlias sourceConfig stepInfo
 
 -------------------------------------------------------------------------------
 -- Step 3: extracting the join index

server/src-lib/Hasura/GraphQL/Transport/HTTP.hs

@@ -38,6 +38,7 @@ import Data.Environment qualified as Env
 import Data.HashMap.Strict.InsOrd qualified as InsOrdHashMap
 import Data.Monoid (Any (..))
 import Data.Text qualified as T
+import Data.Text.Extended ((<>>))
 import Hasura.Backends.DataConnector.Agent.Client (AgentLicenseKey)
 import Hasura.Backends.Postgres.Instances.Transport (runPGMutationTransaction)
 import Hasura.Base.Error
@@ -90,6 +91,7 @@ import Hasura.Server.Types (ReadOnlyMode (..), RequestId (..))
 import Hasura.Services
 import Hasura.Session (SessionVariable, SessionVariableValue, SessionVariables, UserInfo (..), filterSessionVariables)
 import Hasura.Tracing (MonadTrace, attachMetadata)
+import Hasura.Tracing qualified as Tracing
 import Language.GraphQL.Draft.Syntax qualified as G
 import Network.HTTP.Types qualified as HTTP
 import Network.Wai.Extended qualified as Wai
@@ -311,7 +313,7 @@ runGQ env sqlGenCtx sc scVer enableAL readOnlyMode prometheusMetrics logger agen
   let gqlMetrics = pmGraphQLRequestMetrics prometheusMetrics
 
   (totalTime, (response, parameterizedQueryHash, gqlOpType)) <- withElapsedTime $ do
-    (reqParsed, runLimits, queryParts) <- observeGQLQueryError gqlMetrics Nothing $ do
+    (reqParsed, runLimits, queryParts) <- Tracing.newSpan "Parse GraphQL" $ observeGQLQueryError gqlMetrics Nothing $ do
       -- 1. Run system authorization on the 'reqUnparsed :: GQLReqUnparsed' query.
       reqParsed <-
         E.checkGQLExecution userInfo (reqHeaders, ipAddress) enableAL sc reqUnparsed reqId
@@ -378,8 +380,6 @@ runGQ env sqlGenCtx sc scVer enableAL readOnlyMode prometheusMetrics logger agen
       m AnnotatedResponse
     executePlan reqParsed runLimits execPlan = case execPlan of
       E.QueryExecutionPlan queryPlans asts dirMap -> do
-        -- https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/instrumentation/graphql/
-        attachMetadata [("graphql.operation.type", "query")]
         let cachedDirective = runIdentity <$> DM.lookup cached dirMap
         -- Attempt to lookup a cached response in the query cache.
         (cachingHeaders, cachedValue) <- liftEitherM $ cacheLookup queryPlans asts cachedDirective reqParsed userInfo reqHeaders
@@ -424,8 +424,6 @@ runGQ env sqlGenCtx sc scVer enableAL readOnlyMode prometheusMetrics logger agen
                  in -- 4. Return the response.
                     pure $ result {arResponse = addHttpResponseHeaders headers response}
       E.MutationExecutionPlan mutationPlans -> runLimits $ do
-        -- https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/instrumentation/graphql/
-        attachMetadata [("graphql.operation.type", "mutation")]
         {- Note [Backwards-compatible transaction optimisation]
 
            For backwards compatibility, we perform the following optimisation: if all mutation steps
@@ -528,7 +526,7 @@ runGQ env sqlGenCtx sc scVer enableAL readOnlyMode prometheusMetrics logger agen
         _all <- traverse (executeQueryStep fieldName) lst
         pure $ AnnotatedResponsePart 0 Telem.Local (encJFromList (map arpResponse _all)) []
 
-    runRemoteGQ fieldName rsi resultCustomizer gqlReq remoteJoins = do
+    runRemoteGQ fieldName rsi resultCustomizer gqlReq remoteJoins = Tracing.newSpan ("Remote schema query for root field " <>> fieldName) $ do
       (telemTimeIO_DT, remoteResponseHeaders, resp) <-
         doQErr $ E.execRemoteGQ env userInfo reqHeaders (rsDef rsi) gqlReq
       value <- extractFieldFromResponse fieldName resultCustomizer resp

server/src-lib/Hasura/GraphQL/Transport/WebSocket.hs

@@ -38,6 +38,7 @@ import Data.List.NonEmpty qualified as NE
 import Data.String
 import Data.Text qualified as T
 import Data.Text.Encoding qualified as TE
+import Data.Text.Extended ((<>>))
 import Data.Time.Clock
 import Data.Time.Clock qualified as TC
 import Data.Word (Word16)
@@ -469,10 +470,13 @@ onStart enabledLogTypes agentLicenseKey serverEnv wsConn shouldCaptureVariables
   sqlGenCtx <- liftIO $ acSQLGenCtx <$> getAppContext appStateRef
   enableAL <- liftIO $ acEnableAllowlist <$> getAppContext appStateRef
 
-  reqParsedE <- lift $ E.checkGQLExecution userInfo (reqHdrs, ipAddress) enableAL sc q requestId
-  reqParsed <- onLeft reqParsedE (withComplete . preExecErr requestId Nothing)
-  queryPartsE <- runExceptT $ getSingleOperation reqParsed
-  queryParts <- onLeft queryPartsE (withComplete . preExecErr requestId Nothing)
+  (reqParsed, queryParts) <- Tracing.newSpan "Parse GraphQL" $ do
+    reqParsedE <- lift $ E.checkGQLExecution userInfo (reqHdrs, ipAddress) enableAL sc q requestId
+    reqParsed <- onLeft reqParsedE (withComplete . preExecErr requestId Nothing)
+    queryPartsE <- runExceptT $ getSingleOperation reqParsed
+    queryParts <- onLeft queryPartsE (withComplete . preExecErr requestId Nothing)
+    pure (reqParsed, queryParts)
+
   let gqlOpType = G._todType queryParts
       maybeOperationName = _unOperationName <$> _grOperationName reqParsed
   for_ maybeOperationName $ \nm ->
@@ -500,8 +504,6 @@ onStart enabledLogTypes agentLicenseKey serverEnv wsConn shouldCaptureVariables
 
   case execPlan of
     E.QueryExecutionPlan queryPlan asts dirMap -> do
-      -- https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/instrumentation/graphql/
-      Tracing.attachMetadata [("graphql.operation.type", "query")]
       let cachedDirective = runIdentity <$> DM.lookup cached dirMap
 
       -- We ignore the response headers (containing TTL information) because
@@ -571,8 +573,6 @@ onStart enabledLogTypes agentLicenseKey serverEnv wsConn shouldCaptureVariables
 
       liftIO $ sendCompleted (Just requestId) (Just parameterizedQueryHash)
     E.MutationExecutionPlan mutationPlan -> do
-      -- https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/instrumentation/graphql/
-      Tracing.attachMetadata [("graphql.operation.type", "mutation")]
       -- See Note [Backwards-compatible transaction optimisation]
       case coalescePostgresMutations mutationPlan of
         -- we are in the aforementioned case; we circumvent the normal process
@@ -643,8 +643,6 @@ onStart enabledLogTypes agentLicenseKey serverEnv wsConn shouldCaptureVariables
           sendResultFromFragments Telem.Query timerTot requestId conclusion opName parameterizedQueryHash gqlOpType
       liftIO $ sendCompleted (Just requestId) (Just parameterizedQueryHash)
     E.SubscriptionExecutionPlan subExec -> do
-      -- https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/instrumentation/graphql/
-      Tracing.attachMetadata [("graphql.operation.type", "subscription")]
       case subExec of
         E.SEAsyncActionsWithNoRelationships actions -> do
           logQueryLog logger $ QueryLog q Nothing requestId QueryLogKindAction
@@ -782,7 +780,7 @@ onStart enabledLogTypes agentLicenseKey serverEnv wsConn shouldCaptureVariables
       GQLReqOutgoing ->
       Maybe RJ.RemoteJoins ->
       ExceptT (Either GQExecError QErr) (ExceptT () m) AnnotatedResponsePart
-    runRemoteGQ requestId reqUnparsed fieldName userInfo reqHdrs rsi resultCustomizer gqlReq remoteJoins = do
+    runRemoteGQ requestId reqUnparsed fieldName userInfo reqHdrs rsi resultCustomizer gqlReq remoteJoins = Tracing.newSpan ("Remote schema query for root field " <>> fieldName) $ do
       env <- liftIO $ acEnvironment <$> getAppContext appStateRef
       (telemTimeIO_DT, _respHdrs, resp) <-
         doQErr

server/src-lib/Hasura/RQL/Types/SourceConfiguration.hs

@@ -41,3 +41,5 @@ class
   -- | Whether the source configuration specifies the use of a connection
   -- template
   sourceConfigConnectonTemplateEnabled :: SourceConfig b -> Bool
+
+  sourceConfigBackendSourceKind :: SourceConfig b -> BackendSourceKind b

server/src-lib/Hasura/Tracing/Utils.hs

@@ -4,12 +4,15 @@
 -- here in the core engine code.
 module Hasura.Tracing.Utils
   ( traceHTTPRequest,
+    attachSourceConfigAttributes,
   )
 where
 
 import Control.Lens
 import Data.String
+import Data.Text.Extended (toTxt)
 import Hasura.Prelude
+import Hasura.RQL.Types.SourceConfiguration (HasSourceConfiguration (..))
 import Hasura.Tracing.Class
 import Hasura.Tracing.Context
 import Hasura.Tracing.Sampling
@@ -47,3 +50,8 @@ traceHTTPRequest req f = do
           ("X-B3-ParentSpanId",) . spanIdToHex <$> tcCurrentParent,
           ("X-B3-Sampled",) <$> samplingStateToHeader tcSamplingState
         ]
+
+attachSourceConfigAttributes :: forall b m. (HasSourceConfiguration b, MonadTrace m) => SourceConfig b -> m ()
+attachSourceConfigAttributes sourceConfig = do
+  let backendSourceKind = sourceConfigBackendSourceKind @b sourceConfig
+  attachMetadata [("source.kind", toTxt $ backendSourceKind)]