server: fix schema-registry inconsistent metadata bug

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/9670
GitOrigin-RevId: 923b6111c44713956c3b6ee5499712e8b23effea

server/src-lib/Hasura/GraphQL/Schema.hs

@@ -187,9 +187,9 @@ buildGQLContext
     let hasuraContexts = fst <$> contexts
         relayContexts = snd <$> contexts
 
-    (adminErrs, adminIntrospection) <-
+    adminIntrospection <-
       case HashMap.lookup adminRoleName hasuraContexts of
-        Just (_context, errors, introspection) -> pure (errors, introspection)
+        Just (_context, _errors, introspection) -> pure introspection
         Nothing -> throw500 "buildGQLContext failed to build for the admin role"
     (unauthenticated, unauthenticatedRemotesErrors) <- unauthenticatedContext (sqlGen, functionPermissions) sources allRemoteSchemas experimentalFeatures remoteSchemaPermissions
 
@@ -198,22 +198,22 @@ buildGQLContext
         res <- liftIO $ runExceptT $ PG.runTx' (_srpaMetadataDbPoolRef schemaRegistryCtx) selectNowQuery
         case res of
           Left err ->
-            pure $ \_ ->
+            pure $ \_ _ ->
               unLogger logger $ mkGenericLog @Text LevelWarn "schema-registry" ("failed to fetch the time from metadata db correctly: " <> showQErr err)
           Right now -> do
             let schemaRegistryMap = generateSchemaRegistryMap hasuraContexts
-                projectSchemaInfo = \metadataResourceVersion ->
+                projectSchemaInfo = \metadataResourceVersion inconsistentMetadata ->
                   ProjectGQLSchemaInformation
                     schemaRegistryMap
-                    (IsMetadataInconsistent $ checkMdErrs adminErrs)
+                    (IsMetadataInconsistent $ checkMdErrs inconsistentMetadata)
                     (calculateSchemaSDLHash (generateSDL adminIntrospection) adminRoleName)
                     metadataResourceVersion
                     now
             pure
-              $ \metadataResourceVersion ->
+              $ \metadataResourceVersion inconsistentMetadata ->
                 STM.atomically
                   $ STM.writeTQueue (_srpaSchemaRegistryTQueueRef schemaRegistryCtx)
-                  $ projectSchemaInfo metadataResourceVersion
+                  $ projectSchemaInfo metadataResourceVersion inconsistentMetadata
 
     pure
       ( ( adminIntrospection,
@@ -230,7 +230,7 @@ buildGQLContext
         writeToSchemaRegistryAction
       )
     where
-      checkMdErrs = not . Set.null
+      checkMdErrs = not . null
 
       generateSchemaRegistryMap :: HashMap RoleName RoleContextValue -> SchemaRegistryMap
       generateSchemaRegistryMap mpr =

server/src-lib/Hasura/RQL/DDL/Schema/Cache.hs

@@ -272,6 +272,8 @@ instance
     staticConfig <- askCacheStaticConfig
     (RebuildableSchemaCache lastBuiltSC invalidationKeys rule, oldInvalidations, _, _) <- get
     let oldMetadataVersion = scMetadataResourceVersion lastBuiltSC
+        -- We are purposely putting (-1) as the metadata resource version here. This is because we want to
+        -- catch error cases in `withSchemaCache(Read)Update`
         metadataWithVersion = MetadataWithResourceVersion newMetadata $ MetadataResourceVersion (-1)
         newInvalidationKeys = invalidateKeys invalidations invalidationKeys
     storedIntrospection <- loadStoredIntrospection (_cscLogger staticConfig) oldMetadataVersion
@@ -429,7 +431,7 @@ buildSchemaCacheRule ::
   Env.Environment ->
   Maybe SchemaRegistryContext ->
   (MetadataWithResourceVersion, CacheDynamicConfig, InvalidationKeys, Maybe StoredIntrospection) `arr` (SchemaCache, (SourcesIntrospectionStatus, SchemaRegistryAction))
-buildSchemaCacheRule logger env mSchemaRegistryContext = proc (MetadataWithResourceVersion metadataNoDefaults lastKnownMetadataResourceVersion, dynamicConfig, invalidationKeys, storedIntrospection) -> do
+buildSchemaCacheRule logger env mSchemaRegistryContext = proc (MetadataWithResourceVersion metadataNoDefaults interimMetadataResourceVersion, dynamicConfig, invalidationKeys, storedIntrospection) -> do
   invalidationKeysDep <- Inc.newDependency -< invalidationKeys
   let metadataDefaults = _cdcMetadataDefaults dynamicConfig
       metadata@Metadata {..} = overrideMetadataDefaults metadataNoDefaults metadataDefaults
@@ -488,22 +490,6 @@ buildSchemaCacheRule logger env mSchemaRegistryContext = proc (MetadataWithResou
 
       inconsistentQueryCollections = getInconsistentQueryCollections adminIntrospection _metaQueryCollections listedQueryObjects endpoints globalAllowLists
 
-  -- Write the Project Schema information to schema registry service
-  _ <-
-    bindA
-      -< do
-        buildReason <- ask
-        case buildReason of
-          -- If this is a catalog sync then we know for sure that the schema has more chances of being committed as some
-          -- other instance of Hasura has already committed the schema. So we can safely write the schema to the registry
-          -- service.
-          CatalogSync ->
-            for_ schemaRegistryAction $ \action -> do
-              liftIO $ action lastKnownMetadataResourceVersion
-          -- If this is a metadata event then we cannot be sure that the schema will be committed. So we write the schema
-          -- to the registry service only after the schema is committed.
-          CatalogUpdate _ -> pure ()
-
   let schemaCache =
         SchemaCache
           { scSources = _boSources resolvedOutputs,
@@ -536,12 +522,12 @@ buildSchemaCacheRule logger env mSchemaRegistryContext = proc (MetadataWithResou
                 <> inconsistentQueryCollections,
             scApiLimits = _metaApiLimits,
             scMetricsConfig = _metaMetricsConfig,
-            -- Please note that we are setting the metadata resource version to the last known metadata resource
-            -- version. This might not be the final value of the metadata resource version. This is because, if we are
-            -- in the middle of a metadata operation, we might push the metadata to the database and thus this version
-            -- will be of the older metadata. As a fix, we do update the metadata resource version to the latest value
-            -- after the metadata operation is complete (see the usage of `setMetadataResourceVersionInSchemaCache`).
-            scMetadataResourceVersion = lastKnownMetadataResourceVersion,
+            -- Please note that we are setting the metadata resource version to the last known metadata resource version
+            -- for `CatalogSync` or to an invalid metadata resource version (-1) for `CatalogUpdate`.
+            --
+            -- For, CatalogUpdate, we update the metadata resource version to the latest value after the metadata
+            -- operation is complete (see the usage of `setMetadataResourceVersionInSchemaCache`).
+            scMetadataResourceVersion = interimMetadataResourceVersion,
             scSetGraphqlIntrospectionOptions = _metaSetGraphqlIntrospectionOptions,
             scTlsAllowlist = networkTlsAllowlist _metaNetwork,
             scQueryCollections = _metaQueryCollections,
@@ -550,6 +536,23 @@ buildSchemaCacheRule logger env mSchemaRegistryContext = proc (MetadataWithResou
             scSourcePingConfig = buildSourcePingCache _metaSources,
             scOpenTelemetryConfig = openTelemetryInfo
           }
+
+  -- Write the Project Schema information to schema registry service
+  _ <-
+    bindA
+      -< do
+        buildReason <- ask
+        case buildReason of
+          -- If this is a catalog sync then we know for sure that the schema has more chances of being committed as some
+          -- other instance of Hasura has already committed the schema. So we can safely write the schema to the registry
+          -- service.
+          CatalogSync ->
+            for_ schemaRegistryAction $ \action -> do
+              liftIO $ action interimMetadataResourceVersion (scInconsistentObjs schemaCache)
+          -- If this is a metadata event then we cannot be sure that the schema will be committed. So we write the schema
+          -- to the registry service only after the schema is committed.
+          CatalogUpdate _ -> pure ()
+
   returnA -< (schemaCache, (storedIntrospectionStatus, schemaRegistryAction))
   where
     -- See Note [Avoiding GraphQL schema rebuilds when changing irrelevant Metadata]

server/src-lib/Hasura/RQL/DDL/SchemaRegistry.hs

@@ -28,6 +28,7 @@ import Database.PG.Query qualified as PG
 import Hasura.Backends.Postgres.Execute.Types qualified as SQLTypes
 import Hasura.Base.Error
 import Hasura.Prelude
+import Hasura.RQL.Types.Metadata.Object (InconsistentMetadata)
 import Hasura.RQL.Types.Roles
 import Hasura.RQL.Types.SchemaCache (MetadataResourceVersion)
 import Hasura.Server.Utils
@@ -84,7 +85,7 @@ newtype SchemaHash = SchemaHash {_schemaHash :: T.Text}
 
 type SchemaRegistryMap = HashMap RoleName GQLSchemaInformation
 
-type SchemaRegistryAction = Maybe (MetadataResourceVersion -> IO ())
+type SchemaRegistryAction = Maybe (MetadataResourceVersion -> [InconsistentMetadata] -> IO ())
 
 data GQLSchemaInformation = GQLSchemaInformation
   { _gsiSchemaSDL :: SchemaSDL,

server/src-lib/Hasura/Server/API/Metadata.hs

@@ -176,7 +176,7 @@ runMetadataQuery appContext schemaCache closeWebsocketsOnMetadataChange RQLMetad
         Tracing.newSpan "runSchemaRegistryAction"
           $ for_ schemaRegistryAction
           $ \action -> do
-            liftIO $ action newResourceVersion
+            liftIO $ action newResourceVersion (scInconsistentObjs (lastBuiltSchemaCache modSchemaCache))
 
         -- notify schema cache sync
         Tracing.newSpan "notifySchemaCacheSync"

server/src-lib/Hasura/Server/API/Query.hs

@@ -232,7 +232,7 @@ runQuery appContext sc query = do
         saveSourcesIntrospection logger sourcesIntrospection newResourceVersion
         -- run schema registry action
         for_ schemaRegistryAction $ \action -> do
-          liftIO $ action newResourceVersion
+          liftIO $ action newResourceVersion (scInconsistentObjs (lastBuiltSchemaCache modSchemaCache'))
         -- notify schema cache sync
         liftEitherM $ notifySchemaCacheSync newResourceVersion appEnvInstanceId invalidations
 

server/src-lib/Hasura/Server/API/V2Query.hs

@@ -43,7 +43,7 @@ import Hasura.RQL.DML.Update
 import Hasura.RQL.Types.BackendType
 import Hasura.RQL.Types.Common
 import Hasura.RQL.Types.Metadata
-import Hasura.RQL.Types.SchemaCache (MetadataWithResourceVersion (MetadataWithResourceVersion))
+import Hasura.RQL.Types.SchemaCache (MetadataWithResourceVersion (MetadataWithResourceVersion), SchemaCache (scInconsistentObjs))
 import Hasura.RQL.Types.SchemaCache.Build
 import Hasura.RQL.Types.Source
 import Hasura.Server.Types
@@ -151,7 +151,7 @@ runQuery appContext schemaCache rqlQuery = do
         Tracing.newSpan "runSchemaRegistryAction"
           $ for_ schemaRegistryAction
           $ \action -> do
-            liftIO $ action newResourceVersion
+            liftIO $ action newResourceVersion (scInconsistentObjs (lastBuiltSchemaCache modSchemaCache'))
 
         -- notify schema cache sync
         Tracing.newSpan "notifySchemaCacheSync"