server: event triggers should be dropped when a previously present source is dropped in replace_metadata

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/5739
Co-authored-by: paritosh-08 <85472423+paritosh-08@users.noreply.github.com>
GitOrigin-RevId: 772de8bb6a55a0002885e08a778c16f22ea0b113

server/graphql-engine.cabal

@@ -1283,6 +1283,7 @@ test-suite tests-hspec
     Test.EventTrigger.PG.EventTriggersUntrackTableCleanupSpec
     Test.EventTrigger.PG.EventTriggersUniqueNameSpec
     Test.EventTrigger.PG.EventTriggersExtensionSchemaSpec
+    Test.EventTrigger.PG.EventTriggersReplaceMetadataCleanupSpec
     Test.EventTrigger.MSSQL.EventTriggerDropSourceCleanupSpec
     Test.EventTrigger.MSSQL.EventTriggersUntrackTableCleanupSpec
     Test.EventTrigger.MSSQL.EventTiggersUniqueNameSpec

server/src-lib/Hasura/RQL/DDL/Metadata.hs

@@ -152,6 +152,7 @@ runReplaceMetadata ::
   ( CacheRWM m,
     MetadataM m,
     MonadIO m,
+    MonadBaseControl IO m,
     MonadMetadataStorageQueryAPI m,
     MonadReader r m,
     Has (HL.Logger HL.Hasura) r
@@ -167,6 +168,7 @@ runReplaceMetadataV1 ::
     CacheRWM m,
     MetadataM m,
     MonadIO m,
+    MonadBaseControl IO m,
     MonadMetadataStorageQueryAPI m,
     MonadReader r m,
     Has (HL.Logger HL.Hasura) r
@@ -182,6 +184,7 @@ runReplaceMetadataV2 ::
     CacheRWM m,
     MetadataM m,
     MonadIO m,
+    MonadBaseControl IO m,
     MonadMetadataStorageQueryAPI m,
     MonadReader r m,
     Has (HL.Logger HL.Hasura) r
@@ -233,9 +236,20 @@ runReplaceMetadataV2 ReplaceMetadataV2 {..} = do
           mempty
   putMetadata metadata
 
-  -- Check for duplicate trigger names in the new source metadata
   let oldSources = (_metaSources oldMetadata)
   let newSources = (_metaSources metadata)
+
+  -- Clean up the sources that are not present in the new metadata
+  for_ (OMap.toList oldSources) $ \(oldSource, oldSourceBackendMetadata) -> do
+    -- If the source present in old metadata is not present in the new metadata,
+    -- clean that source.
+    onNothing (OMap.lookup oldSource newSources) $ do
+      AB.dispatchAnyBackend @BackendMetadata (unBackendSourceMetadata oldSourceBackendMetadata) \(_oldSourceMetadata :: SourceMetadata b) -> do
+        sourceInfo <- askSourceInfo @b oldSource
+        runPostDropSourceHook oldSource sourceInfo
+        pure (BackendSourceMetadata (AB.mkAnyBackend _oldSourceMetadata))
+
+  -- Check for duplicate trigger names in the new source metadata
   for_ (OMap.toList newSources) $ \(source, newBackendSourceMetadata) -> do
     onJust (OMap.lookup source oldSources) $ \_oldBackendSourceMetadata ->
       dispatch newBackendSourceMetadata \(newSourceMetadata :: SourceMetadata b) -> do

server/src-lib/Hasura/Server/API/Metadata.hs

@@ -617,6 +617,7 @@ runMetadataQueryV1M env currentResourceVersion = \case
 runMetadataQueryV2M ::
   ( MonadIO m,
     CacheRWM m,
+    MonadBaseControl IO m,
     MetadataM m,
     MonadMetadataStorageQueryAPI m,
     MonadReader r m,

server/tests-hspec/Test/EventTrigger/PG/EventTriggersReplaceMetadataCleanupSpec.hs

@@ -0,0 +1,156 @@
+{-# LANGUAGE QuasiQuotes #-}
+{-# LANGUAGE ViewPatterns #-}
+
+-- | Test that when a source is removed via `replace_metadata` API, the cleanup
+--   is done properly.
+module Test.EventTrigger.PG.EventTriggersReplaceMetadataCleanupSpec (spec) where
+
+import Data.List.NonEmpty qualified as NE
+import Data.String (fromString)
+import Database.PostgreSQL.Simple qualified as Postgres
+import Harness.Backend.Postgres qualified as Postgres
+import Harness.Constants as Constants
+import Harness.Exceptions
+import Harness.GraphqlEngine qualified as GraphqlEngine
+import Harness.Quoter.Yaml
+import Harness.Test.Fixture qualified as Fixture
+import Harness.Test.Schema (Table (..), table)
+import Harness.Test.Schema qualified as Schema
+import Harness.TestEnvironment (TestEnvironment, stopServer)
+import Harness.Webhook qualified as Webhook
+import Harness.Yaml (shouldReturnYaml)
+import Hasura.Prelude
+import Test.Hspec (SpecWith, describe, it, shouldBe)
+
+--------------------------------------------------------------------------------
+-- Preamble
+
+spec :: SpecWith TestEnvironment
+spec =
+  Fixture.runWithLocalTestEnvironment
+    ( NE.fromList
+        [ (Fixture.fixture $ Fixture.Backend Fixture.Postgres)
+            { -- setup the webhook server as the local test environment,
+              -- so that the server can be referenced while testing
+              Fixture.mkLocalTestEnvironment = webhookServerMkLocalTestEnvironment,
+              Fixture.setupTeardown = \testEnv ->
+                [ Fixture.SetupAction
+                    { Fixture.setupAction = postgresSetup testEnv,
+                      Fixture.teardownAction = \_ -> postgresTeardown testEnv
+                    }
+                ]
+            }
+        ]
+    )
+    tests
+
+--------------------------------------------------------------------------------
+
+-- * Backend
+
+-- ** Schema
+
+schema :: Text -> [Schema.Table]
+schema authorTableName = [authorsTable authorTableName]
+
+authorsTable :: Text -> Schema.Table
+authorsTable tableName =
+  (table tableName)
+    { tableColumns =
+        [ Schema.column "id" Schema.TInt,
+          Schema.column "name" Schema.TStr
+        ],
+      tablePrimaryKey = ["id"],
+      tableData =
+        [ [Schema.VInt 1, Schema.VStr "Author 1"],
+          [Schema.VInt 2, Schema.VStr "Author 2"]
+        ]
+    }
+
+--------------------------------------------------------------------------------
+-- Tests
+
+tests :: Fixture.Options -> SpecWith (TestEnvironment, (GraphqlEngine.Server, Webhook.EventsQueue))
+tests opts = do
+  cleanupEventTriggersWhenSourceRemoved opts
+
+cleanupEventTriggersWhenSourceRemoved :: Fixture.Options -> SpecWith (TestEnvironment, (GraphqlEngine.Server, Webhook.EventsQueue))
+cleanupEventTriggersWhenSourceRemoved opts =
+  describe "removing a source with event trigger via replace_metadata should also remove the event trigger related stuffs (hdb_catalog.event_log)" do
+    it "remove source via replace_metadata, check that the event_log table is removed as well" $
+      \(testEnvironment, (_, _)) -> do
+        -- `hdb_catalog.event_log` should be existing before (as we have added an event trigger in setup)
+        checkIfPGTableExists "hdb_catalog.event_log" >>= (`shouldBe` True)
+
+        -- remove the source using replace_meatadata API
+        let replaceMetadata =
+              [yaml|
+                type: replace_metadata
+                args:
+                  sources: []
+                  version : 3
+              |]
+
+            expectedResponse =
+              [yaml|
+                message: success
+              |]
+
+        -- Checking if the replace_metadata was successful
+        shouldReturnYaml
+          opts
+          (GraphqlEngine.postMetadata testEnvironment replaceMetadata)
+          expectedResponse
+
+        -- `hdb_catalog.event_log` should not be existing now
+        checkIfPGTableExists "hdb_catalog.event_log" >>= (`shouldBe` False)
+
+--------------------------------------------------------------------------------
+
+-- ** Setup and teardown override
+
+postgresSetup :: (TestEnvironment, (GraphqlEngine.Server, Webhook.EventsQueue)) -> IO ()
+postgresSetup (testEnvironment, (webhookServer, _)) = do
+  Postgres.setup (schema "authors") (testEnvironment, ())
+  let webhookServerEchoEndpoint = GraphqlEngine.serverUrl webhookServer ++ "/echo"
+  GraphqlEngine.postMetadata_ testEnvironment $
+    [yaml|
+      type: bulk
+      args:
+      - type: pg_create_event_trigger
+        args:
+          name: authors_all
+          source: postgres
+          table:
+            name: authors
+            schema: hasura
+          webhook: *webhookServerEchoEndpoint
+          insert:
+            columns: "*"
+    |]
+
+postgresTeardown :: (TestEnvironment, (GraphqlEngine.Server, Webhook.EventsQueue)) -> IO ()
+postgresTeardown (_, (server, _)) = do
+  Postgres.dropTable (authorsTable "authors")
+  stopServer server
+
+webhookServerMkLocalTestEnvironment ::
+  TestEnvironment -> IO (GraphqlEngine.Server, Webhook.EventsQueue)
+webhookServerMkLocalTestEnvironment _ = do
+  Webhook.run
+
+checkIfPGTableExists :: String -> IO Bool
+checkIfPGTableExists tableName = do
+  let sqlQuery = "SELECT to_regclass('" <> tableName <> "')::text;"
+      handleNullExp (Postgres.UnexpectedNull {}) = pure False
+      handleNullExp err = throw err
+  catch
+    ( bracket
+        (Postgres.connectPostgreSQL (fromString Constants.postgresqlConnectionString))
+        Postgres.close
+        ( \conn -> do
+            rows :: [[String]] <- Postgres.query_ conn (fromString sqlQuery)
+            pure (((head . head) rows) == tableName)
+        )
+    )
+    handleNullExp