server: move event trigger schedule cleanup logic

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/7606 Co-authored-by: Krushan Bauva <31391329+krushanbauva@users.noreply.github.com> GitOrigin-RevId: 95819347aff4e6a86b2bc9acf214222e05dba496
2024-12-14 17:02:49 +03:00 · 2023-02-03 17:57:53 +05:30 · 2023-02-03 17:57:53 +05:30 · 775bdc8026
commit 775bdc8026
parent d32e734310
6 changed files with 41 additions and 55 deletions
--- a/server/src-lib/Hasura/App.hs
+++ b/server/src-lib/Hasura/App.hs
@ -1140,6 +1140,7 @@ instance (Monad m) => EB.MonadQueryTags (PGMetadataStorageAppT m) where
 instance (Monad m) => MonadEventLogCleanup (PGMetadataStorageAppT m) where
  runLogCleaner _ = pure $ throw400 NotSupported "Event log cleanup feature is enterprise edition only"
  generateCleanupSchedules _ _ _ = pure $ Right ()
+  updateTriggerCleanupSchedules _ _ _ _ = pure $ Right ()

 runInSeparateTx ::
  (MonadIO m) =>
--- a/server/src-lib/Hasura/RQL/DDL/EventTrigger.hs
+++ b/server/src-lib/Hasura/RQL/DDL/EventTrigger.hs
@ -193,21 +193,52 @@ class Monad m => MonadEventLogCleanup m where
  generateCleanupSchedules ::
    AB.AnyBackend SourceInfo -> TriggerName -> AutoTriggerLogCleanupConfig -> m (Either QErr ())

+  -- | `updateTriggerCleanupSchedules` is primarily used to update the
+  --    cleanup schedules associated with an event trigger in case the cleanup
+  --    config has changed while replacing the metadata.
+  --
+  --    In case,
+  --    i. a source has been dropped -
+  --           We don't need to clear the cleanup schedules
+  --           because the event log cleanup table is dropped as part
+  --           of the post drop source hook.
+  --    ii. a table or an event trigger has been dropped/updated -
+  --           Older cleanup events will be deleted first and in case of
+  --           an update, new cleanup events will be generated and inserted
+  --           into the table.
+  --    iii. a new event trigger with cleanup config has been added -
+  --             Generate the cleanup events and insert it.
+  --    iv. a new source has been added -
+  --           Generate the cleanup events and insert it.
+  --    v. the cron schedule for event trigger cleanup config has changed -
+  --           Delete cleanup events with older cron schedule and generate
+  --           cleanup events with new cron schedule.
+  updateTriggerCleanupSchedules ::
+    L.Logger L.Hasura ->
+    InsOrdHashMap SourceName BackendSourceMetadata ->
+    InsOrdHashMap SourceName BackendSourceMetadata ->
+    SchemaCache ->
+    m (Either QErr ())
+
 instance (MonadEventLogCleanup m) => MonadEventLogCleanup (ReaderT r m) where
  runLogCleaner conf = lift $ runLogCleaner conf
  generateCleanupSchedules sourceInfo triggerName cleanupConfig = lift $ generateCleanupSchedules sourceInfo triggerName cleanupConfig
+  updateTriggerCleanupSchedules logger oldSources newSources schemaCache = lift $ updateTriggerCleanupSchedules logger oldSources newSources schemaCache

 instance (MonadEventLogCleanup m) => MonadEventLogCleanup (ExceptT e m) where
  runLogCleaner conf = lift $ runLogCleaner conf
  generateCleanupSchedules sourceInfo triggerName cleanupConfig = lift $ generateCleanupSchedules sourceInfo triggerName cleanupConfig
+  updateTriggerCleanupSchedules logger oldSources newSources schemaCache = lift $ updateTriggerCleanupSchedules logger oldSources newSources schemaCache

 instance (MonadEventLogCleanup m) => MonadEventLogCleanup (MetadataT m) where
  runLogCleaner conf = lift $ runLogCleaner conf
  generateCleanupSchedules sourceInfo triggerName cleanupConfig = lift $ generateCleanupSchedules sourceInfo triggerName cleanupConfig
+  updateTriggerCleanupSchedules logger oldSources newSources schemaCache = lift $ updateTriggerCleanupSchedules logger oldSources newSources schemaCache

 instance (MonadEventLogCleanup m) => MonadEventLogCleanup (TraceT m) where
  runLogCleaner conf = lift $ runLogCleaner conf
  generateCleanupSchedules sourceInfo triggerName cleanupConfig = lift $ generateCleanupSchedules sourceInfo triggerName cleanupConfig
+  updateTriggerCleanupSchedules logger oldSources newSources schemaCache = lift $ updateTriggerCleanupSchedules logger oldSources newSources schemaCache

 resolveEventTriggerQuery ::
  forall b m.
--- a/server/src-lib/Hasura/RQL/DDL/Metadata.hs
+++ b/server/src-lib/Hasura/RQL/DDL/Metadata.hs
@ -40,7 +40,6 @@ import Data.Text.Encoding qualified as TE
 import Data.Text.Extended (dquoteList, (<<>))
 import Hasura.Base.Error
 import Hasura.EncJSON
-import Hasura.Eventing.EventTrigger (logQErr)
 import Hasura.Logging qualified as HL
 import Hasura.Metadata.Class
 import Hasura.NativeQuery.API
@ -76,7 +75,7 @@ import Hasura.RQL.Types.QueryCollection
 import Hasura.RQL.Types.ScheduledTrigger
 import Hasura.RQL.Types.SchemaCache
 import Hasura.RQL.Types.SchemaCache.Build
-import Hasura.RQL.Types.Source (SourceInfo (..), unsafeSourceInfo)
+import Hasura.RQL.Types.Source (unsafeSourceInfo)
 import Hasura.RQL.Types.SourceCustomization
 import Hasura.SQL.AnyBackend qualified as AB
 import Hasura.SQL.Backend (BackendType (..))
@ -314,7 +313,10 @@ runReplaceMetadataV2 ReplaceMetadataV2 {..} = do
  -- See Note [Cleanup for dropped triggers]
  dropSourceSQLTriggers logger oldSchemaCache (_metaSources oldMetadata) (_metaSources metadata)

-  generateSQLTriggerCleanupSchedules logger (_metaSources oldMetadata) (_metaSources metadata)
+  newSchemaCache <- askSchemaCache
+
+  updateTriggerCleanupSchedules logger (_metaSources oldMetadata) (_metaSources metadata) newSchemaCache
+    >>= (`onLeft` throwError)

  let droppedSources = OMap.difference oldSources newSources

@ -322,7 +324,7 @@ runReplaceMetadataV2 ReplaceMetadataV2 {..} = do
  for_ (OMap.toList droppedSources) $ \(oldSource, oldSourceBackendMetadata) ->
    postDropSourceHookHelper oldSchemaCache oldSource (unBackendSourceMetadata oldSourceBackendMetadata)

-  encJFromJValue . formatInconsistentObjs . scInconsistentObjs <$> askSchemaCache
+  pure . encJFromJValue . formatInconsistentObjs . scInconsistentObjs $ newSchemaCache
  where
    {- Note [Cron triggers behaviour with replace metadata]
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -492,57 +494,6 @@ runReplaceMetadataV2 ReplaceMetadataV2 {..} = do
              <> " For more details, please refer https://hasura.io/docs/latest/graphql/core/event-triggers/index.html "
          )

-    -- \| `generateSQLTriggerCleanupSchedules` is primarily used to update the
-    --    cleanup schedules associated with an event trigger in case the cleanup
-    --    config has changed while replacing the metadata.
-    --
-    --    In case,
-    --    i. a source has been dropped -
-    --           We don't need to clear the cleanup schedules
-    --           because the event log cleanup table is dropped as part
-    --           of the post drop source hook.
-    --    ii. a table or an event trigger has been dropped/updated -
-    --           Older cleanup events will be deleted first and in case of
-    --           an update, new cleanup events will be generated and inserted
-    --           into the table.
-    --    iii. a new event trigger with cleanup config has been added -
-    --             Generate the cleanup events and insert it.
-    generateSQLTriggerCleanupSchedules ::
-      HL.Logger HL.Hasura ->
-      InsOrdHashMap SourceName BackendSourceMetadata ->
-      InsOrdHashMap SourceName BackendSourceMetadata ->
-      m ()
-    generateSQLTriggerCleanupSchedules (HL.Logger logger) oldSources newSources = do
-      -- If there are any event trigger cleanup configs with different cron schedule,
-      -- then delete the older schedules generate cleanup logs for new event trigger
-      -- cleanup config.
-      for_ (OMap.toList newSources) $ \(source, newBackendSourceMetadata) -> do
-        for_ (OMap.lookup source oldSources) $ \oldBackendSourceMetadata ->
-          AB.dispatchAnyBackend @BackendEventTrigger (unBackendSourceMetadata newBackendSourceMetadata) \(newSourceMetadata :: SourceMetadata b) -> do
-            dispatch oldBackendSourceMetadata \oldSourceMetadata -> do
-              sourceInfoMaybe <- askSourceInfoMaybe @b source
-              case sourceInfoMaybe of
-                Nothing ->
-                  logger $
-                    MetadataLog
-                      HL.LevelWarn
-                      ( "Could not cleanup the scheduled autocleanup instances present in the source '"
-                          <> source
-                            <<> "' as it is inconsistent"
-                      )
-                      J.Null
-                Just sourceInfo@(SourceInfo _ _ _ _ sourceConfig _ _) -> do
-                  let getEventMapWithCC sourceMeta = Map.fromList $ concatMap (getAllETWithCleanupConfigInTableMetadata . snd) $ OMap.toList $ _smTables sourceMeta
-                      oldEventTriggersWithCC = getEventMapWithCC oldSourceMetadata
-                      newEventTriggersWithCC = getEventMapWithCC newSourceMetadata
-                      -- event triggers with cleanup config that existed in old metadata but are missing in new metadata
-                      differenceMap = Map.difference oldEventTriggersWithCC newEventTriggersWithCC
-                  for_ (Map.toList differenceMap) $ \(triggerName, cleanupConfig) -> do
-                    deleteAllScheduledCleanups @b sourceConfig triggerName
-                    pure cleanupConfig
-                  for_ (Map.toList newEventTriggersWithCC) $ \(triggerName, cleanupConfig) -> do
-                    (`onLeft` logQErr) =<< generateCleanupSchedules (AB.mkAnyBackend sourceInfo) triggerName cleanupConfig
-
    dispatch (BackendSourceMetadata bs) = AB.dispatchAnyBackend @BackendEventTrigger bs

 -- | Only includes the cron triggers with `included_in_metadata` set to `True`
--- a/server/src-lib/Hasura/RQL/DDL/Schema/Cache.hs
+++ b/server/src-lib/Hasura/RQL/DDL/Schema/Cache.hs
@ -179,6 +179,7 @@ newtype CacheRWT m a
 instance (MonadEventLogCleanup m) => MonadEventLogCleanup (CacheRWT m) where
  runLogCleaner conf = lift $ runLogCleaner conf
  generateCleanupSchedules sourceInfo triggerName cleanupConfig = lift $ generateCleanupSchedules sourceInfo triggerName cleanupConfig
+  updateTriggerCleanupSchedules logger oldSources newSources schemaCache = lift $ updateTriggerCleanupSchedules logger oldSources newSources schemaCache

 runCacheRWT ::
  Functor m =>
--- a/server/src-lib/Hasura/RQL/Types/Run.hs
+++ b/server/src-lib/Hasura/RQL/Types/Run.hs
@ -57,6 +57,7 @@ instance (MonadResolveSource m) => MonadResolveSource (RunT m) where
 instance (MonadEventLogCleanup m) => MonadEventLogCleanup (RunT m) where
  runLogCleaner conf = lift $ runLogCleaner conf
  generateCleanupSchedules sInfo tName cConf = lift $ generateCleanupSchedules sInfo tName cConf
+  updateTriggerCleanupSchedules logger oldSources newSources schemaCache = lift $ updateTriggerCleanupSchedules logger oldSources newSources schemaCache

 peelRun ::
  RunCtx ->
--- a/server/test-postgres/Hasura/Server/MigrateSuite.hs
+++ b/server/test-postgres/Hasura/Server/MigrateSuite.hs
@ -68,6 +68,7 @@ instance (MonadBase IO m) => CacheRM (CacheRefT m) where
 instance (MonadEventLogCleanup m) => MonadEventLogCleanup (CacheRefT m) where
  runLogCleaner conf = lift $ runLogCleaner conf
  generateCleanupSchedules sourceInfo triggerName cleanupConfig = lift $ generateCleanupSchedules sourceInfo triggerName cleanupConfig
+  updateTriggerCleanupSchedules logger oldSources newSources schemaCache = lift $ updateTriggerCleanupSchedules logger oldSources newSources schemaCache

 instance
  ( MonadIO m,