server/postgres: improve fetching tables' and functions' metadata (from database)

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/3778
GitOrigin-RevId: 13bb97bdb7afad265db899f368c74d9f240b214a

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Bug fixes and improvements
 
+- server: improve performance of fetching postgres catalog metadata for tables and functions
 - server: Queries present in query collections, such as allow-list, and rest-endpoints are now validated (against the schema)
 - server: Redesigns internal implementation of webhook transforms.
 - server: improve SQL generation for BigQuery backend queries involving `Orderby`.

server/src-lib/Data/List/Extended.hs

@@ -18,14 +18,15 @@ import Data.HashSet qualified as Set
 import Data.Hashable (Hashable)
 import Data.List qualified as L
 import Data.List.NonEmpty qualified as NE
+import Data.Set qualified as S
 import Prelude
 
 duplicates :: (Eq a, Hashable a) => [a] -> Set.HashSet a
 duplicates =
   Set.fromList . Map.keys . Map.filter (> 1) . Map.fromListWith (+) . map (,1 :: Int)
 
-uniques :: Eq a => [a] -> [a]
-uniques = map NE.head . NE.group
+uniques :: (Ord a) => [a] -> [a]
+uniques = S.toList . S.fromList
 
 getDifference :: (Eq a, Hashable a) => [a] -> [a] -> Set.HashSet a
 getDifference = Set.difference `on` Set.fromList

server/src-lib/Hasura/Backends/BigQuery/Instances/Metadata.hs

@@ -10,7 +10,7 @@ instance BackendMetadata 'BigQuery where
   buildComputedFieldInfo = BigQuery.buildComputedFieldInfo
   fetchAndValidateEnumValues = BigQuery.fetchAndValidateEnumValues
   resolveSourceConfig = BigQuery.resolveSourceConfig
-  resolveDatabaseMetadata = BigQuery.resolveSource
+  resolveDatabaseMetadata _ = BigQuery.resolveSource
   parseBoolExpOperations = BigQuery.parseBoolExpOperations
   buildFunctionInfo = BigQuery.buildFunctionInfo
   updateColumnInEventTrigger = BigQuery.updateColumnInEventTrigger

server/src-lib/Hasura/Backends/MSSQL/Instances/Metadata.hs

@@ -14,7 +14,7 @@ instance BackendMetadata 'MSSQL where
   buildComputedFieldInfo = MSSQL.buildComputedFieldInfo
   fetchAndValidateEnumValues = MSSQL.fetchAndValidateEnumValues
   resolveSourceConfig = MSSQL.resolveSourceConfig
-  resolveDatabaseMetadata = MSSQL.resolveDatabaseMetadata
+  resolveDatabaseMetadata _ = MSSQL.resolveDatabaseMetadata
   parseBoolExpOperations = MSSQL.parseBoolExpOperations
   buildFunctionInfo = MSSQL.buildFunctionInfo
   updateColumnInEventTrigger = MSSQL.updateColumnInEventTrigger

server/src-lib/Hasura/Backends/MySQL/Instances/Metadata.hs

@@ -11,7 +11,7 @@ instance BackendMetadata 'MySQL where
   buildComputedFieldInfo = error "buildComputedFieldInfo: MySQL backend does not support this operation yet."
   fetchAndValidateEnumValues = error "fetchAndValidateEnumValues: MySQL backend does not support this operation yet."
   resolveSourceConfig = MySQL.resolveSourceConfig
-  resolveDatabaseMetadata = MySQL.resolveDatabaseMetadata
+  resolveDatabaseMetadata _ = MySQL.resolveDatabaseMetadata
   parseBoolExpOperations = error "parseBoolExpOperations: MySQL backend does not support this operation yet."
   buildFunctionInfo = error "buildFunctionInfo: MySQL backend does not support this operation yet."
   updateColumnInEventTrigger = error "updateColumnInEventTrigger: MySQL backend does not support this operation yet."

server/src-lib/Hasura/Backends/Postgres/DDL/Function.hs

@@ -15,7 +15,7 @@ import Control.Monad.Validate qualified as MV
 import Data.Sequence qualified as Seq
 import Data.Text qualified as T
 import Data.Text.Extended
-import Hasura.Backends.Postgres.SQL.Types
+import Hasura.Backends.Postgres.SQL.Types hiding (FunctionName)
 import Hasura.Base.Error
 import Hasura.Prelude
 import Hasura.RQL.Types.Backend

server/src-lib/Hasura/Backends/Postgres/DDL/RunSQL.hs

@@ -27,7 +27,7 @@ import Hasura.Backends.Postgres.DDL.Source
     fetchFunctionMetadata,
     fetchTableMetadata,
   )
-import Hasura.Backends.Postgres.SQL.Types
+import Hasura.Backends.Postgres.SQL.Types hiding (FunctionName, TableName)
 import Hasura.Base.Error
 import Hasura.EncJSON
 import Hasura.Prelude
@@ -120,35 +120,35 @@ of queries may not modify the schema at all. As a (fairly stupid) heuristic, we
 check if the query contains any keywords for DDL operations, and if not, we skip
 the metadata check as well. -}
 
-fetchMeta ::
+-- | Fetch metadata of tracked tables/functions and build @'TableMeta'/@'FunctionMeta'
+-- to calculate diff later in @'withMetadataCheck'.
+fetchTablesFunctionsMetadata ::
   (ToMetadataFetchQuery pgKind, BackendMetadata ('Postgres pgKind), MonadTx m) =>
   TableCache ('Postgres pgKind) ->
-  FunctionCache ('Postgres pgKind) ->
+  [TableName ('Postgres pgKind)] ->
+  [FunctionName ('Postgres pgKind)] ->
   m ([TableMeta ('Postgres pgKind)], [FunctionMeta ('Postgres pgKind)])
-fetchMeta tables functions = do
-  tableMetaInfos <- fetchTableMetadata
-  functionMetaInfos <- fetchFunctionMetadata
-
-  let getFunctionMetas function =
-        let mkFunctionMeta rawInfo =
-              FunctionMeta (rfiOid rawInfo) function (rfiFunctionType rawInfo)
-         in maybe [] (map mkFunctionMeta) $ M.lookup function functionMetaInfos
-
-      mkComputedFieldMeta computedField =
-        let function = _cffName $ _cfiFunction computedField
-         in map (ComputedFieldMeta (_cfiName computedField)) $ getFunctionMetas function
-
-      tableMetas = flip map (M.toList tableMetaInfos) $ \(table, tableMetaInfo) ->
+fetchTablesFunctionsMetadata tableCache tables functions = do
+  tableMetaInfos <- fetchTableMetadata tables
+  functionMetaInfos <- fetchFunctionMetadata functions
+  pure (buildTableMeta tableMetaInfos functionMetaInfos, buildFunctionMeta functionMetaInfos)
+  where
+    buildTableMeta tableMetaInfos functionMetaInfos =
+      flip map (M.toList tableMetaInfos) $ \(table, tableMetaInfo) ->
         TableMeta table tableMetaInfo $
-          fromMaybe [] $
-            M.lookup table tables <&> \tableInfo ->
-              let tableCoreInfo = _tiCoreInfo tableInfo
-                  computedFields = getComputedFieldInfos $ _tciFieldInfoMap tableCoreInfo
-               in concatMap mkComputedFieldMeta computedFields
+          foldMap @Maybe (concatMap (mkComputedFieldMeta functionMetaInfos) . getComputedFields) (M.lookup table tableCache)
 
-      functionMetas = concatMap getFunctionMetas $ M.keys functions
+    buildFunctionMeta functionMetaInfos =
+      concatMap (getFunctionMetas functionMetaInfos) functions
 
-  pure (tableMetas, functionMetas)
+    mkComputedFieldMeta functionMetaInfos computedField =
+      let function = _cffName $ _cfiFunction computedField
+       in map (ComputedFieldMeta (_cfiName computedField)) $ getFunctionMetas functionMetaInfos function
+
+    getFunctionMetas functionMetaInfos function =
+      let mkFunctionMeta rawInfo =
+            FunctionMeta (rfiOid rawInfo) function (rfiFunctionType rawInfo)
+       in foldMap @Maybe (map mkFunctionMeta) $ M.lookup function functionMetaInfos
 
 -- | Used as an escape hatch to run raw SQL against a database.
 runRunSQL ::
@@ -188,7 +188,7 @@ runRunSQL q@RunSQL {..} = do
         rawSqlErrHandler txe =
           (err400 PostgresError "query execution failed") {qeInternal = Just $ ExtraInternal $ toJSON txe}
 
--- | @'withMetadataCheck' cascade action@ runs @action@ and checks if the schema changed as a
+-- | @'withMetadataCheck' source cascade txAccess runSQLQuery@ executes @runSQLQuery@ and checks if the schema changed as a
 -- result. If it did, it checks to ensure the changes do not violate any integrity constraints, and
 -- if not, incorporates them into the schema cache.
 -- TODO(antoine): shouldn't this be generalized?
@@ -208,86 +208,243 @@ withMetadataCheck ::
   Q.TxAccess ->
   Q.TxET QErr m a ->
   m a
-withMetadataCheck source cascade txAccess action = do
-  SourceInfo _ preActionTables preActionFunctions sourceConfig _ _ <- askSourceInfo @('Postgres pgKind) source
+withMetadataCheck source cascade txAccess runSQLQuery = do
+  SourceInfo _ tableCache functionCache sourceConfig _ _ <- askSourceInfo @('Postgres pgKind) source
 
-  (actionResult, metadataUpdater) <-
-    liftEitherM $
-      runExceptT $
-        runTx (_pscExecCtx sourceConfig) txAccess $ do
-          -- Drop event triggers so no interference is caused to the sql query
-          forM_ (M.elems preActionTables) $ \tableInfo -> do
-            let eventTriggers = _tiEventTriggerInfoMap tableInfo
-            forM_ (M.keys eventTriggers) (liftTx . dropTriggerQ)
+  let dropTriggersAndRunSQL = do
+        -- We need to drop existing event triggers so that no interference is caused to the sql query execution
+        dropExistingEventTriggers tableCache
+        runSQLQuery
 
-          -- Get the metadata before the sql query, everything, need to filter this
-          (preActionTableMeta, preActionFunctionMeta) <- fetchMeta preActionTables preActionFunctions
-
-          -- Run the action
-          actionResult <- action
-
-          -- Get the metadata after the sql query
-          (postActionTableMeta, postActionFunctionMeta) <- fetchMeta preActionTables preActionFunctions
-
-          let preActionTableMeta' = filter (flip M.member preActionTables . tmTable) preActionTableMeta
-              tablesDiff = getTablesDiff preActionTableMeta' postActionTableMeta
-              FunctionsDiff droppedFuncs alteredFuncs = getFunctionsDiff preActionFunctionMeta postActionFunctionMeta
-              overloadedFuncs = getOverloadedFunctions (M.keys preActionFunctions) postActionFunctionMeta
-
-          -- Do not allow overloading functions
-          unless (null overloadedFuncs) $
-            throw400 NotSupported $
-              "the following tracked function(s) cannot be overloaded: "
-                <> commaSeparated overloadedFuncs
-
-          -- Report back with an error if cascade is not set
-          indirectDeps <- getIndirectDependencies source tablesDiff
-          when (indirectDeps /= [] && not cascade) $ reportDependentObjectsExist indirectDeps
-
-          metadataUpdater <- execWriterT $ do
-            -- Purge all the indirect dependents from state
-            for_ indirectDeps \case
-              SOSourceObj sourceName objectID -> do
-                AB.dispatchAnyBackend @BackendMetadata objectID $ purgeDependentObject sourceName >=> tell
-              _ ->
-                pure ()
-
-            -- Purge all dropped functions
-            let purgedFuncs = flip mapMaybe indirectDeps \case
-                  SOSourceObj _ objectID
-                    | Just (SOIFunction qf) <- AB.unpackAnyBackend @('Postgres pgKind) objectID ->
-                      Just qf
-                  _ -> Nothing
-            for_ (droppedFuncs \\ purgedFuncs) $
-              tell . dropFunctionInMetadata @('Postgres pgKind) source
-
-            -- Process altered functions
-            forM_ alteredFuncs $ \(qf, newTy) -> do
-              when (newTy == FTVOLATILE) $
-                throw400 NotSupported $
-                  "type of function " <> qf <<> " is altered to \"VOLATILE\" which is not supported now"
-
-            -- update the metadata with the changes
-            processTablesDiff source preActionTables tablesDiff
-
-          pure (actionResult, metadataUpdater)
+  -- Run SQL query and metadata checker in a transaction
+  (queryResult, metadataUpdater) <- runTxWithMetadataCheck source sourceConfig txAccess tableCache functionCache cascade dropTriggersAndRunSQL
 
   -- Build schema cache with updated metadata
   withNewInconsistentObjsCheck $
     buildSchemaCacheWithInvalidations mempty {ciSources = HS.singleton source} metadataUpdater
 
-  postActionSchemaCache <- askSchemaCache
+  postRunSQLSchemaCache <- askSchemaCache
 
-  -- Recreate event triggers in hdb_catalog
-  let postActionTables = fromMaybe mempty $ unsafeTableCache @('Postgres pgKind) source $ scSources postActionSchemaCache
-  serverConfigCtx <- askServerConfigCtx
+  -- Recreate event triggers in hdb_catalog. Event triggers are dropped before executing @'runSQLQuery'.
+  recreateEventTriggers sourceConfig postRunSQLSchemaCache
+
+  pure queryResult
+  where
+    dropExistingEventTriggers :: TableCache ('Postgres pgKind) -> Q.TxET QErr m ()
+    dropExistingEventTriggers tableCache =
+      forM_ (M.elems tableCache) $ \tableInfo -> do
+        let eventTriggers = _tiEventTriggerInfoMap tableInfo
+        forM_ (M.keys eventTriggers) (liftTx . dropTriggerQ)
+
+    recreateEventTriggers :: PGSourceConfig -> SchemaCache -> m ()
+    recreateEventTriggers sourceConfig schemaCache = do
+      let tables = fromMaybe mempty $ unsafeTableCache @('Postgres pgKind) source $ scSources schemaCache
+      serverConfigCtx <- askServerConfigCtx
+      liftEitherM $
+        runPgSourceWriteTx sourceConfig $
+          forM_ (M.elems tables) $ \(TableInfo coreInfo _ eventTriggers _) -> do
+            let table = _tciName coreInfo
+                columns = getCols $ _tciFieldInfoMap coreInfo
+            forM_ (M.toList eventTriggers) $ \(triggerName, eti) -> do
+              let opsDefinition = etiOpsDef eti
+              flip runReaderT serverConfigCtx $ mkAllTriggersQ triggerName table columns opsDefinition
+
+-- | @'runTxWithMetadataCheck source sourceConfig txAccess tableCache functionCache cascadeDependencies tx' checks for
+-- changes in GraphQL Engine metadata when a @'tx' is executed on the database alters Postgres
+-- schema of tables and functions. If any indirect dependencies (Eg. remote table dependence of a relationship) are
+-- found and @'cascadeDependencies' is False, then an exception is raised.
+runTxWithMetadataCheck ::
+  forall m a (pgKind :: PostgresKind).
+  ( BackendMetadata ('Postgres pgKind),
+    ToMetadataFetchQuery pgKind,
+    CacheRWM m,
+    MonadIO m,
+    MonadBaseControl IO m,
+    MonadError QErr m
+  ) =>
+  SourceName ->
+  SourceConfig ('Postgres pgKind) ->
+  Q.TxAccess ->
+  TableCache ('Postgres pgKind) ->
+  FunctionCache ('Postgres pgKind) ->
+  Bool ->
+  Q.TxET QErr m a ->
+  m (a, MetadataModifier)
+runTxWithMetadataCheck source sourceConfig txAccess tableCache functionCache cascadeDependencies tx =
   liftEitherM $
-    runPgSourceWriteTx sourceConfig $
-      forM_ (M.elems postActionTables) $ \(TableInfo coreInfo _ eventTriggers _) -> do
-        let table = _tciName coreInfo
-            columns = getCols $ _tciFieldInfoMap coreInfo
-        forM_ (M.toList eventTriggers) $ \(triggerName, eti) -> do
-          let opsDefinition = etiOpsDef eti
-          flip runReaderT serverConfigCtx $ mkAllTriggersQ triggerName table columns opsDefinition
+    runExceptT $
+      runTx (_pscExecCtx sourceConfig) txAccess $ do
+        -- Running in a transaction helps to rollback the @'tx' execution in case of any exceptions
 
-  pure actionResult
+        -- Before running the @'tx', fetch metadata of existing tables and functions from Postgres.
+        let tableNames = M.keys tableCache
+            computedFieldFunctions = concatMap getComputedFieldFunctions (M.elems tableCache)
+            functionNames = M.keys functionCache <> computedFieldFunctions
+        (preTxTablesMeta, preTxFunctionsMeta) <- fetchTablesFunctionsMetadata tableCache tableNames functionNames
+
+        -- Since the @'tx' may alter table/function names we use the OIDs of underlying tables
+        -- (sourced from 'pg_class' for tables and 'pg_proc' for functions), which remain unchanged in the
+        -- case if a table/function is renamed.
+        let tableOids = map (_ptmiOid . tmInfo) preTxTablesMeta
+            functionOids = map fmOid preTxFunctionsMeta
+
+        -- Run the transaction
+        txResult <- tx
+
+        (postTxTablesMeta, postTxFunctionMeta) <-
+          uncurry (fetchTablesFunctionsMetadata tableCache)
+            -- Fetch names of tables and functions using OIDs which also contains renamed items
+            =<< fetchTablesFunctionsFromOids tableOids functionOids
+
+        -- Calculate the tables diff (dropped & altered tables)
+        let tablesDiff = getTablesDiff preTxTablesMeta postTxTablesMeta
+            -- Calculate the functions diff. For calculating diff for functions, only consider
+            -- query/mutation functions and exclude functions underpinning computed fields.
+            -- Computed field functions are being processed under each table diff.
+            -- See @'getTablesDiff' and @'processTablesDiff'
+            excludeComputedFieldFunctions = filter ((`M.member` functionCache) . fmFunction)
+            functionsDiff =
+              getFunctionsDiff
+                (excludeComputedFieldFunctions preTxFunctionsMeta)
+                (excludeComputedFieldFunctions postTxFunctionMeta)
+
+        dontAllowFunctionOverloading $
+          getOverloadedFunctions
+            (M.keys functionCache)
+            (excludeComputedFieldFunctions postTxFunctionMeta)
+
+        -- Update metadata with schema change caused by @'tx'
+        metadataUpdater <- execWriterT do
+          -- Collect indirect dependencies of altered tables
+          tableIndirectDeps <- getIndirectDependencies source tablesDiff
+
+          -- If table indirect dependencies exist and cascading is not enabled then report an exception
+          when (tableIndirectDeps /= [] && not cascadeDependencies) $ reportDependentObjectsExist tableIndirectDeps
+
+          -- Purge all the table dependents
+          purgeDependencies tableIndirectDeps
+
+          -- Collect function names from purged table dependencies
+          let purgedFunctions = collectFunctionsInDeps tableIndirectDeps
+              FunctionsDiff droppedFunctions alteredFunctions = functionsDiff
+
+          -- Drop functions in metadata. Exclude functions that were already dropped as part of table indirect dependencies
+          purgeFunctionsFromMetadata $ droppedFunctions \\ purgedFunctions
+
+          -- If any function type is altered to VOLATILE then raise an exception
+          dontAllowFunctionAlteredVolatile alteredFunctions
+
+          -- Propagate table changes to metadata
+          processTablesDiff source tableCache tablesDiff
+
+        pure (txResult, metadataUpdater)
+  where
+    dontAllowFunctionOverloading ::
+      MonadError QErr n =>
+      [FunctionName ('Postgres pgKind)] ->
+      n ()
+    dontAllowFunctionOverloading overloadedFunctions =
+      unless (null overloadedFunctions) $
+        throw400 NotSupported $
+          "the following tracked function(s) cannot be overloaded: "
+            <> commaSeparated overloadedFunctions
+
+    dontAllowFunctionAlteredVolatile ::
+      MonadError QErr n =>
+      [(FunctionName ('Postgres pgKind), FunctionVolatility)] ->
+      n ()
+    dontAllowFunctionAlteredVolatile alteredFunctions =
+      forM_ alteredFunctions $ \(qf, newTy) -> do
+        when (newTy == FTVOLATILE) $
+          throw400 NotSupported $
+            "type of function " <> qf <<> " is altered to \"VOLATILE\" which is not supported now"
+
+    purgeDependencies ::
+      MonadError QErr n =>
+      [SchemaObjId] ->
+      WriterT MetadataModifier n ()
+    purgeDependencies deps =
+      for_ deps \case
+        SOSourceObj sourceName objectID -> do
+          AB.dispatchAnyBackend @BackendMetadata objectID $ purgeDependentObject sourceName >=> tell
+        _ ->
+          -- Ignore non-source dependencies
+          pure ()
+
+    purgeFunctionsFromMetadata ::
+      Monad n =>
+      [FunctionName ('Postgres pgKind)] ->
+      WriterT MetadataModifier n ()
+    purgeFunctionsFromMetadata functions =
+      for_ functions $ tell . dropFunctionInMetadata @('Postgres pgKind) source
+
+    collectFunctionsInDeps :: [SchemaObjId] -> [FunctionName ('Postgres pgKind)]
+    collectFunctionsInDeps deps =
+      flip mapMaybe deps \case
+        SOSourceObj _ objectID
+          | Just (SOIFunction qf) <- AB.unpackAnyBackend @('Postgres pgKind) objectID ->
+            Just qf
+        _ -> Nothing
+
+-- | Fetch list of tables and functions with provided oids
+fetchTablesFunctionsFromOids ::
+  (MonadIO m) =>
+  [OID] ->
+  [OID] ->
+  Q.TxET QErr m ([TableName ('Postgres pgKind)], [FunctionName ('Postgres pgKind)])
+fetchTablesFunctionsFromOids tableOids functionOids =
+  ((Q.getAltJ *** Q.getAltJ) . Q.getRow)
+    <$> Q.withQE
+      defaultTxErrorHandler
+      [Q.sql|
+    SELECT
+      COALESCE(
+        ( SELECT
+            json_agg(
+              row_to_json(
+                (
+                  SELECT e
+                    FROM ( SELECT "table".relname AS "name",
+                                  "schema".nspname AS "schema"
+                    ) AS e
+                )
+              )
+            ) AS "item"
+            FROM jsonb_to_recordset($1::jsonb) AS oid_table("oid" int)
+                 JOIN pg_catalog.pg_class "table" ON ("table".oid = "oid_table".oid)
+                 JOIN pg_catalog.pg_namespace "schema" ON ("schema".oid = "table".relnamespace)
+        ),
+        '[]'
+      ) AS "tables",
+
+      COALESCE(
+        ( SELECT
+            json_agg(
+              row_to_json(
+                (
+                  SELECT e
+                    FROM ( SELECT "function".proname AS "name",
+                                  "schema".nspname AS "schema"
+                    ) AS e
+                )
+              )
+            ) AS "item"
+            FROM jsonb_to_recordset($2::jsonb) AS oid_table("oid" int)
+                 JOIN pg_catalog.pg_proc "function" ON ("function".oid = "oid_table".oid)
+                 JOIN pg_catalog.pg_namespace "schema" ON ("schema".oid = "function".pronamespace)
+        ),
+        '[]'
+      ) AS "functions"
+  |]
+      (Q.AltJ $ map mkOidObject tableOids, Q.AltJ $ map mkOidObject functionOids)
+      True
+  where
+    mkOidObject oid = object ["oid" .= oid]
+
+------ helpers ------------
+
+getComputedFields :: TableInfo ('Postgres pgKind) -> [ComputedFieldInfo ('Postgres pgKind)]
+getComputedFields = getComputedFieldInfos . _tciFieldInfoMap . _tiCoreInfo
+
+getComputedFieldFunctions :: TableInfo ('Postgres pgKind) -> [FunctionName ('Postgres pgKind)]
+getComputedFieldFunctions = map (_cffName . _cfiFunction) . getComputedFields

server/src-lib/Hasura/Backends/Postgres/DDL/Source.hs

@@ -27,19 +27,23 @@ import Data.Aeson.TH
 import Data.Environment qualified as Env
 import Data.FileEmbed (makeRelativeToProject)
 import Data.HashMap.Strict qualified as Map
+import Data.HashMap.Strict.InsOrd qualified as OMap
+import Data.List.Extended qualified as LE
 import Data.List.NonEmpty qualified as NE
 import Data.Time.Clock (UTCTime)
 import Database.PG.Query qualified as Q
 import Hasura.Backends.Postgres.Connection
 import Hasura.Backends.Postgres.DDL.Source.Version
-import Hasura.Backends.Postgres.SQL.Types
+import Hasura.Backends.Postgres.SQL.Types hiding (FunctionName)
 import Hasura.Base.Error
 import Hasura.Logging
 import Hasura.Prelude
 import Hasura.RQL.Types.Backend
 import Hasura.RQL.Types.Common
+import Hasura.RQL.Types.ComputedField
 import Hasura.RQL.Types.EventTrigger (RecreateEventTriggers (..))
 import Hasura.RQL.Types.Function
+import Hasura.RQL.Types.Metadata (SourceMetadata (..), TableMetadata (..), _cfmDefinition)
 import Hasura.RQL.Types.Source
 import Hasura.RQL.Types.SourceCustomization
 import Hasura.RQL.Types.Table
@@ -134,16 +138,25 @@ logPGSourceCatalogMigrationLockedQueries logger sourceConfig = forever $ do
 resolveDatabaseMetadata ::
   forall pgKind m.
   (Backend ('Postgres pgKind), ToMetadataFetchQuery pgKind, MonadIO m, MonadBaseControl IO m) =>
+  SourceMetadata ('Postgres pgKind) ->
   SourceConfig ('Postgres pgKind) ->
   SourceTypeCustomization ->
   m (Either QErr (ResolvedSource ('Postgres pgKind)))
-resolveDatabaseMetadata sourceConfig sourceCustomization = runExceptT do
+resolveDatabaseMetadata sourceMetadata sourceConfig sourceCustomization = runExceptT do
   (tablesMeta, functionsMeta, pgScalars) <- runTx (_pscExecCtx sourceConfig) Q.ReadOnly $ do
-    tablesMeta <- fetchTableMetadata
-    functionsMeta <- fetchFunctionMetadata
+    tablesMeta <- fetchTableMetadata $ OMap.keys $ _smTables sourceMetadata
+    let allFunctions =
+          OMap.keys (_smFunctions sourceMetadata) -- Tracked functions
+            <> concatMap getComputedFieldFunctionsMetadata (OMap.elems $ _smTables sourceMetadata) -- Computed field functions
+    functionsMeta <- fetchFunctionMetadata allFunctions
     pgScalars <- fetchPgScalars
     pure (tablesMeta, functionsMeta, pgScalars)
   pure $ ResolvedSource sourceConfig sourceCustomization tablesMeta functionsMeta pgScalars
+  where
+    -- A helper function to list all functions underpinning computed fields from a table metadata
+    getComputedFieldFunctionsMetadata :: TableMetadata b -> [FunctionName b]
+    getComputedFieldFunctionsMetadata =
+      map (_cfdFunction . _cfmDefinition) . OMap.elems . _tmComputedFields
 
 -- | Initialise catalog tables for a source, including those required by the event delivery subsystem.
 initCatalogForSource ::
@@ -294,14 +307,15 @@ upMigrationsUntil43 =
 fetchTableMetadata ::
   forall pgKind m.
   (Backend ('Postgres pgKind), ToMetadataFetchQuery pgKind, MonadTx m) =>
+  [QualifiedTable] ->
   m (DBTablesMetadata ('Postgres pgKind))
-fetchTableMetadata = do
+fetchTableMetadata tables = do
   results <-
     liftTx $
       Q.withQE
         defaultTxErrorHandler
         (tableMetadata @pgKind)
-        ()
+        [Q.AltJ $ LE.uniques tables]
         True
   pure $
     Map.fromList $
@@ -309,14 +323,14 @@ fetchTableMetadata = do
         \(schema, table, Q.AltJ info) -> (QualifiedObject schema table, info)
 
 -- | Fetch Postgres metadata for all user functions
-fetchFunctionMetadata :: (MonadTx m) => m (DBFunctionsMetadata ('Postgres pgKind))
-fetchFunctionMetadata = do
+fetchFunctionMetadata :: (MonadTx m) => [QualifiedFunction] -> m (DBFunctionsMetadata ('Postgres pgKind))
+fetchFunctionMetadata functions = do
   results <-
     liftTx $
       Q.withQE
         defaultTxErrorHandler
         $(makeRelativeToProject "src-rsr/pg_function_metadata.sql" >>= Q.sqlFromFile)
-        ()
+        [Q.AltJ $ LE.uniques functions]
         True
   pure $
     Map.fromList $

server/src-lib/Hasura/RQL/DDL/Schema/Cache.hs

@@ -385,7 +385,7 @@ buildSchemaCacheRule logger env = proc (metadata, invalidationKeys) -> do
           metadataObj = MetadataObject (MOSource sourceName) $ toJSON sourceName
           logAndResolveDatabaseMetadata :: SourceConfig b -> SourceTypeCustomization -> m (Either QErr (ResolvedSource b))
           logAndResolveDatabaseMetadata scConfig sType = do
-            resSource <- resolveDatabaseMetadata scConfig sType
+            resSource <- resolveDatabaseMetadata sourceMetadata scConfig sType
             for_ resSource $ liftIO . unLogger logger
             pure resSource
 

server/src-lib/Hasura/RQL/DDL/Schema/Diff.hs

@@ -1,6 +1,7 @@
 module Hasura.RQL.DDL.Schema.Diff
   ( TableMeta (..),
     FunctionMeta (..),
+    TablesDiff (..),
     FunctionsDiff (..),
     ComputedFieldMeta (..),
     getTablesDiff,

server/src-lib/Hasura/RQL/Types/Metadata/Backend.hs

@@ -15,6 +15,7 @@ import Hasura.RQL.Types.Common
 import Hasura.RQL.Types.ComputedField
 import Hasura.RQL.Types.EventTrigger
 import Hasura.RQL.Types.Function
+import Hasura.RQL.Types.Metadata
 import Hasura.RQL.Types.Relationships.Local
 import Hasura.RQL.Types.SchemaCache
 import Hasura.RQL.Types.Source
@@ -60,6 +61,7 @@ class
   -- | Function that introspects a database for tables, columns, functions etc.
   resolveDatabaseMetadata ::
     (MonadIO m, MonadBaseControl IO m, MonadResolveSource m) =>
+    SourceMetadata b ->
     SourceConfig b ->
     SourceTypeCustomization ->
     m (Either QErr (ResolvedSource b))

server/src-rsr/citus_table_metadata.sql

@@ -1,6 +1,6 @@
 SELECT
-  schema.nspname AS table_schema,
-  "table".relname AS table_name,
+  "table".table_schema,
+  "table".table_name,
 
   -- This field corresponds to the `DBTableMetadata` Haskell type
   jsonb_build_object(
@@ -29,10 +29,26 @@ SELECT
       END
   )::json AS info
 
+-- tracked tables
+-- $1 parameter provides JSON array of tracked tables
+FROM
+  ( SELECT "tracked"."name" AS "table_name",
+           "tracked"."schema" AS "table_schema"
+      FROM jsonb_to_recordset($1::jsonb) AS "tracked"("schema" text, "name" text)
+  ) "tracked_table"
+
 -- table & schema
-FROM pg_catalog.pg_class "table"
-JOIN pg_catalog.pg_namespace schema
-  ON schema.oid = "table".relnamespace
+LEFT JOIN
+  ( SELECT "table".oid,
+           "table".relkind,
+           "table".relname AS "table_name",
+           "schema".nspname AS "table_schema"
+      FROM pg_catalog.pg_class "table"
+           JOIN pg_catalog.pg_namespace "schema"
+               ON schema.oid = "table".relnamespace
+  ) "table"
+      ON  "table"."table_name" = "tracked_table"."table_name"
+      AND "table"."table_schema" = "tracked_table"."table_schema"
 
 -- description
 LEFT JOIN pg_catalog.pg_description description
@@ -184,18 +200,18 @@ LEFT JOIN LATERAL
                   AND q.ref_table_id = afc.attrelid
          GROUP BY q.table_schema, q.table_name, q.constraint_name
     ) foreign_key
-    WHERE foreign_key.table_schema = schema.nspname
-      AND foreign_key.table_name = "table".relname
+    WHERE foreign_key.table_schema = "table".table_schema
+      AND foreign_key.table_name = "table".table_name
   ) foreign_key_constraints ON true
 
 LEFT JOIN LATERAL
   ( SELECT citus_table_type, distribution_column, table_name
     FROM citus_tables extraMetadata
     -- DO NOT SUBMIT: Should we compare columns of type 'name' by casting to 'text'?
-    WHERE extraMetadata.table_name::text = "table".relname::text ) extraMetadata ON true
+    WHERE extraMetadata.table_name::text = "table".table_name::text ) extraMetadata ON true
 
 -- all these identify table-like things
 WHERE "table".relkind IN ('r', 't', 'v', 'm', 'f', 'p')
   -- and tables not from any system schemas
-  AND schema.nspname NOT LIKE 'pg_%'
-  AND schema.nspname NOT IN ('information_schema', 'hdb_catalog');
+  AND "table".table_schema NOT LIKE 'pg_%'
+  AND "table".table_schema NOT IN ('information_schema', 'hdb_catalog');

server/src-rsr/pg_function_metadata.sql

@@ -26,34 +26,34 @@ FROM (
     FROM (
       -- Necessary metadata from Postgres
       SELECT
-        p.proname::text AS function_name,
-        pn.nspname::text AS function_schema,
+        "function".function_name,
+        "function".function_schema,
         pd.description,
 
         CASE
-          WHEN (p.provariadic = (0) :: oid) THEN false
+          WHEN ("function".provariadic = (0) :: oid) THEN false
           ELSE true
         END AS has_variadic,
 
         CASE
           WHEN (
-            (p.provolatile) :: text = ('i' :: character(1)) :: text
+            ("function".provolatile) :: text = ('i' :: character(1)) :: text
           ) THEN 'IMMUTABLE' :: text
           WHEN (
-            (p.provolatile) :: text = ('s' :: character(1)) :: text
+            ("function".provolatile) :: text = ('s' :: character(1)) :: text
           ) THEN 'STABLE' :: text
           WHEN (
-            (p.provolatile) :: text = ('v' :: character(1)) :: text
+            ("function".provolatile) :: text = ('v' :: character(1)) :: text
           ) THEN 'VOLATILE' :: text
           ELSE NULL :: text
         END AS function_type,
 
-        pg_get_functiondef(p.oid) AS function_definition,
+        pg_get_functiondef("function".function_oid) AS function_definition,
 
         rtn.nspname::text as return_type_schema,
         rt.typname::text as return_type_name,
         rt.typtype::text as return_type_type,
-        p.proretset AS returns_set,
+        "function".proretset AS returns_set,
         ( SELECT
             COALESCE(json_agg(
               json_build_object('schema', q."schema",
@@ -70,16 +70,16 @@ FROM (
                 pat.ordinality
               FROM
                 unnest(
-                  COALESCE(p.proallargtypes, (p.proargtypes) :: oid [])
+                  COALESCE("function".proallargtypes, ("function".proargtypes) :: oid [])
                 ) WITH ORDINALITY pat(oid, ordinality)
                 LEFT JOIN pg_type pt ON ((pt.oid = pat.oid))
                 LEFT JOIN pg_namespace pns ON (pt.typnamespace = pns.oid)
               ORDER BY pat.ordinality ASC
             ) q
          ) AS input_arg_types,
-        to_json(COALESCE(p.proargnames, ARRAY [] :: text [])) AS input_arg_names,
-        p.pronargdefaults AS default_args,
-        p.oid::integer AS function_oid,
+        to_json(COALESCE("function".proargnames, ARRAY [] :: text [])) AS input_arg_names,
+        "function".pronargdefaults AS default_args,
+        "function".function_oid::integer AS function_oid,
         (exists(
           SELECT
             1
@@ -100,15 +100,23 @@ FROM (
           )
         ) AS returns_table
       FROM
-        pg_proc p
-        JOIN pg_namespace pn ON (pn.oid = p.pronamespace)
-        JOIN pg_type rt ON (rt.oid = p.prorettype)
+        jsonb_to_recordset($1::jsonb) AS tracked("schema" text, "name" text)
+        JOIN
+        ( SELECT   p.oid AS function_oid,
+                   p.*,
+                   p.proname::text AS function_name,
+                   pn.nspname::text AS function_schema
+              FROM pg_proc p
+              JOIN pg_namespace pn ON (pn.oid = p.pronamespace)
+        ) "function" ON "function".function_name = tracked.name
+                     AND "function".function_schema = tracked.schema
+        JOIN pg_type rt ON (rt.oid = "function".prorettype)
         JOIN pg_namespace rtn ON (rtn.oid = rt.typnamespace)
-        LEFT JOIN pg_description pd ON p.oid = pd.objoid
+        LEFT JOIN pg_description pd ON "function".function_oid = pd.objoid
       WHERE
         -- Do not fetch some default functions in public schema
-        p.proname :: text NOT LIKE 'pgp_%'
-        AND p.proname :: text NOT IN
+        "function".function_name NOT LIKE 'pgp_%'
+        AND "function".function_name NOT IN
                           ( 'armor'
                           , 'crypt'
                           , 'dearmor'
@@ -122,15 +130,15 @@ FROM (
                           , 'gen_salt'
                           , 'hmac'
                           )
-        AND pn.nspname :: text NOT LIKE 'pg_%'
-        AND pn.nspname :: text NOT IN ('information_schema', 'hdb_catalog')
+        AND "function".function_schema NOT LIKE 'pg_%'
+        AND "function".function_schema NOT IN ('information_schema', 'hdb_catalog')
         AND (NOT EXISTS (
                 SELECT
                   1
                 FROM
                   pg_aggregate
                 WHERE
-                  ((pg_aggregate.aggfnoid) :: oid = p.oid)
+                  ((pg_aggregate.aggfnoid) :: oid = "function".function_oid)
               )
           )
     ) AS "pg_function"

server/src-rsr/pg_table_metadata.sql

@@ -1,6 +1,6 @@
 SELECT
-  schema.nspname AS table_schema,
-  "table".relname AS table_name,
+  "table".table_schema,
+  "table".table_name,
 
   -- This field corresponds to the `DBTableMetadata` Haskell type
   jsonb_build_object(
@@ -21,10 +21,26 @@ SELECT
     'extra_table_metadata', '[]'::json
   )::json AS info
 
+-- tracked tables
+-- $1 parameter provides JSON array of tracked tables
+FROM
+  ( SELECT "tracked"."name" AS "table_name",
+           "tracked"."schema" AS "table_schema"
+      FROM jsonb_to_recordset($1::jsonb) AS "tracked"("schema" text, "name" text)
+  ) "tracked_table"
+
 -- table & schema
-FROM pg_catalog.pg_class "table"
-JOIN pg_catalog.pg_namespace schema
-  ON schema.oid = "table".relnamespace
+LEFT JOIN
+  ( SELECT "table".oid,
+           "table".relkind,
+           "table".relname AS "table_name",
+           "schema".nspname AS "table_schema"
+      FROM pg_catalog.pg_class "table"
+      JOIN pg_catalog.pg_namespace "schema"
+          ON schema.oid = "table".relnamespace
+  ) "table"
+  ON  "table"."table_name" = "tracked_table"."table_name"
+  AND "table"."table_schema" = "tracked_table"."table_schema"
 
 -- description
 LEFT JOIN pg_catalog.pg_description description
@@ -181,11 +197,11 @@ LEFT JOIN
     ) foreign_key
     GROUP BY foreign_key.table_schema, foreign_key.table_name
   ) foreign_key_constraints
-    ON "table".relname = foreign_key_constraints.table_name
-       AND schema.nspname = foreign_key_constraints.table_schema
+    ON "table".table_name = foreign_key_constraints.table_name
+       AND "table".table_schema = foreign_key_constraints.table_schema
 
 -- all these identify table-like things
 WHERE "table".relkind IN ('r', 't', 'v', 'm', 'f', 'p')
   -- and tables not from any system schemas
-  AND schema.nspname NOT LIKE 'pg_%'
-  AND schema.nspname NOT IN ('information_schema', 'hdb_catalog', 'hdb_lib', '_timescaledb_internal');
+  AND "table".table_schema NOT LIKE 'pg_%'
+  AND "table".table_schema NOT IN ('information_schema', 'hdb_catalog', 'hdb_lib', '_timescaledb_internal');