diff --git a/CHANGELOG.md b/CHANGELOG.md index 303b57548be..a2dd294d852 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Bug fixes and improvements +- server: improve performance of fetching postgres catalog metadata for tables and functions - server: Queries present in query collections, such as allow-list, and rest-endpoints are now validated (against the schema) - server: Redesigns internal implementation of webhook transforms. - server: improve SQL generation for BigQuery backend queries involving `Orderby`. diff --git a/server/src-lib/Data/List/Extended.hs b/server/src-lib/Data/List/Extended.hs index 595cff020af..e7d6ab08498 100644 --- a/server/src-lib/Data/List/Extended.hs +++ b/server/src-lib/Data/List/Extended.hs @@ -18,14 +18,15 @@ import Data.HashSet qualified as Set import Data.Hashable (Hashable) import Data.List qualified as L import Data.List.NonEmpty qualified as NE +import Data.Set qualified as S import Prelude duplicates :: (Eq a, Hashable a) => [a] -> Set.HashSet a duplicates = Set.fromList . Map.keys . Map.filter (> 1) . Map.fromListWith (+) . map (,1 :: Int) -uniques :: Eq a => [a] -> [a] -uniques = map NE.head . NE.group +uniques :: (Ord a) => [a] -> [a] +uniques = S.toList . S.fromList getDifference :: (Eq a, Hashable a) => [a] -> [a] -> Set.HashSet a getDifference = Set.difference `on` Set.fromList diff --git a/server/src-lib/Hasura/Backends/BigQuery/Instances/Metadata.hs b/server/src-lib/Hasura/Backends/BigQuery/Instances/Metadata.hs index 504ae992aa3..df1c15eb11a 100644 --- a/server/src-lib/Hasura/Backends/BigQuery/Instances/Metadata.hs +++ b/server/src-lib/Hasura/Backends/BigQuery/Instances/Metadata.hs @@ -10,7 +10,7 @@ instance BackendMetadata 'BigQuery where buildComputedFieldInfo = BigQuery.buildComputedFieldInfo fetchAndValidateEnumValues = BigQuery.fetchAndValidateEnumValues resolveSourceConfig = BigQuery.resolveSourceConfig - resolveDatabaseMetadata = BigQuery.resolveSource + resolveDatabaseMetadata _ = BigQuery.resolveSource parseBoolExpOperations = BigQuery.parseBoolExpOperations buildFunctionInfo = BigQuery.buildFunctionInfo updateColumnInEventTrigger = BigQuery.updateColumnInEventTrigger diff --git a/server/src-lib/Hasura/Backends/MSSQL/Instances/Metadata.hs b/server/src-lib/Hasura/Backends/MSSQL/Instances/Metadata.hs index 442147a3661..cc4a466a613 100644 --- a/server/src-lib/Hasura/Backends/MSSQL/Instances/Metadata.hs +++ b/server/src-lib/Hasura/Backends/MSSQL/Instances/Metadata.hs @@ -14,7 +14,7 @@ instance BackendMetadata 'MSSQL where buildComputedFieldInfo = MSSQL.buildComputedFieldInfo fetchAndValidateEnumValues = MSSQL.fetchAndValidateEnumValues resolveSourceConfig = MSSQL.resolveSourceConfig - resolveDatabaseMetadata = MSSQL.resolveDatabaseMetadata + resolveDatabaseMetadata _ = MSSQL.resolveDatabaseMetadata parseBoolExpOperations = MSSQL.parseBoolExpOperations buildFunctionInfo = MSSQL.buildFunctionInfo updateColumnInEventTrigger = MSSQL.updateColumnInEventTrigger diff --git a/server/src-lib/Hasura/Backends/MySQL/Instances/Metadata.hs b/server/src-lib/Hasura/Backends/MySQL/Instances/Metadata.hs index f1ffcce1baa..cc7efe7b8f1 100644 --- a/server/src-lib/Hasura/Backends/MySQL/Instances/Metadata.hs +++ b/server/src-lib/Hasura/Backends/MySQL/Instances/Metadata.hs @@ -11,7 +11,7 @@ instance BackendMetadata 'MySQL where buildComputedFieldInfo = error "buildComputedFieldInfo: MySQL backend does not support this operation yet." fetchAndValidateEnumValues = error "fetchAndValidateEnumValues: MySQL backend does not support this operation yet." resolveSourceConfig = MySQL.resolveSourceConfig - resolveDatabaseMetadata = MySQL.resolveDatabaseMetadata + resolveDatabaseMetadata _ = MySQL.resolveDatabaseMetadata parseBoolExpOperations = error "parseBoolExpOperations: MySQL backend does not support this operation yet." buildFunctionInfo = error "buildFunctionInfo: MySQL backend does not support this operation yet." updateColumnInEventTrigger = error "updateColumnInEventTrigger: MySQL backend does not support this operation yet." diff --git a/server/src-lib/Hasura/Backends/Postgres/DDL/Function.hs b/server/src-lib/Hasura/Backends/Postgres/DDL/Function.hs index 3e2cf2b0e84..09e1360d75c 100644 --- a/server/src-lib/Hasura/Backends/Postgres/DDL/Function.hs +++ b/server/src-lib/Hasura/Backends/Postgres/DDL/Function.hs @@ -15,7 +15,7 @@ import Control.Monad.Validate qualified as MV import Data.Sequence qualified as Seq import Data.Text qualified as T import Data.Text.Extended -import Hasura.Backends.Postgres.SQL.Types +import Hasura.Backends.Postgres.SQL.Types hiding (FunctionName) import Hasura.Base.Error import Hasura.Prelude import Hasura.RQL.Types.Backend diff --git a/server/src-lib/Hasura/Backends/Postgres/DDL/RunSQL.hs b/server/src-lib/Hasura/Backends/Postgres/DDL/RunSQL.hs index f9ba4eace78..b326b903cb9 100644 --- a/server/src-lib/Hasura/Backends/Postgres/DDL/RunSQL.hs +++ b/server/src-lib/Hasura/Backends/Postgres/DDL/RunSQL.hs @@ -27,7 +27,7 @@ import Hasura.Backends.Postgres.DDL.Source fetchFunctionMetadata, fetchTableMetadata, ) -import Hasura.Backends.Postgres.SQL.Types +import Hasura.Backends.Postgres.SQL.Types hiding (FunctionName, TableName) import Hasura.Base.Error import Hasura.EncJSON import Hasura.Prelude @@ -120,35 +120,35 @@ of queries may not modify the schema at all. As a (fairly stupid) heuristic, we check if the query contains any keywords for DDL operations, and if not, we skip the metadata check as well. -} -fetchMeta :: +-- | Fetch metadata of tracked tables/functions and build @'TableMeta'/@'FunctionMeta' +-- to calculate diff later in @'withMetadataCheck'. +fetchTablesFunctionsMetadata :: (ToMetadataFetchQuery pgKind, BackendMetadata ('Postgres pgKind), MonadTx m) => TableCache ('Postgres pgKind) -> - FunctionCache ('Postgres pgKind) -> + [TableName ('Postgres pgKind)] -> + [FunctionName ('Postgres pgKind)] -> m ([TableMeta ('Postgres pgKind)], [FunctionMeta ('Postgres pgKind)]) -fetchMeta tables functions = do - tableMetaInfos <- fetchTableMetadata - functionMetaInfos <- fetchFunctionMetadata - - let getFunctionMetas function = - let mkFunctionMeta rawInfo = - FunctionMeta (rfiOid rawInfo) function (rfiFunctionType rawInfo) - in maybe [] (map mkFunctionMeta) $ M.lookup function functionMetaInfos - - mkComputedFieldMeta computedField = - let function = _cffName $ _cfiFunction computedField - in map (ComputedFieldMeta (_cfiName computedField)) $ getFunctionMetas function - - tableMetas = flip map (M.toList tableMetaInfos) $ \(table, tableMetaInfo) -> +fetchTablesFunctionsMetadata tableCache tables functions = do + tableMetaInfos <- fetchTableMetadata tables + functionMetaInfos <- fetchFunctionMetadata functions + pure (buildTableMeta tableMetaInfos functionMetaInfos, buildFunctionMeta functionMetaInfos) + where + buildTableMeta tableMetaInfos functionMetaInfos = + flip map (M.toList tableMetaInfos) $ \(table, tableMetaInfo) -> TableMeta table tableMetaInfo $ - fromMaybe [] $ - M.lookup table tables <&> \tableInfo -> - let tableCoreInfo = _tiCoreInfo tableInfo - computedFields = getComputedFieldInfos $ _tciFieldInfoMap tableCoreInfo - in concatMap mkComputedFieldMeta computedFields + foldMap @Maybe (concatMap (mkComputedFieldMeta functionMetaInfos) . getComputedFields) (M.lookup table tableCache) - functionMetas = concatMap getFunctionMetas $ M.keys functions + buildFunctionMeta functionMetaInfos = + concatMap (getFunctionMetas functionMetaInfos) functions - pure (tableMetas, functionMetas) + mkComputedFieldMeta functionMetaInfos computedField = + let function = _cffName $ _cfiFunction computedField + in map (ComputedFieldMeta (_cfiName computedField)) $ getFunctionMetas functionMetaInfos function + + getFunctionMetas functionMetaInfos function = + let mkFunctionMeta rawInfo = + FunctionMeta (rfiOid rawInfo) function (rfiFunctionType rawInfo) + in foldMap @Maybe (map mkFunctionMeta) $ M.lookup function functionMetaInfos -- | Used as an escape hatch to run raw SQL against a database. runRunSQL :: @@ -188,7 +188,7 @@ runRunSQL q@RunSQL {..} = do rawSqlErrHandler txe = (err400 PostgresError "query execution failed") {qeInternal = Just $ ExtraInternal $ toJSON txe} --- | @'withMetadataCheck' cascade action@ runs @action@ and checks if the schema changed as a +-- | @'withMetadataCheck' source cascade txAccess runSQLQuery@ executes @runSQLQuery@ and checks if the schema changed as a -- result. If it did, it checks to ensure the changes do not violate any integrity constraints, and -- if not, incorporates them into the schema cache. -- TODO(antoine): shouldn't this be generalized? @@ -208,86 +208,243 @@ withMetadataCheck :: Q.TxAccess -> Q.TxET QErr m a -> m a -withMetadataCheck source cascade txAccess action = do - SourceInfo _ preActionTables preActionFunctions sourceConfig _ _ <- askSourceInfo @('Postgres pgKind) source +withMetadataCheck source cascade txAccess runSQLQuery = do + SourceInfo _ tableCache functionCache sourceConfig _ _ <- askSourceInfo @('Postgres pgKind) source - (actionResult, metadataUpdater) <- - liftEitherM $ - runExceptT $ - runTx (_pscExecCtx sourceConfig) txAccess $ do - -- Drop event triggers so no interference is caused to the sql query - forM_ (M.elems preActionTables) $ \tableInfo -> do - let eventTriggers = _tiEventTriggerInfoMap tableInfo - forM_ (M.keys eventTriggers) (liftTx . dropTriggerQ) + let dropTriggersAndRunSQL = do + -- We need to drop existing event triggers so that no interference is caused to the sql query execution + dropExistingEventTriggers tableCache + runSQLQuery - -- Get the metadata before the sql query, everything, need to filter this - (preActionTableMeta, preActionFunctionMeta) <- fetchMeta preActionTables preActionFunctions - - -- Run the action - actionResult <- action - - -- Get the metadata after the sql query - (postActionTableMeta, postActionFunctionMeta) <- fetchMeta preActionTables preActionFunctions - - let preActionTableMeta' = filter (flip M.member preActionTables . tmTable) preActionTableMeta - tablesDiff = getTablesDiff preActionTableMeta' postActionTableMeta - FunctionsDiff droppedFuncs alteredFuncs = getFunctionsDiff preActionFunctionMeta postActionFunctionMeta - overloadedFuncs = getOverloadedFunctions (M.keys preActionFunctions) postActionFunctionMeta - - -- Do not allow overloading functions - unless (null overloadedFuncs) $ - throw400 NotSupported $ - "the following tracked function(s) cannot be overloaded: " - <> commaSeparated overloadedFuncs - - -- Report back with an error if cascade is not set - indirectDeps <- getIndirectDependencies source tablesDiff - when (indirectDeps /= [] && not cascade) $ reportDependentObjectsExist indirectDeps - - metadataUpdater <- execWriterT $ do - -- Purge all the indirect dependents from state - for_ indirectDeps \case - SOSourceObj sourceName objectID -> do - AB.dispatchAnyBackend @BackendMetadata objectID $ purgeDependentObject sourceName >=> tell - _ -> - pure () - - -- Purge all dropped functions - let purgedFuncs = flip mapMaybe indirectDeps \case - SOSourceObj _ objectID - | Just (SOIFunction qf) <- AB.unpackAnyBackend @('Postgres pgKind) objectID -> - Just qf - _ -> Nothing - for_ (droppedFuncs \\ purgedFuncs) $ - tell . dropFunctionInMetadata @('Postgres pgKind) source - - -- Process altered functions - forM_ alteredFuncs $ \(qf, newTy) -> do - when (newTy == FTVOLATILE) $ - throw400 NotSupported $ - "type of function " <> qf <<> " is altered to \"VOLATILE\" which is not supported now" - - -- update the metadata with the changes - processTablesDiff source preActionTables tablesDiff - - pure (actionResult, metadataUpdater) + -- Run SQL query and metadata checker in a transaction + (queryResult, metadataUpdater) <- runTxWithMetadataCheck source sourceConfig txAccess tableCache functionCache cascade dropTriggersAndRunSQL -- Build schema cache with updated metadata withNewInconsistentObjsCheck $ buildSchemaCacheWithInvalidations mempty {ciSources = HS.singleton source} metadataUpdater - postActionSchemaCache <- askSchemaCache + postRunSQLSchemaCache <- askSchemaCache - -- Recreate event triggers in hdb_catalog - let postActionTables = fromMaybe mempty $ unsafeTableCache @('Postgres pgKind) source $ scSources postActionSchemaCache - serverConfigCtx <- askServerConfigCtx + -- Recreate event triggers in hdb_catalog. Event triggers are dropped before executing @'runSQLQuery'. + recreateEventTriggers sourceConfig postRunSQLSchemaCache + + pure queryResult + where + dropExistingEventTriggers :: TableCache ('Postgres pgKind) -> Q.TxET QErr m () + dropExistingEventTriggers tableCache = + forM_ (M.elems tableCache) $ \tableInfo -> do + let eventTriggers = _tiEventTriggerInfoMap tableInfo + forM_ (M.keys eventTriggers) (liftTx . dropTriggerQ) + + recreateEventTriggers :: PGSourceConfig -> SchemaCache -> m () + recreateEventTriggers sourceConfig schemaCache = do + let tables = fromMaybe mempty $ unsafeTableCache @('Postgres pgKind) source $ scSources schemaCache + serverConfigCtx <- askServerConfigCtx + liftEitherM $ + runPgSourceWriteTx sourceConfig $ + forM_ (M.elems tables) $ \(TableInfo coreInfo _ eventTriggers _) -> do + let table = _tciName coreInfo + columns = getCols $ _tciFieldInfoMap coreInfo + forM_ (M.toList eventTriggers) $ \(triggerName, eti) -> do + let opsDefinition = etiOpsDef eti + flip runReaderT serverConfigCtx $ mkAllTriggersQ triggerName table columns opsDefinition + +-- | @'runTxWithMetadataCheck source sourceConfig txAccess tableCache functionCache cascadeDependencies tx' checks for +-- changes in GraphQL Engine metadata when a @'tx' is executed on the database alters Postgres +-- schema of tables and functions. If any indirect dependencies (Eg. remote table dependence of a relationship) are +-- found and @'cascadeDependencies' is False, then an exception is raised. +runTxWithMetadataCheck :: + forall m a (pgKind :: PostgresKind). + ( BackendMetadata ('Postgres pgKind), + ToMetadataFetchQuery pgKind, + CacheRWM m, + MonadIO m, + MonadBaseControl IO m, + MonadError QErr m + ) => + SourceName -> + SourceConfig ('Postgres pgKind) -> + Q.TxAccess -> + TableCache ('Postgres pgKind) -> + FunctionCache ('Postgres pgKind) -> + Bool -> + Q.TxET QErr m a -> + m (a, MetadataModifier) +runTxWithMetadataCheck source sourceConfig txAccess tableCache functionCache cascadeDependencies tx = liftEitherM $ - runPgSourceWriteTx sourceConfig $ - forM_ (M.elems postActionTables) $ \(TableInfo coreInfo _ eventTriggers _) -> do - let table = _tciName coreInfo - columns = getCols $ _tciFieldInfoMap coreInfo - forM_ (M.toList eventTriggers) $ \(triggerName, eti) -> do - let opsDefinition = etiOpsDef eti - flip runReaderT serverConfigCtx $ mkAllTriggersQ triggerName table columns opsDefinition + runExceptT $ + runTx (_pscExecCtx sourceConfig) txAccess $ do + -- Running in a transaction helps to rollback the @'tx' execution in case of any exceptions - pure actionResult + -- Before running the @'tx', fetch metadata of existing tables and functions from Postgres. + let tableNames = M.keys tableCache + computedFieldFunctions = concatMap getComputedFieldFunctions (M.elems tableCache) + functionNames = M.keys functionCache <> computedFieldFunctions + (preTxTablesMeta, preTxFunctionsMeta) <- fetchTablesFunctionsMetadata tableCache tableNames functionNames + + -- Since the @'tx' may alter table/function names we use the OIDs of underlying tables + -- (sourced from 'pg_class' for tables and 'pg_proc' for functions), which remain unchanged in the + -- case if a table/function is renamed. + let tableOids = map (_ptmiOid . tmInfo) preTxTablesMeta + functionOids = map fmOid preTxFunctionsMeta + + -- Run the transaction + txResult <- tx + + (postTxTablesMeta, postTxFunctionMeta) <- + uncurry (fetchTablesFunctionsMetadata tableCache) + -- Fetch names of tables and functions using OIDs which also contains renamed items + =<< fetchTablesFunctionsFromOids tableOids functionOids + + -- Calculate the tables diff (dropped & altered tables) + let tablesDiff = getTablesDiff preTxTablesMeta postTxTablesMeta + -- Calculate the functions diff. For calculating diff for functions, only consider + -- query/mutation functions and exclude functions underpinning computed fields. + -- Computed field functions are being processed under each table diff. + -- See @'getTablesDiff' and @'processTablesDiff' + excludeComputedFieldFunctions = filter ((`M.member` functionCache) . fmFunction) + functionsDiff = + getFunctionsDiff + (excludeComputedFieldFunctions preTxFunctionsMeta) + (excludeComputedFieldFunctions postTxFunctionMeta) + + dontAllowFunctionOverloading $ + getOverloadedFunctions + (M.keys functionCache) + (excludeComputedFieldFunctions postTxFunctionMeta) + + -- Update metadata with schema change caused by @'tx' + metadataUpdater <- execWriterT do + -- Collect indirect dependencies of altered tables + tableIndirectDeps <- getIndirectDependencies source tablesDiff + + -- If table indirect dependencies exist and cascading is not enabled then report an exception + when (tableIndirectDeps /= [] && not cascadeDependencies) $ reportDependentObjectsExist tableIndirectDeps + + -- Purge all the table dependents + purgeDependencies tableIndirectDeps + + -- Collect function names from purged table dependencies + let purgedFunctions = collectFunctionsInDeps tableIndirectDeps + FunctionsDiff droppedFunctions alteredFunctions = functionsDiff + + -- Drop functions in metadata. Exclude functions that were already dropped as part of table indirect dependencies + purgeFunctionsFromMetadata $ droppedFunctions \\ purgedFunctions + + -- If any function type is altered to VOLATILE then raise an exception + dontAllowFunctionAlteredVolatile alteredFunctions + + -- Propagate table changes to metadata + processTablesDiff source tableCache tablesDiff + + pure (txResult, metadataUpdater) + where + dontAllowFunctionOverloading :: + MonadError QErr n => + [FunctionName ('Postgres pgKind)] -> + n () + dontAllowFunctionOverloading overloadedFunctions = + unless (null overloadedFunctions) $ + throw400 NotSupported $ + "the following tracked function(s) cannot be overloaded: " + <> commaSeparated overloadedFunctions + + dontAllowFunctionAlteredVolatile :: + MonadError QErr n => + [(FunctionName ('Postgres pgKind), FunctionVolatility)] -> + n () + dontAllowFunctionAlteredVolatile alteredFunctions = + forM_ alteredFunctions $ \(qf, newTy) -> do + when (newTy == FTVOLATILE) $ + throw400 NotSupported $ + "type of function " <> qf <<> " is altered to \"VOLATILE\" which is not supported now" + + purgeDependencies :: + MonadError QErr n => + [SchemaObjId] -> + WriterT MetadataModifier n () + purgeDependencies deps = + for_ deps \case + SOSourceObj sourceName objectID -> do + AB.dispatchAnyBackend @BackendMetadata objectID $ purgeDependentObject sourceName >=> tell + _ -> + -- Ignore non-source dependencies + pure () + + purgeFunctionsFromMetadata :: + Monad n => + [FunctionName ('Postgres pgKind)] -> + WriterT MetadataModifier n () + purgeFunctionsFromMetadata functions = + for_ functions $ tell . dropFunctionInMetadata @('Postgres pgKind) source + + collectFunctionsInDeps :: [SchemaObjId] -> [FunctionName ('Postgres pgKind)] + collectFunctionsInDeps deps = + flip mapMaybe deps \case + SOSourceObj _ objectID + | Just (SOIFunction qf) <- AB.unpackAnyBackend @('Postgres pgKind) objectID -> + Just qf + _ -> Nothing + +-- | Fetch list of tables and functions with provided oids +fetchTablesFunctionsFromOids :: + (MonadIO m) => + [OID] -> + [OID] -> + Q.TxET QErr m ([TableName ('Postgres pgKind)], [FunctionName ('Postgres pgKind)]) +fetchTablesFunctionsFromOids tableOids functionOids = + ((Q.getAltJ *** Q.getAltJ) . Q.getRow) + <$> Q.withQE + defaultTxErrorHandler + [Q.sql| + SELECT + COALESCE( + ( SELECT + json_agg( + row_to_json( + ( + SELECT e + FROM ( SELECT "table".relname AS "name", + "schema".nspname AS "schema" + ) AS e + ) + ) + ) AS "item" + FROM jsonb_to_recordset($1::jsonb) AS oid_table("oid" int) + JOIN pg_catalog.pg_class "table" ON ("table".oid = "oid_table".oid) + JOIN pg_catalog.pg_namespace "schema" ON ("schema".oid = "table".relnamespace) + ), + '[]' + ) AS "tables", + + COALESCE( + ( SELECT + json_agg( + row_to_json( + ( + SELECT e + FROM ( SELECT "function".proname AS "name", + "schema".nspname AS "schema" + ) AS e + ) + ) + ) AS "item" + FROM jsonb_to_recordset($2::jsonb) AS oid_table("oid" int) + JOIN pg_catalog.pg_proc "function" ON ("function".oid = "oid_table".oid) + JOIN pg_catalog.pg_namespace "schema" ON ("schema".oid = "function".pronamespace) + ), + '[]' + ) AS "functions" + |] + (Q.AltJ $ map mkOidObject tableOids, Q.AltJ $ map mkOidObject functionOids) + True + where + mkOidObject oid = object ["oid" .= oid] + +------ helpers ------------ + +getComputedFields :: TableInfo ('Postgres pgKind) -> [ComputedFieldInfo ('Postgres pgKind)] +getComputedFields = getComputedFieldInfos . _tciFieldInfoMap . _tiCoreInfo + +getComputedFieldFunctions :: TableInfo ('Postgres pgKind) -> [FunctionName ('Postgres pgKind)] +getComputedFieldFunctions = map (_cffName . _cfiFunction) . getComputedFields diff --git a/server/src-lib/Hasura/Backends/Postgres/DDL/Source.hs b/server/src-lib/Hasura/Backends/Postgres/DDL/Source.hs index 6e2871696a4..9dd8b58035c 100644 --- a/server/src-lib/Hasura/Backends/Postgres/DDL/Source.hs +++ b/server/src-lib/Hasura/Backends/Postgres/DDL/Source.hs @@ -27,19 +27,23 @@ import Data.Aeson.TH import Data.Environment qualified as Env import Data.FileEmbed (makeRelativeToProject) import Data.HashMap.Strict qualified as Map +import Data.HashMap.Strict.InsOrd qualified as OMap +import Data.List.Extended qualified as LE import Data.List.NonEmpty qualified as NE import Data.Time.Clock (UTCTime) import Database.PG.Query qualified as Q import Hasura.Backends.Postgres.Connection import Hasura.Backends.Postgres.DDL.Source.Version -import Hasura.Backends.Postgres.SQL.Types +import Hasura.Backends.Postgres.SQL.Types hiding (FunctionName) import Hasura.Base.Error import Hasura.Logging import Hasura.Prelude import Hasura.RQL.Types.Backend import Hasura.RQL.Types.Common +import Hasura.RQL.Types.ComputedField import Hasura.RQL.Types.EventTrigger (RecreateEventTriggers (..)) import Hasura.RQL.Types.Function +import Hasura.RQL.Types.Metadata (SourceMetadata (..), TableMetadata (..), _cfmDefinition) import Hasura.RQL.Types.Source import Hasura.RQL.Types.SourceCustomization import Hasura.RQL.Types.Table @@ -134,16 +138,25 @@ logPGSourceCatalogMigrationLockedQueries logger sourceConfig = forever $ do resolveDatabaseMetadata :: forall pgKind m. (Backend ('Postgres pgKind), ToMetadataFetchQuery pgKind, MonadIO m, MonadBaseControl IO m) => + SourceMetadata ('Postgres pgKind) -> SourceConfig ('Postgres pgKind) -> SourceTypeCustomization -> m (Either QErr (ResolvedSource ('Postgres pgKind))) -resolveDatabaseMetadata sourceConfig sourceCustomization = runExceptT do +resolveDatabaseMetadata sourceMetadata sourceConfig sourceCustomization = runExceptT do (tablesMeta, functionsMeta, pgScalars) <- runTx (_pscExecCtx sourceConfig) Q.ReadOnly $ do - tablesMeta <- fetchTableMetadata - functionsMeta <- fetchFunctionMetadata + tablesMeta <- fetchTableMetadata $ OMap.keys $ _smTables sourceMetadata + let allFunctions = + OMap.keys (_smFunctions sourceMetadata) -- Tracked functions + <> concatMap getComputedFieldFunctionsMetadata (OMap.elems $ _smTables sourceMetadata) -- Computed field functions + functionsMeta <- fetchFunctionMetadata allFunctions pgScalars <- fetchPgScalars pure (tablesMeta, functionsMeta, pgScalars) pure $ ResolvedSource sourceConfig sourceCustomization tablesMeta functionsMeta pgScalars + where + -- A helper function to list all functions underpinning computed fields from a table metadata + getComputedFieldFunctionsMetadata :: TableMetadata b -> [FunctionName b] + getComputedFieldFunctionsMetadata = + map (_cfdFunction . _cfmDefinition) . OMap.elems . _tmComputedFields -- | Initialise catalog tables for a source, including those required by the event delivery subsystem. initCatalogForSource :: @@ -294,14 +307,15 @@ upMigrationsUntil43 = fetchTableMetadata :: forall pgKind m. (Backend ('Postgres pgKind), ToMetadataFetchQuery pgKind, MonadTx m) => + [QualifiedTable] -> m (DBTablesMetadata ('Postgres pgKind)) -fetchTableMetadata = do +fetchTableMetadata tables = do results <- liftTx $ Q.withQE defaultTxErrorHandler (tableMetadata @pgKind) - () + [Q.AltJ $ LE.uniques tables] True pure $ Map.fromList $ @@ -309,14 +323,14 @@ fetchTableMetadata = do \(schema, table, Q.AltJ info) -> (QualifiedObject schema table, info) -- | Fetch Postgres metadata for all user functions -fetchFunctionMetadata :: (MonadTx m) => m (DBFunctionsMetadata ('Postgres pgKind)) -fetchFunctionMetadata = do +fetchFunctionMetadata :: (MonadTx m) => [QualifiedFunction] -> m (DBFunctionsMetadata ('Postgres pgKind)) +fetchFunctionMetadata functions = do results <- liftTx $ Q.withQE defaultTxErrorHandler $(makeRelativeToProject "src-rsr/pg_function_metadata.sql" >>= Q.sqlFromFile) - () + [Q.AltJ $ LE.uniques functions] True pure $ Map.fromList $ diff --git a/server/src-lib/Hasura/RQL/DDL/Schema/Cache.hs b/server/src-lib/Hasura/RQL/DDL/Schema/Cache.hs index 3de3b9c383a..79393d4e7e2 100644 --- a/server/src-lib/Hasura/RQL/DDL/Schema/Cache.hs +++ b/server/src-lib/Hasura/RQL/DDL/Schema/Cache.hs @@ -385,7 +385,7 @@ buildSchemaCacheRule logger env = proc (metadata, invalidationKeys) -> do metadataObj = MetadataObject (MOSource sourceName) $ toJSON sourceName logAndResolveDatabaseMetadata :: SourceConfig b -> SourceTypeCustomization -> m (Either QErr (ResolvedSource b)) logAndResolveDatabaseMetadata scConfig sType = do - resSource <- resolveDatabaseMetadata scConfig sType + resSource <- resolveDatabaseMetadata sourceMetadata scConfig sType for_ resSource $ liftIO . unLogger logger pure resSource diff --git a/server/src-lib/Hasura/RQL/DDL/Schema/Diff.hs b/server/src-lib/Hasura/RQL/DDL/Schema/Diff.hs index 4171769d569..bad0a774d3e 100644 --- a/server/src-lib/Hasura/RQL/DDL/Schema/Diff.hs +++ b/server/src-lib/Hasura/RQL/DDL/Schema/Diff.hs @@ -1,6 +1,7 @@ module Hasura.RQL.DDL.Schema.Diff ( TableMeta (..), FunctionMeta (..), + TablesDiff (..), FunctionsDiff (..), ComputedFieldMeta (..), getTablesDiff, diff --git a/server/src-lib/Hasura/RQL/Types/Metadata/Backend.hs b/server/src-lib/Hasura/RQL/Types/Metadata/Backend.hs index 7547dcfe0ee..27c0bfe8a47 100644 --- a/server/src-lib/Hasura/RQL/Types/Metadata/Backend.hs +++ b/server/src-lib/Hasura/RQL/Types/Metadata/Backend.hs @@ -15,6 +15,7 @@ import Hasura.RQL.Types.Common import Hasura.RQL.Types.ComputedField import Hasura.RQL.Types.EventTrigger import Hasura.RQL.Types.Function +import Hasura.RQL.Types.Metadata import Hasura.RQL.Types.Relationships.Local import Hasura.RQL.Types.SchemaCache import Hasura.RQL.Types.Source @@ -60,6 +61,7 @@ class -- | Function that introspects a database for tables, columns, functions etc. resolveDatabaseMetadata :: (MonadIO m, MonadBaseControl IO m, MonadResolveSource m) => + SourceMetadata b -> SourceConfig b -> SourceTypeCustomization -> m (Either QErr (ResolvedSource b)) diff --git a/server/src-rsr/citus_table_metadata.sql b/server/src-rsr/citus_table_metadata.sql index 4d79f7de55c..04ba2124c50 100644 --- a/server/src-rsr/citus_table_metadata.sql +++ b/server/src-rsr/citus_table_metadata.sql @@ -1,6 +1,6 @@ SELECT - schema.nspname AS table_schema, - "table".relname AS table_name, + "table".table_schema, + "table".table_name, -- This field corresponds to the `DBTableMetadata` Haskell type jsonb_build_object( @@ -29,10 +29,26 @@ SELECT END )::json AS info +-- tracked tables +-- $1 parameter provides JSON array of tracked tables +FROM + ( SELECT "tracked"."name" AS "table_name", + "tracked"."schema" AS "table_schema" + FROM jsonb_to_recordset($1::jsonb) AS "tracked"("schema" text, "name" text) + ) "tracked_table" + -- table & schema -FROM pg_catalog.pg_class "table" -JOIN pg_catalog.pg_namespace schema - ON schema.oid = "table".relnamespace +LEFT JOIN + ( SELECT "table".oid, + "table".relkind, + "table".relname AS "table_name", + "schema".nspname AS "table_schema" + FROM pg_catalog.pg_class "table" + JOIN pg_catalog.pg_namespace "schema" + ON schema.oid = "table".relnamespace + ) "table" + ON "table"."table_name" = "tracked_table"."table_name" + AND "table"."table_schema" = "tracked_table"."table_schema" -- description LEFT JOIN pg_catalog.pg_description description @@ -184,18 +200,18 @@ LEFT JOIN LATERAL AND q.ref_table_id = afc.attrelid GROUP BY q.table_schema, q.table_name, q.constraint_name ) foreign_key - WHERE foreign_key.table_schema = schema.nspname - AND foreign_key.table_name = "table".relname + WHERE foreign_key.table_schema = "table".table_schema + AND foreign_key.table_name = "table".table_name ) foreign_key_constraints ON true LEFT JOIN LATERAL ( SELECT citus_table_type, distribution_column, table_name FROM citus_tables extraMetadata -- DO NOT SUBMIT: Should we compare columns of type 'name' by casting to 'text'? - WHERE extraMetadata.table_name::text = "table".relname::text ) extraMetadata ON true + WHERE extraMetadata.table_name::text = "table".table_name::text ) extraMetadata ON true -- all these identify table-like things WHERE "table".relkind IN ('r', 't', 'v', 'm', 'f', 'p') -- and tables not from any system schemas - AND schema.nspname NOT LIKE 'pg_%' - AND schema.nspname NOT IN ('information_schema', 'hdb_catalog'); + AND "table".table_schema NOT LIKE 'pg_%' + AND "table".table_schema NOT IN ('information_schema', 'hdb_catalog'); diff --git a/server/src-rsr/pg_function_metadata.sql b/server/src-rsr/pg_function_metadata.sql index a0f35712af8..d86b5c338c0 100644 --- a/server/src-rsr/pg_function_metadata.sql +++ b/server/src-rsr/pg_function_metadata.sql @@ -26,34 +26,34 @@ FROM ( FROM ( -- Necessary metadata from Postgres SELECT - p.proname::text AS function_name, - pn.nspname::text AS function_schema, + "function".function_name, + "function".function_schema, pd.description, CASE - WHEN (p.provariadic = (0) :: oid) THEN false + WHEN ("function".provariadic = (0) :: oid) THEN false ELSE true END AS has_variadic, CASE WHEN ( - (p.provolatile) :: text = ('i' :: character(1)) :: text + ("function".provolatile) :: text = ('i' :: character(1)) :: text ) THEN 'IMMUTABLE' :: text WHEN ( - (p.provolatile) :: text = ('s' :: character(1)) :: text + ("function".provolatile) :: text = ('s' :: character(1)) :: text ) THEN 'STABLE' :: text WHEN ( - (p.provolatile) :: text = ('v' :: character(1)) :: text + ("function".provolatile) :: text = ('v' :: character(1)) :: text ) THEN 'VOLATILE' :: text ELSE NULL :: text END AS function_type, - pg_get_functiondef(p.oid) AS function_definition, + pg_get_functiondef("function".function_oid) AS function_definition, rtn.nspname::text as return_type_schema, rt.typname::text as return_type_name, rt.typtype::text as return_type_type, - p.proretset AS returns_set, + "function".proretset AS returns_set, ( SELECT COALESCE(json_agg( json_build_object('schema', q."schema", @@ -70,16 +70,16 @@ FROM ( pat.ordinality FROM unnest( - COALESCE(p.proallargtypes, (p.proargtypes) :: oid []) + COALESCE("function".proallargtypes, ("function".proargtypes) :: oid []) ) WITH ORDINALITY pat(oid, ordinality) LEFT JOIN pg_type pt ON ((pt.oid = pat.oid)) LEFT JOIN pg_namespace pns ON (pt.typnamespace = pns.oid) ORDER BY pat.ordinality ASC ) q ) AS input_arg_types, - to_json(COALESCE(p.proargnames, ARRAY [] :: text [])) AS input_arg_names, - p.pronargdefaults AS default_args, - p.oid::integer AS function_oid, + to_json(COALESCE("function".proargnames, ARRAY [] :: text [])) AS input_arg_names, + "function".pronargdefaults AS default_args, + "function".function_oid::integer AS function_oid, (exists( SELECT 1 @@ -100,15 +100,23 @@ FROM ( ) ) AS returns_table FROM - pg_proc p - JOIN pg_namespace pn ON (pn.oid = p.pronamespace) - JOIN pg_type rt ON (rt.oid = p.prorettype) + jsonb_to_recordset($1::jsonb) AS tracked("schema" text, "name" text) + JOIN + ( SELECT p.oid AS function_oid, + p.*, + p.proname::text AS function_name, + pn.nspname::text AS function_schema + FROM pg_proc p + JOIN pg_namespace pn ON (pn.oid = p.pronamespace) + ) "function" ON "function".function_name = tracked.name + AND "function".function_schema = tracked.schema + JOIN pg_type rt ON (rt.oid = "function".prorettype) JOIN pg_namespace rtn ON (rtn.oid = rt.typnamespace) - LEFT JOIN pg_description pd ON p.oid = pd.objoid + LEFT JOIN pg_description pd ON "function".function_oid = pd.objoid WHERE -- Do not fetch some default functions in public schema - p.proname :: text NOT LIKE 'pgp_%' - AND p.proname :: text NOT IN + "function".function_name NOT LIKE 'pgp_%' + AND "function".function_name NOT IN ( 'armor' , 'crypt' , 'dearmor' @@ -122,15 +130,15 @@ FROM ( , 'gen_salt' , 'hmac' ) - AND pn.nspname :: text NOT LIKE 'pg_%' - AND pn.nspname :: text NOT IN ('information_schema', 'hdb_catalog') + AND "function".function_schema NOT LIKE 'pg_%' + AND "function".function_schema NOT IN ('information_schema', 'hdb_catalog') AND (NOT EXISTS ( SELECT 1 FROM pg_aggregate WHERE - ((pg_aggregate.aggfnoid) :: oid = p.oid) + ((pg_aggregate.aggfnoid) :: oid = "function".function_oid) ) ) ) AS "pg_function" diff --git a/server/src-rsr/pg_table_metadata.sql b/server/src-rsr/pg_table_metadata.sql index 3e4423b9316..7f58f7153aa 100644 --- a/server/src-rsr/pg_table_metadata.sql +++ b/server/src-rsr/pg_table_metadata.sql @@ -1,6 +1,6 @@ SELECT - schema.nspname AS table_schema, - "table".relname AS table_name, + "table".table_schema, + "table".table_name, -- This field corresponds to the `DBTableMetadata` Haskell type jsonb_build_object( @@ -21,10 +21,26 @@ SELECT 'extra_table_metadata', '[]'::json )::json AS info +-- tracked tables +-- $1 parameter provides JSON array of tracked tables +FROM + ( SELECT "tracked"."name" AS "table_name", + "tracked"."schema" AS "table_schema" + FROM jsonb_to_recordset($1::jsonb) AS "tracked"("schema" text, "name" text) + ) "tracked_table" + -- table & schema -FROM pg_catalog.pg_class "table" -JOIN pg_catalog.pg_namespace schema - ON schema.oid = "table".relnamespace +LEFT JOIN + ( SELECT "table".oid, + "table".relkind, + "table".relname AS "table_name", + "schema".nspname AS "table_schema" + FROM pg_catalog.pg_class "table" + JOIN pg_catalog.pg_namespace "schema" + ON schema.oid = "table".relnamespace + ) "table" + ON "table"."table_name" = "tracked_table"."table_name" + AND "table"."table_schema" = "tracked_table"."table_schema" -- description LEFT JOIN pg_catalog.pg_description description @@ -181,11 +197,11 @@ LEFT JOIN ) foreign_key GROUP BY foreign_key.table_schema, foreign_key.table_name ) foreign_key_constraints - ON "table".relname = foreign_key_constraints.table_name - AND schema.nspname = foreign_key_constraints.table_schema + ON "table".table_name = foreign_key_constraints.table_name + AND "table".table_schema = foreign_key_constraints.table_schema -- all these identify table-like things WHERE "table".relkind IN ('r', 't', 'v', 'm', 'f', 'p') -- and tables not from any system schemas - AND schema.nspname NOT LIKE 'pg_%' - AND schema.nspname NOT IN ('information_schema', 'hdb_catalog', 'hdb_lib', '_timescaledb_internal'); + AND "table".table_schema NOT LIKE 'pg_%' + AND "table".table_schema NOT IN ('information_schema', 'hdb_catalog', 'hdb_lib', '_timescaledb_internal');