Allow reading data connector agent URLs from environment variables

GITHUB_PR_NUMBER: 10077
GITHUB_PR_URL: https://github.com/hasura/graphql-engine/pull/10077

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/10585
Co-authored-by: Nick DeGroot <1966472+nickthegroot@users.noreply.github.com>
Co-authored-by: Daniel Chambers <1214352+daniel-chambers@users.noreply.github.com>
Co-authored-by: Vijay Prasanna <11921040+vijayprasanna13@users.noreply.github.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
GitOrigin-RevId: 9169e9250cbbee0fc9d47b503ead5e2219c98bc6

docs/docs/databases/data-connectors/adding-data-connectors.mdx

@@ -58,7 +58,7 @@ Click on the `Add Agent` button.
 </li>
 <li>
 
-Enter the values for name and Agent endpoint. Click `Connect` and you're done!
+Enter the values for agent name and agent URL. You can also choose to put the agent URL in an environment variable instead and use that variable name here. Click `Connect` and you're done!
 
 <Thumbnail
   src="/img/databases/data-connector/connect-final.png"
@@ -88,6 +88,15 @@ dataconnector:
     uri: <data-connector-agent-url>
 ```
 
+Alternatively, you can provide the Data Connector Agent URL via an environment variable:
+
+```yaml
+dataconnector:
+  sqlite:
+    uri:
+      from_env: <data-connector-agent-url-environment-variable-name>
+```
+
 Apply the Metadata by running:
 
 ```yaml
@@ -114,5 +123,23 @@ X-Hasura-Role: admin
 }
 ```
 
+Alternatively, you can provide the Data Connector Agent URL via an environment variable:
+
+```http
+POST /v1/metadata HTTP/1.1
+Content-Type: application/json
+X-Hasura-Role: admin
+
+{
+  "type": "dc_add_agent",
+  "args": {
+      "name": "sqlite",
+      "url": {
+        "from_env": "<data-connector-agent-url-environment-variable-name>"
+      }
+  }
+}
+```
+
 </TabItem>
 </Tabs>

frontend/apps/console-ce/src/css/legacy-boostrap.css

@@ -2325,7 +2325,6 @@ which was not allowing the `cursor-not-allowed` class to take effect
   appearance: none;
 }
 
-:where(.bootstrap-jail) input[type='radio'],
 :where(.bootstrap-jail) input[type='checkbox'] {
   margin: 4px 0 0;
   margin-top: 1px \9;

frontend/libs/console/legacy-ce/src/lib/features/ManageAgents/_test_/useListAvailableAgentsFromMetadata.spec.ts

@@ -45,8 +45,8 @@ describe('useListAvailableAgentsFromMetadata tests: ', () => {
     );
 
     const expectedResult: DcAgent[] = [
-      { name: 'csv', url: 'http://host.docker.internal:8101' },
-      { name: 'sqlite', url: 'http://host.docker.internal:8100' },
+      { name: 'csv', uri: 'http://host.docker.internal:8101' },
+      { name: 'sqlite', uri: 'http://host.docker.internal:8100' },
     ];
 
     await waitFor(() => result.current.isSuccess);

frontend/libs/console/legacy-ce/src/lib/features/ManageAgents/components/AddAgentForm.tsx

@@ -2,25 +2,39 @@ import { Button } from '../../../new-components/Button';
 import { InputField, SimpleForm } from '../../../new-components/Form';
 import { z } from 'zod';
 import { useAddAgent } from '../hooks/useAddAgent';
+import { UrlInput } from './UrlInput';
 
 interface CreateAgentFormProps {
   onClose: () => void;
   onSuccess?: () => void;
 }
 
-const schema = z.object({
+export const schema = z.object({
   name: z.string().min(1, 'Name is required!'),
-  url: z.string().min(1, 'URL is required!'),
+  url: z.discriminatedUnion('type', [
+    z.object({
+      type: z.literal('url'),
+      value: z.string().min(1, 'URL is required!'),
+    }),
+    z.object({
+      type: z.literal('envVar'),
+      value: z.string().min(1, 'ENV variable is required'),
+    }),
+  ]),
 });
 
-type FormValues = z.infer<typeof schema>;
+export type FormValues = z.infer<typeof schema>;
 
 export const AddAgentForm = (props: CreateAgentFormProps) => {
   const { addAgent, isLoading } = useAddAgent();
 
   const handleSubmit = (values: FormValues) => {
     addAgent({
-      ...values,
+      name: values.name,
+      url:
+        values.url.type === 'envVar'
+          ? { from_env: values.url.value }
+          : values.url.value,
     }).then(response => {
       response.makeToast();
       if (response.status === 'added') {
@@ -34,7 +48,9 @@ export const AddAgentForm = (props: CreateAgentFormProps) => {
       schema={schema}
       // something is wrong with type inference with react-hook-form form wrapper. temp until the issue is resolved
       onSubmit={handleSubmit}
-      options={{ defaultValues: { url: '', name: '' } }}
+      options={{
+        defaultValues: { url: { type: 'envVar', value: '' }, name: '' },
+      }}
       className="py-4"
     >
       <div className="bg-white p-6 border border-gray-300 rounded space-y-4 mb-6 max-w-xl">
@@ -51,13 +67,8 @@ export const AddAgentForm = (props: CreateAgentFormProps) => {
           placeholder="Enter the name of the agent"
         />
 
-        <InputField
-          label="URL"
-          name="url"
-          type="text"
-          tooltip="The URL of the data connector agent"
-          placeholder="Enter the URI of the agent"
-        />
+        <UrlInput />
+
         <div className="flex gap-4 justify-end">
           <Button type="submit" mode="primary" isLoading={isLoading}>
             Connect

frontend/libs/console/legacy-ce/src/lib/features/ManageAgents/components/ManageAgentsTable.tsx

@@ -3,6 +3,7 @@ import React from 'react';
 import { FaTrash } from 'react-icons/fa';
 import { useListAvailableAgentsFromMetadata } from '../hooks';
 import { useRemoveAgent } from '../hooks/useRemoveAgent';
+import { DataConnectorUri } from '../../hasura-metadata-types';
 
 export const ManageAgentsTable = () => {
   const { data, isLoading } = useListAvailableAgentsFromMetadata();
@@ -39,7 +40,7 @@ export const ManageAgentsTable = () => {
                   {agent.name}
                 </CardedTable.TableBodyCell>
                 <CardedTable.TableBodyCell>
-                  {agent.url}
+                  <DataConnectorUriLabel uri={agent.uri} />
                 </CardedTable.TableBodyCell>
                 <CardedTable.TableBodyCell>
                   <div className="flex items-center justify-end whitespace-nowrap text-right opacity-0 transition-all duration-200 group-hover:opacity-100">
@@ -62,3 +63,17 @@ export const ManageAgentsTable = () => {
     </div>
   );
 };
+
+const DataConnectorUriLabel = (props: {
+  uri: DataConnectorUri;
+}): JSX.Element => {
+  if (typeof props.uri === 'string') {
+    return <>{props.uri}</>;
+  } else {
+    return (
+      <>
+        Environment Variable: <code>{props.uri.from_env}</code>
+      </>
+    );
+  }
+};

frontend/libs/console/legacy-ce/src/lib/features/ManageAgents/components/UrlInput.tsx

@@ -0,0 +1,42 @@
+import { useFormContext } from 'react-hook-form';
+import { FormValues } from './AddAgentForm';
+import { InputField, Radio } from '../../../new-components/Form';
+
+export const UrlInput = () => {
+  const { watch } = useFormContext<FormValues>();
+  const selectedType = watch('url.type');
+  console.log({ selectedType });
+  return (
+    <div>
+      <Radio
+        label="URL"
+        name="url.type"
+        options={[
+          { label: 'Using URL value', value: 'url' },
+          {
+            label: 'Using Environment Variable (recommended)',
+            value: 'envVar',
+          },
+        ]}
+        orientation="horizontal"
+      />
+      {selectedType === 'url' ? (
+        <InputField
+          label="URL"
+          name="url.value"
+          type="text"
+          tooltip="The URL of the data connector agent"
+          placeholder="Enter the URI of the agent"
+        />
+      ) : (
+        <InputField
+          label="Environment Variable"
+          name="url.value"
+          type="text"
+          tooltip="The Environment variable that contains the URL of the data connector agent"
+          placeholder="DC_AGENT_URL_ENV_VAR"
+        />
+      )}
+    </div>
+  );
+};

frontend/libs/console/legacy-ce/src/lib/features/ManageAgents/hooks/useAddAgent.ts

@@ -11,14 +11,14 @@ type AddAgentConsoleProps = {
   // these are properties added to the response for console use
   error?: Error | null;
   name: string;
-  url: string;
+  url: string | { from_env: string };
   makeToast: () => void;
   status: 'unavailable' | 'error' | 'already-added' | 'added';
 };
 
 export type AddAgentResponse = AddAgentServerResponse & AddAgentConsoleProps;
 
-type AddAgentArgs = { name: string; url: string };
+type AddAgentArgs = { name: string; url: string | { from_env: string } };
 
 const AGENT_UNAVAILABLE_MESSAGE = 'Agent is not available';
 

frontend/libs/console/legacy-ce/src/lib/features/ManageAgents/hooks/useListAvailableAgentsFromMetadata.ts

@@ -25,7 +25,7 @@ export const useListAvailableAgentsFromMetadata = () => {
 
           return {
             name: dcAgentName,
-            url: definition.uri,
+            uri: definition.uri,
           };
         }
       );

frontend/libs/console/legacy-ce/src/lib/features/ManageAgents/types.ts

@@ -1,4 +1,6 @@
+import { DataConnectorUri } from '../hasura-metadata-types';
+
 export type DcAgent = {
   name: string;
-  url: string;
+  uri: DataConnectorUri;
 };

frontend/libs/console/legacy-ce/src/lib/features/hasura-metadata-types/backendConfigs/backendConfigs.ts

@@ -1,3 +1,9 @@
 export type BackendConfigs = {
-  dataconnector: Record<string, { uri: string }>;
+  dataconnector: Record<string, DataConnectorBackendConfig>;
 };
+
+export type DataConnectorBackendConfig = {
+  uri: DataConnectorUri;
+};
+
+export type DataConnectorUri = string | { from_env: string };

metadata.openapi.json

@@ -3754,7 +3754,14 @@
                         "type": "string"
                     },
                     "uri": {
-                        "type": "string"
+                        "oneOf": [
+                            {
+                                "type": "string"
+                            },
+                            {
+                                "$ref": "#/components/schemas/FromEnv"
+                            }
+                        ]
                     }
                 },
                 "required": [

server/src-lib/Hasura/Backends/DataConnector/Adapter/Metadata.hs

@@ -238,8 +238,9 @@ resolveBackendInfo' ::
     ProvidesNetwork m
   ) =>
   Logger Hasura ->
+  Environment ->
   (Inc.Dependency (Maybe (HashMap DC.DataConnectorName Inc.InvalidationKey)), Map.Map DC.DataConnectorName DC.DataConnectorOptions) `arr` HashMap DC.DataConnectorName DC.DataConnectorInfo
-resolveBackendInfo' logger = proc (invalidationKeys, optionsMap) -> do
+resolveBackendInfo' logger env = proc (invalidationKeys, optionsMap) -> do
   maybeDataConnectorCapabilities <-
     (|
       Inc.keyed
@@ -264,11 +265,13 @@ resolveBackendInfo' logger = proc (invalidationKeys, optionsMap) -> do
       HTTP.Manager ->
       ExceptT QErr m (Maybe DC.DataConnectorInfo)
     getDataConnectorCapabilities options@DC.DataConnectorOptions {..} manager =
-      ( ignoreTraceT
-          . flip runAgentClientT (AgentClientContext logger _dcoUri manager Nothing Nothing)
-          $ (Just . mkDataConnectorInfo options)
-          <$> Client.capabilities
-      )
+      do
+        resolvedUri <- DC.resolveDataConnectorUri env _dcoUri
+        ( ignoreTraceT
+            . flip runAgentClientT (AgentClientContext logger resolvedUri manager Nothing Nothing)
+            $ (Just . mkDataConnectorInfo options)
+            <$> Client.capabilities
+          )
         `catchError` ignoreConnectionErrors
 
     -- If we can't connect to a data connector agent to get its capabilities
@@ -305,12 +308,13 @@ resolveSourceConfig'
   env
   manager = runExceptT do
     DC.DataConnectorInfo {_dciOptions = DC.DataConnectorOptions {_dcoUri}, ..} <- getDataConnectorInfo dataConnectorName backendInfo
+    resolvedUri <- DC.resolveDataConnectorUri env _dcoUri
 
     validateConnSourceConfig dataConnectorName sourceName _dciConfigSchemaResponse csc Nothing env
 
     pure
       DC.SourceConfig
-        { _scEndpoint = _dcoUri,
+        { _scEndpoint = resolvedUri,
           _scConfig = originalConfig,
           _scTemplate = _cscTemplate,
           _scTemplateVariables = fromMaybe mempty _cscTemplateVariables,

server/src-lib/Hasura/Backends/DataConnector/Adapter/Types.hs

@@ -18,6 +18,8 @@ module Hasura.Backends.DataConnector.Adapter.Types
     scTemplateVariables,
     scTimeoutMicroseconds,
     scEnvironment,
+    resolveDataConnectorUri,
+    DataConnectorUri (..),
     DataConnectorOptions (..),
     DataConnectorInfo (..),
     TableName (..),
@@ -41,9 +43,9 @@ where
 
 import Autodocodec (HasCodec (codec), optionalField', requiredField', requiredFieldWith')
 import Autodocodec qualified as AC
-import Autodocodec.Extended (baseUrlCodec)
+import Autodocodec.Extended (baseUrlCodec, fromEnvCodec)
 import Control.Lens (makeLenses)
-import Data.Aeson (FromJSON, FromJSONKey, ToJSON, ToJSONKey, genericParseJSON, genericToJSON)
+import Data.Aeson (FromJSON, FromJSONKey, ToJSON, ToJSONKey, genericParseJSON, genericToJSON, parseJSON, toJSON)
 import Data.Aeson qualified as J
 import Data.Aeson.KeyMap qualified as J
 import Data.Aeson.Types (parseEither, toJSONKeyText)
@@ -56,16 +58,18 @@ import Data.OpenApi (ToSchema)
 import Data.Text qualified as Text
 import Data.Text.Extended (ToTxt (..))
 import Hasura.Backends.DataConnector.API qualified as API
+import Hasura.Base.Error
 import Hasura.Base.ErrorValue qualified as ErrorValue
 import Hasura.Base.ToErrorValue (ToErrorValue (..))
 import Hasura.Prelude
 import Hasura.RQL.IR.BoolExp qualified as IR
 import Hasura.RQL.Types.Backend (Backend)
 import Hasura.RQL.Types.BackendType (BackendType (..))
+import Hasura.RQL.Types.Common (getEnv)
 import Hasura.RQL.Types.DataConnector
 import Language.GraphQL.Draft.Syntax qualified as GQL
 import Network.HTTP.Client qualified as HTTP
-import Servant.Client (BaseUrl)
+import Servant.Client (BaseUrl, parseBaseUrl)
 import Witch qualified
 
 --------------------------------------------------------------------------------
@@ -304,17 +308,50 @@ instance AC.HasCodec FunctionReturnType where
 
 ------------
 
+data DataConnectorUri
+  = RawUri BaseUrl
+  | FromEnvironment Text
+  deriving stock (Show, Eq, Generic)
+  deriving (ToJSON, FromJSON) via AC.Autodocodec DataConnectorUri
+
+instance NFData DataConnectorUri
+
+instance HasCodec DataConnectorUri where
+  codec =
+    AC.dimapCodec
+      (either RawUri FromEnvironment)
+      (\case RawUri m -> Left m; FromEnvironment wEnv -> Right wEnv)
+      $ AC.disjointEitherCodec baseUrlCodec fromEnvCodec
+
+resolveDataConnectorUri ::
+  (MonadError QErr m) =>
+  Environment ->
+  DataConnectorUri ->
+  m BaseUrl
+resolveDataConnectorUri env =
+  \case
+    (RawUri uri) -> pure uri
+    (FromEnvironment envVar) -> do
+      envValue <- getEnv env envVar
+      case Text.unpack envValue of
+        uriStr ->
+          onNothing
+            (parseBaseUrl uriStr)
+            (throw400 InvalidParams $ "Invalid URL for " <> envVar)
+
+------------
+
 data DataConnectorOptions = DataConnectorOptions
-  { _dcoUri :: BaseUrl,
+  { _dcoUri :: DataConnectorUri,
     _dcoDisplayName :: Maybe Text
   }
-  deriving stock (Eq, Ord, Show, Generic)
+  deriving stock (Show, Eq, Generic)
 
 instance HasCodec DataConnectorOptions where
   codec =
     AC.object "DataConnectorOptions"
       $ DataConnectorOptions
-      <$> requiredFieldWith' "uri" baseUrlCodec
+      <$> requiredField' "uri"
       AC..= _dcoUri
         <*> optionalField' "display_name"
       AC..= _dcoDisplayName

server/src-lib/Hasura/RQL/DDL/DataConnector.hs

@@ -17,6 +17,7 @@ import Control.Monad.Except
 import Control.Monad.Trans.Control
 import Data.Aeson (FromJSON, ToJSON, (.!=), (.:), (.:?), (.=))
 import Data.Aeson qualified as J
+import Data.Environment (Environment)
 import Data.Has
 import Data.Map.Strict qualified as Map
 import Data.Monoid
@@ -45,7 +46,7 @@ data DCAddAgent = DCAddAgent
   { -- | Source kind, ie., the backend type.
     _gdcaName :: DC.Types.DataConnectorName,
     -- | The Agent URL.
-    _gdcaUrl :: Servant.BaseUrl,
+    _gdcaUrl :: DC.Types.DataConnectorUri,
     -- | Override the display name provided by the Agent.
     _gdcaDisplayName :: Maybe Text,
     -- | Optionally skip the Agent Validation step.
@@ -86,17 +87,19 @@ runAddDataConnectorAgent ::
     MonadIO m,
     MonadBaseControl IO m
   ) =>
+  Environment ->
   DCAddAgent ->
   m EncJSON
-runAddDataConnectorAgent DCAddAgent {..} = do
+runAddDataConnectorAgent env DCAddAgent {..} = do
   let agent :: DC.Types.DataConnectorOptions
       agent = DC.Types.DataConnectorOptions _gdcaUrl _gdcaDisplayName
   sourceKinds <- (:) "postgres" . fmap _skiSourceKind . unSourceKinds <$> agentSourceKinds
   if
     | toTxt _gdcaName `elem` sourceKinds -> Error.throw400 Error.AlreadyExists $ "SourceKind '" <> toTxt _gdcaName <> "' already exists."
     | _gdcaSkipCheck == SkipCheck True -> addAgent _gdcaName agent
-    | otherwise ->
-        checkAgentAvailability _gdcaUrl >>= \case
+    | otherwise -> do
+        dataConnectorUrl <- DC.Types.resolveDataConnectorUri env _gdcaUrl
+        checkAgentAvailability dataConnectorUrl >>= \case
           NotAvailable err ->
             pure
               $ EncJSON.encJFromJValue

server/src-lib/Hasura/RQL/DDL/Schema/Cache.hs

@@ -610,7 +610,7 @@ buildSchemaCacheRule logger env mSchemaRegistryContext = proc (MetadataWithResou
       let backendInvalidationKeys =
             Inc.selectMaybeD #unBackendInvalidationKeysWrapper
               $ BackendMap.lookupD @b backendInvalidationMap
-      backendInfo <- resolveBackendInfo @b logger -< (backendInvalidationKeys, unBackendConfigWrapper backendConfigWrapper)
+      backendInfo <- resolveBackendInfo @b logger env -< (backendInvalidationKeys, unBackendConfigWrapper backendConfigWrapper)
       returnA -< BackendMap.singleton (BackendInfoWrapper @b backendInfo)
 
     resolveBackendCache ::

server/src-lib/Hasura/RQL/Types/Metadata/Backend.hs

@@ -84,14 +84,16 @@ class
       ProvidesNetwork m
     ) =>
     Logger Hasura ->
+    Env.Environment ->
     (Inc.Dependency (Maybe (BackendInvalidationKeys b)), BackendConfig b) `arr` BackendInfo b
   default resolveBackendInfo ::
     ( Arrow arr,
       BackendInfo b ~ ()
     ) =>
     Logger Hasura ->
+    Env.Environment ->
     (Inc.Dependency (Maybe (BackendInvalidationKeys b)), BackendConfig b) `arr` BackendInfo b
-  resolveBackendInfo = const $ arr $ const ()
+  resolveBackendInfo _env = const $ arr $ const ()
 
   -- | Function that resolves the connection related source configuration, and
   -- creates a connection pool (and other related parameters) in the process

server/src-lib/Hasura/Server/API/Metadata.hs

@@ -517,7 +517,7 @@ runMetadataQueryV1M env schemaSampledFeatureFlags remoteSchemaPerms currentResou
   RMUpdateScopeOfCollectionInAllowlist q -> runUpdateScopeOfCollectionInAllowlist q
   RMCreateRestEndpoint q -> runCreateEndpoint q
   RMDropRestEndpoint q -> runDropEndpoint q
-  RMDCAddAgent q -> runAddDataConnectorAgent q
+  RMDCAddAgent q -> runAddDataConnectorAgent env q
   RMDCDeleteAgent q -> runDeleteDataConnectorAgent q
   RMSetCustomTypes q -> runSetCustomTypes q
   RMSetApiLimits q -> runSetApiLimits q

server/src-test/Hasura/SQL/BackendMapSpec.hs

@@ -8,7 +8,7 @@ import Data.Aeson.Types (parseEither)
 import Data.Either.Combinators (fromRight')
 import Data.Map.Strict qualified as Map
 import Data.Maybe (fromJust)
-import Hasura.Backends.DataConnector.Adapter.Types (DataConnectorOptions (..), mkDataConnectorName)
+import Hasura.Backends.DataConnector.Adapter.Types (DataConnectorOptions (..), DataConnectorUri (..), mkDataConnectorName)
 import Hasura.Prelude
 import Hasura.RQL.Types.BackendType (BackendType (..))
 import Hasura.RQL.Types.Metadata.Common (BackendConfigWrapper (BackendConfigWrapper))
@@ -29,7 +29,7 @@ spec = describe "BackendMap" do
                 $ Map.singleton
                   (fromRight' $ mkDataConnectorName $ fromJust $ GQL.mkName "MyConnector")
                   ( DataConnectorOptions
-                      { _dcoUri = fromRight' $ S.parseBaseUrl "https://somehost.org/",
+                      { _dcoUri = fromRight' $ RawUri <$> S.parseBaseUrl "https://somehost.org/",
                         _dcoDisplayName = Just "My Connector"
                       }
                   )