feature (console): UI to manage to DC agents

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/5652
Co-authored-by: Matt Hardman <28978422+mattshardman@users.noreply.github.com>
GitOrigin-RevId: f0f21a680401fb339583c2365da8e0ded6a5ce55

console/src/components/Services/Data/Schema/ManageDatabase.tsx

@@ -2,8 +2,12 @@ import React, { useState, useEffect } from 'react';
 import Helmet from 'react-helmet';
 import { connect, ConnectedProps } from 'react-redux';
 import { FaExclamationTriangle, FaEye, FaTimes } from 'react-icons/fa';
-
+import { ManageAgents } from '@/features/ManageAgents';
 import { Button } from '@/new-components/Button';
+import {
+  availableFeatureFlagIds,
+  useIsFeatureFlagEnabled,
+} from '@/features/FeatureFlags';
 import styles from './styles.module.scss';
 import { Dispatch, ReduxState } from '../../../../types';
 import BreadCrumb from '../../../Common/Layout/BreadCrumb/BreadCrumb';
@@ -215,6 +219,10 @@ const ManageDatabase: React.FC<ManageDatabaseProps> = ({
   const { show: shouldShowVPCBanner, dismiss: dismissVPCBanner } =
     useVPCBannerVisibility();
 
+  const { enabled: isDCAgentsManageUIEnabled } = useIsFeatureFlagEnabled(
+    availableFeatureFlagIds.gdcId
+  );
+
   const crumbs = [
     {
       title: 'Data',
@@ -328,6 +336,12 @@ const ManageDatabase: React.FC<ManageDatabaseProps> = ({
             </table>
           </div>
         </div>
+
+        {isDCAgentsManageUIEnabled ? (
+          <div className="mt-lg">
+            <ManageAgents />
+          </div>
+        ) : null}
       </div>
     </RightContainer>
   );

console/src/features/FeatureFlags/availableFeatureFlags.ts

@@ -1,9 +1,11 @@
 import { FeatureFlagDefinition } from './types';
 
 const relationshipTabTablesId = '0bea35ff-d3e9-45e9-af1b-59923bf82fa9';
+const gdcId = '88436c32-2798-11ed-a261-0242ac120002';
 
 export const availableFeatureFlagIds = {
   relationshipTabTablesId,
+  gdcId,
 };
 
 export const availableFeatureFlags: FeatureFlagDefinition[] = [
@@ -17,4 +19,14 @@ export const availableFeatureFlags: FeatureFlagDefinition[] = [
     defaultValue: false,
     discussionUrl: '',
   },
+  {
+    id: gdcId,
+    title: 'Experimental features for GDC',
+    description:
+      'Try out the very experimental features that are available for GDC on the console',
+    section: 'data',
+    status: 'experimental',
+    defaultValue: false,
+    discussionUrl: '',
+  },
 ];

console/src/features/FeatureFlags/types.ts

@@ -11,7 +11,8 @@ export type FeatureFlagStatus =
   | 'alpha'
   | 'beta'
   | 'release candidate'
-  | 'stable';
+  | 'stable'
+  | 'experimental';
 
 export type FeatureFlagId = string;
 

console/src/features/ManageAgents/_test_/useAddAgent.spec.ts

@@ -0,0 +1,72 @@
+import { renderHook } from '@testing-library/react-hooks';
+import { rest } from 'msw';
+import { setupServer } from 'msw/node';
+import { wrapper } from '../../../hooks/__tests__/common/decorator';
+import { useAddAgent } from '../hooks';
+
+const server = setupServer(
+  rest.post('http://localhost/v1/metadata', (req, res, ctx) => {
+    if ((req.body as Record<string, any>).args.name === 'wrong_payload')
+      return res(ctx.status(400), ctx.json({ message: 'Bad request' }));
+    return res(ctx.status(200), ctx.json({ message: 'success' }));
+  })
+);
+
+describe('useAddAgent tests: ', () => {
+  beforeAll(() => {
+    server.listen();
+    jest.spyOn(console, 'error').mockImplementation(() => null);
+  });
+  afterAll(() => {
+    server.close();
+    jest.spyOn(console, 'error').mockRestore();
+  });
+
+  it('calls the custom success callback after adding a DC agent', async () => {
+    const { result, waitFor } = renderHook(() => useAddAgent(), { wrapper });
+
+    const { addAgent } = result.current;
+
+    const mockCallback = jest.fn(() => {
+      console.log('success');
+    });
+
+    addAgent({
+      name: 'test_dc_agent',
+      url: 'http://localhost:8001',
+      onSuccess: () => {
+        mockCallback();
+      },
+    });
+
+    await waitFor(() => result.current.isSuccess);
+
+    await waitFor(() => {
+      expect(mockCallback).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  it('calls the custom error callback after failing to add a DC agent', async () => {
+    const { result, waitFor } = renderHook(() => useAddAgent(), { wrapper });
+
+    const { addAgent } = result.current;
+
+    const mockCallback = jest.fn(() => {
+      console.log('error');
+    });
+
+    addAgent({
+      name: 'wrong_payload',
+      url: '',
+      onError: () => {
+        mockCallback();
+      },
+    });
+
+    await waitFor(() => result.current.isError);
+
+    await waitFor(() => {
+      expect(mockCallback).toHaveBeenCalledTimes(1);
+    });
+  });
+});

console/src/features/ManageAgents/_test_/useListAvailableAgentsFromMetadata.spec.ts

@@ -0,0 +1,58 @@
+import { renderHook } from '@testing-library/react-hooks';
+import { rest } from 'msw';
+import { setupServer } from 'msw/node';
+import { wrapper } from '../../../hooks/__tests__/common/decorator';
+import { useListAvailableAgentsFromMetadata } from '../hooks';
+import { Metadata } from '../../DataSource';
+import { DcAgent } from '../types';
+
+const metadata: Metadata = {
+  resource_version: 1,
+  metadata: {
+    version: 3,
+    sources: [],
+    backend_configs: {
+      dataconnector: {
+        sqlite: {
+          uri: 'http://host.docker.internal:8100',
+        },
+        csv: {
+          uri: 'http://host.docker.internal:8101',
+        },
+      },
+    },
+  },
+};
+
+const server = setupServer(
+  rest.post('http://localhost/v1/metadata', (req, res, ctx) => {
+    return res(ctx.status(200), ctx.json(metadata));
+  })
+);
+
+describe('useListAvailableAgentsFromMetadata tests: ', () => {
+  beforeAll(() => {
+    server.listen();
+  });
+  afterAll(() => {
+    server.close();
+  });
+
+  it('lists all the dc agents from metadata', async () => {
+    const { result, waitFor } = renderHook(
+      () => useListAvailableAgentsFromMetadata(),
+      { wrapper }
+    );
+
+    const expectedResult: DcAgent[] = [
+      { name: 'csv', url: 'http://host.docker.internal:8101' },
+      { name: 'sqlite', url: 'http://host.docker.internal:8100' },
+    ];
+
+    await waitFor(() => result.current.isSuccess);
+
+    console.log(result.current);
+
+    expect(result.current.data).toEqual(expectedResult);
+  });
+});

console/src/features/ManageAgents/_test_/useRemoveAgent.spec.ts

@@ -0,0 +1,70 @@
+import { renderHook } from '@testing-library/react-hooks';
+import { rest } from 'msw';
+import { setupServer } from 'msw/node';
+import { wrapper } from '../../../hooks/__tests__/common/decorator';
+import { useRemoveAgent } from '../hooks';
+
+const server = setupServer(
+  rest.post('http://localhost/v1/metadata', (req, res, ctx) => {
+    if ((req.body as Record<string, any>).args.name === 'wrong_payload')
+      return res(ctx.status(400), ctx.json({ message: 'Bad request' }));
+    return res(ctx.status(200), ctx.json({ message: 'success' }));
+  })
+);
+
+describe('useRemoveAgent tests: ', () => {
+  beforeAll(() => {
+    server.listen();
+    jest.spyOn(console, 'error').mockImplementation(() => null);
+  });
+  afterAll(() => {
+    server.close();
+    jest.spyOn(console, 'error').mockRestore();
+  });
+
+  it('calls the custom success callback after adding a DC agent', async () => {
+    const { result, waitFor } = renderHook(() => useRemoveAgent(), { wrapper });
+
+    const { removeAgent } = result.current;
+
+    const mockCallback = jest.fn(() => {
+      console.log('success');
+    });
+
+    removeAgent({
+      name: 'test_dc_agent',
+      onSuccess: () => {
+        mockCallback();
+      },
+    });
+
+    await waitFor(() => result.current.isSuccess);
+
+    await waitFor(() => {
+      expect(mockCallback).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  it('calls the custom error callback after failing to add a DC agent', async () => {
+    const { result, waitFor } = renderHook(() => useRemoveAgent(), { wrapper });
+
+    const { removeAgent } = result.current;
+
+    const mockCallback = jest.fn(() => {
+      console.log('error');
+    });
+
+    removeAgent({
+      name: 'wrong_payload',
+      onError: () => {
+        mockCallback();
+      },
+    });
+
+    await waitFor(() => result.current.isError);
+
+    await waitFor(() => {
+      expect(mockCallback).toHaveBeenCalledTimes(1);
+    });
+  });
+});

console/src/features/ManageAgents/components/AddAgentForm.tsx

@@ -0,0 +1,83 @@
+import { Button } from '@/new-components/Button';
+import { Form, InputField } from '@/new-components/Form';
+import React from 'react';
+import { z } from 'zod';
+import { useAddAgent } from '../hooks/useAddAgent';
+
+interface CreateAgentFormProps {
+  onClose: () => void;
+  onSuccess?: () => void;
+}
+
+const schema = z.object({
+  name: z.string().min(1, 'Name is required!'),
+  url: z.string().min(1, 'URL is required!'),
+});
+
+type FormValues = z.infer<typeof schema>;
+
+export const AddAgentForm = (props: CreateAgentFormProps) => {
+  const { addAgent, isLoading } = useAddAgent();
+
+  const handleSubmit = (values: FormValues) => {
+    addAgent({
+      ...values,
+      onSuccess: props.onSuccess,
+    });
+  };
+
+  return (
+    <Form
+      schema={schema}
+      // something is wrong with type inference with react-hook-form form wrapper. temp until the issue is resolved
+      onSubmit={handleSubmit as any}
+      options={{ defaultValues: { url: '', name: '' } }}
+      className="p-0 py-4"
+    >
+      {() => {
+        return (
+          <>
+            <div className="bg-white p-6 border border-gray-300 rounded space-y-4 mb-6 max-w-xl">
+              <p className="text-lg text-gray-600 font-bold">
+                Connect a Data Connector Agent
+              </p>
+              <hr />
+
+              <InputField
+                label="Name"
+                name="name"
+                type="text"
+                tooltip="This value will be used as the source kind in metadata"
+                placeholder="Enter the name of the agent"
+              />
+
+              <InputField
+                label="URL"
+                name="url"
+                type="text"
+                tooltip="The URL of the data connector agent"
+                placeholder="Enter the URI of the agent"
+              />
+              <div className="flex gap-4 justify-end">
+                <Button type="submit" mode="primary" isLoading={isLoading}>
+                  Connect
+                </Button>
+                <Button
+                  onClick={() => {
+                    props.onClose();
+                  }}
+                >
+                  Close
+                </Button>
+              </div>
+            </div>
+          </>
+        );
+      }}
+    </Form>
+  );
+};
+
+AddAgentForm.defaultProps = {
+  onSuccess: () => {},
+};

console/src/features/ManageAgents/components/ManageAgents.tsx

@@ -0,0 +1,25 @@
+import { Button } from '@/new-components/Button';
+import React, { useState } from 'react';
+import { AddAgentForm } from './AddAgentForm';
+import { ManageAgentsTable } from './ManageAgentsTable';
+
+export const ManageAgents = () => {
+  const [showCreateAgentForm, setShowCreateAgentForm] = useState(false);
+
+  return (
+    <div>
+      <p className="text-xl text-gray-600 py-3 font-bold">
+        Data Connector Agents
+      </p>
+      <hr className="m-0" />
+      <ManageAgentsTable />
+      <Button onClick={() => setShowCreateAgentForm(true)}>Add Agent</Button>
+      {showCreateAgentForm ? (
+        <AddAgentForm
+          onClose={() => setShowCreateAgentForm(false)}
+          onSuccess={() => setShowCreateAgentForm(false)} // close the form on successful save
+        />
+      ) : null}
+    </div>
+  );
+};

console/src/features/ManageAgents/components/ManageAgentsTable.tsx

@@ -0,0 +1,64 @@
+import { CardedTable } from '@/new-components/CardedTable';
+import React from 'react';
+import { FaTrash } from 'react-icons/fa';
+import { useListAvailableAgentsFromMetadata } from '../hooks';
+import { useRemoveAgent } from '../hooks/useRemoveAgent';
+
+export const ManageAgentsTable = () => {
+  const { data, isLoading } = useListAvailableAgentsFromMetadata();
+
+  const { removeAgent } = useRemoveAgent();
+
+  if (isLoading) return <>Loading...</>;
+
+  if (!data) return <>Something went wrong while fetching data</>;
+
+  if (!data.length)
+    return (
+      <div className="text-gray-600 my-md">
+        <i>There are no data connector agents connected to Hasura.</i>
+      </div>
+    );
+
+  return (
+    <div className="mt-md">
+      <CardedTable.Table>
+        <CardedTable.TableHead>
+          <CardedTable.TableHeadRow>
+            <CardedTable.TableHeadCell>Agent Name</CardedTable.TableHeadCell>
+            <CardedTable.TableHeadCell>URL</CardedTable.TableHeadCell>
+            <CardedTable.TableHeadCell>Actions</CardedTable.TableHeadCell>
+          </CardedTable.TableHeadRow>
+        </CardedTable.TableHead>
+
+        <CardedTable.TableBody>
+          {data.map((agent, id) => {
+            return (
+              <CardedTable.TableBodyRow key={`${agent.name}-${id}`}>
+                <CardedTable.TableBodyCell>
+                  {agent.name}
+                </CardedTable.TableBodyCell>
+                <CardedTable.TableBodyCell>
+                  {agent.url}
+                </CardedTable.TableBodyCell>
+                <CardedTable.TableBodyCell>
+                  <div className="flex items-center justify-end whitespace-nowrap text-right opacity-0 group-hover:opacity-100">
+                    <button
+                      onClick={() => {
+                        removeAgent({ name: agent.name });
+                      }}
+                      className="flex px-2 py-0.5 items-center font-semibold rounded text-red-700 hover:bg-red-50 focus:bg-red-100"
+                    >
+                      <FaTrash className="fill-current mr-1" />
+                      Remove
+                    </button>
+                  </div>
+                </CardedTable.TableBodyCell>
+              </CardedTable.TableBodyRow>
+            );
+          })}
+        </CardedTable.TableBody>
+      </CardedTable.Table>
+    </div>
+  );
+};

console/src/features/ManageAgents/components/index.ts

@@ -0,0 +1,3 @@
+export { AddAgentForm } from './AddAgentForm';
+export { ManageAgents } from './ManageAgents';
+export { ManageAgentsTable } from './ManageAgentsTable';

console/src/features/ManageAgents/hooks/index.ts

@@ -0,0 +1,3 @@
+export { useListAvailableAgentsFromMetadata } from './useListAvailableAgentsFromMetadata';
+export { useAddAgent } from './useAddAgent';
+export { useRemoveAgent } from './useRemoveAgent';

console/src/features/ManageAgents/hooks/useAddAgent.ts

@@ -0,0 +1,71 @@
+import { useMetadataMigration } from '@/features/MetadataAPI';
+import { useFireNotification } from '@/new-components/Notifications';
+import { useCallback } from 'react';
+import { useQueryClient } from 'react-query';
+
+export const useAddAgent = () => {
+  const { fireNotification } = useFireNotification();
+  const queryClient = useQueryClient();
+  const mutation = useMetadataMigration({
+    onSuccess: async () => {
+      fireNotification({
+        title: 'Success',
+        type: 'success',
+        message: 'Data connector agent added successfully!',
+      });
+      queryClient.refetchQueries(['agent_list'], { exact: true });
+    },
+    onError: err => {
+      fireNotification({
+        title: 'Error',
+        type: 'error',
+        message: JSON.stringify(err),
+      });
+    },
+  });
+
+  const addAgent = useCallback(
+    ({
+      name,
+      url,
+      onSuccess,
+      onError,
+    }: {
+      name: string;
+      url: string;
+      onSuccess?: () => void;
+      onError?: (err: any) => void;
+    }) => {
+      mutation.mutate(
+        {
+          query: {
+            type: 'dc_add_agent',
+            args: {
+              name,
+              url,
+            },
+          },
+        },
+        {
+          onSuccess: () => {
+            if (onSuccess) onSuccess();
+            console.log('called inside hook');
+          },
+          onError: err => {
+            if (onError) onError(err);
+            console.log('called inside hook', err);
+          },
+        }
+      );
+    },
+    [mutation]
+  );
+
+  return {
+    addAgent,
+    isLoading: mutation.isLoading,
+    isSuccess: mutation.isLoading,
+    isError: mutation.isError,
+    error: mutation.error,
+  };
+};

console/src/features/ManageAgents/hooks/useListAvailableAgentsFromMetadata.ts

@@ -0,0 +1,41 @@
+import { exportMetadata } from '@/features/DataSource';
+import { useHttpClient } from '@/features/Network';
+import { useQuery } from 'react-query';
+import { DcAgent } from '../types';
+
+export const useListAvailableAgentsFromMetadata = () => {
+  const httpClient = useHttpClient();
+  return useQuery({
+    queryKey: ['agent_list'],
+    queryFn: async () => {
+      const { metadata } = await exportMetadata({ httpClient });
+
+      if (!metadata)
+        throw Error(
+          'useListAvailableAgentFromMetadata: could not fetch metadata'
+        );
+
+      const backend_configs = metadata.backend_configs;
+
+      if (!backend_configs) return [];
+
+      const values = Object.entries(backend_configs.dataconnector).map<DcAgent>(
+        item => {
+          const [dcAgentName, definition] = item;
+
+          return {
+            name: dcAgentName,
+            url: definition.uri,
+          };
+        }
+      );
+
+      // sort the values by ascending order of name. It's visually easier to read the items.
+      const result = values.sort((a, b) =>
+        a.name > b.name ? 1 : b.name > a.name ? -1 : 0
+      );
+
+      return result;
+    },
+  });
+};

console/src/features/ManageAgents/hooks/useRemoveAgent.ts

@@ -0,0 +1,62 @@
+import { useMetadataMigration } from '@/features/MetadataAPI';
+import { useFireNotification } from '@/new-components/Notifications';
+import { useCallback } from 'react';
+import { useQueryClient } from 'react-query';
+
+export const useRemoveAgent = () => {
+  const { fireNotification } = useFireNotification();
+  const queryClient = useQueryClient();
+  const mutation = useMetadataMigration({
+    onSuccess: () => {
+      fireNotification({
+        title: 'Success',
+        type: 'success',
+        message: 'Data connector agent removed successfully!',
+      });
+      queryClient.refetchQueries(['agent_list'], { exact: true });
+    },
+    onError: () => {},
+  });
+
+  const removeAgent = useCallback(
+    ({
+      name,
+      onSuccess,
+      onError,
+    }: {
+      name: string;
+      onSuccess?: () => void;
+      onError?: (err: any) => void;
+    }) => {
+      mutation.mutate(
+        {
+          query: {
+            type: 'dc_delete_agent',
+            args: {
+              name,
+            },
+          },
+        },
+        {
+          onSuccess: () => {
+            if (onSuccess) onSuccess();
+            console.log('called inside hook');
+          },
+          onError: err => {
+            if (onError) onError(err);
+            console.log('called inside hook', err);
+          },
+        }
+      );
+    },
+    [mutation]
+  );
+
+  return {
+    removeAgent,
+    isLoading: mutation.isLoading,
+    isSuccess: mutation.isLoading,
+    isError: mutation.isError,
+    error: mutation.error,
+  };
+};

console/src/features/ManageAgents/index.ts

@@ -0,0 +1 @@
+export * from './components';

console/src/features/ManageAgents/mocks/handler.mock.ts

@@ -0,0 +1,52 @@
+import { Metadata } from '@/features/DataSource';
+import { rest } from 'msw';
+
+const metadata: Metadata = {
+  resource_version: 1,
+  metadata: {
+    version: 3,
+    sources: [],
+    backend_configs: {
+      dataconnector: {
+        sqlite: {
+          uri: 'http://host.docker.internal:8100',
+        },
+        csv: {
+          uri: 'http://host.docker.internal:8101',
+        },
+      },
+    },
+  },
+};
+
+export const handlers = () => [
+  rest.post(`http://localhost:8080/v1/metadata`, (req, res, ctx) => {
+    const requestBody = req.body as Record<string, any>;
+    if (requestBody.type === 'export_metadata') return res(ctx.json(metadata));
+
+    if (requestBody.type === 'dc_delete_agent') {
+      const agentName = requestBody.args.name;
+      delete metadata.metadata.backend_configs?.dataconnector[agentName];
+      return res(ctx.json(metadata));
+    }
+
+    if (requestBody.type === 'dc_add_agent') {
+      const { name, url: uri } = requestBody.args;
+      metadata.metadata = {
+        ...metadata.metadata,
+        backend_configs: {
+          ...metadata.metadata.backend_configs,
+          dataconnector: {
+            ...metadata.metadata.backend_configs?.dataconnector,
+            [name]: {
+              uri,
+            },
+          },
+        },
+      };
+      return res(ctx.json({ message: 'success' }));
+    }
+
+    return res(ctx.json(metadata));
+  }),
+];

console/src/features/ManageAgents/stories/AddAgentForm.stories.tsx

@@ -0,0 +1,18 @@
+import React from 'react';
+import { ComponentStory, ComponentMeta } from '@storybook/react';
+import { ReactQueryDecorator } from '@/storybook/decorators/react-query';
+import { AddAgentForm } from '../components/AddAgentForm';
+import { handlers } from '../mocks/handler.mock';
+
+export default {
+  title: 'Data/Agents/AddAgentForm',
+  component: AddAgentForm,
+  decorators: [ReactQueryDecorator()],
+  parameters: {
+    msw: handlers(),
+  },
+} as ComponentMeta<typeof AddAgentForm>;
+
+export const Primary: ComponentStory<typeof AddAgentForm> = () => (
+  <AddAgentForm onClose={() => {}} />
+);

console/src/features/ManageAgents/stories/ManageAgents.stories.tsx

@@ -0,0 +1,18 @@
+import React from 'react';
+import { ComponentStory, ComponentMeta } from '@storybook/react';
+import { ReactQueryDecorator } from '@/storybook/decorators/react-query';
+import { ManageAgents } from '../components/ManageAgents';
+import { handlers } from '../mocks/handler.mock';
+
+export default {
+  title: 'Data/Agents/ManageAgents',
+  component: ManageAgents,
+  decorators: [ReactQueryDecorator()],
+  parameters: {
+    msw: handlers(),
+  },
+} as ComponentMeta<typeof ManageAgents>;
+
+export const Primary: ComponentStory<typeof ManageAgents> = () => (
+  <ManageAgents />
+);

console/src/features/ManageAgents/stories/ManageAgentsTable.stories.tsx

@@ -0,0 +1,18 @@
+import React from 'react';
+import { ComponentStory, ComponentMeta } from '@storybook/react';
+import { ReactQueryDecorator } from '@/storybook/decorators/react-query';
+import { ManageAgentsTable } from '../components/ManageAgentsTable';
+import { handlers } from '../mocks/handler.mock';
+
+export default {
+  title: 'Data/Agents/ManageAgentsTable',
+  component: ManageAgentsTable,
+  decorators: [ReactQueryDecorator()],
+  parameters: {
+    msw: handlers(),
+  },
+} as ComponentMeta<typeof ManageAgentsTable>;
+
+export const Primary: ComponentStory<typeof ManageAgentsTable> = () => (
+  <ManageAgentsTable />
+);

console/src/features/ManageAgents/types.ts

@@ -0,0 +1,4 @@
+export type DcAgent = {
+  name: string;
+  url: string;
+};

console/src/metadata/queryUtils.ts

@@ -97,6 +97,8 @@ export const metadataQueryTypes = [
   'drop_rest_endpoint',
   'add_host_to_tls_allowlist',
   'drop_host_from_tls_allowlist',
+  'dc_add_agent',
+  'dc_delete_agent',
 ] as const;
 
 export type MetadataQueryType = typeof metadataQueryTypes[number];