fix (console): make the new tree nav compatible with GDC tables

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/5716
Co-authored-by: Matt Hardman <28978422+mattshardman@users.noreply.github.com>
GitOrigin-RevId: 66e3062dedc05a15e7d46f65e9f901cc6094de0d

console/src/components/Services/Data/DataSources/DataSourceFormWrapper.tsx

@@ -1,7 +1,11 @@
 import React, { FormEvent } from 'react';
 import { LabeledInput } from '@/components/Common/LabeledInput';
 import { Connect, useAvailableDrivers } from '@/features/ConnectDB';
-import { GDC_DB_CONNECTOR_DEV } from '@/utils/featureFlags';
+// import { GDC_DB_CONNECTOR_DEV } from '@/utils/featureFlags';
+import {
+  availableFeatureFlagIds,
+  useIsFeatureFlagEnabled,
+} from '@/features/FeatureFlags';
 import { Button } from '@/new-components/Button';
 import ConnectDatabaseForm, { ConnectDatabaseFormProps } from './ConnectDBForm';
 import styles from './DataSources.module.scss';
@@ -58,6 +62,10 @@ const DataSourceFormWrapper: React.FC<DataSourceFormWrapperProps> = props => {
 
   const { isLoading, data: drivers } = useAvailableDrivers();
 
+  const { enabled: isGDCFeatureFlagEnabled } = useIsFeatureFlagEnabled(
+    availableFeatureFlagIds.gdcId
+  );
+
   const onSampleDBTry = () => {
     if (!sampleDBTrial || !sampleDBTrial.isActive()) return;
 
@@ -86,6 +94,13 @@ const DataSourceFormWrapper: React.FC<DataSourceFormWrapperProps> = props => {
       type: 'UPDATE_DB_DRIVER',
       data: value,
     });
+
+    /**
+     * Early return for gdc drivers when feature flag is enabled
+     */
+    const driver = drivers?.find(d => d.name === value);
+    if (isGDCFeatureFlagEnabled && !driver?.native) return;
+
     if (!isSupported && changeConnectionType) {
       changeConnectionType(driverToLabel[value].defaultConnection);
     }
@@ -101,7 +116,7 @@ const DataSourceFormWrapper: React.FC<DataSourceFormWrapperProps> = props => {
 
   return (
     <>
-      {GDC_DB_CONNECTOR_DEV === 'enabled' &&
+      {isGDCFeatureFlagEnabled &&
       !nativeDrivers.includes(connectionDBState.dbType) ? (
         <div className="max-w-xl">
           <Connect
@@ -158,7 +173,12 @@ const DataSourceFormWrapper: React.FC<DataSourceFormWrapperProps> = props => {
                   disabled={isEditState}
                   data-test="database-type"
                 >
-                  {(drivers ?? []).map(driver => (
+                  {(drivers ?? [])
+                    /**
+                     * Why this filter? if GDC feature flag is not enabled, then I want to see only native sources
+                     */
+                    .filter(driver => driver.native || isGDCFeatureFlagEnabled)
+                    .map(driver => (
                       <option key={driver.name} value={driver.name}>
                         {driver.displayName}{' '}
                         {driver.release === 'GA' ? null : `(${driver.release})`}

console/src/components/Services/Data/DataSubSidebar.js

@@ -22,8 +22,7 @@ import _push from './push';
 import { Button } from '@/new-components/Button';
 import styles from '../../Common/Layout/LeftSubSidebar/LeftSubSidebar.module.scss';
 import Spinner from '../../Common/Spinner/Spinner';
-// import { useGDCTreeClick } from './GDCTree/hooks/useGDCTreeClick';
+import { useGDCTreeItemClick } from './useGDCTreeItemClick';
-import { GDC_TREE_VIEW_DEV } from '@/utils/featureFlags';
 
 const DATA_SIDEBAR_SET_LOADING = 'dataSidebar/DATA_SIDEBAR_SET_LOADING';
 
@@ -203,34 +202,7 @@ const DataSubSidebar = props => {
 
   const [treeViewItems, setTreeViewItems] = useState([]);
 
-  const handleGDCTreeClick = value => {
+  const { handleClick } = useGDCTreeItemClick(dispatch);
-    if (GDC_TREE_VIEW_DEV === 'disabled') return;
-
-    const { database, ...table } = JSON.parse(value[0]);
-
-    const metadataSource = sources.find(source => source.name === database);
-
-    if (!metadataSource)
-      throw Error('useGDCTreeClick: source was not found in metadata');
-
-    /**
-     * Handling click for GDC DBs
-     */
-    const isTableClicked = Object.keys(table).length !== 0;
-    if (isTableClicked) {
-      dispatch(
-        _push(
-          encodeURI(
-            `/data/v2/manage?database=${database}&table=${JSON.stringify(
-              table
-            )}`
-          )
-        )
-      );
-    } else {
-      dispatch(_push(encodeURI(`/data/v2/manage?database=${database}`)));
-    }
-  };
 
   useEffect(() => {
     // skip api call, if the data is there in store
@@ -321,7 +293,7 @@ const DataSubSidebar = props => {
             databaseLoading={databaseLoading}
             schemaLoading={schemaLoading}
             preLoadState={preLoadState}
-            gdcItemClick={handleGDCTreeClick}
+            gdcItemClick={handleClick}
           />
         </div>
       </ul>

console/src/components/Services/Data/GDCTree/GDCTree.tsx

@@ -31,7 +31,7 @@ type Props = {
 /* 
   This component is still very much in development and will be changed once we have an API that tells us about the hierarchy of a GDC source
   Until then, this component is more or less a POC/experminatal in nature and tests for its accompaniying story have not been included for this reason.
-  If you wish to test out this component, head over to src/utils/featureFlags.ts and edit the GDC_TREE_VIEW_DEV to enabled to view it the console with mock data
+  If you wish to test out this component, go to the settings > feature flag and enable "Experimental features for GDC"
 */
 
 export const GDCTree = (props: Props) => {
@@ -39,7 +39,6 @@ export const GDCTree = (props: Props) => {
 
   const activeKey = isGDCRouteActive ? getCurrentActiveKeys() : [];
   const { data: gdcDatabases } = useTreeData();
-
   if (!gdcDatabases || gdcDatabases.length === 0) return null;
 
   return (

console/src/components/Services/Data/GDCTree/hooks/useGDCTreeClick.ts

@@ -1,63 +0,0 @@
-import { exportMetadata } from '@/features/DataSource';
-import { useHttpClient } from '@/features/Network';
-import { useFireNotification } from '@/new-components/Notifications';
-import { Dispatch } from '@/types';
-import { GDC_TREE_VIEW_DEV } from '@/utils/featureFlags';
-import { useCallback } from 'react';
-
-import _push from '../../push';
-import { useIsUnmounted } from '../utils';
-
-export const useGDCTreeClick = (dispatch: Dispatch) => {
-  const isUnmounted = useIsUnmounted();
-  const httpClient = useHttpClient();
-  const { fireNotification } = useFireNotification();
-
-  const handleClick = useCallback(
-    async (value: string[]) => {
-      try {
-        if (isUnmounted()) return;
-
-        if (GDC_TREE_VIEW_DEV === 'disabled') return;
-
-        const { metadata } = await exportMetadata({ httpClient });
-
-        const { database, ...table } = JSON.parse(value[0]);
-
-        const metadataSource = metadata.sources.find(
-          source => source.name === database
-        );
-
-        if (!metadataSource)
-          throw Error('useGDCTreeClick: source was not found in metadata');
-
-        /**
-         * Handling click for GDC DBs
-         */
-        const isTableClicked = Object.keys(table).length !== 0;
-        if (isTableClicked) {
-          dispatch(
-            _push(
-              encodeURI(
-                `/data/v2/manage?database=${database}&table=${JSON.stringify(
-                  table
-                )}`
-              )
-            )
-          );
-        } else {
-          dispatch(_push(encodeURI(`/data/v2/manage?database=${database}`)));
-        }
-      } catch (err) {
-        fireNotification({
-          type: 'error',
-          title: 'Could handle database selection',
-          message: JSON.stringify(err),
-        });
-      }
-    },
-    [dispatch, httpClient, isUnmounted]
-  );
-
-  return handleClick;
-};

console/src/components/Services/Data/GDCTree/hooks/useTreeData.ts

@@ -1,14 +1,42 @@
+import {
+  DataSource,
+  exportMetadata,
+  nativeDrivers,
+} from '@/features/DataSource';
 import { useHttpClient } from '@/features/Network';
+import { DataNode } from 'antd/lib/tree';
 import { useQuery } from 'react-query';
-import { getTreeData } from '../utils';
+
+const isValueDataNode = (value: DataNode | null): value is DataNode =>
+  value !== null;
 
 export const useTreeData = () => {
   const httpClient = useHttpClient();
 
   return useQuery({
-    queryKey: 'treeview',
+    queryKey: ['treeview'],
     queryFn: async () => {
-      return getTreeData({ httpClient });
+      const { metadata } = await exportMetadata({ httpClient });
+
+      if (!metadata) throw Error('Unable to fetch metadata');
+
+      const treeData = metadata.sources
+        /**
+         * NOTE: this filter prevents native drivers from being part of the new tree
+         */
+        .filter(source => !nativeDrivers.includes(source.kind))
+        .map(async source => {
+          const tablesAsTree = await DataSource(
+            httpClient
+          ).getTablesWithHierarchy({ dataSourceName: source.name });
+          return tablesAsTree;
+        });
+
+      const promisesResult = await Promise.all(treeData);
+
+      const filteredResult = promisesResult.filter<DataNode>(isValueDataNode);
+
+      return filteredResult;
     },
   });
 };

console/src/components/Services/Data/GDCTree/utils.tsx

@@ -1,106 +1,4 @@
-import React, { useCallback, useLayoutEffect, useRef } from 'react';
+import { useCallback, useLayoutEffect, useRef } from 'react';
-import { FaTable, FaDatabase, FaFolder } from 'react-icons/fa';
-import { DataSource, exportMetadata, NetworkArgs } from '@/features/DataSource';
-import { DataNode } from 'antd/lib/tree';
-import { GDC_TREE_VIEW_DEV } from '@/utils/featureFlags';
-import { GDCSource } from './types';
-
-const getSources = async ({ httpClient }: NetworkArgs) => {
-  const { metadata } = await exportMetadata({ httpClient });
-  const nativeDrivers = await DataSource(httpClient).getNativeDrivers();
-  return metadata.sources
-    .filter(source => !nativeDrivers.includes(source.kind))
-    .map<GDCSource>(source => ({
-      name: source.name,
-      kind: source.kind,
-      tables: source.tables.map(({ table }) => ({ table })),
-    }));
-};
-
-const nest = (
-  tables: GDCSource['tables'],
-  hierarchy: string[],
-  name: string
-): any => {
-  if (!hierarchy.length) return;
-
-  const key = hierarchy[0];
-
-  function onlyUnique(value: any, index: any, self: string | any[]) {
-    return self.indexOf(value) === index;
-  }
-
-  const levelValues: string[] = tables
-    .map((t: any) => t.table[key])
-    .filter(Boolean);
-
-  const uniqueLevelValues = levelValues.filter(onlyUnique);
-
-  return [
-    ...uniqueLevelValues.map(levelValue => {
-      // eslint-disable-next-line no-underscore-dangle
-      const _key = JSON.stringify({ ...JSON.parse(name), [key]: levelValue });
-      const children = nest(
-        tables.filter((t: any) => t.table[key] === levelValue),
-        hierarchy.slice(1),
-        _key
-      );
-
-      if (!children)
-        return {
-          icon: <FaTable />,
-          title: levelValue,
-          key: _key,
-        };
-
-      return {
-        icon: <FaFolder />,
-        title: levelValue,
-        selectable: false,
-        children,
-        key: _key,
-      };
-    }),
-  ];
-};
-
-export const getTreeData = async ({
-  httpClient,
-}: NetworkArgs): Promise<DataNode[]> => {
-  const sources = await getSources({ httpClient });
-
-  const tree = sources.map(async source => {
-    const tables = source.tables;
-
-    const hierarchy = await DataSource(httpClient).getDatabaseHierarchy({
-      dataSourceName: source.name,
-    });
-
-    // return a node of the tree
-    return {
-      title: (
-        <div className="inline-block">
-          {source.name}
-          <span className="items-center ml-sm px-sm py-0.5 rounded-full text-sm tracking-wide font-semibold bg-indigo-100 text-indigo-800">
-            Experimental
-          </span>
-        </div>
-      ),
-      key: JSON.stringify({ database: source.name }),
-      icon: <FaDatabase />,
-      children: nest(
-        tables,
-        hierarchy,
-        JSON.stringify({ database: source.name })
-      ),
-    };
-  });
-
-  // feature flag to enable tree view
-  if (GDC_TREE_VIEW_DEV === 'enabled') return Promise.all(tree);
-
-  return [];
-};
 
 export function useIsUnmounted() {
   const rIsUnmounted = useRef<'mounting' | 'mounted' | 'unmounted'>('mounting');

console/src/components/Services/Data/Schema/ManageDatabase.tsx

@@ -208,9 +208,17 @@ const ManageDatabase: React.FC<ManageDatabaseProps> = ({
   inconsistentObjects,
   location,
   dataHeaders,
+  sourcesFromMetadata,
 }) => {
   useEffect(() => {
-    if (dataSources.length === 0 && !autoRedirectedToConnectPage) {
+    if (sourcesFromMetadata.length === 0 && !autoRedirectedToConnectPage) {
+      /**
+       * Because the getDataSources() doesn't list the GDC sources, the Data tab will redirect to the /connect page
+       * thinking that are no sources available in Hasura, even if there are GDC sources connected to it. Modifying getDataSources()
+       * to list gdc sources is a huge task that involves modifying redux state variables.
+       * So a quick workaround is to check from the actual metadata if any sources are present -
+       * Combined with checks between getDataSources() and metadata -> we know the remaining sources are GDC sources. In such a case redirect to the manage db route
+       */
       dispatch(_push('/data/manage/connect'));
       autoRedirectedToConnectPage = true;
     }
@@ -356,6 +364,7 @@ const mapStateToProps = (state: ReduxState) => {
     currentSchema: state.tables.currentSchema,
     inconsistentObjects: state.metadata.inconsistentObjects,
     location: state?.routing?.locationBeforeTransitions,
+    sourcesFromMetadata: state?.metadata?.metadataObject?.sources ?? [],
   };
 };
 

console/src/components/Services/Data/TreeView.tsx

@@ -1,6 +1,9 @@
 import React, { useState, useEffect, ReactNode } from 'react';
 import { DownOutlined, RightOutlined } from '@ant-design/icons';
-import { GDC_TREE_VIEW_DEV } from '@/utils/featureFlags';
+import {
+  availableFeatureFlagIds,
+  useIsFeatureFlagEnabled,
+} from '@/features/FeatureFlags'; // Run time flag
 import {
   FaDatabase,
   FaFolder,
@@ -405,7 +408,11 @@ const TreeView: React.FC<TreeViewProps> = ({
     onDatabaseChange(dataSource);
   };
 
-  if (items.length === 0) {
+  const { enabled: isGDCTreeViewEnabled } = useIsFeatureFlagEnabled(
+    availableFeatureFlagIds.gdcId
+  );
+
+  if (items.length === 0 && !isGDCTreeViewEnabled) {
     return preLoadState ? (
       <div className={styles.treeNav}>
         <span className={`${styles.title} ${styles.padd_bottom_small}`}>
@@ -449,7 +456,7 @@ const TreeView: React.FC<TreeViewProps> = ({
           schemaLoading={schemaLoading}
         />
       ))}
-      {GDC_TREE_VIEW_DEV === 'enabled' ? (
+      {isGDCTreeViewEnabled ? (
         <div id="tree-container" className="inline-block">
           <GDCTree onSelect={gdcItemClick} />
         </div>

console/src/components/Services/Data/useGDCTreeItemClick.ts

@@ -0,0 +1,44 @@
+import { exportMetadata } from '@/features/DataSource';
+import { useHttpClient } from '@/features/Network';
+import { Dispatch } from '@/types';
+import { useCallback } from 'react';
+import _push from './push';
+
+export const useGDCTreeItemClick = (dispatch: Dispatch) => {
+  const httpClient = useHttpClient();
+
+  const handleClick = useCallback(
+    async value => {
+      const { metadata } = await exportMetadata({ httpClient });
+
+      const { database, ...table } = JSON.parse(value[0]);
+      const metadataSource = metadata.sources.find(
+        source => source.name === database
+      );
+
+      if (!metadataSource)
+        throw Error('useGDCTreeClick: source was not found in metadata');
+
+      /**
+       * Handling click for GDC DBs
+       */
+      const isTableClicked = Object.keys(table).length !== 0;
+      if (isTableClicked) {
+        dispatch(
+          _push(
+            encodeURI(
+              `/data/v2/manage?database=${database}&table=${JSON.stringify(
+                table
+              )}`
+            )
+          )
+        );
+      } else {
+        dispatch(_push(encodeURI(`/data/v2/manage?database=${database}`)));
+      }
+    },
+    [dispatch, httpClient]
+  );
+
+  return { handleClick };
+};

console/src/features/ConnectDB/components/Fields/ArrayInputs.tsx

@@ -49,7 +49,6 @@ type GetItemsArgs = {
 };
 
 export const getItems = ({ property, otherSchemas }: GetItemsArgs) => {
-  console.log('getItems', property, otherSchemas);
   return isRef(property.items)
     ? get(otherSchemas, property.items.$ref.split('/').slice(2).join('.'))
     : property.items;

console/src/features/Data/ManageContainer.tsx

@@ -3,7 +3,7 @@ import { useTableDefinition } from './hooks';
 import { ManageDatabase } from './ManageDatabase/ManageDatabase';
 
 export const ManageContainer = () => {
-  const urlData = useTableDefinition();
+  const urlData = useTableDefinition(window.location);
 
   if (urlData.querystringParseResult === 'error')
     return <>Something went wrong while parsing the URL parameters</>;

console/src/features/Data/TrackTables/TrackTables.tsx

@@ -23,9 +23,9 @@ export const TrackTables = ({ dataSourceName }: Props) => {
     dataSourceName,
   });
 
-  if (isLoading) return <>Loading...</>;
+  if (isLoading) return <div className="px-md">Loading...</div>;
 
-  if (!data) return <>Something went wrong</>;
+  if (!data) return <div className="px-md">Something went wrong</div>;
 
   const trackedTables = data.filter(({ is_tracked }) => is_tracked);
   const untrackedTables = data.filter(({ is_tracked }) => !is_tracked);

console/src/features/Data/hooks/useTableDefinition.ts

@@ -1,5 +1,3 @@
-import { useMemo } from 'react';
-
 //  TYPES
 export type QueryStringParseResult =
   | {
@@ -30,7 +28,6 @@ const invalidDatabaseDefinitionResult: QueryStringParseResult = {
 //  FUNCTION
 const getTableDefinition = (location: Location): QueryStringParseResult => {
   if (!location.search) return invalidTableDefinitionResult;
-
   // if tableDefinition is present in query params;
   // Idea is to use query params for GDC tables
   const params = new URLSearchParams(location.search);
@@ -58,5 +55,5 @@ const getTableDefinition = (location: Location): QueryStringParseResult => {
 };
 
 export const useTableDefinition = (location = window.location) => {
-  return useMemo(() => getTableDefinition(location), [location]);
+  return getTableDefinition(location);
 };

console/src/features/DataSource/__test__/utils.spec.ts

@@ -1,6 +1,6 @@
-import { RunSQLResponse } from '../../api';
+import { RunSQLResponse } from '../api';
-import { IntrospectedTable, TableColumn } from '../../types';
+import { IntrospectedTable, TableColumn } from '../types';
-import { adaptIntrospectedTables, adaptTableColumns } from '../utils';
+import { adaptIntrospectedTables, adaptTableColumns } from '../common/utils';
 
 describe('adaptIntrospectedTables', () => {
   it('adapts the sql response', () => {

console/src/features/DataSource/bigquery/index.ts

@@ -1,5 +1,9 @@
 import { Database, Feature } from '..';
-import { getTrackableTables, getTableColumns } from './introspection';
+import {
+  getTrackableTables,
+  getTableColumns,
+  getTablesListAsTree,
+} from './introspection';
 
 export type BigQueryTable = { name: string; dataset: string };
 
@@ -19,5 +23,6 @@ export const bigquery: Database = {
     },
     getTableColumns,
     getFKRelationships: async () => Feature.NotImplemented,
+    getTablesListAsTree,
   },
 };

console/src/features/DataSource/bigquery/introspection/getTablesListAsTree.tsx

@@ -0,0 +1,43 @@
+import React from 'react';
+import { FaDatabase } from 'react-icons/fa';
+import { BigQueryTable } from '..';
+import { exportMetadata } from '../../api';
+import { convertToTreeData } from '../../common/utils';
+import { NetworkArgs } from '../../types';
+
+export const getTablesListAsTree = async ({
+  dataSourceName,
+  httpClient,
+}: {
+  dataSourceName: string;
+} & NetworkArgs) => {
+  const hierarchy = ['dataset', 'name'];
+
+  const { metadata } = await exportMetadata({ httpClient });
+
+  if (!metadata) throw Error('Unable to fetch metadata');
+
+  const source = metadata.sources.find(s => s.name === dataSourceName);
+
+  if (!source) throw Error('Unable to fetch metadata source');
+
+  const tables = source.tables.map(table => table.table as BigQueryTable);
+
+  return {
+    title: (
+      <div className="inline-block">
+        {source.name}
+        {/* <span className="items-center ml-sm px-sm py-0.5 rounded-full text-sm tracking-wide font-semibold bg-indigo-100 text-indigo-800">
+          Experimental
+        </span> */}
+      </div>
+    ),
+    key: JSON.stringify({ database: source.name }),
+    icon: <FaDatabase />,
+    children: convertToTreeData(
+      tables,
+      hierarchy,
+      JSON.stringify({ database: source.name })
+    ),
+  };
+};

console/src/features/DataSource/bigquery/introspection/index.ts

@@ -1,2 +1,3 @@
 export { getTableColumns } from './getTableColumns';
 export { getTrackableTables } from './getTrackableTables';
+export { getTablesListAsTree } from './getTablesListAsTree';

console/src/features/DataSource/citus/index.ts

@@ -2,7 +2,11 @@ import { Database, Feature } from '..';
 import { runSQL } from '../api';
 import { adaptIntrospectedTables } from '../common/utils';
 import { GetTrackableTablesProps } from '../types';
-import { getTableColumns, getFKRelationships } from './introspection';
+import {
+  getTableColumns,
+  getFKRelationships,
+  getTablesListAsTree,
+} from './introspection';
 
 export type CitusTable = { name: string; schema: string };
 
@@ -54,5 +58,6 @@ export const citus: Database = {
     },
     getTableColumns,
     getFKRelationships,
+    getTablesListAsTree,
   },
 };

console/src/features/DataSource/citus/introspection/getTablesListAsTree.tsx

@@ -0,0 +1,43 @@
+import React from 'react';
+import { FaDatabase } from 'react-icons/fa';
+import { CitusTable } from '..';
+import { exportMetadata } from '../../api';
+import { convertToTreeData } from '../../common/utils';
+import { NetworkArgs } from '../../types';
+
+export const getTablesListAsTree = async ({
+  dataSourceName,
+  httpClient,
+}: {
+  dataSourceName: string;
+} & NetworkArgs) => {
+  const hierarchy = ['schema', 'name'];
+
+  const { metadata } = await exportMetadata({ httpClient });
+
+  if (!metadata) throw Error('Unable to fetch metadata');
+
+  const source = metadata.sources.find(s => s.name === dataSourceName);
+
+  if (!source) throw Error('Unable to fetch metadata source');
+
+  const tables = source.tables.map(table => table.table as CitusTable);
+
+  return {
+    title: (
+      <div className="inline-block">
+        {source.name}
+        {/* <span className="items-center ml-sm px-sm py-0.5 rounded-full text-sm tracking-wide font-semibold bg-indigo-100 text-indigo-800">
+          Experimental
+        </span> */}
+      </div>
+    ),
+    key: JSON.stringify({ database: source.name }),
+    icon: <FaDatabase />,
+    children: convertToTreeData(
+      tables,
+      hierarchy,
+      JSON.stringify({ database: source.name })
+    ),
+  };
+};

console/src/features/DataSource/citus/introspection/index.ts

@@ -1,2 +1,3 @@
 export { getTableColumns } from './getTableColumns';
 export { getFKRelationships } from './getFKRelationships';
+export { getTablesListAsTree } from './getTablesListAsTree';

console/src/features/DataSource/cockroach/index.ts

@@ -2,7 +2,11 @@ import { Database, Feature } from '..';
 import { runSQL } from '../api';
 import { adaptIntrospectedTables } from '../common/utils';
 import { GetTrackableTablesProps } from '../types';
-import { getTableColumns, getFKRelationships } from './introspection';
+import {
+  getTableColumns,
+  getFKRelationships,
+  getTablesListAsTree,
+} from './introspection';
 
 export type CockroachDBTable = { name: string; schema: string };
 
@@ -52,5 +56,6 @@ export const cockroach: Database = {
     },
     getTableColumns,
     getFKRelationships,
+    getTablesListAsTree,
   },
 };

console/src/features/DataSource/cockroach/introspection/getTablesListAsTree.tsx

@@ -0,0 +1,43 @@
+import React from 'react';
+import { FaDatabase } from 'react-icons/fa';
+import { CockroachDBTable } from '..';
+import { exportMetadata } from '../../api';
+import { convertToTreeData } from '../../common/utils';
+import { NetworkArgs } from '../../types';
+
+export const getTablesListAsTree = async ({
+  dataSourceName,
+  httpClient,
+}: {
+  dataSourceName: string;
+} & NetworkArgs) => {
+  const hierarchy = ['schema', 'name'];
+
+  const { metadata } = await exportMetadata({ httpClient });
+
+  if (!metadata) throw Error('Unable to fetch metadata');
+
+  const source = metadata.sources.find(s => s.name === dataSourceName);
+
+  if (!source) throw Error('Unable to fetch metadata source');
+
+  const tables = source.tables.map(table => table.table as CockroachDBTable);
+
+  return {
+    title: (
+      <div className="inline-block">
+        {source.name}
+        {/* <span className="items-center ml-sm px-sm py-0.5 rounded-full text-sm tracking-wide font-semibold bg-indigo-100 text-indigo-800">
+          Experimental
+        </span> */}
+      </div>
+    ),
+    key: JSON.stringify({ database: source.name }),
+    icon: <FaDatabase />,
+    children: convertToTreeData(
+      tables,
+      hierarchy,
+      JSON.stringify({ database: source.name })
+    ),
+  };
+};

console/src/features/DataSource/cockroach/introspection/index.ts

@@ -1,2 +1,3 @@
 export { getTableColumns } from './getTableColumns';
 export { getFKRelationships } from './getFKRelationships';
+export { getTablesListAsTree } from './getTablesListAsTree';

console/src/features/DataSource/common/utils.tsx

@@ -1,4 +1,7 @@
 import get from 'lodash.get';
+import { FaFolder, FaTable } from 'react-icons/fa';
+import React from 'react';
+import { Table } from '@/features/MetadataAPI';
 import { IntrospectedTable, Property, Ref, TableColumn } from '../types';
 import { RunSQLResponse } from '../api';
 
@@ -63,3 +66,48 @@ export const adaptTableColumns = (
     dataType: row[1],
   }));
 };
+
+export const convertToTreeData = (
+  tables: Table[],
+  hierarchy: string[],
+  name: string
+): any => {
+  if (!hierarchy.length) return;
+
+  const key = hierarchy[0];
+
+  function onlyUnique(value: any, index: any, self: string | any[]) {
+    return self.indexOf(value) === index;
+  }
+
+  const levelValues: string[] = tables.map((t: any) => t[key]).filter(Boolean);
+
+  const uniqueLevelValues = levelValues.filter(onlyUnique);
+
+  return [
+    ...uniqueLevelValues.map(levelValue => {
+      // eslint-disable-next-line no-underscore-dangle
+      const _key = JSON.stringify({ ...JSON.parse(name), [key]: levelValue });
+      const children = convertToTreeData(
+        tables.filter((t: any) => t[key] === levelValue),
+        hierarchy.slice(1),
+        _key
+      );
+
+      if (!children)
+        return {
+          icon: <FaTable />,
+          title: levelValue,
+          key: _key,
+        };
+
+      return {
+        icon: <FaFolder />,
+        title: levelValue,
+        selectable: false,
+        children,
+        key: _key,
+      };
+    }),
+  ];
+};

console/src/features/DataSource/gdc/index.ts

@@ -1,4 +1,7 @@
 import { Database, Feature, Property } from '../index';
+import { getTablesListAsTree } from './introspection/getTablesListAsTree';
+
+export type GDCTable = string[];
 
 export const gdc: Database = {
   introspection: {
@@ -142,5 +145,6 @@ export const gdc: Database = {
       return Feature.NotImplemented;
     },
     getFKRelationships: async () => Feature.NotImplemented,
+    getTablesListAsTree,
   },
 };

console/src/features/DataSource/gdc/introspection/getTablesListAsTree.tsx

@@ -0,0 +1,40 @@
+import React from 'react';
+import { FaDatabase } from 'react-icons/fa';
+import { GDCTable } from '..';
+import { exportMetadata } from '../../api';
+import { NetworkArgs } from '../../types';
+import { convertToTreeData } from './utils';
+
+export const getTablesListAsTree = async ({
+  dataSourceName,
+  httpClient,
+}: {
+  dataSourceName: string;
+} & NetworkArgs) => {
+  const { metadata } = await exportMetadata({ httpClient });
+
+  if (!metadata) throw Error('Unable to fetch metadata');
+
+  const source = metadata.sources.find(s => s.name === dataSourceName);
+
+  if (!source) throw Error('Unable to fetch metadata source');
+
+  const tables = source.tables.map(table => {
+    if (typeof table.table === 'string') return [table.table] as GDCTable;
+    return table.table as GDCTable;
+  });
+
+  return {
+    title: (
+      <div className="inline-block">
+        <span className="font-bold text-lg">{source.name}</span>
+        <span className="items-center ml-sm px-sm py-0.5 rounded-full text-sm tracking-wide font-semibold bg-indigo-100 text-indigo-800">
+          Experimental
+        </span>
+      </div>
+    ),
+    key: JSON.stringify({ database: source.name }),
+    icon: <FaDatabase />,
+    children: convertToTreeData(tables, [], source.name),
+  };
+};

console/src/features/DataSource/gdc/introspection/utils.tsx

@@ -0,0 +1,52 @@
+import { DataNode } from 'antd/lib/tree';
+import React from 'react';
+import { FaTable, FaFolder } from 'react-icons/fa';
+
+export function convertToTreeData(
+  tables: string[][],
+  key: string[],
+  dataSourceName: string
+): DataNode[] {
+  if (tables[0].length === 1) {
+    const leafNodes: DataNode[] = tables.map(table => {
+      return {
+        icon: <FaTable />,
+        key: JSON.stringify({
+          database: dataSourceName,
+          table: [...key, table[0]],
+        }),
+        title: table[0],
+      };
+    });
+
+    return leafNodes;
+  }
+
+  const uniqueLevelValues = Array.from(new Set(tables.map(table => table[0])));
+
+  const acc: DataNode[] = [];
+
+  const values = uniqueLevelValues.reduce<DataNode[]>((_acc, levelValue) => {
+    // eslint-disable-next-line no-underscore-dangle
+    const _childTables = tables
+      .filter(table => table[0] === levelValue)
+      .map<string[]>(table => table.slice(1));
+
+    return [
+      ..._acc,
+      {
+        icon: <FaFolder />,
+        selectable: false,
+        key: JSON.stringify([...key, levelValue[0]]),
+        title: levelValue,
+        children: convertToTreeData(
+          _childTables,
+          [...key, levelValue],
+          dataSourceName
+        ),
+      },
+    ];
+  }, acc);
+
+  return values;
+}

console/src/features/DataSource/index.ts

@@ -1,5 +1,6 @@
 import { AxiosInstance } from 'axios';
 import { z } from 'zod';
+import { DataNode } from 'antd/lib/tree';
 import { SupportedDrivers, Table } from '@/features/MetadataAPI';
 import { postgres } from './postgres';
 import { bigquery } from './bigquery';
@@ -17,6 +18,7 @@ import type {
   TableFkRelationships,
   GetFKRelationshipProps,
   DriverInfoResponse,
+  GetTablesListAsTreeProps,
 } from './types';
 
 import { createZodSchema } from './common/createZodSchema';
@@ -61,6 +63,9 @@ export type Database = {
     getFKRelationships: (
       props: GetFKRelationshipProps
     ) => Promise<TableFkRelationships[] | Feature.NotImplemented>;
+    getTablesListAsTree: (
+      props: GetTablesListAsTreeProps
+    ) => Promise<DataNode | Feature.NotImplemented>;
   };
   query?: {
     getTableData: () => void;
@@ -93,7 +98,9 @@ const getDatabaseMethods = async ({
     );
   }
 
-  return drivers[dataSource.kind];
+  if (nativeDrivers.includes(dataSource.kind)) return drivers[dataSource.kind];
+
+  return drivers.gdc;
 };
 
 export const DataSource = (httpClient: AxiosInstance) => ({
@@ -261,6 +268,28 @@ export const DataSource = (httpClient: AxiosInstance) => ({
 
     return result;
   },
+  getTablesWithHierarchy: async ({
+    dataSourceName,
+  }: {
+    dataSourceName: string;
+  }) => {
+    const database = await getDatabaseMethods({ dataSourceName, httpClient });
+
+    if (!database) return null;
+
+    const introspection = database.introspection;
+
+    if (!introspection) return null;
+
+    const treeData = await introspection.getTablesListAsTree({
+      dataSourceName,
+      httpClient,
+    });
+
+    if (treeData === Feature.NotImplemented) return null;
+
+    return treeData;
+  },
 });
 
 export { exportMetadata, utils, RunSQLResponse, getDriverPrefix };

console/src/features/DataSource/mssql/index.ts

@@ -1,7 +1,11 @@
 import { Database, Feature } from '..';
 import { NetworkArgs, runSQL } from '../api';
 import { adaptIntrospectedTables } from '../common/utils';
-import { getTableColumns, getFKRelationships } from './introspection';
+import {
+  getTableColumns,
+  getFKRelationships,
+  getTablesListAsTree,
+} from './introspection';
 
 export type MssqlTable = { schema: string; name: string };
 
@@ -43,5 +47,6 @@ export const mssql: Database = {
     },
     getTableColumns,
     getFKRelationships,
+    getTablesListAsTree,
   },
 };

console/src/features/DataSource/mssql/introspection/getTablesListAsTree.tsx

@@ -0,0 +1,43 @@
+import React from 'react';
+import { FaDatabase } from 'react-icons/fa';
+import { MssqlTable } from '..';
+import { exportMetadata } from '../../api';
+import { convertToTreeData } from '../../common/utils';
+import { NetworkArgs } from '../../types';
+
+export const getTablesListAsTree = async ({
+  dataSourceName,
+  httpClient,
+}: {
+  dataSourceName: string;
+} & NetworkArgs) => {
+  const hierarchy = ['schema', 'name'];
+
+  const { metadata } = await exportMetadata({ httpClient });
+
+  if (!metadata) throw Error('Unable to fetch metadata');
+
+  const source = metadata.sources.find(s => s.name === dataSourceName);
+
+  if (!source) throw Error('Unable to fetch metadata source');
+
+  const tables = source.tables.map(table => table.table as MssqlTable);
+
+  return {
+    title: (
+      <div className="inline-block">
+        {source.name}
+        {/* <span className="items-center ml-sm px-sm py-0.5 rounded-full text-sm tracking-wide font-semibold bg-indigo-100 text-indigo-800">
+          Experimental
+        </span> */}
+      </div>
+    ),
+    key: JSON.stringify({ database: source.name }),
+    icon: <FaDatabase />,
+    children: convertToTreeData(
+      tables,
+      hierarchy,
+      JSON.stringify({ database: source.name })
+    ),
+  };
+};

console/src/features/DataSource/mssql/introspection/index.ts

@@ -1,2 +1,3 @@
 export { getTableColumns } from './getTableColumns';
 export { getFKRelationships } from './getFKRelationships';
+export { getTablesListAsTree } from './getTablesListAsTree';

console/src/features/DataSource/postgres/index.ts

@@ -4,6 +4,7 @@ import {
   getTrackableTables,
   getTableColumns,
   getFKRelationships,
+  getTablesListAsTree,
 } from './introspection';
 
 export type PostgresTable = { name: string; schema: string };
@@ -22,5 +23,6 @@ export const postgres: Database = {
     },
     getTableColumns,
     getFKRelationships,
+    getTablesListAsTree,
   },
 };

console/src/features/DataSource/postgres/introspection/getTablesListAsTree.tsx

@@ -0,0 +1,43 @@
+import React from 'react';
+import { FaDatabase } from 'react-icons/fa';
+import { PostgresTable } from '..';
+import { exportMetadata } from '../../api';
+import { convertToTreeData } from '../../common/utils';
+import { NetworkArgs } from '../../types';
+
+export const getTablesListAsTree = async ({
+  dataSourceName,
+  httpClient,
+}: {
+  dataSourceName: string;
+} & NetworkArgs) => {
+  const hierarchy = ['schema', 'name'];
+
+  const { metadata } = await exportMetadata({ httpClient });
+
+  if (!metadata) throw Error('Unable to fetch metadata');
+
+  const source = metadata.sources.find(s => s.name === dataSourceName);
+
+  if (!source) throw Error('Unable to fetch metadata source');
+
+  const tables = source.tables.map(table => table.table as PostgresTable);
+
+  return {
+    title: (
+      <div className="inline-block">
+        {source.name}
+        {/* <span className="items-center ml-sm px-sm py-0.5 rounded-full text-sm tracking-wide font-semibold bg-indigo-100 text-indigo-800">
+          Experimental
+        </span> */}
+      </div>
+    ),
+    key: JSON.stringify({ database: source.name }),
+    icon: <FaDatabase />,
+    children: convertToTreeData(
+      tables,
+      hierarchy,
+      JSON.stringify({ database: source.name })
+    ),
+  };
+};

console/src/features/DataSource/postgres/introspection/index.ts

@@ -2,3 +2,4 @@ export { getTrackableTables } from './getTrackableTables';
 export { getDatabaseConfiguration } from './getDatabaseConfiguration';
 export { getTableColumns } from './getTableColumns';
 export { getFKRelationships } from './getFKRelationships';
+export { getTablesListAsTree } from './getTablesListAsTree';

console/src/features/DataSource/types.ts

@@ -115,6 +115,10 @@ export type TableFkRelationships = {
   };
 };
 
+export type GetTablesListAsTreeProps = {
+  dataSourceName: string;
+} & NetworkArgs;
+
 type ReleaseType = 'GA' | 'Beta';
 
 export type DriverInfoResponse = {

console/src/features/ManageAgents/_test_/useListAvailableAgentsFromMetadata.spec.ts

@@ -51,8 +51,6 @@ describe('useListAvailableAgentsFromMetadata tests: ', () => {
 
     await waitFor(() => result.current.isSuccess);
 
-    console.log(result.current);
-
     expect(result.current.data).toEqual(expectedResult);
   });
 });

console/src/features/ManageAgents/hooks/useAddAgent.ts

@@ -49,11 +49,9 @@ export const useAddAgent = () => {
         {
           onSuccess: () => {
             if (onSuccess) onSuccess();
-            console.log('called inside hook');
           },
           onError: err => {
             if (onError) onError(err);
-            console.log('called inside hook', err);
           },
         }
       );

console/src/features/ManageAgents/hooks/useRemoveAgent.ts

@@ -40,11 +40,9 @@ export const useRemoveAgent = () => {
         {
           onSuccess: () => {
             if (onSuccess) onSuccess();
-            console.log('called inside hook');
           },
           onError: err => {
             if (onError) onError(err);
-            console.log('called inside hook', err);
           },
         }
       );

console/src/hooks/__tests__/useUniqueKeys.test.ts

@@ -78,7 +78,6 @@ describe("useUniqueKeys hooks' mssql test", () => {
     await waitForValueToChange(() => result.current.isSuccess);
 
     const firstResult = result.current.data![0];
-    console.log({ firstResult });
 
     expect(firstResult.table_schema).toEqual('dbo');
     expect(firstResult.table_name).toEqual('citizens');

console/src/utils/featureFlags.ts

@@ -2,16 +2,9 @@ type Status = 'enabled' | 'disabled';
 
 export const ENABLE_AUTH_LAYER = true;
 
-/*
-This enables the development code of the GDC tree view on the console to become active. 
-Once enabled, the TreeView.tsx will render a tree nav for non-native DBs (i.e DBs that are no pg, citus, mssql and bq) in the Data Tab.
-This feature is under a development flag because the API and the metadata structure is not decided yet.
-*/
-export const GDC_TREE_VIEW_DEV: Status = 'disabled';
-
 /*
 This enables the development code of the GDC connect database form on the console to become active. 
 Once enabled, the form on the connect db page will be dynamically rendered based on the selected database type.
 This feature is under a development flag because the API and the metadata structure is not decided yet.
 */
-export const GDC_DB_CONNECTOR_DEV: Status = 'disabled';
+export const GDC_DB_CONNECTOR_DEV: Status = 'enabled';