console: add useInstallTemplate hook

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/6077
GitOrigin-RevId: 635477024d968edf37fc4bd1749a320ed9fafe40

console/src/components/Services/Data/DataSources/CreateDataSource/Neon/index.tsx

@@ -32,3 +32,5 @@ export function Neon(props: { allDatabases: string[]; dispatch: Dispatch }) {
 
   return <NeonBanner {...neonBannerProps} />;
 }
+
+export { useNeonIntegration } from './useNeonIntegration';

console/src/features/OnboardingWizard/components/ConnectDBScreen/ConnectDBScreen.tsx

@@ -16,6 +16,12 @@ export function ConnectDBScreen(props: ConnectDBScreenProps) {
   const { skipOnboarding, completeOnboarding } = props;
   const dispatch = useAppDispatch();
 
+  const onError = (error?: string) => {
+    if (error) {
+      throw new Error(error);
+    }
+  };
+
   const onClick = () => {
     // TODO: Due to routing being slow on prod, but wizard closing instantaneously, this causes
     // a flicker of `<Api />` tab before routing to `/data`.
@@ -41,7 +47,7 @@ export function ConnectDBScreen(props: ConnectDBScreenProps) {
             dispatch={dispatch}
             onSkip={skipOnboarding}
             onCompletion={completeOnboarding}
-            onError={() => console.log('error')}
+            onError={onError}
           />
         ) : (
           <>

console/src/features/OnboardingWizard/components/ConnectDBScreen/NeonOnboarding.tsx

@@ -4,29 +4,24 @@ import { useNeonIntegration } from '@/components/Services/Data/DataSources/Creat
 import { transformNeonIntegrationStatusToNeonBannerProps } from '@/components/Services/Data/DataSources/CreateDataSource/Neon/utils';
 import { NeonBanner } from '../NeonConnectBanner/NeonBanner';
 import _push from '../../../../components/Services/Data/push';
-
-const useTemplateGallery = (
-  onSuccess: VoidFunction,
-  onError: VoidFunction,
-  dispatch: Dispatch
-) => {
-  return {
-    install: () => {
-      dispatch(_push(`/data/default`));
-    },
-  };
-};
+import {
+  useInstallTemplate,
+  usePrefetchNeonOnboardingTemplateData,
+} from '../../hooks';
 
 export function NeonOnboarding(props: {
   dispatch: Dispatch;
   onSkip: VoidFunction;
   onCompletion: VoidFunction;
-  onError: VoidFunction;
+  onError: (errorMsg?: string) => void;
 }) {
   const { dispatch, onSkip, onCompletion, onError } = props;
 
-  // Sample function
-  const { install } = useTemplateGallery(onCompletion, onError, dispatch);
+  // Prefetch Neon related template data from github repo
+  usePrefetchNeonOnboardingTemplateData();
+
+  // Memoised function used to install the template
+  const { install } = useInstallTemplate('default', onCompletion, onError);
 
   const neonIntegrationStatus = useNeonIntegration(
     'default',

console/src/features/OnboardingWizard/hooks/constants.ts

@@ -1,5 +1,20 @@
 import { growthExperimentsIds } from '@/features/GrowthExperiments';
 import globals from '@/Globals';
+import { BASE_URL_TEMPLATE } from '@/components/Services/Data/Schema/TemplateGallery/templateGalleryConfig';
+
+// This config is stored in root level index.js, and there is a config file in each directory which stores which
+// stores the directory structure.
+// But in order to save api calls, we directly fetch the migraion and metadata files, without querying config.
+const ROOT_DIR = 'postgres';
+const TEMPLATE_DIR = 'getting-started';
+const NEON_TEMPLATE_BASE_PATH = `${BASE_URL_TEMPLATE}/${ROOT_DIR}/${TEMPLATE_DIR}`;
+
+export const NEON_METADATA_PATH = `${NEON_TEMPLATE_BASE_PATH}/metadata.json`;
+export const NEON_MIGRATIONS_PATH = `${NEON_TEMPLATE_BASE_PATH}/migration.sql`;
+export const NEON_QUERY_PATH = `${NEON_TEMPLATE_BASE_PATH}/sample.graphql`;
+export const NEON_IMAGE_PATH = `${NEON_TEMPLATE_BASE_PATH}/diagram.png`;
+
+export const NEON_ONBOARDING_QUERY_KEY = 'neonOnboarding';
 
 export const experimentId = growthExperimentsIds.onboardingWizardV1;
 
@@ -27,3 +42,6 @@ export const skippedOnboardingVariables = {
   ...mutationVariables,
   kind: 'skipped_onboarding',
 };
+
+// A stale time of 5 minutes for use in useQuery hook
+export const staleTime = 5 * 60 * 1000;

console/src/features/OnboardingWizard/hooks/index.ts

@@ -1 +1,5 @@
 export { useWizardState } from './useWizardState';
+export { usePrefetchNeonOnboardingTemplateData } from './usePrefetchNeonOnboardingTemplateData';
+export { useInstallMigration } from './useInstallMigration';
+export { useInstallMetadata } from './useInstallMetadata';
+export { useInstallTemplate } from './useInstallTemplate';

console/src/features/OnboardingWizard/hooks/useInstallMetadata.ts

@@ -0,0 +1,113 @@
+import { useCallback } from 'react';
+import Endpoints from '@/Endpoints';
+import { Api } from '@/hooks/apiUtils';
+import { isJsonString } from '@/components/Common/utils/jsUtils';
+import { HasuraMetadataV3 } from '@/metadata/types';
+import { MetadataResponse } from '@/features/MetadataAPI';
+import { useAppSelector } from '@/store';
+import { useMutation, useQuery } from 'react-query';
+import { staleTime } from './constants';
+import { fetchTemplateDataQueryFn, transformOldMetadata } from './utils';
+
+type MutationFnArgs = {
+  newMetadata: HasuraMetadataV3;
+  headers: Record<string, string>;
+};
+
+/**
+ * Mutation Function to install the metadata. Calls the `replace_metadata` api with the new
+ * metadata to be replaced.
+ */
+const installMetadataMutationFn = (args: MutationFnArgs) => {
+  const { newMetadata, headers } = args;
+  const payload = {
+    type: 'replace_metadata',
+    args: newMetadata,
+  };
+  return Api.post<Record<string, string>>({
+    url: Endpoints.metadata,
+    headers,
+    body: payload,
+  });
+};
+
+/**
+ * Hook to install metadata from a remote file containing hasura metadata. This will append the new metadata
+ * to the provided data source
+ * @returns A memoised function which can be called imperatively to apply the metadata
+ */
+export function useInstallMetadata(
+  dataSourceName: string,
+  metadataFileUrl: string,
+  onSuccessCb?: () => void,
+  onErrorCb?: (errorMsg?: string) => void
+): { updateMetadata: () => void } | { updateMetadata: undefined } {
+  const headers = useAppSelector(state => state.tables.dataHeaders);
+
+  // Fetch the metadata to be applied from remote file, or return from react-query cache if present
+  const {
+    data: templateMetadata,
+    isLoading,
+    isError,
+  } = useQuery(
+    metadataFileUrl,
+    () => fetchTemplateDataQueryFn<string>(metadataFileUrl, {}),
+    {
+      staleTime,
+    }
+  );
+
+  const mutation = useMutation(
+    (args: MutationFnArgs) => installMetadataMutationFn(args),
+    {
+      onSuccess: onSuccessCb,
+      onError: (error: Error) => {
+        if (onErrorCb) {
+          onErrorCb(error.message ?? 'Failed to apply metadata');
+        }
+      },
+    }
+  );
+
+  const oldMetadata = useAppSelector(state => state.metadata.metadataObject);
+
+  // only do a 'replace_metadata' call if we have the new metadata from the remote url, and current metadata is not null.
+  // otherwise `updateMetadata` will just return an empty function. In that case, error callbacks will have info on what went wrong.
+  const updateMetadata = useCallback(() => {
+    if (templateMetadata && oldMetadata) {
+      let templateMetadataJson: HasuraMetadataV3 | undefined;
+      if (isJsonString(templateMetadata)) {
+        templateMetadataJson = (
+          JSON.parse(templateMetadata) as MetadataResponse
+        )?.metadata;
+      }
+      if (templateMetadataJson) {
+        const transformedMetadata = transformOldMetadata(
+          oldMetadata,
+          templateMetadataJson,
+          dataSourceName
+        );
+
+        mutation.mutate({
+          newMetadata: transformedMetadata,
+          headers,
+        });
+      }
+    }
+  }, [oldMetadata, templateMetadata, headers, dataSourceName]);
+
+  if (isError) {
+    if (onErrorCb) {
+      onErrorCb(
+        `Failed to fetch metadata from the provided Url: ${metadataFileUrl}`
+      );
+    }
+    return { updateMetadata: undefined };
+  }
+
+  if (isLoading) {
+    return { updateMetadata: undefined };
+  }
+
+  return { updateMetadata };
+}

console/src/features/OnboardingWizard/hooks/useInstallMigration.ts

@@ -0,0 +1,93 @@
+import { useCallback } from 'react';
+import { getRunSqlQuery } from '@/components/Common/utils/v1QueryUtils';
+import Endpoints from '@/Endpoints';
+import { RunSQLResponse } from '@/features/DataSource';
+import { Api } from '@/hooks/apiUtils';
+import { useAppSelector } from '@/store';
+import { useMutation, useQuery } from 'react-query';
+import { fetchTemplateDataQueryFn } from './utils';
+import { staleTime } from './constants';
+
+type MutationFnArgs = {
+  sql: string;
+  source: string;
+  headers: Record<string, string>;
+};
+
+/**
+ * Mutation Function to install the migration. Calls the `run_sql` api (assuming postgres driver)
+ */
+const installMigrationMutationFn = (args: MutationFnArgs) => {
+  const { sql, source, headers } = args;
+  const sqlPayload = getRunSqlQuery(sql, source);
+  return Api.post<RunSQLResponse>({
+    url: Endpoints.query,
+    headers,
+    body: sqlPayload,
+  });
+};
+
+/**
+ * Hook to install migration from a remote file containing sql migrations.
+ * @returns A memoised function which can be called imperatively to apply the migrations
+ */
+export function useInstallMigration(
+  dataSourceName: string,
+  migrationFileUrl: string,
+  onSuccessCb?: () => void,
+  onErrorCb?: (errorMsg?: string) => void
+): { performMigration: () => void } | { performMigration: undefined } {
+  const headers = useAppSelector(state => state.tables.dataHeaders);
+
+  // Fetch the migration to be applied from remote file, or return from react-query cache if present
+  const {
+    data: migrationSQL,
+    isLoading,
+    isError,
+  } = useQuery(
+    migrationFileUrl,
+    () => fetchTemplateDataQueryFn<string>(migrationFileUrl, {}),
+    {
+      staleTime,
+    }
+  );
+
+  const mutation = useMutation(
+    (args: MutationFnArgs) => installMigrationMutationFn(args),
+    {
+      onSuccess: onSuccessCb,
+      onError: (error: Error) => {
+        if (onErrorCb) {
+          onErrorCb(error.message ?? 'Failed to apply migration');
+        }
+      },
+    }
+  );
+
+  // only do a 'run_sql' call if we have the migrations file data from the remote url.
+  // otherwise `performMigration` will just return an empty function. In that case, error callbacks will have info on what went wrong.
+  const performMigration = useCallback(() => {
+    if (migrationSQL) {
+      mutation.mutate({
+        sql: migrationSQL,
+        source: dataSourceName,
+        headers,
+      });
+    }
+  }, [dataSourceName, migrationSQL, headers]);
+
+  if (isError) {
+    if (onErrorCb) {
+      onErrorCb(
+        `Failed to fetch migration data from the provided Url: ${migrationFileUrl}`
+      );
+    }
+    return { performMigration: undefined };
+  }
+
+  if (isLoading) {
+    return { performMigration: undefined };
+  }
+
+  return { performMigration };
+}

console/src/features/OnboardingWizard/hooks/useInstallTemplate.ts

@@ -0,0 +1,48 @@
+import { useCallback } from 'react';
+import { useInstallMigration } from './useInstallMigration';
+import { NEON_METADATA_PATH, NEON_MIGRATIONS_PATH } from './constants';
+import { useInstallMetadata } from './useInstallMetadata';
+
+/**
+ * used to install the template, which is a combination of applying migrations,
+ * and then applying metadata to the specified data source.
+ * @return A memoised function which can be called imperatively to initiate the install.
+ */
+export function useInstallTemplate(
+  dataSourceName: string,
+  onSuccessCb: () => void,
+  onErrorCb: (errorMsg?: string) => void
+) {
+  // fetch the function to apply metadata
+  const { updateMetadata } = useInstallMetadata(
+    dataSourceName,
+    NEON_METADATA_PATH,
+    onSuccessCb,
+    onErrorCb
+  );
+
+  // fetch the function to apply migration
+  const { performMigration } = useInstallMigration(
+    dataSourceName,
+    NEON_MIGRATIONS_PATH,
+    // install metadata only if migrations has been applied successfully
+    () => {
+      if (updateMetadata) {
+        updateMetadata();
+      }
+    },
+    onErrorCb
+  );
+
+  const install = useCallback(() => {
+    // only do a template install if both `performMigration` and `updateMetadata` functions are defined.
+    // otherwise `install` will just return an empty function. In that case, error callbacks will have info on what went wrong.
+    if (performMigration && updateMetadata) {
+      // only `performMigration` is called while invoking `install`,
+      // which in turn calls the `updateMetadata` if migration application was successful.
+      performMigration();
+    }
+  }, [performMigration, updateMetadata]);
+
+  return { install };
+}

console/src/features/OnboardingWizard/hooks/usePrefetchNeonOnboardingTemplateData.ts

@@ -0,0 +1,33 @@
+import { useEffect } from 'react';
+import { useQueryClient } from 'react-query';
+import {
+  NEON_METADATA_PATH,
+  NEON_MIGRATIONS_PATH,
+  NEON_IMAGE_PATH,
+  NEON_QUERY_PATH,
+} from './constants';
+import { fetchTemplateDataQueryFn } from './utils';
+
+/**
+ * Prefetch migrations and metadata file contents for NEON onboarding. Use it to get data early
+ * so data is ready by the time we need it. For example this could be fired while Neon
+ * DB is being created.
+ */
+export const usePrefetchNeonOnboardingTemplateData = () => {
+  const queryClient = useQueryClient();
+
+  useEffect(() => {
+    queryClient.prefetchQuery(NEON_MIGRATIONS_PATH, () =>
+      fetchTemplateDataQueryFn(NEON_MIGRATIONS_PATH, {})
+    );
+    queryClient.prefetchQuery(NEON_METADATA_PATH, () =>
+      fetchTemplateDataQueryFn(NEON_METADATA_PATH, {})
+    );
+    queryClient.prefetchQuery(NEON_IMAGE_PATH, () =>
+      fetchTemplateDataQueryFn(NEON_IMAGE_PATH, {})
+    );
+    queryClient.prefetchQuery(NEON_QUERY_PATH, () =>
+      fetchTemplateDataQueryFn(NEON_QUERY_PATH, {})
+    );
+  }, []);
+};

console/src/features/OnboardingWizard/hooks/utils.ts

@@ -1,4 +1,6 @@
 import { ExperimentConfig } from '@/features/GrowthExperiments';
+import { Api } from '@/hooks/apiUtils';
+import { HasuraMetadataV3 } from '@/metadata/types';
 
 export function isExperimentActive(
   experimentsData: ExperimentConfig[],
@@ -25,3 +27,57 @@ export function shouldShowOnboarding(
   }
   return true;
 }
+
+/**
+ * Utility function to be used as a react query QueryFn, which does a `GET` request to
+ * fetch our requested object, and returns a promise.
+ */
+export function fetchTemplateDataQueryFn<
+  ResponseData,
+  TransformedData = ResponseData
+>(
+  dataUrl: string,
+  headers: Record<string, string>,
+  transformFn?: (data: ResponseData) => TransformedData
+) {
+  return Api.get<ResponseData, TransformedData>(
+    {
+      url: dataUrl,
+      headers,
+    },
+    transformFn
+  );
+}
+
+/**
+ * Utility function which merges the old and additional metadata objects
+ * to create a new metadata object and returns it.
+ */
+export const transformOldMetadata = (
+  oldMetadata: HasuraMetadataV3,
+  additionalMetadata: HasuraMetadataV3,
+  source: string
+) => {
+  const newMetadata: HasuraMetadataV3 = {
+    ...oldMetadata,
+    sources:
+      oldMetadata?.sources?.map(oldSource => {
+        if (oldSource.name !== source) {
+          return oldSource;
+        }
+        const metadataObject = additionalMetadata?.sources?.[0];
+        if (!metadataObject) {
+          return oldSource;
+        }
+        return {
+          ...oldSource,
+          tables: [...oldSource.tables, ...(metadataObject.tables ?? [])],
+          functions: [
+            ...(oldSource.functions ?? []),
+            ...(metadataObject.functions ?? []),
+          ],
+        };
+      }) ?? [],
+  };
+  return newMetadata;
+};