console: decouple row fetch and count queries in data browser (close #3793) (#4269)

CHANGELOG.md

@@ -14,6 +14,7 @@ The order and collapsed state of columns is now persisted across page navigation
 - cli: template assets path in console HTML for unversioned builds
 - console: allow customising graphql field names for columns of views (close #3689) (#4255)
 - console: fix clone permission migrations (close #3985) (#4277)
+- console: decouple data rows and count fetch in data browser to account for really large tables (close #3793) (#4269)
 - docs: add One-Click Render deployment guide (close #3683) (#4209)
 - server: reserved keywords in column references break parser (fix #3597) #3927
 

console/src/components/Common/Spinner/Spinner.js

@@ -1,10 +1,10 @@
 import React from 'react';
 
-const Spinner = () => {
+const Spinner = ({ className = '' }) => {
   const styles = require('./Spinner.scss');
 
   return (
-    <div className={styles.sk_circle}>
+    <div className={styles.sk_circle + ' ' + className}>
       <div className={styles.sk_circle1 + ' ' + styles.sk_child} />
       <div className={styles.sk_circle2 + ' ' + styles.sk_child} />
       <div className={styles.sk_circle3 + ' ' + styles.sk_child} />

console/src/components/Common/TableCommon/DragFoldTable.js

@@ -1,9 +1,10 @@
 /* eslint-disable */
 
-import React, { Component, Fragment } from 'react';
+import React, { Component } from 'react';
 import ReactTable from 'react-table';
 import 'react-table/react-table.css';
 import FoldableHoc from './foldableTable';
+
 import { isObject, isNotDefined } from '../utils/jsUtils';
 
 class DragFoldTable extends Component {

console/src/components/Common/utils/v1QueryUtils.js

@@ -279,3 +279,38 @@ export const resetMetadataQuery = {
   type: 'clear_metadata',
   args: {},
 };
+
+export const generateSelectQuery = (
+  type,
+  tableDef,
+  { where, limit, offset, order_by, columns }
+) => ({
+  type,
+  args: {
+    columns,
+    where,
+    limit,
+    offset,
+    order_by,
+    table: tableDef,
+  },
+});
+
+export const getFetchManualTriggersQuery = tableName => ({
+  type: 'select',
+  args: {
+    table: {
+      name: 'event_triggers',
+      schema: 'hdb_catalog',
+    },
+    columns: ['*'],
+    order_by: {
+      column: 'name',
+      type: 'asc',
+      nulls: 'last',
+    },
+    where: {
+      table_name: tableName,
+    },
+  },
+});

console/src/components/Services/Data/TableBrowseRows/FilterActions.js

@@ -1,6 +1,6 @@
 // import Endpoints, {globalCookiePolicy} from '../../Endpoints';
 import { defaultCurFilter } from '../DataState';
-import { vMakeRequest } from './ViewActions';
+import { vMakeTableRequests } from './ViewActions';
 import { Integers, Reals } from '../constants';
 
 const LOADING = 'ViewTable/FilterQuery/LOADING';
@@ -108,7 +108,7 @@ const runQuery = tableSchema => {
       delete newQuery.order_by;
     }
     dispatch({ type: 'ViewTable/V_SET_QUERY_OPTS', queryStuff: newQuery });
-    dispatch(vMakeRequest());
+    dispatch(vMakeTableRequests());
   };
 };
 

console/src/components/Services/Data/TableBrowseRows/ViewActions.js

@@ -2,14 +2,21 @@ import { defaultViewState } from '../DataState';
 import Endpoints, { globalCookiePolicy } from '../../../../Endpoints';
 import requestAction from 'utils/requestAction';
 import filterReducer from './FilterActions';
-import { findTableFromRel } from '../utils';
+import { findTableFromRel, getEstimateCountQuery } from '../utils';
 import {
   showSuccessNotification,
   showErrorNotification,
 } from '../../Common/Notification';
 import dataHeaders from '../Common/Headers';
 import { getConfirmation } from '../../../Common/utils/jsUtils';
-import { getBulkDeleteQuery } from '../../../Common/utils/v1QueryUtils';
+import {
+  getBulkDeleteQuery,
+  generateSelectQuery,
+  getFetchManualTriggersQuery,
+  getDeleteQuery,
+  getRunSqlQuery,
+} from '../../../Common/utils/v1QueryUtils';
+import { generateTableDef } from '../../../Common/utils/pgUtils';
 
 /* ****************** View actions *************/
 const V_SET_DEFAULTS = 'ViewTable/V_SET_DEFAULTS';
@@ -22,6 +29,8 @@ const V_REQUEST_PROGRESS = 'ViewTable/V_REQUEST_PROGRESS';
 const V_EXPAND_ROW = 'ViewTable/V_EXPAND_ROW';
 const V_COLLAPSE_ROW = 'ViewTable/V_COLLAPSE_ROW';
 
+const V_COUNT_REQUEST_SUCCESS = 'ViewTable/V_COUNT_REQUEST_SUCCESS';
+
 const FETCHING_MANUAL_TRIGGER = 'ViewTable/FETCHING_MANUAL_TRIGGER';
 const FETCH_MANUAL_TRIGGER_SUCCESS = 'ViewTable/FETCH_MANUAL_TRIGGER_SUCCESS';
 const FETCH_MANUAL_TRIGGER_FAIL = 'ViewTable/FETCH_MANUAL_TRIGGER_SUCCESS';
@@ -49,36 +58,26 @@ const vCollapseRow = () => ({
 
 const vSetDefaults = () => ({ type: V_SET_DEFAULTS });
 
-const vMakeRequest = () => {
+const vMakeRowsRequest = () => {
   return (dispatch, getState) => {
-    const state = getState();
+    const {
+      currentTable: originalTable,
+      currentSchema,
+      view,
+    } = getState().tables;
+
     const url = Endpoints.query;
-    const originalTable = getState().tables.currentTable;
     dispatch({ type: V_REQUEST_PROGRESS, data: true });
 
     const requestBody = {
       type: 'bulk',
       args: [
-        {
+        generateSelectQuery(
-          type: 'select',
+          'select',
-          args: {
+          generateTableDef(originalTable, currentSchema),
-            ...state.tables.view.query,
+          view.query
-            table: {
+        ),
-              name: state.tables.currentTable,
+        getRunSqlQuery(getEstimateCountQuery(currentSchema, originalTable)),
-              schema: getState().tables.currentSchema,
-            },
-          },
-        },
-        {
-          type: 'count',
-          args: {
-            ...state.tables.view.query,
-            table: {
-              name: state.tables.currentTable,
-              schema: getState().tables.currentSchema,
-            },
-          },
-        },
       ],
     };
     const options = {
@@ -90,12 +89,14 @@ const vMakeRequest = () => {
     return dispatch(requestAction(url, options)).then(
       data => {
         const currentTable = getState().tables.currentTable;
-        if (originalTable === currentTable) {
+
+        // in case table has changed before count load
+        if (currentTable === originalTable) {
           Promise.all([
             dispatch({
               type: V_REQUEST_SUCCESS,
               data: data[0],
-              count: data[1].count,
+              estimatedCount: data[1].result[1],
             }),
             dispatch({ type: V_REQUEST_PROGRESS, data: false }),
           ]);
@@ -113,27 +114,58 @@ const vMakeRequest = () => {
   };
 };
 
+const vMakeCountRequest = () => {
+  return (dispatch, getState) => {
+    const {
+      currentTable: originalTable,
+      currentSchema,
+      view,
+    } = getState().tables;
+    const url = Endpoints.query;
+
+    const requestBody = generateSelectQuery(
+      'count',
+      generateTableDef(originalTable, currentSchema),
+      view.query
+    );
+
+    const options = {
+      method: 'POST',
+      body: JSON.stringify(requestBody),
+      headers: dataHeaders(getState),
+      credentials: globalCookiePolicy,
+    };
+
+    return dispatch(requestAction(url, options)).then(
+      data => {
+        const currentTable = getState().tables.currentTable;
+
+        // in case table has changed before count load
+        if (currentTable === originalTable) {
+          dispatch({
+            type: V_COUNT_REQUEST_SUCCESS,
+            count: data.count,
+          });
+        }
+      },
+      error => {
+        dispatch(
+          showErrorNotification('Count query failed!', error.error, error)
+        );
+      }
+    );
+  };
+};
+
+const vMakeTableRequests = () => dispatch => {
+  dispatch(vMakeRowsRequest());
+  dispatch(vMakeCountRequest());
+};
+
 const fetchManualTriggers = tableName => {
   return (dispatch, getState) => {
     const url = Endpoints.getSchema;
-    const body = {
+    const body = getFetchManualTriggersQuery(tableName);
-      type: 'select',
-      args: {
-        table: {
-          name: 'event_triggers',
-          schema: 'hdb_catalog',
-        },
-        columns: ['*'],
-        order_by: {
-          column: 'name',
-          type: 'asc',
-          nulls: 'last',
-        },
-        where: {
-          table_name: tableName,
-        },
-      },
-    };
 
     const options = {
       credentials: globalCookiePolicy,
@@ -178,16 +210,12 @@ const deleteItem = pkClause => {
     const state = getState();
 
     const url = Endpoints.query;
-    const reqBody = {
+    const reqBody = getDeleteQuery(
-      type: 'delete',
+      pkClause,
-      args: {
+      state.tables.currentTable,
-        table: {
+      state.tables.currentSchema
-          name: state.tables.currentTable,
+    );
-          schema: state.tables.currentSchema,
+
-        },
-        where: pkClause,
-      },
-    };
     const options = {
       method: 'POST',
       body: JSON.stringify(reqBody),
@@ -196,7 +224,7 @@ const deleteItem = pkClause => {
     };
     dispatch(requestAction(url, options)).then(
       data => {
-        dispatch(vMakeRequest());
+        dispatch(vMakeTableRequests());
         dispatch(
           showSuccessNotification(
             'Row deleted!',
@@ -238,7 +266,7 @@ const deleteItems = pkClauses => {
     dispatch(requestAction(Endpoints.query, options)).then(
       data => {
         const affected = data.reduce((acc, d) => acc + d.affected_rows, 0);
-        dispatch(vMakeRequest());
+        dispatch(vMakeTableRequests());
         dispatch(
           showSuccessNotification('Rows deleted!', 'Affected rows: ' + affected)
         );
@@ -257,7 +285,7 @@ const vExpandRel = (path, relname, pk) => {
     // Modify the query (UI will automatically change)
     dispatch({ type: V_EXPAND_REL, path, relname, pk });
     // Make a request
-    return dispatch(vMakeRequest());
+    return dispatch(vMakeTableRequests());
   };
 };
 const vCloseRel = (path, relname) => {
@@ -265,7 +293,7 @@ const vCloseRel = (path, relname) => {
     // Modify the query (UI will automatically change)
     dispatch({ type: V_CLOSE_REL, path, relname });
     // Make a request
-    return dispatch(vMakeRequest());
+    return dispatch(vMakeTableRequests());
   };
 };
 /* ************ helpers ************************/
@@ -543,9 +571,15 @@ const viewReducer = (tableName, currentSchema, schemas, viewState, action) => {
         ),
       };
     case V_REQUEST_SUCCESS:
-      return { ...viewState, rows: action.data, count: action.count };
+      return {
+        ...viewState,
+        rows: action.data,
+        estimatedCount: action.estimatedCount,
+      };
     case V_REQUEST_PROGRESS:
       return { ...viewState, isProgressing: action.data };
+    case V_COUNT_REQUEST_SUCCESS:
+      return { ...viewState, count: action.count };
     case V_EXPAND_ROW:
       return {
         ...viewState,
@@ -595,7 +629,6 @@ export default viewReducer;
 export {
   fetchManualTriggers,
   vSetDefaults,
-  vMakeRequest,
   vExpandRel,
   vCloseRel,
   vExpandRow,
@@ -605,4 +638,5 @@ export {
   deleteItems,
   UPDATE_TRIGGER_ROW,
   UPDATE_TRIGGER_FUNCTION,
+  vMakeTableRequests,
 };

console/src/components/Services/Data/TableBrowseRows/ViewRows.js

@@ -913,6 +913,7 @@ const ViewRows = ({
       if (curFilter.offset !== page * curFilter.limit) {
         dispatch(setOffset(page * curFilter.limit));
         dispatch(runQuery(tableSchema));
+        setSelectedRows([]);
       }
     };
 
@@ -921,6 +922,7 @@ const ViewRows = ({
         dispatch(setLimit(size));
         dispatch(setOffset(0));
         dispatch(runQuery(tableSchema));
+        setSelectedRows([]);
       }
     };
 

console/src/components/Services/Data/TableBrowseRows/ViewTable.js

@@ -2,17 +2,18 @@ import PropTypes from 'prop-types';
 import React, { Component } from 'react';
 import {
   vSetDefaults,
-  vMakeRequest,
   // vExpandHeading,
   fetchManualTriggers,
   UPDATE_TRIGGER_ROW,
   UPDATE_TRIGGER_FUNCTION,
+  vMakeTableRequests,
 } from './ViewActions';
 import { setTable } from '../DataActions';
 import TableHeader from '../TableCommon/TableHeader';
 import ViewRows from './ViewRows';
 
 import { NotFoundError } from '../../../Error/PageNotFound';
+import { exists } from '../../../Common/utils/jsUtils';
 
 /*
 const genHeadings = headings => {
@@ -47,6 +48,7 @@ const genHeadings = headings => {
   throw 'Incomplete pattern match'; // eslint-disable-line no-throw-literal
 };
 
+
 const genRow = (row, headings) => {
   if (headings.length === 0) {
     return [];
@@ -94,7 +96,7 @@ class ViewTable extends Component {
     Promise.all([
       dispatch(setTable(tableName)),
       dispatch(vSetDefaults(tableName)),
-      dispatch(vMakeRequest()),
+      dispatch(vMakeTableRequests()),
       dispatch(fetchManualTriggers(tableName)),
     ]);
   }
@@ -162,6 +164,7 @@ class ViewTable extends Component {
       triggeredRow,
       triggeredFunction,
       location,
+      estimatedCount,
     } = this.props;
 
     // check if table exists
@@ -197,7 +200,7 @@ class ViewTable extends Component {
         lastSuccess={lastSuccess}
         schemas={schemas}
         curDepth={0}
-        count={count}
+        count={exists(count) ? count : estimatedCount}
         dispatch={dispatch}
         expandedRow={expandedRow}
         manualTriggers={manualTriggers}

console/src/components/Services/Data/utils.js

@@ -655,3 +655,15 @@ const postgresFunctionTester = /.*\(\)$/gm;
 
 export const isPostgresFunction = str =>
   new RegExp(postgresFunctionTester).test(str);
+
+export const getEstimateCountQuery = (schemaName, tableName) => {
+  return `
+SELECT
+  reltuples::BIGINT
+FROM
+  pg_class
+WHERE
+  oid = (quote_ident('${schemaName}') || '.' || quote_ident('${tableName}'))::regclass::oid
+  AND relname = '${tableName}';
+`;
+};