diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b28f515f48..e58d40539f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The order and collapsed state of columns is now persisted across page navigation - cli: template assets path in console HTML for unversioned builds - console: allow customising graphql field names for columns of views (close #3689) (#4255) - console: fix clone permission migrations (close #3985) (#4277) +- console: decouple data rows and count fetch in data browser to account for really large tables (close #3793) (#4269) - docs: add One-Click Render deployment guide (close #3683) (#4209) - server: reserved keywords in column references break parser (fix #3597) #3927 diff --git a/console/src/components/Common/Spinner/Spinner.js b/console/src/components/Common/Spinner/Spinner.js index 0e4dd4feb36..ab73cc8d7bf 100644 --- a/console/src/components/Common/Spinner/Spinner.js +++ b/console/src/components/Common/Spinner/Spinner.js @@ -1,10 +1,10 @@ import React from 'react'; -const Spinner = () => { +const Spinner = ({ className = '' }) => { const styles = require('./Spinner.scss'); return ( -
+
diff --git a/console/src/components/Common/TableCommon/DragFoldTable.js b/console/src/components/Common/TableCommon/DragFoldTable.js index 21c00862478..7a732b43969 100644 --- a/console/src/components/Common/TableCommon/DragFoldTable.js +++ b/console/src/components/Common/TableCommon/DragFoldTable.js @@ -1,9 +1,10 @@ /* eslint-disable */ -import React, { Component, Fragment } from 'react'; +import React, { Component } from 'react'; import ReactTable from 'react-table'; import 'react-table/react-table.css'; import FoldableHoc from './foldableTable'; + import { isObject, isNotDefined } from '../utils/jsUtils'; class DragFoldTable extends Component { diff --git a/console/src/components/Common/utils/v1QueryUtils.js b/console/src/components/Common/utils/v1QueryUtils.js index 665912c058d..d648fe8865c 100644 --- a/console/src/components/Common/utils/v1QueryUtils.js +++ b/console/src/components/Common/utils/v1QueryUtils.js @@ -279,3 +279,38 @@ export const resetMetadataQuery = { type: 'clear_metadata', args: {}, }; + +export const generateSelectQuery = ( + type, + tableDef, + { where, limit, offset, order_by, columns } +) => ({ + type, + args: { + columns, + where, + limit, + offset, + order_by, + table: tableDef, + }, +}); + +export const getFetchManualTriggersQuery = tableName => ({ + type: 'select', + args: { + table: { + name: 'event_triggers', + schema: 'hdb_catalog', + }, + columns: ['*'], + order_by: { + column: 'name', + type: 'asc', + nulls: 'last', + }, + where: { + table_name: tableName, + }, + }, +}); diff --git a/console/src/components/Services/Data/TableBrowseRows/FilterActions.js b/console/src/components/Services/Data/TableBrowseRows/FilterActions.js index ad0e4bcec77..a2cc7c39787 100644 --- a/console/src/components/Services/Data/TableBrowseRows/FilterActions.js +++ b/console/src/components/Services/Data/TableBrowseRows/FilterActions.js @@ -1,6 +1,6 @@ // import Endpoints, {globalCookiePolicy} from '../../Endpoints'; import { defaultCurFilter } from '../DataState'; -import { vMakeRequest } from './ViewActions'; +import { vMakeTableRequests } from './ViewActions'; import { Integers, Reals } from '../constants'; const LOADING = 'ViewTable/FilterQuery/LOADING'; @@ -108,7 +108,7 @@ const runQuery = tableSchema => { delete newQuery.order_by; } dispatch({ type: 'ViewTable/V_SET_QUERY_OPTS', queryStuff: newQuery }); - dispatch(vMakeRequest()); + dispatch(vMakeTableRequests()); }; }; diff --git a/console/src/components/Services/Data/TableBrowseRows/ViewActions.js b/console/src/components/Services/Data/TableBrowseRows/ViewActions.js index 0b968f1f369..4b10b987914 100644 --- a/console/src/components/Services/Data/TableBrowseRows/ViewActions.js +++ b/console/src/components/Services/Data/TableBrowseRows/ViewActions.js @@ -2,14 +2,21 @@ import { defaultViewState } from '../DataState'; import Endpoints, { globalCookiePolicy } from '../../../../Endpoints'; import requestAction from 'utils/requestAction'; import filterReducer from './FilterActions'; -import { findTableFromRel } from '../utils'; +import { findTableFromRel, getEstimateCountQuery } from '../utils'; import { showSuccessNotification, showErrorNotification, } from '../../Common/Notification'; import dataHeaders from '../Common/Headers'; import { getConfirmation } from '../../../Common/utils/jsUtils'; -import { getBulkDeleteQuery } from '../../../Common/utils/v1QueryUtils'; +import { + getBulkDeleteQuery, + generateSelectQuery, + getFetchManualTriggersQuery, + getDeleteQuery, + getRunSqlQuery, +} from '../../../Common/utils/v1QueryUtils'; +import { generateTableDef } from '../../../Common/utils/pgUtils'; /* ****************** View actions *************/ const V_SET_DEFAULTS = 'ViewTable/V_SET_DEFAULTS'; @@ -22,6 +29,8 @@ const V_REQUEST_PROGRESS = 'ViewTable/V_REQUEST_PROGRESS'; const V_EXPAND_ROW = 'ViewTable/V_EXPAND_ROW'; const V_COLLAPSE_ROW = 'ViewTable/V_COLLAPSE_ROW'; +const V_COUNT_REQUEST_SUCCESS = 'ViewTable/V_COUNT_REQUEST_SUCCESS'; + const FETCHING_MANUAL_TRIGGER = 'ViewTable/FETCHING_MANUAL_TRIGGER'; const FETCH_MANUAL_TRIGGER_SUCCESS = 'ViewTable/FETCH_MANUAL_TRIGGER_SUCCESS'; const FETCH_MANUAL_TRIGGER_FAIL = 'ViewTable/FETCH_MANUAL_TRIGGER_SUCCESS'; @@ -49,36 +58,26 @@ const vCollapseRow = () => ({ const vSetDefaults = () => ({ type: V_SET_DEFAULTS }); -const vMakeRequest = () => { +const vMakeRowsRequest = () => { return (dispatch, getState) => { - const state = getState(); + const { + currentTable: originalTable, + currentSchema, + view, + } = getState().tables; + const url = Endpoints.query; - const originalTable = getState().tables.currentTable; dispatch({ type: V_REQUEST_PROGRESS, data: true }); const requestBody = { type: 'bulk', args: [ - { - type: 'select', - args: { - ...state.tables.view.query, - table: { - name: state.tables.currentTable, - schema: getState().tables.currentSchema, - }, - }, - }, - { - type: 'count', - args: { - ...state.tables.view.query, - table: { - name: state.tables.currentTable, - schema: getState().tables.currentSchema, - }, - }, - }, + generateSelectQuery( + 'select', + generateTableDef(originalTable, currentSchema), + view.query + ), + getRunSqlQuery(getEstimateCountQuery(currentSchema, originalTable)), ], }; const options = { @@ -90,12 +89,14 @@ const vMakeRequest = () => { return dispatch(requestAction(url, options)).then( data => { const currentTable = getState().tables.currentTable; - if (originalTable === currentTable) { + + // in case table has changed before count load + if (currentTable === originalTable) { Promise.all([ dispatch({ type: V_REQUEST_SUCCESS, data: data[0], - count: data[1].count, + estimatedCount: data[1].result[1], }), dispatch({ type: V_REQUEST_PROGRESS, data: false }), ]); @@ -113,27 +114,58 @@ const vMakeRequest = () => { }; }; +const vMakeCountRequest = () => { + return (dispatch, getState) => { + const { + currentTable: originalTable, + currentSchema, + view, + } = getState().tables; + const url = Endpoints.query; + + const requestBody = generateSelectQuery( + 'count', + generateTableDef(originalTable, currentSchema), + view.query + ); + + const options = { + method: 'POST', + body: JSON.stringify(requestBody), + headers: dataHeaders(getState), + credentials: globalCookiePolicy, + }; + + return dispatch(requestAction(url, options)).then( + data => { + const currentTable = getState().tables.currentTable; + + // in case table has changed before count load + if (currentTable === originalTable) { + dispatch({ + type: V_COUNT_REQUEST_SUCCESS, + count: data.count, + }); + } + }, + error => { + dispatch( + showErrorNotification('Count query failed!', error.error, error) + ); + } + ); + }; +}; + +const vMakeTableRequests = () => dispatch => { + dispatch(vMakeRowsRequest()); + dispatch(vMakeCountRequest()); +}; + const fetchManualTriggers = tableName => { return (dispatch, getState) => { const url = Endpoints.getSchema; - const body = { - type: 'select', - args: { - table: { - name: 'event_triggers', - schema: 'hdb_catalog', - }, - columns: ['*'], - order_by: { - column: 'name', - type: 'asc', - nulls: 'last', - }, - where: { - table_name: tableName, - }, - }, - }; + const body = getFetchManualTriggersQuery(tableName); const options = { credentials: globalCookiePolicy, @@ -178,16 +210,12 @@ const deleteItem = pkClause => { const state = getState(); const url = Endpoints.query; - const reqBody = { - type: 'delete', - args: { - table: { - name: state.tables.currentTable, - schema: state.tables.currentSchema, - }, - where: pkClause, - }, - }; + const reqBody = getDeleteQuery( + pkClause, + state.tables.currentTable, + state.tables.currentSchema + ); + const options = { method: 'POST', body: JSON.stringify(reqBody), @@ -196,7 +224,7 @@ const deleteItem = pkClause => { }; dispatch(requestAction(url, options)).then( data => { - dispatch(vMakeRequest()); + dispatch(vMakeTableRequests()); dispatch( showSuccessNotification( 'Row deleted!', @@ -238,7 +266,7 @@ const deleteItems = pkClauses => { dispatch(requestAction(Endpoints.query, options)).then( data => { const affected = data.reduce((acc, d) => acc + d.affected_rows, 0); - dispatch(vMakeRequest()); + dispatch(vMakeTableRequests()); dispatch( showSuccessNotification('Rows deleted!', 'Affected rows: ' + affected) ); @@ -257,7 +285,7 @@ const vExpandRel = (path, relname, pk) => { // Modify the query (UI will automatically change) dispatch({ type: V_EXPAND_REL, path, relname, pk }); // Make a request - return dispatch(vMakeRequest()); + return dispatch(vMakeTableRequests()); }; }; const vCloseRel = (path, relname) => { @@ -265,7 +293,7 @@ const vCloseRel = (path, relname) => { // Modify the query (UI will automatically change) dispatch({ type: V_CLOSE_REL, path, relname }); // Make a request - return dispatch(vMakeRequest()); + return dispatch(vMakeTableRequests()); }; }; /* ************ helpers ************************/ @@ -543,9 +571,15 @@ const viewReducer = (tableName, currentSchema, schemas, viewState, action) => { ), }; case V_REQUEST_SUCCESS: - return { ...viewState, rows: action.data, count: action.count }; + return { + ...viewState, + rows: action.data, + estimatedCount: action.estimatedCount, + }; case V_REQUEST_PROGRESS: return { ...viewState, isProgressing: action.data }; + case V_COUNT_REQUEST_SUCCESS: + return { ...viewState, count: action.count }; case V_EXPAND_ROW: return { ...viewState, @@ -595,7 +629,6 @@ export default viewReducer; export { fetchManualTriggers, vSetDefaults, - vMakeRequest, vExpandRel, vCloseRel, vExpandRow, @@ -605,4 +638,5 @@ export { deleteItems, UPDATE_TRIGGER_ROW, UPDATE_TRIGGER_FUNCTION, + vMakeTableRequests, }; diff --git a/console/src/components/Services/Data/TableBrowseRows/ViewRows.js b/console/src/components/Services/Data/TableBrowseRows/ViewRows.js index 618a6e513b1..0e1d5c7e43b 100644 --- a/console/src/components/Services/Data/TableBrowseRows/ViewRows.js +++ b/console/src/components/Services/Data/TableBrowseRows/ViewRows.js @@ -913,6 +913,7 @@ const ViewRows = ({ if (curFilter.offset !== page * curFilter.limit) { dispatch(setOffset(page * curFilter.limit)); dispatch(runQuery(tableSchema)); + setSelectedRows([]); } }; @@ -921,6 +922,7 @@ const ViewRows = ({ dispatch(setLimit(size)); dispatch(setOffset(0)); dispatch(runQuery(tableSchema)); + setSelectedRows([]); } }; diff --git a/console/src/components/Services/Data/TableBrowseRows/ViewTable.js b/console/src/components/Services/Data/TableBrowseRows/ViewTable.js index 5ccd0aca6d0..8cb663fa2fa 100644 --- a/console/src/components/Services/Data/TableBrowseRows/ViewTable.js +++ b/console/src/components/Services/Data/TableBrowseRows/ViewTable.js @@ -2,17 +2,18 @@ import PropTypes from 'prop-types'; import React, { Component } from 'react'; import { vSetDefaults, - vMakeRequest, // vExpandHeading, fetchManualTriggers, UPDATE_TRIGGER_ROW, UPDATE_TRIGGER_FUNCTION, + vMakeTableRequests, } from './ViewActions'; import { setTable } from '../DataActions'; import TableHeader from '../TableCommon/TableHeader'; import ViewRows from './ViewRows'; import { NotFoundError } from '../../../Error/PageNotFound'; +import { exists } from '../../../Common/utils/jsUtils'; /* const genHeadings = headings => { @@ -47,6 +48,7 @@ const genHeadings = headings => { throw 'Incomplete pattern match'; // eslint-disable-line no-throw-literal }; + const genRow = (row, headings) => { if (headings.length === 0) { return []; @@ -94,7 +96,7 @@ class ViewTable extends Component { Promise.all([ dispatch(setTable(tableName)), dispatch(vSetDefaults(tableName)), - dispatch(vMakeRequest()), + dispatch(vMakeTableRequests()), dispatch(fetchManualTriggers(tableName)), ]); } @@ -162,6 +164,7 @@ class ViewTable extends Component { triggeredRow, triggeredFunction, location, + estimatedCount, } = this.props; // check if table exists @@ -197,7 +200,7 @@ class ViewTable extends Component { lastSuccess={lastSuccess} schemas={schemas} curDepth={0} - count={count} + count={exists(count) ? count : estimatedCount} dispatch={dispatch} expandedRow={expandedRow} manualTriggers={manualTriggers} diff --git a/console/src/components/Services/Data/utils.js b/console/src/components/Services/Data/utils.js index 99993f52f58..c73c596956b 100644 --- a/console/src/components/Services/Data/utils.js +++ b/console/src/components/Services/Data/utils.js @@ -655,3 +655,15 @@ const postgresFunctionTester = /.*\(\)$/gm; export const isPostgresFunction = str => new RegExp(postgresFunctionTester).test(str); + +export const getEstimateCountQuery = (schemaName, tableName) => { + return ` +SELECT + reltuples::BIGINT +FROM + pg_class +WHERE + oid = (quote_ident('${schemaName}') || '.' || quote_ident('${tableName}'))::regclass::oid + AND relname = '${tableName}'; +`; +};