console: decouple row fetch and count queries in data browser (close #3793) (#4269)

This commit is contained in:
Aleksandra Sikora 2020-04-07 13:17:13 +02:00 committed by GitHub
parent de41438e8a
commit f615efd37d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 157 additions and 69 deletions

View File

@ -14,6 +14,7 @@ The order and collapsed state of columns is now persisted across page navigation
- cli: template assets path in console HTML for unversioned builds
- console: allow customising graphql field names for columns of views (close #3689) (#4255)
- console: fix clone permission migrations (close #3985) (#4277)
- console: decouple data rows and count fetch in data browser to account for really large tables (close #3793) (#4269)
- docs: add One-Click Render deployment guide (close #3683) (#4209)
- server: reserved keywords in column references break parser (fix #3597) #3927

View File

@ -1,10 +1,10 @@
import React from 'react';
const Spinner = () => {
const Spinner = ({ className = '' }) => {
const styles = require('./Spinner.scss');
return (
<div className={styles.sk_circle}>
<div className={styles.sk_circle + ' ' + className}>
<div className={styles.sk_circle1 + ' ' + styles.sk_child} />
<div className={styles.sk_circle2 + ' ' + styles.sk_child} />
<div className={styles.sk_circle3 + ' ' + styles.sk_child} />

View File

@ -1,9 +1,10 @@
/* eslint-disable */
import React, { Component, Fragment } from 'react';
import React, { Component } from 'react';
import ReactTable from 'react-table';
import 'react-table/react-table.css';
import FoldableHoc from './foldableTable';
import { isObject, isNotDefined } from '../utils/jsUtils';
class DragFoldTable extends Component {

View File

@ -279,3 +279,38 @@ export const resetMetadataQuery = {
type: 'clear_metadata',
args: {},
};
export const generateSelectQuery = (
type,
tableDef,
{ where, limit, offset, order_by, columns }
) => ({
type,
args: {
columns,
where,
limit,
offset,
order_by,
table: tableDef,
},
});
export const getFetchManualTriggersQuery = tableName => ({
type: 'select',
args: {
table: {
name: 'event_triggers',
schema: 'hdb_catalog',
},
columns: ['*'],
order_by: {
column: 'name',
type: 'asc',
nulls: 'last',
},
where: {
table_name: tableName,
},
},
});

View File

@ -1,6 +1,6 @@
// import Endpoints, {globalCookiePolicy} from '../../Endpoints';
import { defaultCurFilter } from '../DataState';
import { vMakeRequest } from './ViewActions';
import { vMakeTableRequests } from './ViewActions';
import { Integers, Reals } from '../constants';
const LOADING = 'ViewTable/FilterQuery/LOADING';
@ -108,7 +108,7 @@ const runQuery = tableSchema => {
delete newQuery.order_by;
}
dispatch({ type: 'ViewTable/V_SET_QUERY_OPTS', queryStuff: newQuery });
dispatch(vMakeRequest());
dispatch(vMakeTableRequests());
};
};

View File

@ -2,14 +2,21 @@ import { defaultViewState } from '../DataState';
import Endpoints, { globalCookiePolicy } from '../../../../Endpoints';
import requestAction from 'utils/requestAction';
import filterReducer from './FilterActions';
import { findTableFromRel } from '../utils';
import { findTableFromRel, getEstimateCountQuery } from '../utils';
import {
showSuccessNotification,
showErrorNotification,
} from '../../Common/Notification';
import dataHeaders from '../Common/Headers';
import { getConfirmation } from '../../../Common/utils/jsUtils';
import { getBulkDeleteQuery } from '../../../Common/utils/v1QueryUtils';
import {
getBulkDeleteQuery,
generateSelectQuery,
getFetchManualTriggersQuery,
getDeleteQuery,
getRunSqlQuery,
} from '../../../Common/utils/v1QueryUtils';
import { generateTableDef } from '../../../Common/utils/pgUtils';
/* ****************** View actions *************/
const V_SET_DEFAULTS = 'ViewTable/V_SET_DEFAULTS';
@ -22,6 +29,8 @@ const V_REQUEST_PROGRESS = 'ViewTable/V_REQUEST_PROGRESS';
const V_EXPAND_ROW = 'ViewTable/V_EXPAND_ROW';
const V_COLLAPSE_ROW = 'ViewTable/V_COLLAPSE_ROW';
const V_COUNT_REQUEST_SUCCESS = 'ViewTable/V_COUNT_REQUEST_SUCCESS';
const FETCHING_MANUAL_TRIGGER = 'ViewTable/FETCHING_MANUAL_TRIGGER';
const FETCH_MANUAL_TRIGGER_SUCCESS = 'ViewTable/FETCH_MANUAL_TRIGGER_SUCCESS';
const FETCH_MANUAL_TRIGGER_FAIL = 'ViewTable/FETCH_MANUAL_TRIGGER_SUCCESS';
@ -49,36 +58,26 @@ const vCollapseRow = () => ({
const vSetDefaults = () => ({ type: V_SET_DEFAULTS });
const vMakeRequest = () => {
const vMakeRowsRequest = () => {
return (dispatch, getState) => {
const state = getState();
const {
currentTable: originalTable,
currentSchema,
view,
} = getState().tables;
const url = Endpoints.query;
const originalTable = getState().tables.currentTable;
dispatch({ type: V_REQUEST_PROGRESS, data: true });
const requestBody = {
type: 'bulk',
args: [
{
type: 'select',
args: {
...state.tables.view.query,
table: {
name: state.tables.currentTable,
schema: getState().tables.currentSchema,
},
},
},
{
type: 'count',
args: {
...state.tables.view.query,
table: {
name: state.tables.currentTable,
schema: getState().tables.currentSchema,
},
},
},
generateSelectQuery(
'select',
generateTableDef(originalTable, currentSchema),
view.query
),
getRunSqlQuery(getEstimateCountQuery(currentSchema, originalTable)),
],
};
const options = {
@ -90,12 +89,14 @@ const vMakeRequest = () => {
return dispatch(requestAction(url, options)).then(
data => {
const currentTable = getState().tables.currentTable;
if (originalTable === currentTable) {
// in case table has changed before count load
if (currentTable === originalTable) {
Promise.all([
dispatch({
type: V_REQUEST_SUCCESS,
data: data[0],
count: data[1].count,
estimatedCount: data[1].result[1],
}),
dispatch({ type: V_REQUEST_PROGRESS, data: false }),
]);
@ -113,27 +114,58 @@ const vMakeRequest = () => {
};
};
const vMakeCountRequest = () => {
return (dispatch, getState) => {
const {
currentTable: originalTable,
currentSchema,
view,
} = getState().tables;
const url = Endpoints.query;
const requestBody = generateSelectQuery(
'count',
generateTableDef(originalTable, currentSchema),
view.query
);
const options = {
method: 'POST',
body: JSON.stringify(requestBody),
headers: dataHeaders(getState),
credentials: globalCookiePolicy,
};
return dispatch(requestAction(url, options)).then(
data => {
const currentTable = getState().tables.currentTable;
// in case table has changed before count load
if (currentTable === originalTable) {
dispatch({
type: V_COUNT_REQUEST_SUCCESS,
count: data.count,
});
}
},
error => {
dispatch(
showErrorNotification('Count query failed!', error.error, error)
);
}
);
};
};
const vMakeTableRequests = () => dispatch => {
dispatch(vMakeRowsRequest());
dispatch(vMakeCountRequest());
};
const fetchManualTriggers = tableName => {
return (dispatch, getState) => {
const url = Endpoints.getSchema;
const body = {
type: 'select',
args: {
table: {
name: 'event_triggers',
schema: 'hdb_catalog',
},
columns: ['*'],
order_by: {
column: 'name',
type: 'asc',
nulls: 'last',
},
where: {
table_name: tableName,
},
},
};
const body = getFetchManualTriggersQuery(tableName);
const options = {
credentials: globalCookiePolicy,
@ -178,16 +210,12 @@ const deleteItem = pkClause => {
const state = getState();
const url = Endpoints.query;
const reqBody = {
type: 'delete',
args: {
table: {
name: state.tables.currentTable,
schema: state.tables.currentSchema,
},
where: pkClause,
},
};
const reqBody = getDeleteQuery(
pkClause,
state.tables.currentTable,
state.tables.currentSchema
);
const options = {
method: 'POST',
body: JSON.stringify(reqBody),
@ -196,7 +224,7 @@ const deleteItem = pkClause => {
};
dispatch(requestAction(url, options)).then(
data => {
dispatch(vMakeRequest());
dispatch(vMakeTableRequests());
dispatch(
showSuccessNotification(
'Row deleted!',
@ -238,7 +266,7 @@ const deleteItems = pkClauses => {
dispatch(requestAction(Endpoints.query, options)).then(
data => {
const affected = data.reduce((acc, d) => acc + d.affected_rows, 0);
dispatch(vMakeRequest());
dispatch(vMakeTableRequests());
dispatch(
showSuccessNotification('Rows deleted!', 'Affected rows: ' + affected)
);
@ -257,7 +285,7 @@ const vExpandRel = (path, relname, pk) => {
// Modify the query (UI will automatically change)
dispatch({ type: V_EXPAND_REL, path, relname, pk });
// Make a request
return dispatch(vMakeRequest());
return dispatch(vMakeTableRequests());
};
};
const vCloseRel = (path, relname) => {
@ -265,7 +293,7 @@ const vCloseRel = (path, relname) => {
// Modify the query (UI will automatically change)
dispatch({ type: V_CLOSE_REL, path, relname });
// Make a request
return dispatch(vMakeRequest());
return dispatch(vMakeTableRequests());
};
};
/* ************ helpers ************************/
@ -543,9 +571,15 @@ const viewReducer = (tableName, currentSchema, schemas, viewState, action) => {
),
};
case V_REQUEST_SUCCESS:
return { ...viewState, rows: action.data, count: action.count };
return {
...viewState,
rows: action.data,
estimatedCount: action.estimatedCount,
};
case V_REQUEST_PROGRESS:
return { ...viewState, isProgressing: action.data };
case V_COUNT_REQUEST_SUCCESS:
return { ...viewState, count: action.count };
case V_EXPAND_ROW:
return {
...viewState,
@ -595,7 +629,6 @@ export default viewReducer;
export {
fetchManualTriggers,
vSetDefaults,
vMakeRequest,
vExpandRel,
vCloseRel,
vExpandRow,
@ -605,4 +638,5 @@ export {
deleteItems,
UPDATE_TRIGGER_ROW,
UPDATE_TRIGGER_FUNCTION,
vMakeTableRequests,
};

View File

@ -913,6 +913,7 @@ const ViewRows = ({
if (curFilter.offset !== page * curFilter.limit) {
dispatch(setOffset(page * curFilter.limit));
dispatch(runQuery(tableSchema));
setSelectedRows([]);
}
};
@ -921,6 +922,7 @@ const ViewRows = ({
dispatch(setLimit(size));
dispatch(setOffset(0));
dispatch(runQuery(tableSchema));
setSelectedRows([]);
}
};

View File

@ -2,17 +2,18 @@ import PropTypes from 'prop-types';
import React, { Component } from 'react';
import {
vSetDefaults,
vMakeRequest,
// vExpandHeading,
fetchManualTriggers,
UPDATE_TRIGGER_ROW,
UPDATE_TRIGGER_FUNCTION,
vMakeTableRequests,
} from './ViewActions';
import { setTable } from '../DataActions';
import TableHeader from '../TableCommon/TableHeader';
import ViewRows from './ViewRows';
import { NotFoundError } from '../../../Error/PageNotFound';
import { exists } from '../../../Common/utils/jsUtils';
/*
const genHeadings = headings => {
@ -47,6 +48,7 @@ const genHeadings = headings => {
throw 'Incomplete pattern match'; // eslint-disable-line no-throw-literal
};
const genRow = (row, headings) => {
if (headings.length === 0) {
return [];
@ -94,7 +96,7 @@ class ViewTable extends Component {
Promise.all([
dispatch(setTable(tableName)),
dispatch(vSetDefaults(tableName)),
dispatch(vMakeRequest()),
dispatch(vMakeTableRequests()),
dispatch(fetchManualTriggers(tableName)),
]);
}
@ -162,6 +164,7 @@ class ViewTable extends Component {
triggeredRow,
triggeredFunction,
location,
estimatedCount,
} = this.props;
// check if table exists
@ -197,7 +200,7 @@ class ViewTable extends Component {
lastSuccess={lastSuccess}
schemas={schemas}
curDepth={0}
count={count}
count={exists(count) ? count : estimatedCount}
dispatch={dispatch}
expandedRow={expandedRow}
manualTriggers={manualTriggers}

View File

@ -655,3 +655,15 @@ const postgresFunctionTester = /.*\(\)$/gm;
export const isPostgresFunction = str =>
new RegExp(postgresFunctionTester).test(str);
export const getEstimateCountQuery = (schemaName, tableName) => {
return `
SELECT
reltuples::BIGINT
FROM
pg_class
WHERE
oid = (quote_ident('${schemaName}') || '.' || quote_ident('${tableName}'))::regclass::oid
AND relname = '${tableName}';
`;
};