console: BigQuery filters not working

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/6115
GitOrigin-RevId: 13866a5eaa12fbba33a06e9b38c496e46293d9be
This commit is contained in:
Luca Restagno 2022-09-30 12:44:10 +02:00 committed by hasura-bot
parent 8c725acac8
commit 736a8e25ce
18 changed files with 176 additions and 310 deletions

View File

@ -181,7 +181,12 @@ type ColumnExpValue = Record<validOperators | string, any>;
type ColumnExp = Record<string, ColumnExpValue>;
type BoolExp = AndExp | OrExp | NotExp | ColumnExp;
export type WhereClause = BoolExp | Record<string, any>;
type ColumnName = string;
type Operator = string;
type ClauseValue = string | string[] | number | number[];
type OperatorValue = Record<Operator, ClauseValue>;
export type WhereClauseRecord = Record<ColumnName, OperatorValue>;
export type WhereClause = BoolExp | WhereClauseRecord;
export const makeOrderBy = (
column: string,

View File

@ -1013,11 +1013,13 @@ const ViewRows = props => {
return (
<div className={isVisible ? '' : 'hide '}>
{isFilterSectionVisible && (
<FilterSectionContainer
dataSourceName={currentSource}
table={{ schema: tableSchema.table_schema, name: curTableName }}
onRunQuery={newUserQuery => setUserQuery(newUserQuery)}
/>
<div className="mt-4">
<FilterSectionContainer
dataSourceName={currentSource}
table={{ schema: tableSchema.table_schema, name: curTableName }}
onRunQuery={newUserQuery => setUserQuery(newUserQuery)}
/>
</div>
)}
<div className="w-fit ml-0 mt-md">
{getSelectedRowsSection()}

View File

@ -1,7 +1,7 @@
import { CustomRootFields, TableConfig } from './../../metadata/types';
import {
OrderBy,
WhereClause,
WhereClauseRecord,
} from '../../components/Common/utils/v1QueryUtils';
import { ReduxState } from '../../types';
import { BaseTableColumn, Relationship, Table } from '../types';
@ -82,25 +82,29 @@ export const RqlToGraphQlOp = (op: string) => {
};
export const generateWhereClauseQueryString = (
wheres: WhereClause[],
wheres: WhereClauseRecord[],
columnTypeInfo: BaseTableColumn[],
tableConfiguration: TableConfig,
getFormattedValue: (type: string, value: any) => string | number | undefined
): string | null => {
const columnConfig = tableConfiguration?.column_config ?? {};
const whereClausesArr = wheres.map((i: Record<string, any>) => {
const columnName = Object.keys(i)[0];
const RqlOperator = Object.keys(i[columnName])[0];
const value = i[columnName][RqlOperator];
const type = columnTypeInfo?.find(
const whereClausesArr = wheres.map(whereClause => {
const columnName = Object.keys(whereClause)[0];
const rqlOperator = Object.keys(whereClause[columnName])[0];
const value = whereClause[columnName][rqlOperator];
const currentColumnInfo = columnTypeInfo?.find(
c => c.column_name === columnName
)?.data_type_name;
return `${
columnConfig[columnName]?.custom_name ?? columnName
}: {${RqlToGraphQlOp(RqlOperator)}: ${getFormattedValue(
type || 'varchar',
value
)} }`;
);
// NOTE: the usage of `data_type` has been introduced as a workaround to support BigQuery, for which `data_type_name` is not defined here
const columnType =
currentColumnInfo?.data_type_name || currentColumnInfo?.data_type;
const queryColumnName = columnConfig[columnName]?.custom_name ?? columnName;
const operator = RqlToGraphQlOp(rqlOperator);
const formattedValue = getFormattedValue(columnType || 'varchar', value);
return `${queryColumnName}: {${operator}: ${formattedValue} }`;
});
return whereClausesArr.length
? `where: {${whereClausesArr.join(',')}}`
@ -128,7 +132,7 @@ export const getGraphQLQueryBase = ({
throw new Error('Error in finding column info for table');
}
let whereConditions: WhereClause[] = [];
let whereConditions: WhereClauseRecord[] = [];
let isRelationshipView = false;
if (view.query?.where) {
if (view.query.where.$and) {

View File

@ -58,10 +58,19 @@ const columnDataTypes = {
const operators = [
{ name: 'equals', value: '$eq', graphqlOp: '_eq' },
{ name: 'not equals', value: '$ne', graphqlOp: '_neq' },
{ name: 'in', value: '$in', graphqlOp: '_in', defaultValue: '[]' },
{ name: 'nin', value: '$nin', graphqlOp: '_nin', defaultValue: '[]' },
{ name: '>', value: '$gt', graphqlOp: '_gt' },
{ name: '<', value: '$lt', graphqlOp: '_lt' },
{ name: '>=', value: '$gte', graphqlOp: '_gte' },
{ name: '<=', value: '$lte', graphqlOp: '_lte' },
{ name: 'like', value: '$like', graphqlOp: '_like', defaultValue: '%%' },
{
name: 'not like',
value: '$nlike',
graphqlOp: '_nlike',
defaultValue: '%%',
},
];
// createSQLRegex matches one or more sql for creating view, table or functions, and extracts the type, schema, name and also if it is a partition.

View File

@ -3,26 +3,13 @@ import {
getGraphQLQueryBase,
QueryBody,
} from '@/dataSources/common';
import {
OrderBy,
WhereClause,
} from '../../../components/Common/utils/v1QueryUtils';
import Endpoints from '../../../Endpoints';
import { TableConfig } from '../../../metadata/types';
import { ReduxState } from '../../../types';
import { BaseTableColumn, Relationship, Table } from '../../types';
import { Relationship } from '../../types';
import { isEmpty } from '../../../components/Common/utils/jsUtils';
type Tables = ReduxState['tables'];
interface GetGraphQLQuery {
allSchemas: Table[];
view: Tables['view'];
originalTable: string;
currentSchema: string;
isExport?: boolean;
tableConfiguration: TableConfig;
defaultSchema: string;
}
export const BigQueryDataTypes = {
character: ['STRING'],
@ -56,59 +43,23 @@ const getFormattedValue = (
if (
BigQueryDataTypes.character.includes(type) ||
BigQueryDataTypes.dateTime.includes(type)
)
) {
if (Array.isArray(value)) {
return JSON.stringify(value);
}
return `"${value}"`;
}
if (BigQueryDataTypes.numeric.includes(type)) return value;
if (BigQueryDataTypes.numeric.includes(type)) {
if (Array.isArray(value)) {
return JSON.stringify(value);
}
return value;
}
return value;
};
const RqlToGraphQlOp = (op: string) => {
if (!op || !op?.startsWith('$')) return 'none';
return (
operators.find(_op => _op.value === op)?.graphqlOp ?? op.replace('$', '_')
);
};
const generateWhereClauseQueryString = (
wheres: WhereClause[],
columnTypeInfo: BaseTableColumn[],
tableConfiguration: TableConfig
): string | null => {
const columnConfig = tableConfiguration?.column_config ?? {};
const whereClausesArr = wheres.map((i: Record<string, any>) => {
const columnName = Object.keys(i)[0];
const RqlOperator = Object.keys(i[columnName])[0];
const value = i[columnName][RqlOperator];
const type = columnTypeInfo?.find(
c => c.column_name === columnName
)?.data_type_name;
return `${
columnConfig[columnName]?.custom_name ?? columnName
}: {${RqlToGraphQlOp(RqlOperator)}: ${getFormattedValue(
type || 'varchar',
value
)} }`;
});
return whereClausesArr.length
? `where: {${whereClausesArr.join(',')}}`
: null;
};
const generateSortClauseQueryString = (
sorts: OrderBy[],
tableConfiguration: TableConfig
): string | null => {
const columnConfig = tableConfiguration?.column_config ?? {};
const sortClausesArr = sorts.map((i: OrderBy) => {
return `${columnConfig[i.column]?.custom_name ?? i.column}: ${i.type}`;
});
return sortClausesArr.length
? `order_by: {${sortClausesArr.join(',')}}`
: null;
};
const getColQuery = (
cols: (string | { name: string; columns: string[] })[],
limit: number,
@ -128,79 +79,6 @@ const getColQuery = (
});
};
export const getGraphQLQueryForBrowseRows = ({
allSchemas,
view,
originalTable,
currentSchema,
isExport = false,
tableConfiguration,
defaultSchema,
}: GetGraphQLQuery) => {
const currentTable: Table | undefined = allSchemas?.find(
(t: Table) =>
t.table_name === originalTable && t.table_schema === currentSchema
);
const columnTypeInfo: BaseTableColumn[] = currentTable?.columns || [];
const relationshipInfo: Relationship[] = currentTable?.relationships || [];
if (!columnTypeInfo) {
throw new Error('Error in finding column info for table');
}
let whereConditions: WhereClause[] = [];
let isRelationshipView = false;
if (view.query.where) {
if (view.query.where.$and) {
whereConditions = view.query.where.$and;
} else {
isRelationshipView = true;
whereConditions = Object.keys(view.query.where)
.filter(k => view.query.where[k])
.map(k => {
const obj = {} as any;
obj[k] = { $eq: view.query.where[k] };
return obj;
});
}
}
const sortConditions: OrderBy[] = [];
if (view.query.order_by) {
sortConditions.push(...view.query.order_by);
}
const limit = isExport ? null : `limit: ${view.curFilter.limit}`;
const offset = isExport
? null
: `offset: ${!isRelationshipView ? view.curFilter.offset : 0}`;
const clauses = `${[
generateWhereClauseQueryString(
whereConditions,
columnTypeInfo,
tableConfiguration
),
generateSortClauseQueryString(sortConditions, tableConfiguration),
limit,
offset,
]
.filter(Boolean)
.join(',')}`;
return `query TableRows {
${
currentSchema === defaultSchema
? originalTable
: `${currentSchema}_${originalTable}`
} ${clauses && `(${clauses})`} {
${getColQuery(
view.query.columns,
view.curFilter.limit,
relationshipInfo,
tableConfiguration
).join('\n')}
}
}`;
};
export const getTableRowRequestBody = ({
tables,
isExport,

View File

@ -85,10 +85,18 @@ const getFormattedValue = (
CitusDataTypes.character.includes(type) ||
CitusDataTypes.dateTime.includes(type)
) {
if (Array.isArray(value)) {
return JSON.stringify(value);
}
return `"${value}"`;
}
if (CitusDataTypes.numeric.includes(type)) return value;
if (CitusDataTypes.numeric.includes(type)) {
if (Array.isArray(value)) {
return JSON.stringify(value);
}
return value;
}
return undefined;
};

View File

@ -84,10 +84,19 @@ const getFormattedValue = (
if (
CockroachDataTypes.character.includes(type) ||
CockroachDataTypes.dateTime.includes(type)
)
) {
if (Array.isArray(value)) {
return JSON.stringify(value);
}
return `"${value}"`;
}
if (CockroachDataTypes.numeric.includes(type)) return value;
if (CockroachDataTypes.numeric.includes(type)) {
if (Array.isArray(value)) {
return JSON.stringify(value);
}
return value;
}
return value;
};

View File

@ -4,25 +4,12 @@ import {
QueryBody,
} from './../../common';
import { TableConfig } from './../../../metadata/types';
import {
OrderBy,
WhereClause,
} from '../../../components/Common/utils/v1QueryUtils';
import Endpoints from '../../../Endpoints';
import { ReduxState } from '../../../types';
import { BaseTableColumn, Relationship, Table } from '../../types';
import { Relationship } from '../../types';
import { isEmpty } from '../../../components/Common/utils/jsUtils';
type Tables = ReduxState['tables'];
interface GetGraphQLQuery {
allSchemas: Table[];
view: Tables['view'];
originalTable: string;
currentSchema: string;
isExport?: boolean;
tableConfiguration: TableConfig;
defaultSchema: string;
}
export const SQLServerTypes = {
character: [
@ -78,58 +65,21 @@ const getFormattedValue = (
SQLServerTypes.character.includes(type) ||
SQLServerTypes.dateTime.includes(type)
) {
if (Array.isArray(value)) {
return JSON.stringify(value);
}
return `"${value}"`;
}
if (SQLServerTypes.numeric.includes(type)) return value;
if (SQLServerTypes.numeric.includes(type)) {
if (Array.isArray(value)) {
return JSON.stringify(value);
}
return value;
}
return undefined;
};
const RqlToGraphQlOp = (op: string) => {
if (!op || !op?.startsWith('$')) return 'none';
return (
operators.find(_op => _op.value === op)?.graphqlOp ?? op.replace('$', '_')
);
};
const generateWhereClauseQueryString = (
wheres: WhereClause[],
columnTypeInfo: BaseTableColumn[],
tableConfiguration: TableConfig
): string | null => {
const columnConfig = tableConfiguration?.column_config ?? {};
const whereClausesArr = wheres.map((i: Record<string, any>) => {
const columnName = Object.keys(i)[0];
const RqlOperator = Object.keys(i[columnName])[0];
const value = i[columnName][RqlOperator];
const type = columnTypeInfo?.find(
c => c.column_name === columnName
)?.data_type_name;
return `${
columnConfig[columnName]?.custom_name ?? columnName
}: {${RqlToGraphQlOp(RqlOperator)}: ${getFormattedValue(
type || 'varchar',
value
)} }`;
});
return whereClausesArr.length
? `where: {${whereClausesArr.join(',')}}`
: null;
};
const generateSortClauseQueryString = (
sorts: OrderBy[],
tableConfiguration: TableConfig
): string | null => {
const columnConfig = tableConfiguration?.column_config ?? {};
const sortClausesArr = sorts.map((i: OrderBy) => {
return `${columnConfig[i.column]?.custom_name ?? i.column}: ${i.type}`;
});
return sortClausesArr.length
? `order_by: {${sortClausesArr.join(',')}}`
: null;
};
const getColQuery = (
cols: (string | { name: string; columns: string[] })[],
limit: number,
@ -149,79 +99,6 @@ const getColQuery = (
});
};
export const getGraphQLQueryForBrowseRows = ({
allSchemas,
view,
originalTable,
currentSchema,
isExport = false,
tableConfiguration,
defaultSchema,
}: GetGraphQLQuery) => {
const currentTable: Table | undefined = allSchemas?.find(
(t: Table) =>
t.table_name === originalTable && t.table_schema === currentSchema
);
const columnTypeInfo: BaseTableColumn[] = currentTable?.columns || [];
const relationshipInfo: Relationship[] = currentTable?.relationships || [];
if (!columnTypeInfo) {
throw new Error('Error in finding column info for table');
}
let whereConditions: WhereClause[] = [];
let isRelationshipView = false;
if (view.query.where) {
if (view.query.where.$and) {
whereConditions = view.query.where.$and;
} else {
isRelationshipView = true;
whereConditions = Object.keys(view.query.where)
.filter(k => view.query.where[k])
.map(k => {
const obj = {} as any;
obj[k] = { $eq: view.query.where[k] };
return obj;
});
}
}
const sortConditions: OrderBy[] = [];
if (view.query.order_by) {
sortConditions.push(...view.query.order_by);
}
const limit = isExport ? null : `limit: ${view.curFilter.limit}`;
const offset = isExport
? null
: `offset: ${!isRelationshipView ? view.curFilter.offset : 0}`;
const clauses = `${[
generateWhereClauseQueryString(
whereConditions,
columnTypeInfo,
tableConfiguration
),
generateSortClauseQueryString(sortConditions, tableConfiguration),
limit,
offset,
]
.filter(Boolean)
.join(',')}`;
return `query TableRows {
${
currentSchema === defaultSchema
? originalTable
: `${currentSchema}_${originalTable}`
} ${clauses && `(${clauses})`} {
${getColQuery(
view.query.columns,
view.curFilter.limit,
relationshipInfo,
tableConfiguration
).join('\n')}
}
}`;
};
export const getTableRowRequestBody = ({
tables,
isExport,

View File

@ -18,14 +18,15 @@ import {
} from './FiltersSectionContainer.utils';
import { FiltersSection, sortOptions } from './FiltersSection';
import { FiltersAndSortFormValues, UserQuery } from './types';
import { Table, useTableSchema } from './hooks/useTableSchema';
import { useTableSchema } from './hooks/useTableSchema';
import type { BrowseRowsTable } from './hooks/useTableSchema.utils';
import { useTableColumns } from './hooks/useTableColumns';
import { useTableName } from './hooks/useTableName';
type FilterSectionContainerProps = {
onRunQuery: (userQuery: UserQuery) => void | null;
dataSourceName: string;
table: Table;
table: BrowseRowsTable;
};
const replaceAllDotsWithUnderscore = (text: string) => text.replace(/\./g, '_');
@ -65,6 +66,11 @@ export const FilterSectionContainer = ({
}, []);
const onSubmit = (userQuery: UserQuery) => {
if (!tableSchema) {
console.error('tableSchema is not defined', tableSchema);
return;
}
dispatch(setOffset(0));
setUrlParams(userQuery.where.$and, userQuery.order_by);
dispatch(

View File

@ -4,7 +4,7 @@ import { createHistory } from 'history';
import { ThunkDispatch } from 'redux-thunk';
import { AnyAction } from 'redux';
import { ReduxState } from '@/types';
import { TableSchema } from '@/components/Services/Data/TableBrowseRows/utils';
import { NormalizedTable } from '@/dataSources/types';
import { Integers, Reals } from '../../../components/Services/Data/constants';
import { vMakeTableRequests } from '../../../components/Services/Data/TableBrowseRows/ViewActions';
import { sortPlaceholder } from './FiltersSection';
@ -163,7 +163,7 @@ export const filterValidUserQuery = (userQuery: UserQuery): UserQuery => {
};
type RunFilterQuery = {
tableSchema: TableSchema;
tableSchema: NormalizedTable;
whereAnd: UserQuery['where']['$and'];
orderBy: UserQuery['order_by'];
limit: number;

View File

@ -1,14 +1,18 @@
import { NormalizedTable } from '@/dataSources/types';
import { useAppSelector } from '@/store';
import { getTableSchemaName } from './useTableSchema.utils';
import type { BrowseRowsTable } from './useTableSchema.utils';
// NOTE: temporary type, to be replaced when GDC is ready
export type Table = { schema: string; name: string };
export const useTableSchema = (table: Table) => {
const tableSchema = useAppSelector(state =>
export const useTableSchema = (table: BrowseRowsTable) => {
const tableSchema: NormalizedTable | null = useAppSelector(state =>
state.tables.allSchemas.find((schema: NormalizedTable) => {
const tableSchemaName = getTableSchemaName(table);
if (!tableSchemaName) {
return null;
}
return (
schema.table_schema === table?.schema &&
schema.table_schema === tableSchemaName &&
schema.table_name === table?.name
);
})

View File

@ -0,0 +1,19 @@
import type { BigQueryTable } from '../../../DataSource';
import { getTableSchemaName } from './useTableSchema.utils';
import type { SqlTable } from './useTableSchema.utils';
describe('getTableSchemaName', () => {
describe('when a SQLTable is provided', () => {
it('returns the schema', () => {
const table: SqlTable = { schema: 'aSchema', name: 'aName' };
expect(getTableSchemaName(table)).toBe('aSchema');
});
});
describe('when a BigQueryTable is provided', () => {
it('returns the schema', () => {
const table: BigQueryTable = { dataset: 'aDataset', name: 'aName' };
expect(getTableSchemaName(table)).toBe('aDataset');
});
});
});

View File

@ -0,0 +1,28 @@
import { Table } from '@/features/MetadataAPI';
import { BigQueryTable } from '@/features/DataSource';
export type SqlTable = { schema: string; name: string };
export type BrowseRowsTable = SqlTable | BigQueryTable;
export const getTableSchemaName = (table: Table) => {
const isObject =
table && typeof table === 'object' && Object.keys(table).length > 0;
if (isObject && 'schema' in table) {
const sqlTable = table as SqlTable;
return sqlTable?.schema;
}
if (isObject && 'dataset' in table) {
const bigQueryTable = table as BigQueryTable;
return bigQueryTable?.dataset;
}
console.error(
`Invalid Table object provided (missing schema or dataset), ${JSON.stringify(
table
)}`
);
return undefined;
};

View File

@ -1,3 +1,4 @@
export { runFilterQuery } from './FiltersSection/FiltersSectionContainer.utils';
export type { UserQuery } from './FiltersSection/types';
export { getTableSchemaName } from './FiltersSection/hooks/useTableSchema.utils';
export type { BrowseRowsTable } from './FiltersSection/hooks/useTableSchema.utils';

View File

@ -1,4 +1,4 @@
import { BigQueryTable } from '..';
import { getTableSchemaName, BrowseRowsTable } from '@/features/BrowseRows';
import { runSQL } from '../../api';
import { adaptTableColumns } from '../../common/utils';
import { GetTableColumnsProps } from '../../types';
@ -8,9 +8,21 @@ export const getTableColumns = async ({
table,
httpClient,
}: GetTableColumnsProps) => {
const { dataset, name } = table as BigQueryTable;
const innerTable = table as BrowseRowsTable;
const sql = `SELECT column_name, data_type FROM ${dataset}.INFORMATION_SCHEMA.COLUMNS WHERE table_name = '${name}';`;
if ('schema' in innerTable) {
console.warn(
'BigQuery: received key "schema" while expecting a table object with "dataset"',
innerTable
);
}
const dataset = getTableSchemaName(innerTable);
if (!dataset) {
return Promise.resolve([]);
}
const sql = `SELECT column_name, data_type FROM ${dataset}.INFORMATION_SCHEMA.COLUMNS WHERE table_name = '${innerTable?.name}';`;
const tables = await runSQL({
source: {

View File

@ -14,6 +14,8 @@ import {
import { NetworkArgs } from './api';
export type { BigQueryTable } from './bigquery';
export type AllowedTableRelationships =
| Legacy_SourceToRemoteSchemaRelationship
| SourceToRemoteSchemaRelationship

View File

@ -57,9 +57,11 @@ export const bigquerySqlQueries: DatasourceSqlQueries = {
-- test_id = ${'schemas' in options ? 'multi' : 'single'}_foreign_key
select []`;
},
getTableColumnsSql({ name, schema }: QualifiedTable): string {
if (!schema || !name) throw Error('empty parameters are not allowed!');
getTableColumnsSql(table: QualifiedTable): string {
const { name } = table;
const schemaName = table?.dataset || table.schema;
if (!schemaName || !name) throw Error('empty parameters are not allowed!');
return `SELECT * FROM ${schema}.INFORMATION_SCHEMA.COLUMNS WHERE table_name = '${name}';`;
return `SELECT * FROM ${schemaName}.INFORMATION_SCHEMA.COLUMNS WHERE table_name = '${name}';`;
},
};

View File

@ -3,7 +3,7 @@ import { persistPageSizeChange } from '@/components/Services/Data/TableBrowseRow
import { runFilterQuery } from '@/features/BrowseRows';
import type { UserQuery } from '@/features/BrowseRows';
import { useAppDispatch } from '@/store';
import { TableSchema } from '@/components/Services/Data/TableBrowseRows/utils';
import { NormalizedTable } from '@/dataSources/types';
import {
setLimit,
setOffset,
@ -20,7 +20,7 @@ type PaginationWithOnlyNavContainerProps = {
onChangePageSize: PaginationWithOnlyNavProps['changePageSize'];
pageSize: number;
rows: PaginationWithOnlyNavProps['rows'];
tableSchema: TableSchema;
tableSchema: NormalizedTable;
userQuery: UserQuery;
};