hasura/graphql-engine
1
0
Fork 0
mirror of https://github.com/hasura/graphql-engine.git synced 2024-09-20 06:58:39 +03:00
Code Issues Projects Releases Wiki Activity

console: bigquery support

Browse Source 
Co-authored-by: Abhijeet Singh Khangarot <26903230+abhi40308@users.noreply.github.com>
Co-authored-by: Aleksandra Sikora <9019397+beerose@users.noreply.github.com>
GitOrigin-RevId: 177f57bde4694ab22798e2afd97d489af875e6f7
This commit is contained in:
Vijay Prasanna 2021-04-16 22:51:10 +05:30 committed by hasura-bot
parent 7f17b2683d
commit e3fd4d395a
41 changed files with 1454 additions and 281 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -3,6 +3,7 @@
## Next release
(Add entries here in the order of: server, console, cli, docs, others)
- console: add bigquery support (#1000)
## v2.0.0-alpha.8

42
console/package-lock.json generated
View File

@ -6183,16 +6183,6 @@
"integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==",
"dev": true
},
"case-sensitive-paths-webpack-plugin": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.3.0.tgz",
"integrity": "sha512-/4YgnZS8y1UXXmC02xD5rRrBEu6T5ub+mQHLNRj0fzTRbgdBYhsNo2V5EqwgqrExjxsjtF/OpAKAMkKsxbD5XQ=="
},
"caseless": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
},
"ansi-html": {
"version": "0.0.7",
"resolved": "https://registry.npmjs.org/ansi-html/-/ansi-html-0.0.7.tgz",
@ -7796,6 +7786,11 @@
"rsvp": "^4.8.4"
}
},
"case-sensitive-paths-webpack-plugin": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.3.0.tgz",
"integrity": "sha512-/4YgnZS8y1UXXmC02xD5rRrBEu6T5ub+mQHLNRj0fzTRbgdBYhsNo2V5EqwgqrExjxsjtF/OpAKAMkKsxbD5XQ=="
},
"caseless": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
@ -13133,12 +13128,11 @@
},
"dependencies": {
"ansi-styles": {
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
"integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"dev": true,
"requires": {
"@types/color-name": "^1.1.1",
"color-convert": "^2.0.1"
}
},
@ -13152,12 +13146,6 @@
"supports-color": "^7.1.0"
}
},
"ci-info": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
"integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==",
"dev": true
},
"color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
@ -13223,9 +13211,9 @@
}
},
"supports-color": {
"version": "7.1.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz",
"integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==",
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
"dev": true,
"requires": {
"has-flag": "^4.0.0"
@ -19221,14 +19209,14 @@
},
"onetime": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz",
"resolved": "http://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz",
"integrity": "sha1-ofeDj4MUxRbwXs78vEzP4EtO14k=",
"dev": true
},
"opencollective-postinstall": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.2.tgz",
"integrity": "sha512-pVOEP16TrAO2/fjej1IdOyupJY8KDUM1CvsaScRbw6oddvpQoOfGk4ywha0HKKVAD6RkW4x6Q+tNBwhf3Bgpuw==",
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz",
"integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==",
"dev": true
},
"optimize-css-assets-webpack-plugin": {

2
console/package.json
View File

@ -198,7 +198,7 @@
"font-awesome": "4.7.0",
"font-awesome-webpack": "0.0.4",
"fork-ts-checker-webpack-plugin": "4.1.3",
"husky": "4.2.3",
"husky": "^4.2.3",
"identity-obj-proxy": "^3.0.0",
"ignore-loader": "0.1.2",
"jest": "26.6.3",

1
console/src/components/Common/NotSupportedNote/index.tsx
View File

@ -5,6 +5,7 @@ const driverToLabel: Record<Driver, string> = {
mysql: 'MySQL',
postgres: 'PostgreSQL',
mssql: 'MS Server',
bigquery: 'Big Query',
};
type NotSupportedNoteProps = {

4
console/src/components/Common/utils/v1QueryUtils.ts
View File

@ -16,8 +16,8 @@ export const getRunSqlQuery = (
driver = currentDriver
) => {
let type = 'run_sql';
if (driver === 'mssql') {
type = 'mssql_run_sql';
if (driver === 'mssql' || driver === 'bigquery') {
type = `${driver}_run_sql`;
}
return {

72
console/src/components/Services/Data/Add/AddExistingTableViewActions.js
View File

@ -8,8 +8,12 @@ import {
getTableModifyRoute,
getFunctionModifyRoute,
} from '../../../Common/utils/routesUtils';
import { dataSource } from '../../../../dataSources';
import { findTable, escapeTableColumns } from '../../../../dataSources/common';
import { dataSource, currentDriver } from '../../../../dataSources';
import {
findTable,
escapeTableColumns,
getQualifiedTableDef,
} from '../../../../dataSources/common';
import { exportMetadata } from '../../../../metadata/actions';
import {
getUntrackTableQuery,
@ -39,21 +43,24 @@ const addExistingTableSql = (name, customSchema, skipRouting = false) => {
: getState().tables.currentSchema;
const currentDataSource = getState().tables.currentDataSource;
const tableName = name ? name : state.tableName.trim();
const tableDef = { name: tableName, schema: currentSchema };
const tableDef = getQualifiedTableDef(
{
name: tableName,
schema: currentSchema,
},
currentDriver
);
const table = findTable(getState().tables.allSchemas, tableDef);
const requestBodyUp = getTrackTableQuery({
tableDef,
source: currentDataSource,
customColumnNames: escapeTableColumns(table),
});
const requestBodyDown = getUntrackTableQuery(
{
name: tableName,
schema: currentSchema,
},
currentDataSource
);
const requestBodyDown = getUntrackTableQuery(tableDef, currentDataSource);
const migrationName = `add_existing_table_or_view_${currentSchema}_${tableName}`;
@ -67,19 +74,20 @@ const addExistingTableSql = (name, customSchema, skipRouting = false) => {
t => t.table_name === tableName && t.table_schema === currentSchema
);
const isTableType = dataSource.isTable(newTable);
const nextRoute = isTableType
? getTableModifyRoute(
currentSchema,
currentDataSource,
tableName,
isTableType
)
: getTableBrowseRoute(
currentSchema,
currentDataSource,
tableName,
isTableType
);
const nextRoute =
isTableType && currentDriver !== 'bigquery'
? getTableModifyRoute(
currentSchema,
currentDataSource,
tableName,
isTableType
)
: getTableBrowseRoute(
currentSchema,
currentDataSource,
tableName,
isTableType
);
if (!skipRouting) {
dispatch(_push(nextRoute));
}
@ -178,12 +186,17 @@ const addAllUntrackedTablesSql = tableList => {
dispatch(showSuccessNotification('Adding...'));
const bulkQueryUp = [];
const bulkQueryDown = [];
for (let i = 0; i < tableList.length; i++) {
if (tableList[i].table_name !== 'schema_migrations') {
const tableDef = {
name: tableList[i].table_name,
schema: currentSchema,
};
const tableDef = getQualifiedTableDef(
{
name: tableList[i].table_name,
schema: currentSchema,
},
currentDriver
);
const table = findTable(getState().tables.allSchemas, tableDef);
bulkQueryUp.push(
getTrackTableQuery({
@ -197,7 +210,9 @@ const addAllUntrackedTablesSql = tableList => {
{
table: {
name: tableList[i].table_name,
schema: currentSchema,
[currentDriver === 'bigquery'
? 'dataset'
: 'schema']: currentSchema,
},
},
currentDataSource
@ -205,6 +220,7 @@ const addAllUntrackedTablesSql = tableList => {
);
}
}
const migrationName = 'add_all_existing_table_or_view_' + currentSchema;
const requestMsg = 'Adding existing table/view...';

35
console/src/components/Services/Data/DataActions.js
View File

@ -26,7 +26,11 @@ import {
cascadeUpQueries,
getDependencyError,
} from './utils';
import { mergeDataMssql, mergeLoadSchemaDataPostgres } from './mergeData';
import {
mergeDataMssql,
mergeLoadSchemaDataPostgres,
mergeDataBigQuery,
} from './mergeData';
import _push from './push';
import { convertArrayToJson } from './TableModify/utils';
import { CLI_CONSOLE_MODE, SERVER_CONSOLE_MODE } from '../../../constants';
@ -146,7 +150,9 @@ const loadSchema = configOptions => {
(!configOptions.tables || configOptions.tables.length === 0))
) {
configOptions = {
schemas: [getState().tables.currentSchema],
schemas: [
getState().tables.currentSchema || getState().tables.schemaList[0],
],
};
}
@ -171,7 +177,6 @@ const loadSchema = configOptions => {
);
}
}
const body = {
type: 'bulk',
source,
@ -225,6 +230,9 @@ const loadSchema = configOptions => {
case 'mssql':
mergedData = mergeDataMssql(data, metadataTables);
break;
case 'bigquery':
mergedData = mergeDataBigQuery(data, metadataTables);
break;
default:
}
@ -308,9 +316,14 @@ const setConsistentSchema = data => ({
const fetchDataInit = (source, driver) => (dispatch, getState) => {
const url = Endpoints.query;
const { schemaFilter } = getState().tables;
const currentSource = source || getState().tables.currentDataSource;
let { schemaFilter } = getState().tables;
if (driver === 'bigquery')
schemaFilter = getState().metadata.metadataObject.sources.find(
x => x.name === source
).configuration.datasets;
const currentSource = source || getState().tables.currentDataSource;
const query = getRunSqlQuery(
dataSource.schemaListSql(schemaFilter),
currentSource,
@ -340,7 +353,6 @@ const fetchDataInit = (source, driver) => (dispatch, getState) => {
type: FETCH_SCHEMA_LIST,
schemaList,
});
let newSchema = '';
if (schemaList.length) {
newSchema =
@ -350,7 +362,6 @@ const fetchDataInit = (source, driver) => (dispatch, getState) => {
: schemaList.sort(Intl.Collator().compare)[0];
}
dispatch({ type: UPDATE_CURRENT_SCHEMA, currentSchema: newSchema });
return dispatch(updateSchemaInfo()); // TODO
},
error => {
@ -558,7 +569,6 @@ export const getDatabaseTableTypeInfo = (
resolve({});
});
}
const url = Endpoints.query;
const sql = services[sourceType].getTableInfo(tables);
const query = getRunSqlQuery(sql, sourceName, false, false, sourceType);
@ -582,13 +592,18 @@ export const getDatabaseTableTypeInfo = (
try {
if (currentDriver === 'mssql') {
res = JSON.parse(result.slice(1).join());
} else if (currentDriver === 'bigquery') {
res = result.slice(1).map(t => ({
table_name: t[0],
table_schema: t[1],
table_type: t[2],
}));
} else {
res = JSON.parse(result[1]);
}
} catch {
} catch (err) {
res = [];
}
res.forEach(i => {
if (
!trackedTables.some(

4
console/src/components/Services/Data/DataSourceContainer.tsx
View File

@ -83,6 +83,7 @@ const DataSourceContainer = ({
dispatch({ type: UPDATE_CURRENT_SCHEMA, currentSchema: schema });
return;
}
// eslint-disable-next-line no-useless-return
if (!dataLoaded) return;
let newSchema = '';
@ -94,11 +95,10 @@ const DataSourceContainer = ({
? dataSource.defaultRedirectSchema
: schemaList.sort(Intl.Collator().compare)[0];
}
if (location.pathname.includes('schema')) {
dispatch(push(`/data/${source}/schema/${newSchema}`));
}
}, [dispatch, schema, schemaList, source, location, dataLoaded]);
}, [dispatch, schema, schemaList, location, source, dataLoaded]);
useEffect(() => {
const driver = getSourceDriver(dataSources, currentSource);

94
console/src/components/Services/Data/DataSources/ConnectDBForm.tsx
View File

@ -2,7 +2,9 @@ import React, { ChangeEvent, Dispatch, useState } from 'react';
import { ConnectDBActions, ConnectDBState, connectionTypes } from './state';
import { LabeledInput } from '../../../Common/LabeledInput';
import Tooltip from '../../../Common/Tooltip/Tooltip';
import { Driver } from '../../../../dataSources';
import { readFile } from './utils';
import styles from './DataSources.scss';
@ -41,6 +43,7 @@ const dbTypePlaceholders: Record<Driver, string> = {
mssql:
'Driver={ODBC Driver 17 for SQL Server};Server=serveraddress;Database=dbname;Uid=username;Pwd=password;',
mysql: 'MySQL connection string',
bigquery: 'SERVICE_ACCOUNT_KEY_FROM_ENV',
};
const defaultTitle = 'Connect Database Via';
@ -59,6 +62,23 @@ const ConnectDatabaseForm: React.FC<ConnectDatabaseFormProps> = ({
const toggleConnectionParams = (value: boolean) => () => {
toggleConnectionParamState(value);
};
const handleFileUpload = (e: React.ChangeEvent<HTMLInputElement>) => {
const file = e.target.files![0];
const addFileQueries = (content: string) => {
try {
connectionDBStateDispatch({
type: 'UPDATE_DB_BIGQUERY_SERVICE_ACCOUNT_FILE',
data: content,
});
} catch (error) {
console.log(error);
}
};
readFile(file, addFileQueries);
};
return (
<>
<h4 className={`${styles.remove_pad_bottom} ${styles.connect_db_header}`}>
@ -132,12 +152,16 @@ const ConnectDatabaseForm: React.FC<ConnectDatabaseFormProps> = ({
<option key="mssql" value="mssql">
MS Server
</option>
<option key="bigquery" value="bigquery">
BigQuery
</option>
</select>
</>
)}
{connectionTypeState.includes(connectionTypes.DATABASE_URL) ||
(connectionTypeState.includes(connectionTypes.CONNECTION_PARAMS) &&
connectionDBState.dbType === 'mssql') ? (
{(connectionTypeState.includes(connectionTypes.DATABASE_URL) ||
(connectionTypeState.includes(connectionTypes.CONNECTION_PARAMS) &&
connectionDBState.dbType === 'mssql')) &&
connectionDBState.dbType !== 'bigquery' ? (
<LabeledInput
label="Database URL"
onChange={e =>
@ -152,7 +176,8 @@ const ConnectDatabaseForm: React.FC<ConnectDatabaseFormProps> = ({
// disabled={isEditState}
/>
) : null}
{connectionTypeState.includes(connectionTypes.ENV_VAR) ? (
{connectionTypeState.includes(connectionTypes.ENV_VAR) &&
connectionDBState.dbType !== 'bigquery' ? (
<LabeledInput
label="Environment Variable"
placeholder="HASURA_GRAPHQL_DB_URL_FROM_ENV"
@ -162,12 +187,69 @@ const ConnectDatabaseForm: React.FC<ConnectDatabaseFormProps> = ({
data: e.target.value,
})
}
value={connectionDBState.envVarURLState.envVarURL}
value={connectionDBState.envVarState.envVar}
data-test="database-url-env"
/>
) : null}
{(connectionTypeState.includes(connectionTypes.DATABASE_URL) ||
connectionTypeState.includes(connectionTypes.CONNECTION_PARAMS) ||
connectionTypeState.includes(connectionTypes.ENV_VAR)) &&
connectionDBState.dbType === 'bigquery' ? (
<>
{connectionTypeState.includes(connectionTypes.ENV_VAR) ? (
<LabeledInput
label="Environment Variable"
placeholder={dbTypePlaceholders[connectionDBState.dbType]}
onChange={e =>
connectionDBStateDispatch({
type: 'UPDATE_DB_URL_ENV_VAR',
data: e.target.value,
})
}
value={connectionDBState.envVarState.envVar}
data-test="service-account-env-var"
/>
) : (
<div className={styles.add_mar_bottom_mid}>
<div className={styles.add_mar_bottom_mid}>
<b>Service Account File:</b>
<Tooltip message="Service account key file for bigquery db" />
</div>
<input
type="file"
className={`form-control input-sm ${styles.inline_block}`}
onChange={handleFileUpload}
/>
</div>
)}
<LabeledInput
label="Project Id"
onChange={e =>
connectionDBStateDispatch({
type: 'UPDATE_DB_BIGQUERY_PROJECT_ID',
data: e.target.value,
})
}
value={connectionDBState.databaseURLState.projectId}
placeholder="project_id"
data-test="project-id"
/>
<LabeledInput
label="Datasets"
onChange={e =>
connectionDBStateDispatch({
type: 'UPDATE_DB_BIGQUERY_DATASETS',
data: e.target.value,
})
}
value={connectionDBState.databaseURLState.datasets}
placeholder="dataset1, dataset2"
data-test="datasets"
/>
</>
) : null}
{connectionTypeState.includes(connectionTypes.CONNECTION_PARAMS) &&
connectionDBState.dbType !== 'mssql' ? (
connectionDBState.dbType === 'postgres' ? (
<>
<LabeledInput
label="Host"

39
console/src/components/Services/Data/DataSources/ConnectDatabase.tsx
View File

@ -28,6 +28,7 @@ import ConnectDatabaseForm from './ConnectDBForm';
import ReadReplicaForm from './ReadReplicaForm';
import styles from './DataSources.scss';
import { getSupportedDrivers } from '../../../../dataSources';
interface ConnectDatabaseProps extends InjectedProps {}
@ -128,7 +129,10 @@ const ConnectDatabase: React.FC<ConnectDatabaseProps> = props => {
(connectionType === connectionTypes.CONNECTION_PARAMS &&
connectDBInputState.dbType === 'mssql')
) {
if (!connectDBInputState.databaseURLState.dbURL.trim()) {
if (
!connectDBInputState.databaseURLState.dbURL.trim() &&
connectDBInputState.dbType !== 'bigquery'
) {
dispatch(
showErrorNotification(
'Database URL is a mandatory field',
@ -152,7 +156,10 @@ const ConnectDatabase: React.FC<ConnectDatabaseProps> = props => {
}
if (connectionType === connectionTypes.ENV_VAR) {
if (!connectDBInputState.envVarURLState.envVarURL.trim()) {
if (
!connectDBInputState.envVarState.envVar.trim() &&
connectDBInputState.dbType !== 'bigquery'
) {
dispatch(
showErrorNotification(
'Environment Variable is a mandatory field',
@ -184,17 +191,19 @@ const ConnectDatabase: React.FC<ConnectDatabaseProps> = props => {
database,
} = connectDBInputState.connectionParamState;
if (!host || !port || !username || !database) {
const errorMessage = getErrorMessageFromMissingFields(
host,
port,
username,
database
);
dispatch(
showErrorNotification('Required fields are missing', errorMessage)
);
return;
if (connectDBInputState.dbType !== 'bigquery') {
if (!host || !port || !username || !database) {
const errorMessage = getErrorMessageFromMissingFields(
host,
port,
username,
database
);
dispatch(
showErrorNotification('Required fields are missing', errorMessage)
);
return;
}
}
setLoading(true);
connectDataSource(
@ -259,7 +268,9 @@ const ConnectDatabase: React.FC<ConnectDatabaseProps> = props => {
updateConnectionTypeRadio={onChangeConnectionType}
/>
{/* Should be rendered only on Pro and Cloud Console */}
{connectDBInputState.dbType !== 'mssql' &&
{getSupportedDrivers('connectDbForm.read_replicas').includes(
connectDBInputState.dbType
) &&
(window.__env.consoleId || window.__env.userRole) && (
<ReadReplicaForm
readReplicaState={readReplicasState}

4
console/src/components/Services/Data/DataSources/ReadReplicaForm.tsx
View File

@ -31,7 +31,7 @@ const checkIfFieldsAreEmpty = (
}
if (
currentReadReplicaConnectionType === connectionTypes.ENV_VAR &&
!currentReadReplicaState?.envVarURLState?.envVarURL
!currentReadReplicaState?.envVarState?.envVar
) {
return true;
}
@ -149,7 +149,7 @@ const ReadReplicaListItem: React.FC<ReadReplicaListItemProps> = ({
>
Remove
</Button>
<p>{isFromEnvVar ? currentState.envVarURLState.envVarURL : host}</p>
<p>{isFromEnvVar ? currentState.envVarState.envVar : host}</p>
{/* The connection string is redundant if it's provided via ENV VAR */}
{!isFromEnvVar && (
<span

83
console/src/components/Services/Data/DataSources/state.ts
View File

@ -1,4 +1,4 @@
import { Driver } from '../../../../dataSources';
import { Driver, getSupportedDrivers } from '../../../../dataSources';
import { makeConnectionStringFromConnectionParams } from './ManageDBUtils';
import { addDataSource } from '../../../../metadata/actions';
import { Dispatch } from '../../../../types';
@ -30,9 +30,12 @@ export type ConnectDBState = {
connectionParamState: ConnectionParams;
databaseURLState: {
dbURL: string;
serviceAccountFile: string;
projectId: string;
datasets: string;
};
envVarURLState: {
envVarURL: string;
envVarState: {
envVar: string;
};
connectionSettings: ConnectionSettings;
};
@ -49,9 +52,12 @@ export const defaultState: ConnectDBState = {
},
databaseURLState: {
dbURL: '',
serviceAccountFile: '',
projectId: '',
datasets: '',
},
envVarURLState: {
envVarURL: '',
envVarState: {
envVar: '',
},
connectionSettings: {},
};
@ -69,10 +75,11 @@ export const getDefaultState = (props?: DefaultStateProps): ConnectDBState => {
...defaultState,
displayName: props?.dbConnection.dbName || '',
databaseURLState: {
...defaultState.databaseURLState,
dbURL: props?.dbConnection.dbURL || '',
},
envVarURLState: {
envVarURL: props?.dbConnection.envVar || '',
envVarState: {
envVar: props?.dbConnection.envVar || '',
},
};
};
@ -88,12 +95,23 @@ export const connectDataSource = (
cb: () => void,
replicas?: Omit<SourceConnectionInfo, 'connection_string'>[]
) => {
let databaseURL:
| string
| { from_env: string } = currentState.databaseURLState.dbURL.trim();
if (typeConnection === connectionTypes.ENV_VAR) {
databaseURL = { from_env: currentState.envVarURLState.envVarURL.trim() };
} else if (typeConnection === connectionTypes.CONNECTION_PARAMS) {
let databaseURL: string | { from_env: string } =
currentState.dbType === 'bigquery'
? currentState.databaseURLState.serviceAccountFile.trim()
: currentState.databaseURLState.dbURL.trim();
if (
typeConnection === connectionTypes.ENV_VAR &&
getSupportedDrivers('connectDbForm.environmentVariable').includes(
currentState.dbType
)
) {
databaseURL = { from_env: currentState.envVarState.envVar.trim() };
} else if (
typeConnection === connectionTypes.CONNECTION_PARAMS &&
getSupportedDrivers('connectDbForm.connectionParameters').includes(
currentState.dbType
)
) {
databaseURL = makeConnectionStringFromConnectionParams({
dbType: currentState.dbType,
...currentState.connectionParamState,
@ -108,6 +126,10 @@ export const connectDataSource = (
name: currentState.displayName.trim(),
dbUrl: databaseURL,
connection_pool_settings: currentState.connectionSettings,
bigQuery: {
projectId: currentState.databaseURLState.projectId,
datasets: currentState.databaseURLState.datasets,
},
},
},
cb,
@ -128,6 +150,9 @@ export type ConnectDBActions =
}
| { type: 'UPDATE_DISPLAY_NAME'; data: string }
| { type: 'UPDATE_DB_URL'; data: string }
| { type: 'UPDATE_DB_BIGQUERY_SERVICE_ACCOUNT_FILE'; data: string }
| { type: 'UPDATE_DB_BIGQUERY_PROJECT_ID'; data: string }
| { type: 'UPDATE_DB_BIGQUERY_DATASETS'; data: string }
| { type: 'UPDATE_DB_URL_ENV_VAR'; data: string }
| { type: 'UPDATE_DB_HOST'; data: string }
| { type: 'UPDATE_DB_PORT'; data: string }
@ -152,6 +177,7 @@ export const connectDBReducer = (
displayName: action.data.name,
dbType: action.data.driver,
databaseURLState: {
...state.databaseURLState,
dbURL: action.data.databaseUrl,
},
connectionSettings: action.data.connectionSettings,
@ -170,14 +196,15 @@ export const connectDBReducer = (
return {
...state,
databaseURLState: {
...state.databaseURLState,
dbURL: action.data,
},
};
case 'UPDATE_DB_URL_ENV_VAR':
return {
...state,
envVarURLState: {
envVarURL: action.data,
envVarState: {
envVar: action.data,
},
};
case 'UPDATE_DB_HOST':
@ -253,6 +280,30 @@ export const connectDBReducer = (
...state,
connectionSettings: action.data,
};
case 'UPDATE_DB_BIGQUERY_SERVICE_ACCOUNT_FILE':
return {
...state,
databaseURLState: {
...state.databaseURLState,
serviceAccountFile: action.data,
},
};
case 'UPDATE_DB_BIGQUERY_DATASETS':
return {
...state,
databaseURLState: {
...state.databaseURLState,
datasets: action.data,
},
};
case 'UPDATE_DB_BIGQUERY_PROJECT_ID':
return {
...state,
databaseURLState: {
...state.databaseURLState,
projectId: action.data,
},
};
default:
return state;
}
@ -310,7 +361,7 @@ export const makeReadReplicaConnectionObject = (
database_url = stateVal.databaseURLState?.dbURL?.trim() ?? '';
} else if (stateVal.chosenConnectionType === connectionTypes.ENV_VAR) {
database_url = {
from_env: stateVal.envVarURLState?.envVarURL?.trim() ?? '',
from_env: stateVal.envVarState?.envVar?.trim() ?? '',
};
} else {
database_url = makeConnectionStringFromConnectionParams({

17
console/src/components/Services/Data/DataSources/utils.ts
View File

@ -34,3 +34,20 @@ export const getDatasourceURL = (
}
return link.from_env.toString();
};
export const readFile = (
file: File | null,
callback: (content: string) => void
) => {
const reader = new FileReader();
reader.onload = event => {
const content = event.target!.result as string;
callback(content);
};
reader.onerror = event => {
console.error(`File could not be read! Code ${event.target!.error!.code}`);
};
if (file) reader.readAsText(file);
};

6
console/src/components/Services/Data/DataSubSidebar.js
View File

@ -86,9 +86,9 @@ const DataSubSidebar = props => {
type: UPDATE_CURRENT_DATA_SOURCE,
source: newSourceName,
});
setDriver(driver);
dispatch(_push(`/data/${newSourceName}/`));
dispatch(fetchDataInit()).finally(() => {
setDriver(driver);
dispatch(fetchDataInit(newSourceName, driver)).finally(() => {
setDatabaseLoading(false);
});
};
@ -201,7 +201,7 @@ const DataSubSidebar = props => {
sources.forEach(source => {
const currentSourceTables = sources
.filter(i => i.name === source.name)[0]
.tables.map(i => `'${i.table.name}'`);
.tables.map(t => t.table);
schemaPromises.push(
dispatch(
getDatabaseTableTypeInfo(

26
console/src/components/Services/Data/RawSQL/RawSQL.js
View File

@ -32,6 +32,8 @@ import DropDownSelector from './DropDownSelector';
import { getSourceDriver } from '../utils';
import { getDataSources } from '../../../../metadata/selector';
import { services } from '../../../../dataSources/services';
import { isFeatureSupported } from '../../../../dataSources';
/**
* # RawSQL React FC
* ## renders raw SQL page on route `/data/sql`
@ -332,17 +334,19 @@ const RawSQL = ({
return (
<div className={styles.add_mar_top}>
<label>
<input
checked={isTableTrackChecked}
className={`${styles.add_mar_right_small} ${styles.cursorPointer}`}
id="track-checkbox"
type="checkbox"
onChange={dispatchTrackThis}
data-test="raw-sql-track-check"
/>
Track this
</label>
{isFeatureSupported('rawSQL.tracking') && (
<label>
<input
checked={isTableTrackChecked}
className={`${styles.add_mar_right_small} ${styles.cursorPointer}`}
id="track-checkbox"
type="checkbox"
onChange={dispatchTrackThis}
data-test="raw-sql-track-check"
/>
Track this
</label>
)}
<Tooltip
message={
'If you are creating tables, views or functions, checking this will also expose them over the GraphQL API as top level fields'

1
console/src/components/Services/Data/Schema/ManageDatabase.tsx
View File

@ -24,6 +24,7 @@ const driverToLabel: Record<Driver, string> = {
mysql: 'MySQL',
postgres: 'PostgreSQL',
mssql: 'MS Server',
bigquery: 'Big Query',
};
type DatabaseListItemProps = {

65
console/src/components/Services/Data/Schema/Schema.js
View File

@ -34,6 +34,7 @@ import {
getUntrackedTables,
dataSource,
currentDriver,
isFeatureSupported,
} from '../../../../dataSources';
import { isEmpty } from '../../../Common/utils/jsUtils';
import { getConfirmation } from '../../../Common/utils/jsUtils';
@ -47,21 +48,6 @@ import { TrackableFunctionsList } from './FunctionsList';
import { getTrackableFunctions } from './utils';
import BreadCrumb from '../../../Common/Layout/BreadCrumb/BreadCrumb';
const FEATURES = {
UNTRACKED_TABLES: 'UNTRACKED_TABLES',
UNTRACKED_RELATIONS: 'UNTRACKED_RELATIONS',
UNTRACKED_FUNCTION: 'UNTRACKED_FUNCTION',
NON_TRACKABLE_FUNCTIONS: 'NON_TRACKABLE_FUNCTIONS',
};
const supportedFeaturesByDriver = {
postgres: Object.values(FEATURES),
mssql: [FEATURES.UNTRACKED_TABLES, FEATURES.UNTRACKED_RELATIONS],
};
const isAllowed = (driver, feature) =>
supportedFeaturesByDriver[driver].includes(feature);
const DeleteSchemaButton = ({ dispatch, migrationMode, currentDataSource }) => {
const successCb = () => {
dispatch(updateCurrentSchema('public', currentDataSource));
@ -140,17 +126,21 @@ const CreateSchemaSection = React.forwardRef(
}) =>
migrationMode && (
<div className={`${styles.display_flex}`}>
{createSchemaOpen ? (
<OpenCreateSection
ref={ref}
value={schemaNameEdit}
handleInputChange={handleSchemaNameChange}
handleCreate={handleCreateClick}
handleCancelCreate={handleCancelCreateNewSchema}
/>
) : (
<ClosedCreateSection onClick={handleCreateNewClick} />
)}
{isFeatureSupported('schemas.create.enabled') ? (
<span>
{createSchemaOpen ? (
<OpenCreateSection
ref={ref}
value={schemaNameEdit}
handleInputChange={handleSchemaNameChange}
handleCreate={handleCreateClick}
handleCancelCreate={handleCancelCreateNewSchema}
/>
) : (
<ClosedCreateSection onClick={handleCreateNewClick} />
)}
</span>
) : null}
<SchemaPermissionsButton schema={schema} source={currentDataSource} />
</div>
)
@ -261,7 +251,7 @@ class Schema extends Component {
const getCreateBtn = () => {
let createBtn = null;
if (migrationMode && currentDriver === 'postgres') {
if (migrationMode && isFeatureSupported('tables.create.enabled')) {
const handleClick = e => {
e.preventDefault();
@ -295,12 +285,14 @@ class Schema extends Component {
</div>
<div className={`${styles.display_inline} ${styles.add_mar_left}`}>
<div style={{ display: 'flex', alignItems: 'center' }}>
<DeleteSchemaButton
dispatch={dispatch}
migrationMode={migrationMode}
currentDataSource={currentDataSource}
schemaList={this.props.schemaList}
/>
{isFeatureSupported('schemas.delete.enabled') ? (
<DeleteSchemaButton
dispatch={dispatch}
migrationMode={migrationMode}
currentDataSource={currentDataSource}
schemaList={this.props.schemaList}
/>
) : null}
<CreateSchemaSection
ref={this.schemaNameInputRef}
migrationMode={migrationMode}
@ -682,11 +674,12 @@ class Schema extends Component {
{getCurrentSchemaSection()}
<hr />
{getUntrackedTablesSection()}
{getUntrackedRelationsSection()}
{isFeatureSupported('tables.relationships.track') &&
getUntrackedRelationsSection()}
{getUntrackedFunctionsSection(
isAllowed(currentDriver, FEATURES.NON_TRACKABLE_FUNCTIONS)
isFeatureSupported('functions.track.enabled')
)}
{isAllowed(currentDriver, FEATURES.NON_TRACKABLE_FUNCTIONS) &&
{isFeatureSupported('functions.nonTrackableFunctions.enabled') &&
getNonTrackableFunctionsSection()}
<hr />
</div>

126
console/src/components/Services/Data/SourceView.tsx
View File

@ -12,6 +12,7 @@ import {
getSchemaPermissionsRoute,
} from '../../Common/utils/routesUtils';
import _push from './push';
import { isFeatureSupported } from '../../../dataSources';
import BreadCrumb from '../../Common/Layout/BreadCrumb/BreadCrumb';
interface Props {
@ -71,55 +72,59 @@ const SourceView: React.FC<Props> = props => {
<h2 className={`${styles.headerText} ${styles.display_inline}`}>
{currentDataSource}
</h2>
{!isCreateActive ? (
<Button
data-test="data-create-schema"
color="yellow"
size="sm"
className={styles.add_mar_left}
onClick={() => setIsCreateActive(true)}
>
Create Schema
</Button>
) : (
<div
className={styles.display_inline}
style={{ paddingLeft: '10px' }}
>
<div className={styles.display_inline}>
<input
type="text"
placeholder="Enter Schema name"
className={`form-control input-sm ${styles.display_inline}`}
value={createSchemaName}
onChange={(e: any) => {
e.persist();
setCreateSchemaName(e.target.value);
}}
/>
</div>
<Button
data-test="data-create-schema"
color="yellow"
size="sm"
className={styles.add_mar_left}
onClick={handleCreateSchema}
>
Create Schema
</Button>
<Button
color="white"
size="xs"
className={styles.add_mar_left_mid}
onClick={() => {
setIsCreateActive(false);
setCreateSchemaName('');
}}
>
Cancel
</Button>
</div>
)}
{isFeatureSupported('schemas.create.enabled') ? (
<span>
{!isCreateActive ? (
<Button
data-test="data-create-schema"
color="yellow"
size="sm"
className={styles.add_mar_left}
onClick={() => setIsCreateActive(true)}
>
Create Schema
</Button>
) : (
<div
className={styles.display_inline}
style={{ paddingLeft: '10px' }}
>
<div className={styles.display_inline}>
<input
type="text"
placeholder="Enter Schema name"
className={`form-control input-sm ${styles.display_inline}`}
value={createSchemaName}
onChange={(e: any) => {
e.persist();
setCreateSchemaName(e.target.value);
}}
/>
</div>
<Button
data-test="data-create-schema"
color="yellow"
size="sm"
className={styles.add_mar_left}
onClick={handleCreateSchema}
>
Create Schema
</Button>
<Button
color="white"
size="xs"
className={styles.add_mar_left_mid}
onClick={() => {
setIsCreateActive(false);
setCreateSchemaName('');
}}
>
Cancel
</Button>
</div>
)}
</span>
) : null}
</div>
<div>
<hr />
@ -127,11 +132,12 @@ const SourceView: React.FC<Props> = props => {
{schemaList.length ? (
schemaList.map((schema, key: number) => {
return (
<div className={styles.padd_small}>
<div
className={`${styles.padd_small} ${styles.padd_left_remove}`}
>
<Button
color="white"
size="xs"
className={styles.mar_small_left}
onClick={() => handleView(schema)}
>
View
@ -144,14 +150,16 @@ const SourceView: React.FC<Props> = props => {
>
Permissions Summary
</Button>
<Button
color="white"
size="xs"
className={styles.mar_small_left}
onClick={() => handleDelete(schema)}
>
<i className="fa fa-trash" aria-hidden="true" />
</Button>
{isFeatureSupported('schemas.delete.enabled') ? (
<Button
color="white"
size="xs"
className={styles.mar_small_left}
onClick={() => handleDelete(schema)}
>
<i className="fa fa-trash" aria-hidden="true" />
</Button>
) : null}
<div
key={key}
className={`${styles.display_inline} ${styles.padd_small_left}`}

2
console/src/components/Services/Data/TableBrowseRows/ViewRows.js
View File

@ -772,8 +772,8 @@ const ViewRows = props => {
if (isObjectRel) {
childRows = [childRows];
}
const childTableDef = getRelationshipRefTable(tableSchema, rel);
const childTable = findTable(schemas, childTableDef);
return (

6
console/src/components/Services/Data/TableCommon/TableHeader.js
View File

@ -116,7 +116,8 @@ const TableHeader = ({
}`,
'table-browse-rows'
)}
{!readOnlyMode &&
{isFeatureSupported('tables.insert.enabled') &&
!readOnlyMode &&
isTableType &&
getTab(
'insert',
@ -129,7 +130,8 @@ const TableHeader = ({
'Insert Row',
'table-insert-rows'
)}
{migrationMode &&
{isFeatureSupported('tables.modify.enabled') &&
migrationMode &&
getTab(
'modify',
getTableModifyRoute(

58
console/src/components/Services/Data/TablePermissions/Actions.js
View File

@ -7,6 +7,7 @@ import {
getTableDef,
getTablePermissions,
generateTableDef,
getQualifiedTableDef,
} from '../../../../dataSources';
import { capitalize } from '../../../Common/utils/jsUtils';
import { exportMetadata } from '../../../../metadata/actions';
@ -15,6 +16,7 @@ import {
getDropPermissionQuery,
} from '../../../../metadata/queryUtils';
import Migration from '../../../../utils/migration/Migration';
import { currentDriver } from '../../../../dataSources';
export const PERM_OPEN_EDIT = 'ModifyTable/PERM_OPEN_EDIT';
export const PERM_SET_FILTER_TYPE = 'ModifyTable/PERM_SET_FILTER_TYPE';
@ -267,17 +269,25 @@ const permRemoveRole = (tableSchema, roleName) => {
const permissionsUpQueries = [];
const permissionsDownQueries = [];
const tableDef = getQualifiedTableDef(
{
name: table,
schema: currentSchema,
},
currentDriver
);
if (currRolePermissions && currRolePermissions.permissions) {
Object.keys(currRolePermissions.permissions).forEach(type => {
const deleteQuery = getDropPermissionQuery(
type,
{ name: table, schema: currentSchema },
tableDef,
role,
currentDataSource
);
const createQuery = getCreatePermissionQuery(
type,
{ name: table, schema: currentSchema },
tableDef,
role,
currRolePermissions.permissions[type],
currentDataSource
@ -333,6 +343,14 @@ const permRemoveMultipleRoles = tableSchema => {
const permissionsUpQueries = [];
const permissionsDownQueries = [];
const tableDef = getQualifiedTableDef(
{
name: table,
schema: currentSchema,
},
currentDriver
);
roles.map(role => {
const currentRolePermission = currentPermissions.filter(el => {
return el.role_name === role;
@ -340,13 +358,13 @@ const permRemoveMultipleRoles = tableSchema => {
Object.keys(currentRolePermission[0].permissions).forEach(type => {
const deleteQuery = getDropPermissionQuery(
type,
{ name: table, schema: currentSchema },
tableDef,
role,
currentDataSource
);
const createQuery = getCreatePermissionQuery(
type,
{ name: table, schema: currentSchema },
tableDef,
role,
currentRolePermission[0].permissions[type],
currentDataSource
@ -401,6 +419,14 @@ const applySamePermissionsBulk = (tableSchema, arePermissionsModified) => {
applyTo => applyTo.table && applyTo.action && applyTo.role
);
const tableDef = getQualifiedTableDef(
{
name: table,
schema: currentSchema,
},
currentDriver
);
if (arePermissionsModified) {
const mainApplyTo = {
table: table,
@ -431,14 +457,14 @@ const applySamePermissionsBulk = (tableSchema, arePermissionsModified) => {
// existing permission is there. so drop and recreate for down migrations
const deleteQuery = getDropPermissionQuery(
applyTo.action,
{ name: applyTo.table, schema: currentSchema },
tableDef,
applyTo.role,
currentDataSource
);
const createQuery = getCreatePermissionQuery(
applyTo.action,
{ name: applyTo.table, schema: currentSchema },
tableDef,
applyTo.role,
currentPermPermission.permissions[applyTo.action],
currentDataSource
@ -463,14 +489,14 @@ const applySamePermissionsBulk = (tableSchema, arePermissionsModified) => {
// now add normal create and drop permissions
const createQuery = getCreatePermissionQuery(
applyTo.action,
{ name: applyTo.table, schema: currentSchema },
tableDef,
applyTo.role,
sanitizedPermission,
currentDataSource
);
const deleteQuery = getDropPermissionQuery(
applyTo.action,
{ name: applyTo.table, schema: currentSchema },
tableDef,
applyTo.role,
currentDataSource
);
@ -726,19 +752,27 @@ const permChangePermissions = changeType => {
delete permissionsState[query].limit;
}
const tableDef = getQualifiedTableDef(
{
name: table,
schema: currentSchema,
},
currentDriver
);
const permissionsUpQueries = [];
const permissionsDownQueries = [];
if (currRolePermissions && currRolePermissions.permissions[query]) {
const deleteQuery = getDropPermissionQuery(
query,
{ name: table, schema: currentSchema },
tableDef,
role,
currentDataSource
);
const createQuery = getCreatePermissionQuery(
query,
{ name: table, schema: currentSchema },
tableDef,
role,
prevPermissionsState[query],
currentDataSource
@ -750,14 +784,14 @@ const permChangePermissions = changeType => {
if (changeType === permChangeTypes.save) {
const createQuery = getCreatePermissionQuery(
query,
{ name: table, schema: currentSchema },
tableDef,
role,
permissionsState[query],
currentDataSource
);
const deleteQuery = getDropPermissionQuery(
query,
{ name: table, schema: currentSchema },
tableDef,
role,
currentDataSource
);

64
console/src/components/Services/Data/TableRelationships/Actions.js
View File

@ -19,6 +19,7 @@ import {
getAddRelationshipQuery,
} from '../../../../metadata/queryUtils';
import Migration from '../../../../utils/migration/Migration';
import { currentDriver, getQualifiedTableDef } from '../../../../dataSources';
export const SET_MANUAL_REL_ADD = 'ModifyTable/SET_MANUAL_REL_ADD';
export const MANUAL_REL_SET_TYPE = 'ModifyTable/MANUAL_REL_SET_TYPE';
@ -234,27 +235,20 @@ const saveRenameRelationship = (oldName, newName, tableName, callback) => {
return (dispatch, getState) => {
const currentSchema = getState().tables.currentSchema;
const currentSource = getState().tables.currentDataSource;
const tableDef = getQualifiedTableDef(
{
name: tableName,
schema: currentSchema,
},
currentDriver
);
const migrateUp = [
getRenameRelationshipQuery(
{
name: tableName,
schema: currentSchema,
},
oldName,
newName,
currentSource
),
getRenameRelationshipQuery(tableDef, oldName, newName, currentSource),
];
const migrateDown = [
getRenameRelationshipQuery(
{
name: tableName,
schema: currentSchema,
},
newName,
oldName,
currentSource
),
getRenameRelationshipQuery(tableDef, newName, oldName, currentSource),
];
// Apply migrations
const migrationName = `rename_relationship_${oldName}_to_${newName}_schema_${currentSchema}_table_${tableName}`;
@ -323,7 +317,13 @@ const generateRelationshipsQuery = (relMeta, currentDataSource) => {
}
_downQuery = getDropRelationshipQuery(
{ name: relMeta.lTable, schema: relMeta.lSchema },
getQualifiedTableDef(
{
name: relMeta.lTable,
schema: relMeta.lSchema,
},
currentDriver
),
relMeta.relName,
currentDataSource
);
@ -368,7 +368,13 @@ const generateRelationshipsQuery = (relMeta, currentDataSource) => {
}
_downQuery = getDropRelationshipQuery(
{ name: relMeta.lTable, schema: relMeta.lSchema },
getQualifiedTableDef(
{
name: relMeta.lTable,
schema: relMeta.lSchema,
},
currentDriver
),
relMeta.relName,
currentDataSource
);
@ -495,8 +501,22 @@ const addRelViewMigrate = (tableSchema, toggleEditor) => (
}
columnMapping[colMap.column] = colMap.refColumn;
});
const tableInfo = { name: currentTableName, schema: currentTableSchema };
const remoteTableInfo = { name: rTable, schema: rSchema };
const tableInfo = getQualifiedTableDef(
{
name: currentTableName,
schema: currentTableSchema,
},
currentDriver
);
const remoteTableInfo = getQualifiedTableDef(
{
name: rTable,
schema: rSchema,
},
currentDriver
);
const relChangesUp = [
getAddRelationshipQuery(

16
console/src/components/Services/Data/TableRelationships/Relationships.js
View File

@ -426,13 +426,15 @@ const Relationships = ({
if (relAdd.isActive) {
addRelSection = (
<div className={styles.activeEdit}>
<AddRelationship
tableName={tableName}
currentSchema={currentSchema}
allSchemas={allSchemas}
cachedRelationshipData={relAdd}
dispatch={dispatch}
/>
{isFeatureSupported('tables.relationships.track') && (
<AddRelationship
tableName={tableName}
currentSchema={currentSchema}
allSchemas={allSchemas}
cachedRelationshipData={relAdd}
dispatch={dispatch}
/>
)}
<AddManualRelationship
tableSchema={tableSchema}
allSchemas={allSchemas}

5
console/src/components/Services/Data/TableRelationships/RelationshipsView.js
View File

@ -9,7 +9,7 @@ import { NotFoundError } from '../../../Error/PageNotFound';
import RemoteRelationships from './RemoteRelationships/RemoteRelationships';
import ToolTip from '../../../Common/Tooltip/Tooltip';
import KnowMoreLink from '../../../Common/KnowMoreLink/KnowMoreLink';
import { findAllFromRel } from '../../../../dataSources';
import { findAllFromRel, isFeatureSupported } from '../../../../dataSources';
import { getRemoteSchemasSelector } from '../../../../metadata/selector';
import { RightContainer } from '../../../Common/Layout/RightContainer';
@ -176,7 +176,8 @@ class RelationshipsView extends Component {
/>
</div>
</div>
{remoteRelationshipsSection()}
{isFeatureSupported('tables.relationships.track') &&
remoteRelationshipsSection()}
</div>
<div className={`${styles.fixed} hidden`}>{alert}</div>
</div>

2
console/src/components/Services/Data/TreeView.tsx
View File

@ -189,7 +189,7 @@ const SchemaItemsView: React.FC<SchemaItemsViewProps> = ({
) : (
<li>
<span
className={`${styles.title} ${styles.titleClosed} ${styles.padd_bottom_small}`}
className={`${styles.sidebarTablePadding} ${styles.padd_bottom_small}`}
>
<i className="fa fa-table" />
<span className={styles.loaderBar} />

120
console/src/components/Services/Data/mergeData.ts
View File

@ -377,3 +377,123 @@ export const mergeLoadSchemaDataPostgres = (
return _mergedTableData;
};
type BigQueryTable = {
columns: Array<{
column_name: string;
data_type: string;
data_type_name: string;
is_nullable: 'YES' | 'NO';
ordinal_position: number;
table_name: string;
table_schema: string;
}>;
comment: string;
table_name: string;
table_schema: string;
table_type: 'TABLE' | 'VIEW' | 'EXTERNAL';
};
export const mergeDataBigQuery = (
data: Array<{ result: string[] }>,
metadataTables: TableEntry[]
): Table[] => {
const result = [] as Table[];
const tables = [] as BigQueryTable[];
data[0].result.slice(1).forEach(row => {
try {
tables.push({
table_schema: row[0],
table_name: row[1],
table_type: row[2] as BigQueryTable['table_type'],
comment: row[3],
columns: JSON.parse(row[4]),
});
// eslint-disable-next-line no-empty
} catch (err) {
console.log(err);
}
});
tables.forEach(table => {
const metadataTable = metadataTables?.find(
t =>
t.table.schema === table.table_schema &&
t.table.name === table.table_name
);
const relationships = [] as Table['relationships'];
metadataTable?.array_relationships?.forEach(rel => {
relationships.push({
rel_def: rel.using,
rel_name: rel.name,
table_name: table.table_name,
table_schema: table.table_schema,
rel_type: 'array',
});
});
metadataTable?.object_relationships?.forEach(rel => {
relationships.push({
rel_def: rel.using,
rel_name: rel.name,
table_name: table.table_name,
table_schema: table.table_schema,
rel_type: 'object',
});
});
const rolePermMap = permKeys.reduce((rpm: Record<string, any>, key) => {
if (metadataTable) {
metadataTable[key]?.forEach(
(perm: { role: string; permission: Record<string, any> }) => {
rpm[perm.role] = {
permissions: {
...(rpm[perm.role] && rpm[perm.role].permissions),
[keyToPermission[key]]: perm.permission,
},
};
}
);
}
return rpm;
}, {});
const permissions: Table['permissions'] = Object.keys(rolePermMap).map(
role => ({
role_name: role,
permissions: rolePermMap[role].permissions,
table_name: table.table_name,
table_schema: table.table_schema,
})
);
const mergedInfo = {
table_schema: table.table_schema,
table_name: table.table_name,
table_type: table.table_type,
is_table_tracked: metadataTables.some(
t =>
t.table.name === table.table_name &&
t.table.schema === table.table_schema
),
columns: table.columns,
comment: '',
triggers: [],
primary_key: null,
relationships,
permissions,
unique_constraints: [],
check_constraints: [],
foreign_key_constraints: [] as Table['foreign_key_constraints'],
opp_foreign_key_constraints: [] as Table['foreign_key_constraints'],
view_info: null,
remote_relationships: [],
is_enum: false,
configuration: undefined,
computed_fields: [],
};
result.push(mergedInfo);
});
return result;
};

15
console/src/dataSources/common.ts
View File

@ -33,8 +33,19 @@ export const getTableDef = (table: Table) => {
return generateTableDef(table.table_name, table.table_schema);
};
export const getQualifiedTableDef = (tableDef: QualifiedTable | string) => {
return typeof tableDef === 'string' ? generateTableDef(tableDef) : tableDef;
export const getQualifiedTableDef = (
tableDef: QualifiedTable | string,
driver?: string
) => {
if (typeof tableDef === 'string') return generateTableDef(tableDef);
if (driver === 'bigquery')
return {
name: tableDef.name,
dataset: tableDef.schema,
};
return tableDef;
};
export const getTableNameWithSchema = (

11
console/src/dataSources/index.ts
View File

@ -20,8 +20,9 @@ import { QualifiedTable } from '../metadata/types';
import { supportedFeatures as PGSupportedFeatures } from './services/postgresql';
import { supportedFeatures as MssqlSupportedFeatures } from './services/mssql';
import { supportedFeatures as BigQuerySupportedFeatures } from './services/bigquery';
export const drivers = ['postgres', 'mysql', 'mssql'] as const;
export const drivers = ['postgres', 'mysql', 'mssql', 'bigquery'] as const;
export type Driver = typeof drivers[number];
export type ColumnsInfoResult = {
@ -304,7 +305,7 @@ export interface DataSourcesAPI {
eventId: string
) => string;
getDatabaseInfo: string;
getTableInfo?: (tables: string[]) => string;
getTableInfo?: (tables: QualifiedTable[]) => string;
generateTableRowRequest?: () => generateTableRowRequestType;
getDatabaseVersionSql?: string;
permissionColumnDataTypes: Partial<PermissionColumnCategories> | null;
@ -331,7 +332,11 @@ export const getSupportedDrivers = (
return get(supportedFeatures, feature) || false;
};
return [PGSupportedFeatures, MssqlSupportedFeatures]
return [
PGSupportedFeatures,
MssqlSupportedFeatures,
BigQuerySupportedFeatures,
]
.filter(d => isEnabled(d))
.map(d => d.driver.name) as Driver[];
};

376
console/src/dataSources/services/bigquery/index.tsx Normal file
View File

@ -0,0 +1,376 @@
import React from 'react';
import { DataSourcesAPI } from '../..';
import { QualifiedTable } from '../../../metadata/types';
import {
TableColumn,
Table,
BaseTableColumn,
SupportedFeaturesType,
} from '../../types';
import { generateTableRowRequest } from './utils';
const permissionColumnDataTypes = {
character: [
'STRING',
'INT64',
'NUMERIC',
'DECIMAL',
'BIGNUMERIC',
'BIGDECIMAL',
'FLOAT64',
'INTEGER',
],
numeric: [],
dateTime: ['DATETIME', 'TIME', 'TIMESTAMP'],
user_defined: [],
};
const supportedColumnOperators = [
'_is_null',
'_eq',
'_neq',
'_gt',
'_lt',
'_gte',
'_lte',
];
const isTable = (table: Table) => {
if (!table.table_type) return true; // todo
return table.table_type === 'TABLE' || table.table_type === 'BASE TABLE';
};
const columnDataTypes = {
INTEGER: 'integer',
BIGINT: 'bigint',
GUID: 'guid',
JSONDTYPE: 'nvarchar',
DATETIMEOFFSET: 'timestamp with time zone',
NUMERIC: 'numeric',
DATE: 'date',
TIME: 'time',
TEXT: 'text',
};
const operators = [
{ name: 'equals', value: '$eq', graphqlOp: '_eq' },
{ name: 'not equals', value: '$ne', graphqlOp: '_neq' },
{ name: '>', value: '$gt', graphqlOp: '_gt' },
{ name: '<', value: '$lt', graphqlOp: '_lt' },
{ name: '>=', value: '$gte', graphqlOp: '_gte' },
{ name: '<=', value: '$lte', graphqlOp: '_lte' },
];
// eslint-disable-next-line no-useless-escape
const createSQLRegex = /create\s*(?:|or\s*replace)\s*(view|table|function)\s*(?:\s*if*\s*not\s*exists\s*)?((\"?\w+\"?)\.(\"?\w+\"?)|(\"?\w+\"?))/g;
export const displayTableName = (table: Table) => {
const tableName = table.table_name;
return isTable(table) ? <span>{tableName}</span> : <i>{tableName}</i>;
};
export const isJsonColumn = (column: BaseTableColumn): boolean => {
return column.data_type_name === 'json' || column.data_type_name === 'jsonb';
};
export const supportedFeatures: SupportedFeaturesType = {
driver: {
name: 'bigquery',
},
schemas: {
create: {
enabled: false,
},
delete: {
enabled: false,
},
},
tables: {
create: {
enabled: false,
},
browse: {
enabled: true,
customPagination: true,
aggregation: false,
},
insert: {
enabled: false,
},
modify: {
enabled: false,
},
relationships: {
enabled: true,
track: false,
},
permissions: {
enabled: true,
},
track: {
enabled: true,
},
},
functions: {
enabled: true,
track: {
enabled: false,
},
nonTrackableFunctions: {
enabled: false,
},
},
events: {
triggers: {
enabled: true,
add: false,
},
},
actions: {
enabled: true,
relationships: false,
},
rawSQL: {
enabled: true,
tracking: false,
},
connectDbForm: {
connectionParameters: false,
databaseURL: true,
environmentVariable: true,
read_replicas: false,
},
};
export const bigquery: DataSourcesAPI = {
isTable,
isJsonColumn,
displayTableName,
operators,
generateTableRowRequest,
getFunctionSchema: () => {
return '';
},
getFunctionDefinition: () => {
return '';
},
getSchemaFunctions: () => {
return [];
},
findFunction: () => {
return undefined;
},
getGroupedTableComputedFields: () => {
return { scalar: [], table: [] };
},
isColumnAutoIncrement: () => {
return false;
},
getTableSupportedQueries: () => {
// since only subscriptions and queries are supported on MSSQL atm.
return ['select'];
},
getColumnType: (col: TableColumn) => col.data_type_name ?? col.data_type,
arrayToPostgresArray: () => {
return '';
},
schemaListSql: (schemaFilter: string[]) => {
if (schemaFilter.length)
return `select schema_name from INFORMATION_SCHEMA.SCHEMATA where schema_name in (${schemaFilter
.map(s => `'${s}'`)
.join(',')})`;
return `select schema_name from INFORMATION_SCHEMA.SCHEMATA`;
},
parseColumnsInfoResult: () => {
return {};
},
columnDataTypes,
getFetchTablesListQuery: ({ schemas, tables }) => {
let datasets = [];
if (schemas) {
datasets = schemas;
} else {
datasets = tables.map(t => t.table_schema);
}
const query = (dataset: string) => `
select
t.table_schema as table_schema,
t.table_name as table_name,
t.table_type as table_type,
opts.option_value as comment,
CONCAT("[", c.json_data ,"]") as columns
FROM ${dataset}.INFORMATION_SCHEMA.TABLES as t
LEFT JOIN
(
with x as (
select table_name, table_schema, column_name, ordinal_position, is_nullable, data_type from ${dataset}.INFORMATION_SCHEMA.COLUMNS
) select x.table_name as table_name, x.table_schema as table_schema, STRING_AGG(TO_JSON_STRING(x)) as json_data from x group by x.table_name,x.table_schema
) as c
ON c.table_name = t.table_name and t.table_schema = c.table_schema
LEFT JOIN ${dataset}.INFORMATION_SCHEMA.TABLE_OPTIONS as opts
ON opts.table_name = t.table_name and opts.table_schema = t.table_schema and opts.option_name = "description"
`;
return datasets
.map(dataset => {
return query(dataset);
})
.join('union all');
},
commonDataTypes: [],
fetchColumnTypesQuery: '',
fetchColumnDefaultFunctions: () => {
return '';
},
isSQLFunction: () => {
return false;
},
getEstimateCountQuery: () => {
return '';
},
isColTypeString: () => {
return false;
},
cascadeSqlQuery: () => {
return '';
},
dependencyErrorCode: '',
getCreateTableQueries: () => {
return [];
},
getDropTableSql: () => {
return '';
},
createSQLRegex,
getDropSchemaSql: (schema: string) => {
return `drop schema ${schema};`;
},
getCreateSchemaSql: (schema: string) => {
return `create schema ${schema};`;
},
isTimeoutError: () => {
return false;
},
getAlterForeignKeySql: () => {
return '';
},
getCreateFKeySql: () => {
return '';
},
getDropConstraintSql: () => {
return '';
},
getRenameTableSql: () => {
return '';
},
getDropTriggerSql: () => {
return '';
},
getCreateTriggerSql: () => {
return '';
},
getDropSql: () => {
return '';
},
getViewDefinitionSql: () => {
return '';
},
getDropColumnSql: () => {
return '';
},
getAddColumnSql: () => {
return '';
},
getAddUniqueConstraintSql: () => {
return '';
},
getDropNotNullSql: () => {
return '';
},
getSetCommentSql: () => {
return '';
},
getSetColumnDefaultSql: () => {
return '';
},
getSetNotNullSql: () => {
return '';
},
getAlterColumnTypeSql: () => {
return '';
},
getDropColumnDefaultSql: () => {
return '';
},
getRenameColumnQuery: () => {
return '';
},
fetchColumnCastsQuery: '',
checkSchemaModification: () => {
return false;
},
getCreateCheckConstraintSql: () => {
return '';
},
getCreatePkSql: () => {
return '';
},
getFunctionDefinitionSql: null,
primaryKeysInfoSql: () => {
return 'select []';
},
checkConstraintsSql: () => {
return 'select []';
},
uniqueKeysSql: () => {
return 'select []';
},
frequentlyUsedColumns: [],
getFKRelations: () => {
return 'select []';
},
getReferenceOption: () => {
return '';
},
deleteFunctionSql: () => {
return '';
},
getEventInvocationInfoByIDSql: () => {
return '';
},
getDatabaseInfo: '',
getTableInfo: (tables: QualifiedTable[]) => {
if (!tables.length) return 'select []';
const schemaMap = {} as Record<string, Array<string>>;
tables.forEach(t => {
if (!schemaMap[t.schema]) schemaMap[t.schema] = [t.name];
else schemaMap[t.schema].push(t.name);
});
let query = '';
Object.keys(schemaMap).forEach((schema, index) => {
query += ` select
table_name,
table_schema,
case
when table_type = 'VIEW' then 'view'
else 'table'
end as table_type
from ${schema}.INFORMATION_SCHEMA.TABLES where table_name in (${schemaMap[
schema
]
.map(t => `'${t}'`)
.join(',')})`;
if (index !== Object.keys(schemaMap).length - 1) query += ` union all`;
});
return query;
},
supportedFeatures,
getDatabaseVersionSql: 'SELECT @@VERSION;',
permissionColumnDataTypes,
viewsSupported: false,
supportedColumnOperators,
aggregationPermissionsAllowed: false,
};

204
console/src/dataSources/services/bigquery/utils.ts Normal file
View File

@ -0,0 +1,204 @@
import {
OrderBy,
WhereClause,
} from '../../../components/Common/utils/v1QueryUtils';
import Endpoints from '../../../Endpoints';
import { TableEntry } from '../../../metadata/types';
import { ReduxState } from '../../../types';
import { BaseTableColumn, Relationship, Table } from '../../types';
type Tables = ReduxState['tables'];
interface GetGraphQLQuery {
allSchemas: Table[];
view: Tables['view'];
originalTable: string;
currentSchema: string;
isExport?: boolean;
}
export const BigQueryDataTypes = {
character: ['STRING'],
numeric: [
'INT64',
'NUMERIC',
'DECIMAL',
'BIGNUMERIC',
'BIGDECIMAL',
'FLOAT64',
],
dateTime: ['DATETIME', 'TIME', 'TIMESTAMP', 'DATE'],
user_defined: [],
};
const getFormattedValue = (
type: string,
value: any
): string | number | undefined => {
if (
BigQueryDataTypes.character.includes(type) ||
BigQueryDataTypes.dateTime.includes(type)
)
return `"${value}"`;
if (BigQueryDataTypes.numeric.includes(type)) return value;
return value;
};
const RqlToGraphQlOp = (op: string) => {
if (!op || !op?.startsWith('$')) return 'none';
return op.replace('$', '_');
};
const generateWhereClauseQueryString = (
wheres: WhereClause[],
columnTypeInfo: BaseTableColumn[]
): string | null => {
const whereClausesArr = wheres.map((i: Record<string, any>) => {
const columnName = Object.keys(i)[0];
const RqlOperator = Object.keys(i[columnName])[0];
const value = i[columnName][RqlOperator];
const type = columnTypeInfo?.find(c => c.column_name === columnName)
?.data_type;
return `${columnName}: {${RqlToGraphQlOp(RqlOperator)}: ${getFormattedValue(
type || 'varchar',
value
)} }`;
});
return whereClausesArr.length
? `where: {${whereClausesArr.join(',')}}`
: null;
};
const generateSortClauseQueryString = (sorts: OrderBy[]): string | null => {
const sortClausesArr = sorts.map((i: OrderBy) => {
return `${i.column}: ${i.type}`;
});
return sortClausesArr.length
? `order_by: {${sortClausesArr.join(',')}}`
: null;
};
const getColQuery = (
cols: (string | { name: string; columns: string[] })[],
limit: number,
relationships: Relationship[]
): string[] => {
return cols.map(c => {
if (typeof c === 'string') return c;
const rel = relationships.find((r: any) => r.rel_name === c.name);
return `${c.name} ${
rel?.rel_type === 'array' ? `(limit: ${limit})` : ''
} { ${getColQuery(c.columns, limit, relationships).join('\n')} }`;
});
};
export const getGraphQLQueryForBrowseRows = ({
allSchemas,
view,
originalTable,
currentSchema,
isExport,
}: GetGraphQLQuery) => {
const currentTable: Table | undefined = allSchemas?.find(
(t: Table) =>
t.table_name === originalTable && t.table_schema === currentSchema
);
const columnTypeInfo: BaseTableColumn[] = currentTable?.columns || [];
const relationshipInfo: Relationship[] = currentTable?.relationships || [];
if (!columnTypeInfo) {
throw new Error('Error in finding column info for table');
}
let whereConditions: WhereClause[] = [];
let isRelationshipView = false;
if (view.query.where) {
if (view.query.where.$and) {
whereConditions = view.query.where.$and;
} else {
isRelationshipView = true;
whereConditions = Object.keys(view.query.where)
.filter(k => view.query.where[k])
.map(k => {
const obj = {} as any;
obj[k] = { $eq: view.query.where[k] };
return obj;
});
}
}
const sortConditions: OrderBy[] = [];
if (view.query.order_by) {
sortConditions.push(...view.query.order_by);
}
const limit = isExport ? null : `limit: ${view.curFilter.limit}`;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const offset = isExport
? null
: `offset: ${!isRelationshipView ? view.curFilter.offset : 0}`;
const clauses = `${[
generateWhereClauseQueryString(whereConditions, columnTypeInfo),
generateSortClauseQueryString(sortConditions),
limit,
// offset,
]
.filter(Boolean)
.join(',')}`;
return `query TableRows {
${`${currentSchema}_${originalTable}`} ${clauses && `(${clauses})`} {
${getColQuery(
view.query.columns,
view.curFilter.limit,
relationshipInfo
).join('\n')}
}
}
`;
};
const getTableRowRequestBody = ({
tables,
isExport,
}: {
tables: Tables;
isExport?: boolean;
tableConfiguration?: TableEntry['configuration'];
}) => {
const {
currentTable: originalTable,
view,
allSchemas,
currentSchema,
} = tables;
return {
query: getGraphQLQueryForBrowseRows({
allSchemas,
view,
originalTable,
currentSchema,
isExport,
}),
variables: null,
operationName: 'TableRows',
};
};
const processTableRowData = (
data: any,
config?: { originalTable: string; currentSchema: string }
) => {
const { originalTable, currentSchema } = config!;
const rows = data.data[`${currentSchema}_${originalTable}`];
return { estimatedCount: rows.length, rows };
};
export const generateTableRowRequest = () => {
return {
endpoint: Endpoints.graphQLUrl,
getTableRowRequestBody,
processTableRowData,
};
};

3
console/src/dataSources/services/index.ts
View File

@ -1,5 +1,6 @@
import { postgres } from './postgresql';
import { mysql } from './mysql';
import { mssql } from './mssql';
import { bigquery } from './bigquery';
export const services = { postgres, mysql, mssql };
export const services = { postgres, mysql, mssql, bigquery };

36
console/src/dataSources/services/mssql/index.tsx
View File

@ -1,5 +1,6 @@
import React from 'react';
import { DataSourcesAPI } from '../..';
import { QualifiedTable } from '../../../metadata/types';
import {
TableColumn,
Table,
@ -83,6 +84,14 @@ export const supportedFeatures: SupportedFeaturesType = {
driver: {
name: 'mssql',
},
schemas: {
create: {
enabled: true,
},
delete: {
enabled: true,
},
},
tables: {
create: {
enabled: false,
@ -100,10 +109,23 @@ export const supportedFeatures: SupportedFeaturesType = {
},
relationships: {
enabled: true,
track: true,
},
permissions: {
enabled: true,
},
track: {
enabled: false,
},
},
functions: {
enabled: true,
track: {
enabled: false,
},
nonTrackableFunctions: {
enabled: false,
},
},
events: {
triggers: {
@ -115,6 +137,16 @@ export const supportedFeatures: SupportedFeaturesType = {
enabled: true,
relationships: false,
},
rawSQL: {
enabled: true,
tracking: true,
},
connectDbForm: {
connectionParameters: false,
databaseURL: true,
environmentVariable: true,
read_replicas: false,
},
};
export const isJsonColumn = (column: BaseTableColumn): boolean => {
@ -389,7 +421,7 @@ INNER JOIN sys.schemas sch2
return '';
},
getDatabaseInfo: '',
getTableInfo: (tables: string[]) => `
getTableInfo: (tables: QualifiedTable[]) => `
SELECT
o.name AS table_name,
s.name AS table_schema,
@ -405,7 +437,7 @@ FROM
sys.objects AS o
JOIN sys.schemas AS s ON (o.schema_id = s.schema_id)
WHERE
o.name in (${tables.join(',')}) for json path;
o.name in (${tables.map(t => `'${t.name}'`).join(',')}) for json path;
`,
getDatabaseVersionSql: 'SELECT @@VERSION;',
permissionColumnDataTypes,

31
console/src/dataSources/services/postgresql/index.tsx
View File

@ -507,6 +507,14 @@ export const supportedFeatures: SupportedFeaturesType = {
driver: {
name: 'postgres',
},
schemas: {
create: {
enabled: true,
},
delete: {
enabled: true,
},
},
tables: {
create: {
enabled: true,
@ -524,10 +532,23 @@ export const supportedFeatures: SupportedFeaturesType = {
relationships: {
enabled: true,
remoteRelationships: true,
track: true,
},
permissions: {
enabled: true,
},
track: {
enabled: false,
},
},
functions: {
enabled: true,
track: {
enabled: true,
},
nonTrackableFunctions: {
enabled: true,
},
},
events: {
triggers: {
@ -539,6 +560,16 @@ export const supportedFeatures: SupportedFeaturesType = {
enabled: true,
relationships: true,
},
rawSQL: {
enabled: true,
tracking: true,
},
connectDbForm: {
connectionParameters: true,
databaseURL: true,
environmentVariable: true,
read_replicas: true,
},
};
const defaultRedirectSchema = 'public';

4
console/src/dataSources/services/postgresql/sqlUtils.ts
View File

@ -1210,7 +1210,7 @@ FROM (
table_schema) AS info;
`;
export const getTableInfo = (tables: string[]) => `
export const getTableInfo = (tables: QualifiedTable[]) => `
SELECT
COALESCE(json_agg(row_to_json(info)), '[]'::JSON)
FROM (
@ -1226,7 +1226,7 @@ FROM (
join pg_catalog.pg_namespace n
on n.oid = pgclass.relnamespace
where
pgclass.relname in (${tables.join(',')})
pgclass.relname in (${tables.map(t => `'${t.name}'`).join(',')})
) AS info;
`;

34
console/src/dataSources/types.ts
View File

@ -109,7 +109,8 @@ export interface Table extends BaseTable {
| 'FOREIGN TABLE'
| 'PARTITIONED TABLE'
| 'BASE TABLE'
| 'TABLE'; // specific to SQL Server
| 'TABLE' // specific to SQL Server
| 'EXTERNAL'; // specific to Big Query
primary_key: {
table_name: string;
table_schema: string;
@ -195,6 +196,14 @@ export type SupportedFeaturesType = {
driver: {
name: string;
};
schemas: {
create: {
enabled: boolean;
};
delete: {
enabled: boolean;
};
};
tables: {
create: {
enabled: boolean;
@ -213,10 +222,23 @@ export type SupportedFeaturesType = {
relationships: {
enabled: boolean;
remoteRelationships?: boolean;
track: boolean;
};
permissions: {
enabled: boolean;
};
track: {
enabled: boolean;
};
};
functions: {
enabled: boolean;
track: {
enabled: boolean;
};
nonTrackableFunctions: {
enabled: boolean;
};
};
events: {
triggers: {
@ -228,6 +250,16 @@ export type SupportedFeaturesType = {
enabled: boolean;
relationships: boolean;
};
rawSQL: {
enabled: boolean;
tracking: boolean;
};
connectDbForm: {
connectionParameters: boolean;
databaseURL: boolean;
environmentVariable: boolean;
read_replicas: boolean;
};
};
type Tables = ReduxState['tables'];

4
console/src/metadata/actions.ts
View File

@ -123,6 +123,10 @@ export interface AddDataSourceRequest {
idle_timeout?: number; // in seconds
retries?: number;
};
bigQuery: {
projectId: string;
datasets: string;
};
};
};
}

7
console/src/metadata/queryUtils.ts
View File

@ -3,6 +3,7 @@ import {
ActionDefinition,
CustomTypes,
QualifiedTable,
QualifiedTableBigQuery,
HasuraMetadataV3,
QualifiedFunction,
RestEndpointEntry,
@ -121,6 +122,9 @@ export const getMetadataQuery = (
case 'mssql':
prefix = 'mssql_';
break;
case 'bigquery':
prefix = 'bigquery_';
break;
case 'postgres':
default:
prefix = 'pg_';
@ -306,7 +310,7 @@ export const getTrackTableQuery = ({
driver,
customColumnNames,
}: {
tableDef: QualifiedTable;
tableDef: QualifiedTable | QualifiedTableBigQuery;
source: string;
driver: Driver;
customColumnNames?: Record<string, string>;
@ -319,6 +323,7 @@ export const getTrackTableQuery = ({
custom_column_names: customColumnNames,
},
};
return getMetadataQuery('track_table', source, args, driver);
};

65
console/src/metadata/reducer.ts
View File

@ -30,6 +30,66 @@ const defaultState: MetadataState = {
inheritedRoles: [],
};
const renameSourceAttributes = (sources: HasuraMetadataV3['sources']) =>
sources.map((s: any) => {
let tables = s.tables;
if (s.kind === 'bigquery') {
tables = s.tables.map((t: any) => {
let object_relationships = [];
if (t.object_relationships) {
object_relationships = t.object_relationships.map((objRel: any) => {
return {
...objRel,
using: {
...objRel.using,
manual_configuration: {
...objRel.using.manual_configuration,
remote_table: {
schema:
objRel.using.manual_configuration.remote_table.dataset,
name: objRel.using.manual_configuration.remote_table.name,
},
},
},
};
});
}
let array_relationships = [];
if (t.array_relationships) {
array_relationships = t.array_relationships.map((objRel: any) => {
return {
...objRel,
using: {
...objRel.using,
manual_configuration: {
...objRel.using.manual_configuration,
remote_table: {
schema:
objRel.using.manual_configuration.remote_table.dataset,
name: objRel.using.manual_configuration.remote_table.name,
},
},
},
};
});
}
return {
object_relationships,
array_relationships,
table: {
name: t.table.name,
schema: t.table.dataset,
},
select_permissions: t.select_permissions,
};
});
}
return { ...s, tables };
});
export const metadataReducer = (
state = defaultState,
action: MetadataActions
@ -40,7 +100,10 @@ export const metadataReducer = (
'metadata' in action.data ? action.data.metadata : action.data;
return {
...state,
metadataObject: metadata,
metadataObject: {
...metadata,
sources: renameSourceAttributes(metadata.sources),
},
resourceVersion:
'resource_version' in action.data ? action.data.resource_version : 1,
allowedQueries: setAllowedQueries(

12
console/src/metadata/selector.ts
View File

@ -359,11 +359,17 @@ export const getInheritedRoles = (state: ReduxState) =>
export const getDataSources = createSelector(getMetadata, metadata => {
const sources: DataSource[] = [];
metadata?.sources.forEach(source => {
let url: string | { from_env: string } = '';
if (source.kind === 'bigquery') {
url = source.configuration?.service_account?.from_env || '';
} else {
url = source.configuration?.connection_info?.connection_string
? source.configuration?.connection_info.connection_string
: source.configuration?.connection_info?.database_url || '';
}
sources.push({
name: source.name,
url: source.configuration?.connection_info?.connection_string
? source.configuration?.connection_info.connection_string
: source.configuration?.connection_info?.database_url || '',
url,
connection_pool_settings: source.configuration?.connection_info
?.pool_settings || {
retries: 1,

21
console/src/metadata/sourcesUtils.ts
View File

@ -11,6 +11,10 @@ export const addSource = (
idle_timeout?: number;
retries?: number;
};
bigQuery: {
projectId: string;
datasets: string;
};
},
// supported only for PG sources at the moment
replicas?: Omit<SourceConnectionInfo, 'connection_string'>[]
@ -30,6 +34,20 @@ export const addSource = (
};
}
if (driver === 'bigquery') {
return {
type: 'bigquery_add_source',
args: {
name: payload.name,
configuration: {
service_account: payload.dbUrl,
project_id: payload.bigQuery.projectId,
datasets: payload.bigQuery.datasets.split(',').map(d => d.trim()),
},
},
};
}
return {
type: 'pg_add_source',
args: {
@ -54,6 +72,9 @@ export const removeSource = (driver: Driver, name: string) => {
case 'mysql':
prefix = 'mysql_';
break;
case 'bigquery':
prefix = 'bigquery_';
break;
default:
prefix = 'pg_';
}

17
console/src/metadata/types.ts
View File

@ -44,6 +44,11 @@ export interface QualifiedTable {
schema: string;
}
export interface QualifiedTableBigQuery {
name: string;
dataset: string;
}
/**
* Configuration for the table/view
* https://hasura.io/docs/latest/graphql/core/api-reference/schema-metadata-api/table-view.html#table-config
@ -854,6 +859,13 @@ export interface RestEndpointDefinition {
};
}
export interface BigQueryServiceAccount {
project_id?: string;
client_email?: string;
private_key?: string;
from_env?: string;
}
export interface RestEndpointEntry {
name: string;
url: string;
@ -903,11 +915,14 @@ export interface HasuraMetadataV2 {
export interface MetadataDataSource {
name: string;
kind?: 'postgres' | 'mysql' | 'mssql';
kind?: 'postgres' | 'mysql' | 'mssql' | 'bigquery';
configuration?: {
connection_info?: SourceConnectionInfo;
// pro-only feature
read_replicas?: SourceConnectionInfo[];
service_account?: BigQueryServiceAccount;
project_id?: string;
datasets?: string[];
};
tables: TableEntry[];
functions?: Array<{