hasura/graphql-engine
1
0
Fork 0
mirror of https://github.com/hasura/graphql-engine.git synced 2024-12-14 17:02:49 +03:00
Code Issues Projects Releases Wiki Activity

fix: fix performance issue for event triggers invocation logs

Browse Source 
PR-URL: https://github.com/hasura/graphql-engine-mono/pull/7155
GitOrigin-RevId: c72512a1638944d64354b36e57dd4ca77626f276
This commit is contained in:
Daniele Cammareri 2022-12-20 10:47:23 +01:00 committed by hasura-bot
parent d0f31e59e8
commit 6d64c67b4a
20 changed files with 20 additions and 252 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
console/src/components/Common/FilterQuery/FilterQuery.tsx
View File

@ -44,7 +44,7 @@ const FilterQuery: React.FC<Props> = props => {
triggerType,
} = props;
const { rows, count, runQuery, state, setState } = useFilterQuery(
const { rows, runQuery, state, setState } = useFilterQuery(
generateTableDef(table.table_name, table.table_schema),
dispatch,
presets,
@ -58,7 +58,7 @@ const FilterQuery: React.FC<Props> = props => {
return (
<Analytics name="EventFilterQuery" {...REDACT_EVERYTHING}>
<div className={styles.add_mar_top}>
{render(rows, count, state, setState, runQuery)}
{render(rows, state, setState, runQuery)}
</div>
</Analytics>
);

18
console/src/components/Common/FilterQuery/state.ts
View File

@ -44,7 +44,6 @@ export const useFilterQuery = (
) => {
const [state, setState] = React.useState(defaultState);
const [rows, setRows] = React.useState<any[]>([]);
const [count, setCount] = React.useState<number>();
const [loading, setLoading] = React.useState(false);
const [error, setError] = React.useState(false);
@ -97,16 +96,6 @@ export const useFilterQuery = (
if (triggerName) {
query = {
args: [
getRunSqlQuery(
dataSource?.getDataTriggerLogsCountQuery?.(
triggerName,
triggerOp
) ?? '',
currentSource ?? 'default',
false,
false,
currentDriver
),
getRunSqlQuery(
dataSource?.getDataTriggerLogsQuery?.(
triggerOp,
@ -138,10 +127,9 @@ export const useFilterQuery = (
).then(
(data: any) => {
if (triggerType === 'data') {
setCount(Number(data?.[0].result?.[1]?.[0]));
// formatting of the data
const formattedData: Record<string, any>[] = parseEventsSQLResp(
data?.[1]?.result ?? []
data?.[0]?.result ?? []
);
setRows(formattedData);
} else if (triggerOp !== 'invocation') {
@ -163,9 +151,6 @@ export const useFilterQuery = (
sorts: newSorts,
}));
}
if (triggerType !== 'data') {
setCount(data?.count ?? 10);
}
},
() => {
setError(true);
@ -223,7 +208,6 @@ export const useFilterQuery = (
error,
runQuery,
state,
count,
setState: setter,
};
};

1
console/src/components/Common/FilterQuery/types.ts
View File

@ -139,7 +139,6 @@ export type RunQuery = (options?: RunQueryOptions) => void;
export type FilterRenderProp = (
rows: any[],
count: number | undefined,
state: FilterState,
setState: SetFilterState,
runQuery: RunQuery

2
console/src/components/Services/Events/AdhocEvents/Logs/Logs.tsx
View File

@ -15,7 +15,6 @@ const InvocationLogs: React.FC<Props> = props => {
const renderRows: FilterRenderProp = (
rows,
count,
filterState,
setFilterState,
runQuery
@ -23,7 +22,6 @@ const InvocationLogs: React.FC<Props> = props => {
<InvocationLogsTable
rows={rows}
filterState={filterState}
count={count}
setFilterState={setFilterState}
runQuery={runQuery}
columns={['id', 'status', 'event_id', 'created_at']}

2
console/src/components/Services/Events/AdhocEvents/PendingEvents/PendingEvents.tsx
View File

@ -37,14 +37,12 @@ const PendingEvents: React.FC<Props> = props => {
const renderRows: FilterRenderProp = (
rows,
count,
filterState,
setFilterState,
runQuery
) => (
<EventsTable
rows={rows}
count={count}
filterState={filterState}
setFilterState={setFilterState}
runQuery={runQuery}

2
console/src/components/Services/Events/AdhocEvents/ProcessedEvents/ProcessedEvents.tsx
View File

@ -18,14 +18,12 @@ const ProcessedEvents: React.FC<Props> = props => {
const renderRows: FilterRenderProp = (
rows,
count,
filterState,
setFilterState,
runQuery
) => (
<EventsTable
rows={rows}
count={count}
filterState={filterState}
setFilterState={setFilterState}
runQuery={runQuery}

2
console/src/components/Services/Events/CronTriggers/Logs/Logs.tsx
View File

@ -23,7 +23,6 @@ const InvocationLogs: React.FC<Props> = props => {
const renderRows: FilterRenderProp = (
rows,
count,
filterState,
setFilterState,
runQuery
@ -31,7 +30,6 @@ const InvocationLogs: React.FC<Props> = props => {
<InvocationLogsTable
rows={rows}
filterState={filterState}
count={count}
setFilterState={setFilterState}
runQuery={runQuery}
columns={['id', 'status', 'event_id', 'created_at']}

2
console/src/components/Services/Events/CronTriggers/PendingEvents/PendingEvents.tsx
View File

@ -40,14 +40,12 @@ const PendingEvents: React.FC<Props> = props => {
const renderRows: FilterRenderProp = (
rows,
count,
filterState,
setFilterState,
runQuery
) => (
<EventsTable
rows={rows}
count={count}
filterState={filterState}
setFilterState={setFilterState}
runQuery={runQuery}

3
console/src/components/Services/Events/CronTriggers/ProcessedEvents/ProcessedEvents.tsx
View File

@ -22,14 +22,13 @@ const ProcessedEvents: React.FC<Props> = props => {
const renderRows: FilterRenderProp = (
rows,
count,
filterState,
setFilterState,
runQuery
) => (
<EventsTable
rows={rows}
count={count}
filterState={filterState}
setFilterState={setFilterState}
runQuery={runQuery}

3
console/src/components/Services/Events/EventTriggers/InvocationLogs/InvocationLogs.tsx
View File

@ -20,7 +20,7 @@ const InvocationLogs: React.FC<Props> = props => {
const { dispatch, triggerName, currentTrigger, readOnlyMode } = props;
const renderRows: FilterRenderProp = (
rows,
count,
filterState,
setFilterState,
runQuery
@ -28,7 +28,6 @@ const InvocationLogs: React.FC<Props> = props => {
<InvocationLogsTable
rows={rows}
filterState={filterState}
count={count}
setFilterState={setFilterState}
runQuery={runQuery}
columns={['id', 'redeliver', 'status', 'event_id', 'created_at']}

3
console/src/components/Services/Events/EventTriggers/PendingEvents/PendingEvents.tsx
View File

@ -20,14 +20,13 @@ const PendingEvents: React.FC<Props> = props => {
const renderRows: FilterRenderProp = (
rows,
count,
filterState,
setFilterState,
runQuery
) => (
<EventsTable
rows={rows}
count={count}
filterState={filterState}
setFilterState={setFilterState}
runQuery={runQuery}

3
console/src/components/Services/Events/EventTriggers/ProcessedEvents/ProcessedEvents.tsx
View File

@ -21,14 +21,13 @@ const ProcessedEvents: React.FC<Props> = props => {
const renderRows: FilterRenderProp = (
rows,
count,
filterState,
setFilterState,
runQuery
) => (
<EventsTable
rows={rows}
count={count}
filterState={filterState}
setFilterState={setFilterState}
runQuery={runQuery}

4
console/src/dataSources/index.ts
View File

@ -433,10 +433,6 @@ export interface DataSourcesAPI {
generateBulkDeleteRowRequest?: () => GenerateBulkDeleteRowRequest;
// New Simple Queries to fetch just what we need at a time
schemaListQuery: string;
getDataTriggerLogsCountQuery?: (
triggerName: string,
triggerOp: TriggerOperation
) => string;
getDataTriggerLogsQuery?: (
triggerOp: TriggerOperation,
triggerName: string,

2
console/src/dataSources/services/cockroach/index.tsx
View File

@ -54,7 +54,6 @@ import {
getAlterColumnCommentSql,
getAlterFunctionCommentSql,
getDataTriggerInvocations,
getDataTriggerLogsCountQuery,
getDataTriggerLogsQuery,
} from './sqlUtils';
import {
@ -215,7 +214,6 @@ export const cockroach: DataSourcesAPI = {
getAlterColumnCommentSql,
getAlterFunctionCommentSql,
getDataTriggerInvocations,
getDataTriggerLogsCountQuery,
getDataTriggerLogsQuery,
createIndexSql: getCreateIndexSql,
dropIndexSql: getDropIndexSql,

47
console/src/dataSources/services/cockroach/sqlUtils.ts
View File

@ -1292,46 +1292,6 @@ WHERE
AND schema_name NOT LIKE 'pg_temp_%';
`;
export const getDataTriggerLogsCountQuery = (
triggerName: string,
triggerOp: TriggerOperation
): string => {
const triggerTypes = {
pending: 'pending',
processed: 'processed',
invocation: 'invocation',
};
const eventRelTable = `"hdb_catalog"."event_log"`;
const eventInvTable = `"hdb_catalog"."event_invocation_logs"`;
let logsCountQuery = `SELECT
COUNT(*)
FROM ${eventRelTable} data_table
WHERE data_table.trigger_name = '${triggerName}' `;
switch (triggerOp) {
case triggerTypes.pending:
logsCountQuery += `AND delivered=false AND error=false AND archived=false;`;
break;
case triggerTypes.processed:
logsCountQuery += `AND (delivered=true OR error=true) AND archived=false;`;
break;
case triggerTypes.invocation:
logsCountQuery = `SELECT
COUNT(*)
FROM ${eventInvTable} original_table
LEFT JOIN ${eventRelTable} data_table
ON original_table.event_id = data_table.id
WHERE data_table.trigger_name = '${triggerName}' OR original_table.trigger_name = '${triggerName}'`;
break;
default:
break;
}
return logsCountQuery;
};
export const getDataTriggerLogsQuery = (
triggerOp: TriggerOperation,
triggerName: string,
@ -1364,10 +1324,9 @@ export const getDataTriggerLogsQuery = (
case triggerTypes.invocation:
sql = `
SELECT original_table.*, data_table
FROM ${eventInvTable} original_table
LEFT JOIN ${eventRelTable} data_table ON original_table.event_id = data_table.id
WHERE data_table.trigger_name = '${triggerName}' OR original_table.trigger_name = '${triggerName}'
SELECT *
FROM ${eventInvTable} data_table
WHERE data_table.trigger_name = '${triggerName}'
ORDER BY original_table.created_at DESC NULLS LAST`;
break;
default:

50
console/src/dataSources/services/mssql/index.tsx
View File

@ -1021,43 +1021,7 @@ WHERE
},
// temporary workaround SQL query (till we get an API) as ODBC server library does not support some data types
// https://github.com/hasura/graphql-engine-mono/issues/4641
getDataTriggerLogsCountQuery: (
triggerName: string,
triggerOp: TriggerOperation
): string => {
const triggerTypes = {
pending: 'pending',
processed: 'processed',
invocation: 'invocation',
};
const eventRelTable = `"hdb_catalog"."event_log"`;
const eventInvTable = `"hdb_catalog"."event_invocation_logs"`;
let logsCountQuery = `SELECT
COUNT(*)
FROM ${eventRelTable} data_table
WHERE data_table.trigger_name = '${triggerName}' `;
switch (triggerOp) {
case triggerTypes.pending:
logsCountQuery += `AND delivered=0 AND error=0 AND archived=0;`;
break;
case triggerTypes.processed:
logsCountQuery += `AND (delivered=1 OR error=1) AND archived=0;`;
break;
case triggerTypes.invocation:
logsCountQuery = `SELECT
COUNT(*)
FROM ${eventInvTable} original_table JOIN ${eventRelTable} data_table
ON original_table.event_id = data_table.id
WHERE data_table.trigger_name = '${triggerName}' `;
break;
default:
break;
}
return logsCountQuery;
},
// temporary workaround SQL query (till we get an API) as ODBC server library does not support some data types
// https://github.com/hasura/graphql-engine-mono/issues/4641
getDataTriggerLogsQuery: (
@ -1095,15 +1059,13 @@ WHERE
break;
case triggerTypes.invocation:
sql = `SELECT CONVERT(varchar(MAX), original_table.id) AS "id", CONVERT(varchar(MAX), original_table.event_id) AS "event_id",
original_table.status, CONVERT(varchar(MAX), original_table.request) AS "request", CONVERT(varchar(MAX), original_table.response) AS "response",
CONVERT(varchar(MAX), CAST(original_table.created_at as datetime2)) AS "created_at", CONVERT(varchar(MAX), data_table.id) AS "id", data_table.schema_name,
data_table.table_name, data_table.trigger_name, CONVERT(varchar(MAX), data_table.payload) AS "payload", data_table.delivered, data_table.error,
data_table.tries, CONVERT(varchar(MAX), CAST(data_table.created_at as datetime2)) AS "created_at", CONVERT(varchar(MAX), data_table.locked) AS "locked",
CONVERT(varchar(MAX), data_table.next_retry_at) AS "next_retry_at", data_table.archived
FROM ${eventInvTable} AS original_table JOIN ${eventRelTable} AS data_table ON original_table.event_id = data_table.id
sql = `SELECT CONVERT(varchar(MAX), data_table.id) AS "id", CONVERT(varchar(MAX), data_table.event_id) AS "event_id",
data_table.status, CONVERT(varchar(MAX), data_table.request) AS "request", CONVERT(varchar(MAX), data_table.response) AS "response",
CONVERT(varchar(MAX), CAST(data_table.created_at as datetime2)) AS "created_at", CONVERT(varchar(MAX), data_table.id) AS "id",
data_table.trigger_name, CONVERT(varchar(MAX), CAST(data_table.created_at as datetime2)) AS "created_at"
FROM ${eventInvTable} AS data_table
WHERE data_table.trigger_name = '${triggerName}'
ORDER BY original_table.created_at DESC `;
ORDER BY data_table.created_at DESC `;
break;
default:
break;

23
console/src/dataSources/services/postgresql/__tests__/__snapshots__/postgresql.test.ts.snap
View File

@ -41,29 +41,6 @@ exports[`postgresql datasource tests getDataTriggerInvocations should generate S
ORDER BY created_at DESC NULLS LAST;"
`;
exports[`postgresql datasource tests getDataTriggerLogsCountQuery should generate SQL query for invocation event count 1`] = `
"SELECT
COUNT(*)
FROM \\"hdb_catalog\\".\\"event_invocation_logs\\" original_table
LEFT JOIN \\"hdb_catalog\\".\\"event_log\\" data_table
ON original_table.event_id = data_table.id
WHERE data_table.trigger_name = 'test_event' OR original_table.trigger_name = 'test_event' "
`;
exports[`postgresql datasource tests getDataTriggerLogsCountQuery should generate SQL query for pending event count 1`] = `
"SELECT
COUNT(*)
FROM \\"hdb_catalog\\".\\"event_log\\" data_table
WHERE data_table.trigger_name = 'new_user' AND delivered=false AND error=false AND archived=false;"
`;
exports[`postgresql datasource tests getDataTriggerLogsCountQuery should generate SQL query for processed event count 1`] = `
"SELECT
COUNT(*)
FROM \\"hdb_catalog\\".\\"event_log\\" data_table
WHERE data_table.trigger_name = 'new_user' AND (delivered=true OR error=true) AND archived=false;"
`;
exports[`postgresql datasource tests getDataTriggerLogsQuery should generate SQL query for event invocation logs 1`] = `
"SELECT *
FROM \\"hdb_catalog\\".\\"event_log\\" data_table

50
console/src/dataSources/services/postgresql/__tests__/postgresql.test.ts
View File

@ -7,7 +7,6 @@ const {
getAlterViewCommentSql,
getAlterFunctionCommentSql,
createIndexSql,
getDataTriggerLogsCountQuery,
getDataTriggerLogsQuery,
getDataTriggerInvocations,
} = postgres;
@ -173,55 +172,6 @@ describe('postgresql datasource tests', () => {
});
});
describe('getDataTriggerLogsCountQuery', () => {
if (getDataTriggerLogsCountQuery) {
it('should generate SQL query for pending event count ', () => {
const pendingCountQuery = getDataTriggerLogsCountQuery(
'new_user',
'pending'
);
expect(pendingCountQuery).toContain(
'delivered=false AND error=false AND archived=false'
);
expect(pendingCountQuery).toContain(
"data_table.trigger_name = 'new_user'"
);
expect(pendingCountQuery).toContain('FROM "hdb_catalog"."event_log"');
expect(pendingCountQuery).toMatchSnapshot();
});
it('should generate SQL query for processed event count', () => {
const processedCountQuery = getDataTriggerLogsCountQuery(
'new_user',
'processed'
);
expect(processedCountQuery).toContain(
'AND (delivered=true OR error=true) AND archived=false'
);
expect(processedCountQuery).toContain(
"data_table.trigger_name = 'new_user'"
);
expect(processedCountQuery).toContain('FROM "hdb_catalog"."event_log"');
expect(processedCountQuery).toMatchSnapshot();
});
it('should generate SQL query for invocation event count', () => {
const invocationCountQuery = getDataTriggerLogsCountQuery(
'test_event',
'invocation'
);
expect(invocationCountQuery).toContain(
"data_table.trigger_name = 'test_event'"
);
expect(invocationCountQuery).toContain(
'FROM "hdb_catalog"."event_invocation_logs"'
);
expect(invocationCountQuery).toMatchSnapshot();
});
}
});
describe('getDataTriggerLogsQuery', () => {
if (getDataTriggerLogsQuery) {
it('should generate SQL query for pending event logs', () => {

2
console/src/dataSources/services/postgresql/index.tsx
View File

@ -81,7 +81,6 @@ import {
getAlterViewCommentSql,
getAlterFunctionCommentSql,
getDataTriggerInvocations,
getDataTriggerLogsCountQuery,
getDataTriggerLogsQuery,
} from './sqlUtils';
import globals from '../../../Globals';
@ -895,6 +894,5 @@ export const postgres: DataSourcesAPI = {
getAlterViewCommentSql,
getAlterFunctionCommentSql,
getDataTriggerInvocations,
getDataTriggerLogsCountQuery,
getDataTriggerLogsQuery,
};

49
console/src/dataSources/services/postgresql/sqlUtils.ts
View File

@ -1339,46 +1339,6 @@ WHERE
AND schema_name NOT LIKE 'pg_temp_%';
`;
export const getDataTriggerLogsCountQuery = (
triggerName: string,
triggerOp: TriggerOperation
): string => {
const triggerTypes = {
pending: 'pending',
processed: 'processed',
invocation: 'invocation',
};
const eventRelTable = `"hdb_catalog"."event_log"`;
const eventInvTable = `"hdb_catalog"."event_invocation_logs"`;
let logsCountQuery = `SELECT
COUNT(*)
FROM ${eventRelTable} data_table
WHERE data_table.trigger_name = '${triggerName}' `;
switch (triggerOp) {
case triggerTypes.pending:
logsCountQuery += `AND delivered=false AND error=false AND archived=false;`;
break;
case triggerTypes.processed:
logsCountQuery += `AND (delivered=true OR error=true) AND archived=false;`;
break;
case triggerTypes.invocation:
logsCountQuery = `SELECT
COUNT(*)
FROM ${eventInvTable} original_table
LEFT JOIN ${eventRelTable} data_table
ON original_table.event_id = data_table.id
WHERE data_table.trigger_name = '${triggerName}' OR original_table.trigger_name = '${triggerName}' `;
break;
default:
break;
}
return logsCountQuery;
};
export const getDataTriggerLogsQuery = (
triggerOp: TriggerOperation,
triggerName: string,
@ -1411,11 +1371,10 @@ export const getDataTriggerLogsQuery = (
case triggerTypes.invocation:
sql = `
SELECT original_table.*, data_table
FROM ${eventInvTable} original_table
LEFT JOIN ${eventRelTable} data_table ON original_table.event_id = data_table.id
WHERE data_table.trigger_name = '${triggerName}' OR original_table.trigger_name = '${triggerName}'
ORDER BY original_table.created_at DESC NULLS LAST `;
SELECT data_table.*
FROM ${eventInvTable} data_table
WHERE data_table.trigger_name = '${triggerName}'
ORDER BY data_table.created_at DESC NULLS LAST`;
break;
default:
break;