From 3cad1319c40133e593fd7481093f01699cf9f2d0 Mon Sep 17 00:00:00 2001 From: Tirumarai Selvan Date: Wed, 13 Nov 2019 12:59:19 +0530 Subject: [PATCH] improve event fetch query (#3236) --- .../Services/EventTrigger/EventActions.js | 115 +++++++++++------- .../EventTrigger/PendingEvents/ViewActions.js | 13 ++ .../ProcessedEvents/ViewActions.js | 13 ++ .../EventTrigger/RunningEvents/ViewActions.js | 13 ++ .../EventTrigger/StreamingLogs/LogActions.js | 26 ++++ console/src/helpers/versionUtils.js | 2 + server/src-lib/Hasura/Events/Lib.hs | 5 +- server/src-lib/Hasura/RQL/DDL/EventTrigger.hs | 13 +- .../src-lib/Hasura/Server/Migrate/Version.hs | 2 +- server/src-rsr/initialise.sql | 4 +- server/src-rsr/migrations/26_to_27.sql | 9 ++ 11 files changed, 164 insertions(+), 51 deletions(-) create mode 100644 server/src-rsr/migrations/26_to_27.sql diff --git a/console/src/components/Services/EventTrigger/EventActions.js b/console/src/components/Services/EventTrigger/EventActions.js index f5d3ca9ac25..abbeebb2f20 100644 --- a/console/src/components/Services/EventTrigger/EventActions.js +++ b/console/src/components/Services/EventTrigger/EventActions.js @@ -21,6 +21,7 @@ import { getEventTriggersQuery } from './utils'; import { CLI_CONSOLE_MODE, SERVER_CONSOLE_MODE } from '../../../constants'; import { REQUEST_COMPLETE, REQUEST_ONGOING } from './Modify/Actions'; +import { IMPROVED_EVENT_FETCH_QUERY } from '../../../helpers/versionUtils'; const SET_TRIGGER = 'Event/SET_TRIGGER'; const LOAD_TRIGGER_LIST = 'Event/LOAD_TRIGGER_LIST'; @@ -91,32 +92,41 @@ const loadTriggers = triggerNames => (dispatch, getState) => { const loadPendingEvents = () => (dispatch, getState) => { const url = Endpoints.getSchema; + const body = { + type: 'select', + args: { + table: { + name: 'event_triggers', + schema: 'hdb_catalog', + }, + columns: [ + '*', + { + name: 'events', + columns: [ + '*', + { name: 'logs', columns: ['*'], order_by: ['-created_at'] }, + ], + where: { delivered: false, error: false, tries: 0 }, + order_by: ['-created_at'], + limit: 10, + }, + ], + }, + }; + + if ( + globals.featuresCompatibility && + globals.featuresCompatibility[IMPROVED_EVENT_FETCH_QUERY] + ) { + body.args.columns[1].where.archived = false; + } + const options = { credentials: globalCookiePolicy, method: 'POST', headers: dataHeaders(getState), - body: JSON.stringify({ - type: 'select', - args: { - table: { - name: 'event_triggers', - schema: 'hdb_catalog', - }, - columns: [ - '*', - { - name: 'events', - columns: [ - '*', - { name: 'logs', columns: ['*'], order_by: ['-created_at'] }, - ], - where: { delivered: false, error: false, tries: 0 }, - order_by: ['-created_at'], - limit: 10, - }, - ], - }, - }), + body: JSON.stringify(body), }; return dispatch(requestAction(url, options)).then( data => { @@ -130,32 +140,41 @@ const loadPendingEvents = () => (dispatch, getState) => { const loadRunningEvents = () => (dispatch, getState) => { const url = Endpoints.getSchema; + const body = { + type: 'select', + args: { + table: { + name: 'event_triggers', + schema: 'hdb_catalog', + }, + columns: [ + '*', + { + name: 'events', + columns: [ + '*', + { name: 'logs', columns: ['*'], order_by: ['-created_at'] }, + ], + where: { delivered: false, error: false, tries: { $gt: 0 } }, + order_by: ['-created_at'], + limit: 10, + }, + ], + }, + }; + + if ( + globals.featuresCompatibility && + globals.featuresCompatibility[IMPROVED_EVENT_FETCH_QUERY] + ) { + body.args.columns[1].where.archived = false; + } + const options = { credentials: globalCookiePolicy, method: 'POST', headers: dataHeaders(getState), - body: JSON.stringify({ - type: 'select', - args: { - table: { - name: 'event_triggers', - schema: 'hdb_catalog', - }, - columns: [ - '*', - { - name: 'events', - columns: [ - '*', - { name: 'logs', columns: ['*'], order_by: ['-created_at'] }, - ], - where: { delivered: false, error: false, tries: { $gt: 0 } }, - order_by: ['-created_at'], - limit: 10, - }, - ], - }, - }), + body: JSON.stringify(body), }; return dispatch(requestAction(url, options)).then( data => { @@ -214,6 +233,14 @@ const loadEventLogs = triggerName => (dispatch, getState) => { }, ], }; + + if ( + globals.featuresCompatibility && + globals.featuresCompatibility[IMPROVED_EVENT_FETCH_QUERY] + ) { + body.args[0].args.where.event.archived = false; + } + const logOptions = { credentials: globalCookiePolicy, method: 'POST', diff --git a/console/src/components/Services/EventTrigger/PendingEvents/ViewActions.js b/console/src/components/Services/EventTrigger/PendingEvents/ViewActions.js index 515651d9d3b..4f9c9d2b19f 100644 --- a/console/src/components/Services/EventTrigger/PendingEvents/ViewActions.js +++ b/console/src/components/Services/EventTrigger/PendingEvents/ViewActions.js @@ -4,6 +4,8 @@ import requestAction from '../../../../utils/requestAction'; import pendingFilterReducer from './FilterActions'; import { findTableFromRel } from '../utils'; import dataHeaders from '../Common/Headers'; +import globals from '../../../../Globals'; +import { IMPROVED_EVENT_FETCH_QUERY } from '../../../../helpers/versionUtils'; /* ****************** View actions *************/ const V_SET_DEFAULTS = 'PendingEvents/V_SET_DEFAULTS'; @@ -66,6 +68,17 @@ const vMakeRequest = () => { }; } + if ( + globals.featuresCompatibility && + globals.featuresCompatibility[IMPROVED_EVENT_FETCH_QUERY] + ) { + if (currentQuery.columns[1]) { + currentQuery.columns[1].where = currentQuery.columns[1].where || {}; + currentQuery.columns[1].where.archived = false; + } + countQuery.where.archived = false; + } + // order_by for relationship const currentOrderBy = state.triggers.view.query.order_by; if (currentOrderBy) { diff --git a/console/src/components/Services/EventTrigger/ProcessedEvents/ViewActions.js b/console/src/components/Services/EventTrigger/ProcessedEvents/ViewActions.js index ceba1245f6b..d12ac2cc7d4 100644 --- a/console/src/components/Services/EventTrigger/ProcessedEvents/ViewActions.js +++ b/console/src/components/Services/EventTrigger/ProcessedEvents/ViewActions.js @@ -9,6 +9,8 @@ import { } from '../../Common/Notification'; import dataHeaders from '../Common/Headers'; import { getConfirmation } from '../../../Common/utils/jsUtils'; +import globals from '../../../../Globals'; +import { IMPROVED_EVENT_FETCH_QUERY } from '../../../../helpers/versionUtils'; /* ****************** View actions *************/ const V_SET_DEFAULTS = 'ProcessedEvents/V_SET_DEFAULTS'; @@ -80,6 +82,17 @@ const vMakeRequest = () => { }; } + if ( + globals.featuresCompatibility && + globals.featuresCompatibility[IMPROVED_EVENT_FETCH_QUERY] + ) { + if (currentQuery.columns[1]) { + currentQuery.columns[1].where = currentQuery.columns[1].where || {}; + currentQuery.columns[1].where.archived = false; + } + countQuery.where.archived = false; + } + // order_by for relationship const currentOrderBy = state.triggers.view.query.order_by; if (currentOrderBy) { diff --git a/console/src/components/Services/EventTrigger/RunningEvents/ViewActions.js b/console/src/components/Services/EventTrigger/RunningEvents/ViewActions.js index d2b0a747d3a..bef89db04ee 100644 --- a/console/src/components/Services/EventTrigger/RunningEvents/ViewActions.js +++ b/console/src/components/Services/EventTrigger/RunningEvents/ViewActions.js @@ -4,6 +4,8 @@ import requestAction from 'utils/requestAction'; import pendingFilterReducer from './FilterActions'; import { findTableFromRel } from '../utils'; import dataHeaders from '../Common/Headers'; +import globals from '../../../../Globals'; +import { IMPROVED_EVENT_FETCH_QUERY } from '../../../../helpers/versionUtils'; /* ****************** View actions *************/ const V_SET_DEFAULTS = 'RunningEvents/V_SET_DEFAULTS'; @@ -72,6 +74,17 @@ const vMakeRequest = () => { }; } + if ( + globals.featuresCompatibility && + globals.featuresCompatibility[IMPROVED_EVENT_FETCH_QUERY] + ) { + if (currentQuery.columns[1]) { + currentQuery.columns[1].where = currentQuery.columns[1].where || {}; + currentQuery.columns[1].where.archived = false; + } + countQuery.where.archived = false; + } + // order_by for relationship const currentOrderBy = state.triggers.view.query.order_by; if (currentOrderBy) { diff --git a/console/src/components/Services/EventTrigger/StreamingLogs/LogActions.js b/console/src/components/Services/EventTrigger/StreamingLogs/LogActions.js index cf8683da24b..5d0794bb347 100644 --- a/console/src/components/Services/EventTrigger/StreamingLogs/LogActions.js +++ b/console/src/components/Services/EventTrigger/StreamingLogs/LogActions.js @@ -2,6 +2,8 @@ import { defaultLogState } from '../EventState'; import Endpoints, { globalCookiePolicy } from '../../../../Endpoints'; import requestAction from 'utils/requestAction'; import dataHeaders from '../Common/Headers'; +import globals from '../../../../Globals'; +import { IMPROVED_EVENT_FETCH_QUERY } from '../../../../helpers/versionUtils'; /* ****************** View actions *************/ const V_SET_DEFAULTS = 'StreamingLogs/V_SET_DEFAULTS'; @@ -42,6 +44,14 @@ const vMakeRequest = triggerName => { currentQuery.where = { event: { trigger_name: triggerName } }; + if ( + globals.featuresCompatibility && + globals.featuresCompatibility[IMPROVED_EVENT_FETCH_QUERY] + ) { + currentQuery.where.event.archived = false; + countQuery.where.event.archived = false; + } + // order_by for relationship currentQuery.order_by = ['-created_at']; @@ -112,6 +122,14 @@ const loadNewerEvents = (latestTimestamp, triggerName) => { created_at: { $gt: latestTimestamp }, }; + if ( + globals.featuresCompatibility && + globals.featuresCompatibility[IMPROVED_EVENT_FETCH_QUERY] + ) { + currentQuery.where.event.archived = false; + countQuery.where.event.archived = false; + } + // order_by for relationship currentQuery.order_by = ['-created_at']; @@ -202,6 +220,14 @@ const loadOlderEvents = (oldestTimestamp, triggerName) => { created_at: { $lt: oldestTimestamp }, }; + if ( + globals.featuresCompatibility && + globals.featuresCompatibility[IMPROVED_EVENT_FETCH_QUERY] + ) { + currentQuery.where.event.archived = false; + countQuery.where.event.archived = false; + } + // order_by for relationship currentQuery.order_by = ['-created_at']; diff --git a/console/src/helpers/versionUtils.js b/console/src/helpers/versionUtils.js index 340d3a6e0f0..ee535486760 100644 --- a/console/src/helpers/versionUtils.js +++ b/console/src/helpers/versionUtils.js @@ -10,6 +10,7 @@ export const TABLE_ENUMS_SUPPORT = 'tableEnumsSupport'; export const EXISTS_PERMISSION_SUPPORT = 'existsPermissionSupport'; export const CUSTOM_GRAPHQL_FIELDS_SUPPORT = 'customGraphQLFieldsSupport'; export const COMPUTED_FIELDS_SUPPORT = 'computedFieldsSupport'; +export const IMPROVED_EVENT_FETCH_QUERY = 'improvedEventFetchQuery'; // list of feature launch versions const featureLaunchVersions = { @@ -21,6 +22,7 @@ const featureLaunchVersions = { [EXISTS_PERMISSION_SUPPORT]: 'v1.0.0-beta.7', [CUSTOM_GRAPHQL_FIELDS_SUPPORT]: 'v1.0.0-beta.8', [COMPUTED_FIELDS_SUPPORT]: 'v1.0.0-beta.8', + [IMPROVED_EVENT_FETCH_QUERY]: 'v1.0.0-beta.10', }; export const checkValidServerVersion = version => { diff --git a/server/src-lib/Hasura/Events/Lib.hs b/server/src-lib/Hasura/Events/Lib.hs index 2d069e06d07..c80ea607353 100644 --- a/server/src-lib/Hasura/Events/Lib.hs +++ b/server/src-lib/Hasura/Events/Lib.hs @@ -441,10 +441,9 @@ fetchEvents = SET locked = 't' WHERE id IN ( SELECT l.id FROM hdb_catalog.event_log l - JOIN hdb_catalog.event_triggers e - ON (l.trigger_name = e.name) - WHERE l.delivered ='f' and l.error = 'f' and l.locked = 'f' + WHERE l.delivered = 'f' and l.error = 'f' and l.locked = 'f' and (l.next_retry_at is NULL or l.next_retry_at <= now()) + and l.archived = 'f' FOR UPDATE SKIP LOCKED LIMIT 100 ) RETURNING id, schema_name, table_name, trigger_name, payload::json, tries, created_at diff --git a/server/src-lib/Hasura/RQL/DDL/EventTrigger.hs b/server/src-lib/Hasura/RQL/DDL/EventTrigger.hs index 4464e5af011..dd5c8cf45fe 100644 --- a/server/src-lib/Hasura/RQL/DDL/EventTrigger.hs +++ b/server/src-lib/Hasura/RQL/DDL/EventTrigger.hs @@ -135,7 +135,7 @@ addEventTriggerToCatalog qt allCols strfyNum etc = do INSERT into hdb_catalog.event_triggers (name, type, schema_name, table_name, configuration) VALUES ($1, 'table', $2, $3, $4) - |] (name, sn, tn, Q.AltJ $ toJSON etc) True + |] (name, sn, tn, Q.AltJ $ toJSON etc) False mkAllTriggersQ name qt allCols strfyNum fullspec where @@ -148,8 +148,17 @@ delEventTriggerFromCatalog trn = do DELETE FROM hdb_catalog.event_triggers WHERE name = $1 - |] (Identity trn) True + |] (Identity trn) False delTriggerQ trn + archiveEvents trn + +archiveEvents:: TriggerName -> Q.TxE QErr () +archiveEvents trn = do + Q.unitQE defaultTxErrorHandler [Q.sql| + UPDATE hdb_catalog.event_log + SET archived = 't' + WHERE trigger_name = $1 + |] (Identity trn) False updateEventTriggerToCatalog :: QualifiedTable diff --git a/server/src-lib/Hasura/Server/Migrate/Version.hs b/server/src-lib/Hasura/Server/Migrate/Version.hs index aa75f6b1d78..a094e0a13fd 100644 --- a/server/src-lib/Hasura/Server/Migrate/Version.hs +++ b/server/src-lib/Hasura/Server/Migrate/Version.hs @@ -12,7 +12,7 @@ import Hasura.Prelude import qualified Data.Text as T latestCatalogVersion :: Integer -latestCatalogVersion = 26 +latestCatalogVersion = 27 latestCatalogVersionString :: T.Text latestCatalogVersionString = T.pack $ show latestCatalogVersion diff --git a/server/src-rsr/initialise.sql b/server/src-rsr/initialise.sql index 54774f85d34..24317b0b4db 100644 --- a/server/src-rsr/initialise.sql +++ b/server/src-rsr/initialise.sql @@ -280,11 +280,13 @@ CREATE TABLE hdb_catalog.event_log tries INTEGER NOT NULL DEFAULT 0, created_at TIMESTAMP DEFAULT NOW(), locked BOOLEAN NOT NULL DEFAULT FALSE, - next_retry_at TIMESTAMP + next_retry_at TIMESTAMP, + archived BOOLEAN NOT NULL DEFAULT FALSE ); CREATE INDEX ON hdb_catalog.event_log (trigger_name); CREATE INDEX ON hdb_catalog.event_log (locked); +CREATE INDEX ON hdb_catalog.event_log (delivered); CREATE TABLE hdb_catalog.event_invocation_logs ( diff --git a/server/src-rsr/migrations/26_to_27.sql b/server/src-rsr/migrations/26_to_27.sql new file mode 100644 index 00000000000..85b54fcde6f --- /dev/null +++ b/server/src-rsr/migrations/26_to_27.sql @@ -0,0 +1,9 @@ +ALTER TABLE hdb_catalog.event_log + ADD COLUMN archived BOOLEAN NOT NULL DEFAULT FALSE; + +UPDATE hdb_catalog.event_log + SET archived = 't' + WHERE + trigger_name NOT IN (SELECT name from hdb_catalog.event_triggers); + +CREATE INDEX ON hdb_catalog.event_log (delivered);