diff --git a/AGHTechDoc.md b/AGHTechDoc.md index 60fc986f..c6e70391 100644 --- a/AGHTechDoc.md +++ b/AGHTechDoc.md @@ -43,6 +43,7 @@ Contents: * API: Set statistics parameters * API: Get statistics parameters * Query logs + * API: Get query log * API: Set querylog parameters * API: Get querylog parameters * Filtering @@ -1007,6 +1008,92 @@ Response: ## Query logs +When a new DNS request is received and processed, we store information about this event in "query log". It is a file on disk in JSON format: + + { + "Question":"..."," + Answer":"...", + "Result":{ + "IsFiltered":true, + "Reason":3, + "Rule":"...", + "FilterID":1 + }, + "Time":"...", + "Elapsed":12345, + "IP":"127.0.0.1" + } + + +### Adding new data + +First, new data is stored in a memory region. When this array is filled to a particular amount of entries (e.g. 5000), we flush this data to a file and clear the array. + + +### Getting data + +When UI asks for data from query log (see "API: Get query log"), server reads the newest entries from memory array and the file. The maximum number of items returned per one request is limited by configuration. + + +### Removing old data + +We store data for a limited amount of time - the log file is automatically rotated. + + +### API: Get query log + +Request: + + POST /control/querylog + + { + older_than: "2006-01-02T15:04:05.999999999Z07:00" // must be "" for the first request + + filter:{ + domain: "..." + client: "..." + question_type: "A" | "AAAA" + response_status: "" | "filtered" + } + } + +If `older_than` value is set, server returns the next chunk of entries that are older than this time stamp. This setting is used for paging. UI sets this value to `""` on the first request and gets the latest log entries. To get the older entries, UI sets this value to the timestamp of the last (the oldest) entry from the previous response from Server. + +If "filter" settings are set, server returns only entries that match the specified request. + +For `filter.domain` and `filter.client` the server matches substrings by default: `adguard.com` matches `www.adguard.com`. Strict matching can be enabled by enclosing the value in double quotes: `"adguard.com"` matches `adguard.com` but doesn't match `www.adguard.com`. + +Response: + + [ + { + "answer":[ + { + "ttl":10, + "type":"AAAA", + "value":"::" + } + ... + ], + "client":"127.0.0.1", + "elapsedMs":"0.098403", + "filterId":1, + "question":{ + "class":"IN", + "host":"doubleclick.net", + "type":"AAAA" + }, + "reason":"FilteredBlackList", + "rule":"||doubleclick.net^", + "status":"NOERROR", + "time":"2006-01-02T15:04:05.999999999Z07:00" + } + ... + ] + +The most recent entries are at the top of list. + + ### API: Set querylog parameters Request: diff --git a/client/package-lock.json b/client/package-lock.json index b2070cdb..8419dae8 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -9868,12 +9868,20 @@ "dev": true }, "prop-types": { - "version": "15.6.2", - "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.6.2.tgz", - "integrity": "sha512-3pboPvLiWD7dkI3qf3KbUe6hKFKa52w+AE0VCqECtf+QHAKgOL37tTaNCnuX1nAAQ4ZhyP+kYVKf8rLmJ/feDQ==", + "version": "15.7.2", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz", + "integrity": "sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==", "requires": { - "loose-envify": "^1.3.1", - "object-assign": "^4.1.1" + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.8.1" + }, + "dependencies": { + "react-is": { + "version": "16.9.0", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.9.0.tgz", + "integrity": "sha512-tJBzzzIgnnRfEm046qRcURvwQnZVXmuCbscxUO5RWrGTXpon2d4c8mI0D8WE6ydVIm29JiLB6+RslkIvym9Rjw==" + } } }, "proxy-addr": { @@ -10212,9 +10220,9 @@ } }, "react-table": { - "version": "6.8.6", - "resolved": "https://registry.npmjs.org/react-table/-/react-table-6.8.6.tgz", - "integrity": "sha1-oK2LSDkxkFLVvvwBJgP7Fh5S7eM=", + "version": "6.10.3", + "resolved": "https://registry.npmjs.org/react-table/-/react-table-6.10.3.tgz", + "integrity": "sha512-sVlq2/rxVaQJywGD95+qGiMr/SMHFIFnXdx619BLOWE/Os5FOGtV6pQJNAjZixbQZiOu7dmBO1kME28uxh6wmA==", "requires": { "classnames": "^2.2.5" } diff --git a/client/package.json b/client/package.json index 0e8b6d4d..7c2b1d18 100644 --- a/client/package.json +++ b/client/package.json @@ -17,7 +17,7 @@ "i18next-browser-languagedetector": "^2.2.3", "lodash": "^4.17.15", "nanoid": "^1.2.3", - "prop-types": "^15.6.1", + "prop-types": "^15.7.2", "react": "^16.4.0", "react-click-outside": "^3.0.1", "react-dom": "^16.4.0", @@ -27,7 +27,7 @@ "react-redux-loading-bar": "^4.0.7", "react-router-dom": "^4.2.2", "react-router-hash-link": "^1.2.2", - "react-table": "^6.8.6", + "react-table": "^6.10.3", "react-transition-group": "^2.4.0", "redux": "^4.0.0", "redux-actions": "^2.4.0", diff --git a/client/src/__locales/en.json b/client/src/__locales/en.json index e4e1c1af..d3700869 100644 --- a/client/src/__locales/en.json +++ b/client/src/__locales/en.json @@ -163,7 +163,6 @@ "show_filtered_type": "Show filtered", "no_logs_found": "No logs found", "refresh_btn": "Refresh", - "last_dns_queries": "Last 5000 DNS queries", "previous_btn": "Previous", "next_btn": "Next", "loading_table_status": "Loading...", @@ -182,6 +181,7 @@ "query_log_enable": "Enable log", "query_log_configuration": "Logs configuration", "query_log_disabled": "The query log is disabled and can be configured in the <0>settings", + "query_log_strict_search": "Use double quotes for strict search", "source_label": "Source", "found_in_known_domain_db": "Found in the known domains database.", "category_label": "Category", diff --git a/client/src/actions/queryLogs.js b/client/src/actions/queryLogs.js index c68ddf15..20672cf6 100644 --- a/client/src/actions/queryLogs.js +++ b/client/src/actions/queryLogs.js @@ -4,15 +4,19 @@ import apiClient from '../api/Api'; import { addErrorToast, addSuccessToast } from './index'; import { normalizeLogs } from '../helpers/helpers'; +export const setLogsPagination = createAction('LOGS_PAGINATION'); +export const setLogsFilter = createAction('LOGS_FILTER'); + export const getLogsRequest = createAction('GET_LOGS_REQUEST'); export const getLogsFailure = createAction('GET_LOGS_FAILURE'); export const getLogsSuccess = createAction('GET_LOGS_SUCCESS'); -export const getLogs = () => async (dispatch) => { +export const getLogs = config => async (dispatch) => { dispatch(getLogsRequest()); try { - const logs = normalizeLogs(await apiClient.getQueryLog()); - dispatch(getLogsSuccess(logs)); + const { filter, lastRowTime: older_than } = config; + const logs = normalizeLogs(await apiClient.getQueryLog({ filter, older_than })); + dispatch(getLogsSuccess({ logs, ...config })); } catch (error) { dispatch(addErrorToast({ error })); dispatch(getLogsFailure(error)); diff --git a/client/src/api/Api.js b/client/src/api/Api.js index b7a7d045..9cd0e650 100644 --- a/client/src/api/Api.js +++ b/client/src/api/Api.js @@ -482,14 +482,18 @@ class Api { } // Query log - GET_QUERY_LOG = { path: 'querylog', method: 'GET' }; + GET_QUERY_LOG = { path: 'querylog', method: 'POST' }; QUERY_LOG_CONFIG = { path: 'querylog_config', method: 'POST' }; QUERY_LOG_INFO = { path: 'querylog_info', method: 'GET' }; QUERY_LOG_CLEAR = { path: 'querylog_clear', method: 'POST' }; - getQueryLog() { + getQueryLog(data) { const { path, method } = this.GET_QUERY_LOG; - return this.makeRequest(path, method); + const config = { + data, + headers: { 'Content-Type': 'application/json' }, + }; + return this.makeRequest(path, method, config); } getQueryLogInfo() { diff --git a/client/src/components/Logs/Logs.css b/client/src/components/Logs/Logs.css index 931e8694..5a79ed85 100644 --- a/client/src/components/Logs/Logs.css +++ b/client/src/components/Logs/Logs.css @@ -107,6 +107,11 @@ border: 1px solid rgba(0, 40, 100, 0.12); } +.logs__table .rt-thead.-filters select { + background: #fff url("data:image/svg+xml;base64,PHN2ZyB4bWxucz0naHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmcnIHZpZXdCb3g9JzAgMCAxMCA1Jz48cGF0aCBmaWxsPScjOTk5JyBkPSdNMCAwTDEwIDBMNSA1TDAgMCcvPjwvc3ZnPg==") no-repeat right 0.75rem center; + background-size: 8px 10px; +} + .logs__table .rt-thead.-filters input:focus, .logs__table .rt-thead.-filters select:focus { border-color: #1991eb; @@ -130,6 +135,21 @@ overflow: hidden; } +.logs__input-wrap { + position: relative; +} + +.logs__notice { + position: absolute; + z-index: 1; + top: 8px; + right: 10px; + margin-top: 3px; + font-size: 12px; + text-align: left; + color: #a5a5a5; +} + .logs__whois { display: inline; } diff --git a/client/src/components/Logs/index.js b/client/src/components/Logs/index.js index 994a4e18..a0bd2db7 100644 --- a/client/src/components/Logs/index.js +++ b/client/src/components/Logs/index.js @@ -5,9 +5,14 @@ import escapeRegExp from 'lodash/escapeRegExp'; import endsWith from 'lodash/endsWith'; import { Trans, withNamespaces } from 'react-i18next'; import { HashLink as Link } from 'react-router-hash-link'; +import debounce from 'lodash/debounce'; -import { formatTime, formatDateTime } from '../../helpers/helpers'; -import { SERVICES, FILTERED_STATUS } from '../../helpers/constants'; +import { + formatTime, + formatDateTime, + isValidQuestionType, +} from '../../helpers/helpers'; +import { SERVICES, FILTERED_STATUS, DEBOUNCE_TIMEOUT, DEFAULT_LOGS_FILTER } from '../../helpers/constants'; import { getTrackerData } from '../../helpers/trackers/trackers'; import { formatClientCell } from '../../helpers/formatClientCell'; @@ -16,8 +21,12 @@ import Card from '../ui/Card'; import Loading from '../ui/Loading'; import PopoverFiltered from '../ui/PopoverFilter'; import Popover from '../ui/Popover'; +import Tooltip from '../ui/Tooltip'; import './Logs.css'; +const TABLE_FIRST_PAGE = 0; +const TABLE_DEFAULT_PAGE_SIZE = 50; +const INITIAL_REQUEST_DATA = ['', DEFAULT_LOGS_FILTER, TABLE_FIRST_PAGE, TABLE_DEFAULT_PAGE_SIZE]; const FILTERED_REASON = 'Filtered'; const RESPONSE_FILTER = { ALL: 'all', @@ -26,26 +35,34 @@ const RESPONSE_FILTER = { class Logs extends Component { componentDidMount() { - this.getLogs(); + this.getLogs(...INITIAL_REQUEST_DATA); this.props.getFilteringStatus(); this.props.getClients(); this.props.getLogsConfig(); } - componentDidUpdate(prevProps) { - // get logs when queryLog becomes enabled - if (this.props.queryLogs.enabled && !prevProps.queryLogs.enabled) { - this.props.getLogs(); - } - } - - getLogs = () => { - // get logs on initialization if queryLogIsEnabled + getLogs = (lastRowTime, filter, page, pageSize, filtered) => { if (this.props.queryLogs.enabled) { - this.props.getLogs(); + this.props.getLogs({ + lastRowTime, filter, page, pageSize, filtered, + }); } }; + refreshLogs = () => { + window.location.reload(); + }; + + handleLogsFiltering = debounce((lastRowTime, filter, page, pageSize, filtered) => { + this.props.getLogs({ + lastRowTime, + filter, + page, + pageSize, + filtered, + }); + }, DEBOUNCE_TIMEOUT); + renderTooltip = (isFiltered, rule, filter, service) => isFiltered && ; @@ -215,8 +232,75 @@ class Logs extends Component { ); }; - renderLogs(logs) { - const { t } = this.props; + getFilterInput = ({ filter, onChange }) => ( + +
+ onChange(event.target.value)} + value={filter ? filter.value : ''} + /> + + + +
+ + ); + + getFilters = (filtered) => { + const filteredObj = filtered.reduce((acc, cur) => ({ ...acc, [cur.id]: cur.value }), {}); + const { + domain, client, type, response, + } = filteredObj; + + return { + domain: domain || '', + client: client || '', + question_type: isValidQuestionType(type) ? type.toUpperCase() : '', + response_status: response === RESPONSE_FILTER.FILTERED ? response : '', + }; + }; + + fetchData = (state) => { + const { pageSize, page, pages } = state; + const { allLogs, filter } = this.props.queryLogs; + const isLastPage = page + 1 === pages; + + if (isLastPage) { + const lastRow = allLogs[allLogs.length - 1]; + const lastRowTime = (lastRow && lastRow.time) || ''; + this.getLogs(lastRowTime, filter, page, pageSize, true); + } else { + this.props.setLogsPagination({ page, pageSize }); + } + }; + + handleFilterChange = (filtered) => { + const filters = this.getFilters(filtered); + this.props.setLogsFilter(filters); + this.handleLogsFiltering('', filters, TABLE_FIRST_PAGE, TABLE_DEFAULT_PAGE_SIZE, true); + } + + showTotalPagesCount = (pages) => { + const { total, isEntireLog } = this.props.queryLogs; + const showEllipsis = !isEntireLog && total >= 500; + + return ( + + {pages || 1}{showEllipsis && '…' } + + ); + } + + renderLogs() { + const { queryLogs, dashboard, t } = this.props; + const { processingClients } = dashboard; + const { + processingGetLogs, processingGetConfig, logs, pages, + } = queryLogs; + const isLoading = processingGetLogs || processingClients || processingGetConfig; + const columns = [ { Header: t('time_table_header'), @@ -230,6 +314,7 @@ class Logs extends Component { accessor: 'domain', minWidth: 180, Cell: this.getDomainCell, + Filter: this.getFilterInput, }, { Header: t('type_table_header'), @@ -251,7 +336,7 @@ class Logs extends Component { }, Filter: ({ filter, onChange }) => (