diff --git a/CHANGELOG.md b/CHANGELOG.md index 007b41fc..8407d083 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to ### Added +- Search by clients' names in the query log ([#1273]). - Verbose version output with `-v --version` ([#2416]). - The ability to set a custom TLD for known local-network hosts ([#2393]). - The ability to serve DNS queries on multiple hosts and interfaces ([#1401]). @@ -44,6 +45,7 @@ and this project adheres to - Go 1.14 support. +[#1273]: https://github.com/AdguardTeam/AdGuardHome/issues/1273 [#1401]: https://github.com/AdguardTeam/AdGuardHome/issues/1401 [#2385]: https://github.com/AdguardTeam/AdGuardHome/issues/2385 [#2393]: https://github.com/AdguardTeam/AdGuardHome/issues/2393 diff --git a/client/src/actions/queryLogs.js b/client/src/actions/queryLogs.js index 076a9008..99da2cb0 100644 --- a/client/src/actions/queryLogs.js +++ b/client/src/actions/queryLogs.js @@ -1,23 +1,12 @@ import { createAction } from 'redux-actions'; import apiClient from '../api/Api'; -import { normalizeLogs, getParamsForClientsSearch, addClientInfo } from '../helpers/helpers'; +import { normalizeLogs } from '../helpers/helpers'; import { DEFAULT_LOGS_FILTER, FORM_NAME, QUERY_LOGS_PAGE_LIMIT, } from '../helpers/constants'; import { addErrorToast, addSuccessToast } from './toasts'; -const enrichWithClientInfo = async (logs) => { - const clientsParams = getParamsForClientsSearch(logs, 'client', 'client_id'); - - if (Object.keys(clientsParams).length > 0) { - const clients = await apiClient.findClients(clientsParams); - return addClientInfo(logs, clients, 'client_id', 'client'); - } - - return logs; -}; - const getLogsWithParams = async (config) => { const { older_than, filter, ...values } = config; const rawLogs = await apiClient.getQueryLog({ @@ -25,11 +14,9 @@ const getLogsWithParams = async (config) => { older_than, }); const { data, oldest } = rawLogs; - const normalizedLogs = normalizeLogs(data); - const logs = await enrichWithClientInfo(normalizedLogs); return { - logs, + logs: normalizeLogs(data), oldest, older_than, filter, @@ -92,10 +79,8 @@ export const updateLogs = () => async (dispatch, getState) => { try { const { logs, oldest, older_than } = getState().queryLogs; - const enrichedLogs = await enrichWithClientInfo(logs); - dispatch(getLogsSuccess({ - logs: enrichedLogs, + logs, oldest, older_than, })); diff --git a/client/src/components/Logs/Cells/ClientCell.js b/client/src/components/Logs/Cells/ClientCell.js index 863cd726..fa579d46 100644 --- a/client/src/components/Logs/Cells/ClientCell.js +++ b/client/src/components/Logs/Cells/ClientCell.js @@ -17,11 +17,8 @@ import { updateLogs } from '../../../actions/queryLogs'; const ClientCell = ({ client, client_id, + client_info, domain, - info, - info: { - name, whois_info, disallowed, disallowed_rule, - }, reason, }) => { const { t } = useTranslation(); @@ -33,18 +30,22 @@ const ClientCell = ({ const autoClient = autoClients.find((autoClient) => autoClient.name === client); const source = autoClient?.source; - const whoisAvailable = whois_info && Object.keys(whois_info).length > 0; - const clientName = name || client_id; - const clientInfo = { ...info, name: clientName }; + const whoisAvailable = client_info && Object.keys(client_info.whois).length > 0; + const clientName = client_info?.name || client_id; + const clientInfo = client_info && { + ...client_info, + whois_info: client_info?.whois, + name: clientName, + }; const id = nanoid(); const data = { address: client, name: clientName, - country: whois_info?.country, - city: whois_info?.city, - network: whois_info?.orgname, + country: client_info?.whois?.country, + city: client_info?.whois?.city, + network: client_info?.whois?.orgname, source_label: source, }; @@ -53,7 +54,7 @@ const ClientCell = ({ const isFiltered = checkFiltered(reason); const nameClass = classNames('w-90 o-hidden d-flex flex-column', { - 'mt-2': isDetailed && !name && !whoisAvailable, + 'mt-2': isDetailed && !client_info?.name && !whoisAvailable, 'white-space--nowrap': isDetailed, }); @@ -69,7 +70,11 @@ const ClientCell = ({ confirmMessage, buttonKey: blockingClientKey, isNotInAllowedList, - } = getBlockClientInfo(client, disallowed, disallowed_rule); + } = getBlockClientInfo( + client, + client_info?.disallowed || false, + client_info?.disallowed_rule || '', + ); const blockingForClientKey = isFiltered ? 'unblock_for_this_client_only' : 'block_for_this_client_only'; const clientNameBlockingFor = getBlockingClientName(clients, client); @@ -85,7 +90,11 @@ const ClientCell = ({ name: blockingClientKey, onClick: async () => { if (window.confirm(confirmMessage)) { - await dispatch(toggleClientBlock(client, disallowed, disallowed_rule)); + await dispatch(toggleClientBlock( + client, + client_info?.disallowed || false, + client_info?.disallowed_rule || '', + )); await dispatch(updateLogs()); } }, @@ -199,20 +208,18 @@ const ClientCell = ({ ClientCell.propTypes = { client: propTypes.string.isRequired, client_id: propTypes.string, + client_info: propTypes.shape({ + ids: propTypes.arrayOf(propTypes.string).isRequired, + name: propTypes.string.isRequired, + whois: propTypes.shape({ + country: propTypes.string, + city: propTypes.string, + orgname: propTypes.string, + }).isRequired, + disallowed: propTypes.bool.isRequired, + disallowed_rule: propTypes.string.isRequired, + }), domain: propTypes.string.isRequired, - info: propTypes.oneOfType([ - propTypes.string, - propTypes.shape({ - name: propTypes.string.isRequired, - whois_info: propTypes.shape({ - country: propTypes.string, - city: propTypes.string, - orgname: propTypes.string, - }), - disallowed: propTypes.bool.isRequired, - disallowed_rule: propTypes.string.isRequired, - }), - ]), reason: propTypes.string.isRequired, }; diff --git a/client/src/components/Logs/Cells/index.js b/client/src/components/Logs/Cells/index.js index 5ec64f1f..5740bbd0 100644 --- a/client/src/components/Logs/Cells/index.js +++ b/client/src/components/Logs/Cells/index.js @@ -29,11 +29,12 @@ import DateCell from './DateCell'; import DomainCell from './DomainCell'; import ResponseCell from './ResponseCell'; import ClientCell from './ClientCell'; -import '../Logs.css'; import { toggleClientBlock } from '../../../actions/access'; import { getBlockClientInfo, BUTTON_PREFIX } from './helpers'; import { updateLogs } from '../../../actions/queryLogs'; +import '../Logs.css'; + const Row = memo(({ style, rowProps, @@ -61,9 +62,7 @@ const Row = memo(({ client, domain, elapsedMs, - info, - info: { disallowed, disallowed_rule }, - reason, + client_info, response, time, tracker, @@ -82,11 +81,6 @@ const Row = memo(({ const autoClient = autoClients .find((autoClient) => autoClient.name === client); - const { whois_info } = info; - const country = whois_info?.country; - const city = whois_info?.city; - const network = whois_info?.orgname; - const source = autoClient?.source; const formattedElapsedMs = formatElapsedMs(elapsedMs, t); @@ -111,7 +105,11 @@ const Row = memo(({ confirmMessage, buttonKey: blockingClientKey, isNotInAllowedList, - } = getBlockClientInfo(client, disallowed, disallowed_rule); + } = getBlockClientInfo( + client, + client_info?.disallowed || false, + client_info?.disallowed_rule || '', + ); const blockingForClientKey = isFiltered ? 'unblock_for_this_client_only' : 'block_for_this_client_only'; const clientNameBlockingFor = getBlockingClientName(clients, client); @@ -122,7 +120,13 @@ const Row = memo(({ const onBlockingClientClick = async () => { if (window.confirm(confirmMessage)) { - await dispatch(toggleClientBlock(client, disallowed, disallowed_rule)); + await dispatch( + toggleClientBlock( + client, + client_info?.disallowed || false, + client_info?.disallowed_rule || '', + ), + ); await dispatch(updateLogs()); setModalOpened(false); } @@ -177,10 +181,10 @@ const Row = memo(({ response_code: status, client_details: 'title', ip_address: client, - name: info?.name || client_id, - country, - city, - network, + name: client_info?.name || client_id, + country: client_info?.whois?.country, + city: client_info?.whois?.city, + network: client_info?.whois?.orgname, source_label: source, validated_with_dnssec: dnssec_enabled ? Boolean(answer_dnssec) : false, original_response: originalResponse?.join('\n'), @@ -219,15 +223,6 @@ Row.propTypes = { client: propTypes.string.isRequired, domain: propTypes.string.isRequired, elapsedMs: propTypes.string.isRequired, - info: propTypes.oneOfType([ - propTypes.string, - propTypes.shape({ - whois_info: propTypes.shape({ - country: propTypes.string, - city: propTypes.string, - orgname: propTypes.string, - }), - })]), response: propTypes.array.isRequired, time: propTypes.string.isRequired, tracker: propTypes.object, @@ -235,6 +230,17 @@ Row.propTypes = { type: propTypes.string.isRequired, client_proto: propTypes.string.isRequired, client_id: propTypes.string, + client_info: propTypes.shape({ + ids: propTypes.arrayOf(propTypes.string).isRequired, + name: propTypes.string.isRequired, + whois: propTypes.shape({ + country: propTypes.string, + city: propTypes.string, + orgname: propTypes.string, + }).isRequired, + disallowed: propTypes.bool.isRequired, + disallowed_rule: propTypes.string.isRequired, + }), rules: propTypes.arrayOf(propTypes.shape({ text: propTypes.string.isRequired, filter_list_id: propTypes.number.isRequired, diff --git a/client/src/components/Logs/InfiniteTable.js b/client/src/components/Logs/InfiniteTable.js index 55c9db6d..d419ac3d 100644 --- a/client/src/components/Logs/InfiniteTable.js +++ b/client/src/components/Logs/InfiniteTable.js @@ -56,13 +56,13 @@ const InfiniteTable = ({ }, []); const renderRow = (row, idx) => ; + key={idx} + rowProps={row} + isSmallScreen={isSmallScreen} + setDetailedDataCurrent={setDetailedDataCurrent} + setButtonType={setButtonType} + setModalOpened={setModalOpened} + />; const isNothingFound = items.length === 0 && !processingGetLogs; diff --git a/client/src/helpers/helpers.js b/client/src/helpers/helpers.js index f8f276c8..8be106ab 100644 --- a/client/src/helpers/helpers.js +++ b/client/src/helpers/helpers.js @@ -63,6 +63,7 @@ export const normalizeLogs = (logs) => logs.map((log) => { client, client_proto, client_id, + client_info, elapsedMs, question, reason, @@ -101,6 +102,7 @@ export const normalizeLogs = (logs) => logs.map((log) => { client, client_proto, client_id, + client_info, /* TODO 'filterId' and 'rule' are deprecated, will be removed in 0.106 */ filterId, rule, diff --git a/internal/dnsforward/dnsforward.go b/internal/dnsforward/dnsforward.go index adab01a3..5a6045b5 100644 --- a/internal/dnsforward/dnsforward.go +++ b/internal/dnsforward/dnsforward.go @@ -375,5 +375,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { // IsBlockedIP - return TRUE if this client should be blocked func (s *Server) IsBlockedIP(ip net.IP) (bool, string) { + if ip == nil { + return false, "" + } + return s.access.IsBlockedIP(ip) } diff --git a/internal/home/clients.go b/internal/home/clients.go index 9e0d0fd3..ad119b00 100644 --- a/internal/home/clients.go +++ b/internal/home/clients.go @@ -15,6 +15,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/dhcpd" "github.com/AdguardTeam/AdGuardHome/internal/dnsfilter" "github.com/AdguardTeam/AdGuardHome/internal/dnsforward" + "github.com/AdguardTeam/AdGuardHome/internal/querylog" "github.com/AdguardTeam/AdGuardHome/internal/util" "github.com/AdguardTeam/dnsproxy/proxy" "github.com/AdguardTeam/dnsproxy/upstream" @@ -60,19 +61,26 @@ const ( ClientSourceHostsFile ) -// ClientHost information -type ClientHost struct { +// RuntimeClient information +type RuntimeClient struct { Host string Source clientSource - WhoisInfo [][]string // [[key,value], ...] + WhoisInfo *RuntimeClientWhoisInfo +} + +// RuntimeClientWhoisInfo is the filtered WHOIS data for a runtime client. +type RuntimeClientWhoisInfo struct { + City string `json:"city,omitempty"` + Country string `json:"country,omitempty"` + Orgname string `json:"orgname,omitempty"` } type clientsContainer struct { // TODO(a.garipov): Perhaps use a number of separate indices for // different types (string, net.IP, and so on). - list map[string]*Client // name -> client - idIndex map[string]*Client // ID -> client - ipHost map[string]*ClientHost // IP -> Hostname + list map[string]*Client // name -> client + idIndex map[string]*Client // ID -> client + ipToRC map[string]*RuntimeClient // IP -> runtime client lock sync.Mutex allTags map[string]bool @@ -97,7 +105,7 @@ func (clients *clientsContainer) Init(objects []clientObject, dhcpServer *dhcpd. } clients.list = make(map[string]*Client) clients.idIndex = make(map[string]*Client) - clients.ipHost = make(map[string]*ClientHost) + clients.ipToRC = make(map[string]*RuntimeClient) clients.allTags = make(map[string]bool) for _, t := range clientTags { @@ -128,7 +136,7 @@ func (clients *clientsContainer) Start() { } } -// Reload - reload auto-clients +// Reload reloads runtime clients. func (clients *clientsContainer) Reload() { clients.addFromSystemARP() } @@ -248,21 +256,70 @@ func (clients *clientsContainer) Exists(id string, source clientSource) (ok bool return true } - var ch *ClientHost - ch, ok = clients.ipHost[id] + var rc *RuntimeClient + rc, ok = clients.ipToRC[id] if !ok { return false } // Return false if the new source has higher priority. - return source <= ch.Source + return source <= rc.Source } func copyStrings(a []string) (b []string) { return append(b, a...) } -// Find searches for a client by its ID. +func toQueryLogWhois(wi *RuntimeClientWhoisInfo) (cw *querylog.ClientWhois) { + if wi == nil { + return &querylog.ClientWhois{} + } + + return &querylog.ClientWhois{ + City: wi.City, + Country: wi.Country, + Orgname: wi.Orgname, + } +} + +// findMultiple is a wrapper around Find to make it a valid client finder for +// the query log. err is always nil. +func (clients *clientsContainer) findMultiple(ids []string) (c *querylog.Client, err error) { + for _, id := range ids { + var name string + var foundIDs []string + whois := &querylog.ClientWhois{} + + c, ok := clients.Find(id) + if ok { + name = c.Name + foundIDs = c.IDs + } else { + var rc RuntimeClient + rc, ok = clients.FindRuntimeClient(id) + if !ok { + continue + } + + foundIDs = []string{rc.Host} + whois = toQueryLogWhois(rc.WhoisInfo) + } + + ip := net.ParseIP(id) + disallowed, disallowedRule := clients.dnsServer.IsBlockedIP(ip) + + return &querylog.Client{ + Name: name, + DisallowedRule: disallowedRule, + Whois: whois, + IDs: foundIDs, + Disallowed: disallowed, + }, nil + } + + return nil, nil +} + func (clients *clientsContainer) Find(id string) (c *Client, ok bool) { clients.lock.Lock() defer clients.lock.Unlock() @@ -361,21 +418,22 @@ func (clients *clientsContainer) findLocked(id string) (c *Client, ok bool) { return nil, false } -// FindAutoClient - search for an auto-client by IP -func (clients *clientsContainer) FindAutoClient(ip string) (ClientHost, bool) { +// FindRuntimeClient finds a runtime client by their IP. +func (clients *clientsContainer) FindRuntimeClient(ip string) (RuntimeClient, bool) { ipAddr := net.ParseIP(ip) if ipAddr == nil { - return ClientHost{}, false + return RuntimeClient{}, false } clients.lock.Lock() defer clients.lock.Unlock() - ch, ok := clients.ipHost[ip] + rc, ok := clients.ipToRC[ip] if ok { - return *ch, true + return *rc, true } - return ClientHost{}, false + + return RuntimeClient{}, false } // check validates the client. @@ -558,9 +616,7 @@ func (clients *clientsContainer) Update(name string, c *Client) (err error) { } // SetWhoisInfo sets the WHOIS information for a client. -// -// TODO(a.garipov): Perhaps replace [][]string with map[string]string. -func (clients *clientsContainer) SetWhoisInfo(ip string, info [][]string) { +func (clients *clientsContainer) SetWhoisInfo(ip string, wi *RuntimeClientWhoisInfo) { clients.lock.Lock() defer clients.lock.Unlock() @@ -570,21 +626,24 @@ func (clients *clientsContainer) SetWhoisInfo(ip string, info [][]string) { return } - ch, ok := clients.ipHost[ip] + rc, ok := clients.ipToRC[ip] if ok { - ch.WhoisInfo = info - log.Debug("clients: set whois info for auto-client %s: %q", ch.Host, info) + rc.WhoisInfo = wi + log.Debug("clients: set whois info for runtime client %s: %+v", rc.Host, wi) return } - // Create a ClientHost implicitly so that we don't do this check again - ch = &ClientHost{ + // Create a RuntimeClient implicitly so that we don't do this check + // again. + rc = &RuntimeClient{ Source: ClientSourceWHOIS, } - ch.WhoisInfo = info - clients.ipHost[ip] = ch - log.Debug("clients: set whois info for auto-client with IP %s: %q", ip, info) + + rc.WhoisInfo = wi + clients.ipToRC[ip] = rc + + log.Debug("clients: set whois info for runtime client with ip %s: %+v", ip, wi) } // AddHost adds a new IP-hostname pairing. The priorities of the sources is @@ -600,24 +659,25 @@ func (clients *clientsContainer) AddHost(ip, host string, src clientSource) (ok // addHostLocked adds a new IP-hostname pairing. For internal use only. func (clients *clientsContainer) addHostLocked(ip, host string, src clientSource) (ok bool) { - var ch *ClientHost - ch, ok = clients.ipHost[ip] + var rc *RuntimeClient + rc, ok = clients.ipToRC[ip] if ok { - if ch.Source > src { + if rc.Source > src { return false } - ch.Source = src + rc.Source = src } else { - ch = &ClientHost{ - Host: host, - Source: src, + rc = &RuntimeClient{ + Host: host, + Source: src, + WhoisInfo: &RuntimeClientWhoisInfo{}, } - clients.ipHost[ip] = ch + clients.ipToRC[ip] = rc } - log.Debug("clients: added %q -> %q [%d]", ip, host, len(clients.ipHost)) + log.Debug("clients: added %q -> %q [%d]", ip, host, len(clients.ipToRC)) return true } @@ -625,9 +685,9 @@ func (clients *clientsContainer) addHostLocked(ip, host string, src clientSource // rmHostsBySrc removes all entries that match the specified source. func (clients *clientsContainer) rmHostsBySrc(src clientSource) { n := 0 - for k, v := range clients.ipHost { + for k, v := range clients.ipToRC { if v.Source == src { - delete(clients.ipHost, k) + delete(clients.ipToRC, k) n++ } } diff --git a/internal/home/clients_test.go b/internal/home/clients_test.go index 85eb866e..aa040a58 100644 --- a/internal/home/clients_test.go +++ b/internal/home/clients_test.go @@ -163,17 +163,20 @@ func TestClientsWhois(t *testing.T) { testing: true, } clients.Init(nil, nil, nil) - whois := [][]string{{"orgname", "orgname-val"}, {"country", "country-val"}} + whois := &RuntimeClientWhoisInfo{ + Country: "AU", + Orgname: "Example Org", + } t.Run("new_client", func(t *testing.T) { clients.SetWhoisInfo("1.1.1.255", whois) - require.NotNil(t, clients.ipHost["1.1.1.255"]) - h := clients.ipHost["1.1.1.255"] + require.NotNil(t, clients.ipToRC["1.1.1.255"]) - require.Len(t, h.WhoisInfo, 2) - require.Len(t, h.WhoisInfo[0], 2) - assert.Equal(t, "orgname-val", h.WhoisInfo[0][1]) + h := clients.ipToRC["1.1.1.255"] + require.NotNil(t, h) + + assert.Equal(t, h.WhoisInfo, whois) }) t.Run("existing_auto-client", func(t *testing.T) { @@ -183,12 +186,11 @@ func TestClientsWhois(t *testing.T) { clients.SetWhoisInfo("1.1.1.1", whois) - require.NotNil(t, clients.ipHost["1.1.1.1"]) - h := clients.ipHost["1.1.1.1"] + require.NotNil(t, clients.ipToRC["1.1.1.1"]) + h := clients.ipToRC["1.1.1.1"] + require.NotNil(t, h) - require.Len(t, h.WhoisInfo, 2) - require.Len(t, h.WhoisInfo[0], 2) - assert.Equal(t, "orgname-val", h.WhoisInfo[0][1]) + assert.Equal(t, h.WhoisInfo, whois) }) t.Run("can't_set_manually-added", func(t *testing.T) { @@ -200,7 +202,7 @@ func TestClientsWhois(t *testing.T) { assert.True(t, ok) clients.SetWhoisInfo("1.1.1.2", whois) - require.Nil(t, clients.ipHost["1.1.1.2"]) + require.Nil(t, clients.ipToRC["1.1.1.2"]) assert.True(t, clients.Del("client1")) }) } diff --git a/internal/home/clientshttp.go b/internal/home/clientshttp.go index 23930411..681e9c55 100644 --- a/internal/home/clientshttp.go +++ b/internal/home/clientshttp.go @@ -22,7 +22,7 @@ type clientJSON struct { Upstreams []string `json:"upstreams"` - WhoisInfo map[string]string `json:"whois_info"` + WhoisInfo *RuntimeClientWhoisInfo `json:"whois_info"` // Disallowed - if true -- client's IP is not disallowed // Otherwise, it is blocked. @@ -34,18 +34,18 @@ type clientJSON struct { DisallowedRule string `json:"disallowed_rule"` } -type clientHostJSON struct { +type runtimeClientJSON struct { + WhoisInfo *RuntimeClientWhoisInfo `json:"whois_info"` + IP string `json:"ip"` Name string `json:"name"` Source string `json:"source"` - - WhoisInfo map[string]string `json:"whois_info"` } type clientListJSON struct { - Clients []clientJSON `json:"clients"` - AutoClients []clientHostJSON `json:"auto_clients"` - Tags []string `json:"supported_tags"` + Clients []clientJSON `json:"clients"` + RuntimeClients []runtimeClientJSON `json:"auto_clients"` + Tags []string `json:"supported_tags"` } // respond with information about configured clients @@ -53,18 +53,21 @@ func (clients *clientsContainer) handleGetClients(w http.ResponseWriter, _ *http data := clientListJSON{} clients.lock.Lock() + defer clients.lock.Unlock() + for _, c := range clients.list { cj := clientToJSON(c) data.Clients = append(data.Clients, cj) } - for ip, ch := range clients.ipHost { - cj := clientHostJSON{ - IP: ip, - Name: ch.Host, + for ip, rc := range clients.ipToRC { + cj := runtimeClientJSON{ + IP: ip, + Name: rc.Host, + WhoisInfo: rc.WhoisInfo, } cj.Source = "etc/hosts" - switch ch.Source { + switch rc.Source { case ClientSourceDHCP: cj.Source = "DHCP" case ClientSourceRDNS: @@ -75,14 +78,8 @@ func (clients *clientsContainer) handleGetClients(w http.ResponseWriter, _ *http cj.Source = "WHOIS" } - cj.WhoisInfo = map[string]string{} - for _, wi := range ch.WhoisInfo { - cj.WhoisInfo[wi[0]] = wi[1] - } - - data.AutoClients = append(data.AutoClients, cj) + data.RuntimeClients = append(data.RuntimeClients, cj) } - clients.lock.Unlock() data.Tags = clientTags @@ -129,21 +126,21 @@ func clientToJSON(c *Client) clientJSON { BlockedServices: c.BlockedServices, Upstreams: c.Upstreams, + + WhoisInfo: &RuntimeClientWhoisInfo{}, } + return cj } -// Convert ClientHost object to JSON -func clientHostToJSON(ip string, ch ClientHost) clientJSON { - cj := clientJSON{ - Name: ch.Host, - IDs: []string{ip}, +// runtimeClientToJSON converts a RuntimeClient into a JSON struct. +func runtimeClientToJSON(ip string, rc RuntimeClient) (cj clientJSON) { + cj = clientJSON{ + Name: rc.Host, + IDs: []string{ip}, + WhoisInfo: rc.WhoisInfo, } - cj.WhoisInfo = map[string]string{} - for _, wi := range ch.WhoisInfo { - cj.WhoisInfo[wi[0]] = wi[1] - } return cj } @@ -268,7 +265,7 @@ func (clients *clientsContainer) findTemporary(ip net.IP, idStr string) (cj clie return cj, false } - ch, ok := clients.FindAutoClient(idStr) + rc, ok := clients.FindRuntimeClient(idStr) if !ok { // It is still possible that the IP used to be in the runtime // clients list, but then the server was reloaded. So, check @@ -284,12 +281,13 @@ func (clients *clientsContainer) findTemporary(ip net.IP, idStr string) (cj clie IDs: []string{idStr}, Disallowed: disallowed, DisallowedRule: rule, + WhoisInfo: &RuntimeClientWhoisInfo{}, } return cj, true } - cj = clientHostToJSON(idStr, ch) + cj = runtimeClientToJSON(idStr, rc) cj.Disallowed, cj.DisallowedRule = clients.dnsServer.IsBlockedIP(ip) return cj, true diff --git a/internal/home/config.go b/internal/home/config.go index 58970520..da181153 100644 --- a/internal/home/config.go +++ b/internal/home/config.go @@ -285,7 +285,7 @@ func (c *configuration) write() error { Context.queryLog.WriteDiskConfig(&dc) config.DNS.QueryLogEnabled = dc.Enabled config.DNS.QueryLogFileEnabled = dc.FileEnabled - config.DNS.QueryLogInterval = dc.Interval + config.DNS.QueryLogInterval = dc.RotationIvl config.DNS.QueryLogMemSize = dc.MemSize config.DNS.AnonymizeClientIP = dc.AnonymizeClientIP } diff --git a/internal/home/dns.go b/internal/home/dns.go index 5e629d4d..d6a4aecc 100644 --- a/internal/home/dns.go +++ b/internal/home/dns.go @@ -42,15 +42,17 @@ func initDNSServer() error { if err != nil { return fmt.Errorf("couldn't initialize statistics module") } + conf := querylog.Config{ - Enabled: config.DNS.QueryLogEnabled, - FileEnabled: config.DNS.QueryLogFileEnabled, - BaseDir: baseDir, - Interval: config.DNS.QueryLogInterval, - MemSize: config.DNS.QueryLogMemSize, - AnonymizeClientIP: config.DNS.AnonymizeClientIP, ConfigModified: onConfigModified, HTTPRegister: httpRegister, + FindClient: Context.clients.findMultiple, + BaseDir: baseDir, + RotationIvl: config.DNS.QueryLogInterval, + MemSize: config.DNS.QueryLogMemSize, + Enabled: config.DNS.QueryLogEnabled, + FileEnabled: config.DNS.QueryLogFileEnabled, + AnonymizeClientIP: config.DNS.AnonymizeClientIP, } Context.queryLog = querylog.New(conf) diff --git a/internal/home/rdns_test.go b/internal/home/rdns_test.go index 0e313ef6..d89c5b11 100644 --- a/internal/home/rdns_test.go +++ b/internal/home/rdns_test.go @@ -84,7 +84,7 @@ func TestRDNS_Begin(t *testing.T) { clients: &clientsContainer{ list: map[string]*Client{}, idIndex: tc.cliIDIndex, - ipHost: map[string]*ClientHost{}, + ipToRC: map[string]*RuntimeClient{}, allTags: map[string]bool{}, }, } @@ -229,7 +229,7 @@ func TestRDNS_WorkerLoop(t *testing.T) { cc := &clientsContainer{ list: map[string]*Client{}, idIndex: map[string]*Client{}, - ipHost: map[string]*ClientHost{}, + ipToRC: map[string]*RuntimeClient{}, allTags: map[string]bool{}, } ch := make(chan net.IP) diff --git a/internal/home/whois.go b/internal/home/whois.go index 20f035e2..f2923815 100644 --- a/internal/home/whois.go +++ b/internal/home/whois.go @@ -182,29 +182,31 @@ func (w *Whois) queryAll(ctx context.Context, target string) (string, error) { } // Request WHOIS information -func (w *Whois) process(ctx context.Context, ip net.IP) [][]string { - data := [][]string{} +func (w *Whois) process(ctx context.Context, ip net.IP) (wi *RuntimeClientWhoisInfo) { resp, err := w.queryAll(ctx, ip.String()) if err != nil { log.Debug("Whois: error: %s IP:%s", err, ip) - return data + + return nil } log.Debug("Whois: IP:%s response: %d bytes", ip, len(resp)) m := whoisParse(resp) - keys := []string{"orgname", "country", "city"} - for _, k := range keys { - v, found := m[k] - if !found { - continue - } - pair := []string{k, v} - data = append(data, pair) + wi = &RuntimeClientWhoisInfo{ + City: m["city"], + Country: m["country"], + Orgname: m["orgname"], } - return data + // Don't return an empty struct so that the frontend doesn't get + // confused. + if *wi == (RuntimeClientWhoisInfo{}) { + return nil + } + + return wi } // Begin - begin requesting WHOIS info @@ -234,11 +236,9 @@ func (w *Whois) Begin(ip net.IP) { // workerLoop processes the IP addresses it got from the channel and associates // the retrieving WHOIS info with a client. func (w *Whois) workerLoop() { - for { - ip := <-w.ipChan - + for ip := range w.ipChan { info := w.process(context.Background(), ip) - if len(info) == 0 { + if info == nil { continue } diff --git a/internal/querylog/client.go b/internal/querylog/client.go new file mode 100644 index 00000000..101a8cd4 --- /dev/null +++ b/internal/querylog/client.go @@ -0,0 +1,33 @@ +package querylog + +// Client is the information required by the query log to match against clients +// during searches. +type Client struct { + Name string `json:"name"` + DisallowedRule string `json:"disallowed_rule"` + Whois *ClientWhois `json:"whois,omitempty"` + IDs []string `json:"ids"` + Disallowed bool `json:"disallowed"` +} + +// ClientWhois is the filtered WHOIS data for the client. +// +// TODO(a.garipov): Merge with home.RuntimeClientWhoisInfo after the +// refactoring is done. +type ClientWhois struct { + City string `json:"city,omitempty"` + Country string `json:"country,omitempty"` + Orgname string `json:"orgname,omitempty"` +} + +// clientCacheKey is the key by which a cached client information is found. +type clientCacheKey struct { + clientID string + ip string +} + +// clientCache is the cache of client information found throughout a request to +// the query log API. It is used both to speed up the lookup, as well as to +// make sure that changes in client data between two lookups don't create +// discrepancies in our response. +type clientCache map[clientCacheKey]*Client diff --git a/internal/querylog/http.go b/internal/querylog/http.go index 4f71c21d..d3fcd63e 100644 --- a/internal/querylog/http.go +++ b/internal/querylog/http.go @@ -68,7 +68,7 @@ func (l *queryLog) handleQueryLogClear(_ http.ResponseWriter, _ *http.Request) { func (l *queryLog) handleQueryLogInfo(w http.ResponseWriter, r *http.Request) { resp := qlogConfig{} resp.Enabled = l.conf.Enabled - resp.Interval = l.conf.Interval + resp.Interval = l.conf.RotationIvl resp.AnonymizeClientIP = l.conf.AnonymizeClientIP jsonVal, err := json.Marshal(resp) @@ -104,7 +104,7 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request) conf.Enabled = d.Enabled } if req.Exists("interval") { - conf.Interval = d.Interval + conf.RotationIvl = d.Interval } if req.Exists("anonymize_client_ip") { conf.AnonymizeClientIP = d.AnonymizeClientIP diff --git a/internal/querylog/json.go b/internal/querylog/json.go index 306a6bcb..217c2d17 100644 --- a/internal/querylog/json.go +++ b/internal/querylog/json.go @@ -71,6 +71,7 @@ func (l *queryLog) logEntryToJSONEntry(entry *logEntry) (jsonEntry jobject) { "elapsedMs": strconv.FormatFloat(entry.Elapsed.Seconds()*1000, 'f', -1, 64), "time": entry.Time.Format(time.RFC3339Nano), "client": l.getClientIP(entry.IP), + "client_info": entry.client, "client_proto": entry.ClientProto, "upstream": entry.Upstream, "question": jobject{ diff --git a/internal/querylog/qlog.go b/internal/querylog/qlog.go index 4726f075..14cbd1f4 100644 --- a/internal/querylog/qlog.go +++ b/internal/querylog/qlog.go @@ -6,7 +6,6 @@ import ( "fmt" "net" "os" - "path/filepath" "strings" "sync" "time" @@ -22,12 +21,17 @@ const ( // queryLog is a structure that writes and reads the DNS query log type queryLog struct { + findClient func(ids []string) (c *Client, err error) + conf *Config lock sync.Mutex logFile string // path to the log file - bufferLock sync.RWMutex - buffer []*logEntry + // bufferLock protects buffer. + bufferLock sync.RWMutex + // buffer contains recent log entries. + buffer []*logEntry + fileFlushLock sync.Mutex // synchronize a file-flushing goroutine and main thread flushPending bool // don't start another goroutine while the previous one is still running fileWriteLock sync.Mutex @@ -64,6 +68,9 @@ func NewClientProto(s string) (cp ClientProto, err error) { // logEntry - represents a single log entry type logEntry struct { + // client is the found client information, if any. + client *Client + IP net.IP `json:"IP"` // Client IP Time time.Time `json:"T"` @@ -82,18 +89,6 @@ type logEntry struct { Upstream string `json:",omitempty"` // if empty, means it was cached } -// create a new instance of the query log -func newQueryLog(conf Config) *queryLog { - l := queryLog{} - l.logFile = filepath.Join(conf.BaseDir, queryLogFileName) - l.conf = &Config{} - *l.conf = conf - if !checkInterval(l.conf.Interval) { - l.conf.Interval = 1 - } - return &l -} - func (l *queryLog) Start() { if l.conf.HTTPRegister != nil { l.initWeb() @@ -138,12 +133,16 @@ func (l *queryLog) clear() { } func (l *queryLog) Add(params AddParams) { + var err error + if !l.conf.Enabled { return } - if params.Question == nil || len(params.Question.Question) != 1 || len(params.Question.Question[0].Name) == 0 || - params.ClientIP == nil { + err = params.validate() + if err != nil { + log.Error("querylog: adding record: %s, skipping", err) + return } @@ -168,20 +167,26 @@ func (l *queryLog) Add(params AddParams) { entry.QClass = dns.Class(q.Qclass).String() if params.Answer != nil { - a, err := params.Answer.Pack() + var a []byte + a, err = params.Answer.Pack() if err != nil { - log.Info("Querylog: Answer.Pack(): %s", err) + log.Error("querylog: Answer.Pack(): %s", err) + return } + entry.Answer = a } if params.OrigAnswer != nil { - a, err := params.OrigAnswer.Pack() + var a []byte + a, err = params.OrigAnswer.Pack() if err != nil { - log.Info("Querylog: OrigAnswer.Pack(): %s", err) + log.Error("querylog: OrigAnswer.Pack(): %s", err) + return } + entry.OrigAnswer = a } diff --git a/internal/querylog/qlog_test.go b/internal/querylog/qlog_test.go index f8b8f3c9..880da49e 100644 --- a/internal/querylog/qlog_test.go +++ b/internal/querylog/qlog_test.go @@ -26,7 +26,7 @@ func TestQueryLog(t *testing.T) { l := newQueryLog(Config{ Enabled: true, FileEnabled: true, - Interval: 1, + RotationIvl: 1, MemSize: 100, BaseDir: t.TempDir(), }) @@ -127,10 +127,10 @@ func TestQueryLog(t *testing.T) { func TestQueryLogOffsetLimit(t *testing.T) { l := newQueryLog(Config{ - Enabled: true, - Interval: 1, - MemSize: 100, - BaseDir: t.TempDir(), + Enabled: true, + RotationIvl: 1, + MemSize: 100, + BaseDir: t.TempDir(), }) const ( @@ -202,7 +202,7 @@ func TestQueryLogMaxFileScanEntries(t *testing.T) { l := newQueryLog(Config{ Enabled: true, FileEnabled: true, - Interval: 1, + RotationIvl: 1, MemSize: 100, BaseDir: t.TempDir(), }) @@ -230,7 +230,7 @@ func TestQueryLogFileDisabled(t *testing.T) { l := newQueryLog(Config{ Enabled: true, FileEnabled: false, - Interval: 1, + RotationIvl: 1, MemSize: 2, BaseDir: t.TempDir(), }) diff --git a/internal/querylog/querylog.go b/internal/querylog/querylog.go index 98b8959d..0eaa7189 100644 --- a/internal/querylog/querylog.go +++ b/internal/querylog/querylog.go @@ -3,9 +3,12 @@ package querylog import ( "net" "net/http" + "path/filepath" "time" + "github.com/AdguardTeam/AdGuardHome/internal/agherr" "github.com/AdguardTeam/AdGuardHome/internal/dnsfilter" + "github.com/AdguardTeam/golibs/log" "github.com/miekg/dns" ) @@ -25,18 +28,37 @@ type QueryLog interface { // Config - configuration object type Config struct { - Enabled bool // enable the module - FileEnabled bool // write logs to file - BaseDir string // directory where log file is stored - Interval uint32 // interval to rotate logs (in days) - MemSize uint32 // number of entries kept in memory before they are flushed to disk - AnonymizeClientIP bool // anonymize clients' IP addresses - - // Called when the configuration is changed by HTTP request + // ConfigModified is called when the configuration is changed, for + // example by HTTP requests. ConfigModified func() - // Register an HTTP handler + // HTTPRegister registers an HTTP handler. HTTPRegister func(string, string, func(http.ResponseWriter, *http.Request)) + + // FindClient returns client information by their IDs. + FindClient func(ids []string) (c *Client, err error) + + // BaseDir is the base directory for log files. + BaseDir string + + // RotationIvl is the interval for log rotation, in days. After that + // period, the old log file will be renamed, NOT deleted, so the actual + // log retention time is twice the interval. + RotationIvl uint32 + + // MemSize is the number of entries kept in a memory buffer before they + // are flushed to disk. + MemSize uint32 + + // Enabled tells if the query log is enabled. + Enabled bool + + // FileEnabled tells if the query log writes logs to files. + FileEnabled bool + + // AnonymizeClientIP tells if the query log should anonymize clients' IP + // addresses. + AnonymizeClientIP bool } // AddParams - parameters for Add() @@ -52,7 +74,52 @@ type AddParams struct { ClientProto ClientProto } -// New - create a new instance of the query log -func New(conf Config) QueryLog { +// validate returns an error if the parameters aren't valid. +func (p *AddParams) validate() (err error) { + switch { + case p.Question == nil: + return agherr.Error("question is nil") + case len(p.Question.Question) != 1: + return agherr.Error("more than one question") + case len(p.Question.Question[0].Name) == 0: + return agherr.Error("no host in question") + case p.ClientIP == nil: + return agherr.Error("no client ip") + default: + return nil + } +} + +// New creates a new instance of the query log. +func New(conf Config) (ql QueryLog) { return newQueryLog(conf) } + +// newQueryLog crates a new queryLog. +func newQueryLog(conf Config) (l *queryLog) { + findClient := conf.FindClient + if findClient == nil { + findClient = func(_ []string) (_ *Client, _ error) { + return nil, nil + } + } + + l = &queryLog{ + findClient: findClient, + + logFile: filepath.Join(conf.BaseDir, queryLogFileName), + } + + l.conf = &Config{} + *l.conf = conf + + if !checkInterval(conf.RotationIvl) { + log.Info( + "querylog: warning: unsupported rotation interval %d, setting to 1 day", + conf.RotationIvl, + ) + l.conf.RotationIvl = 1 + } + + return l +} diff --git a/internal/querylog/querylogfile.go b/internal/querylog/querylogfile.go index a0fd165b..6ba859bb 100644 --- a/internal/querylog/querylogfile.go +++ b/internal/querylog/querylogfile.go @@ -129,7 +129,7 @@ func (l *queryLog) readFileFirstTimeValue() int64 { } func (l *queryLog) periodicRotate() { - intervalSeconds := uint64(l.conf.Interval) * 24 * 60 * 60 + intervalSeconds := uint64(l.conf.RotationIvl) * 24 * 60 * 60 for { oldest := l.readFileFirstTimeValue() if uint64(oldest)+intervalSeconds <= uint64(time.Now().Unix()) { diff --git a/internal/querylog/search.go b/internal/querylog/search.go index 216f9167..264b3acd 100644 --- a/internal/querylog/search.go +++ b/internal/querylog/search.go @@ -8,6 +8,67 @@ import ( "github.com/AdguardTeam/golibs/log" ) +// client finds the client info, if any, by its client ID and IP address, +// optionally checking the provided cache. It will use the IP address +// regardless of if the IP anonymization is enabled now, because the +// anonymization could have been disabled in the past, and client will try to +// find those records as well. +func (l *queryLog) client(clientID, ip string, cache clientCache) (c *Client, err error) { + cck := clientCacheKey{clientID: clientID, ip: ip} + if c = cache[cck]; c != nil { + return c, nil + } + + var ids []string + if clientID != "" { + ids = append(ids, clientID) + } + + if ip != "" { + ids = append(ids, ip) + } + + c, err = l.findClient(ids) + if err != nil { + return nil, err + } + + if cache != nil { + cache[cck] = c + } + + return c, nil +} + +// searchMemory looks up log records which are currently in the in-memory +// buffer. It optionally uses the client cache, if provided. It also returns +// the total amount of records in the buffer at the moment of searching. +func (l *queryLog) searchMemory(params *searchParams, cache clientCache) (entries []*logEntry, total int) { + l.bufferLock.Lock() + defer l.bufferLock.Unlock() + + // Go through the buffer in the reverse order, from newer to older. + var err error + for i := len(l.buffer) - 1; i >= 0; i-- { + e := l.buffer[i] + + e.client, err = l.client(e.ClientID, e.IP.String(), cache) + if err != nil { + msg := "querylog: enriching memory record at time %s" + + " for client %q (client id %q): %s" + log.Error(msg, e.Time, e.IP, e.ClientID, err) + + // Go on and try to match anyway. + } + + if params.match(e) { + entries = append(entries, e) + } + } + + return entries, len(l.buffer) +} + // search - searches log entries in the query log using specified parameters // returns the list of entries found + time of the oldest entry func (l *queryLog) search(params *searchParams) ([]*logEntry, time.Time) { @@ -17,26 +78,11 @@ func (l *queryLog) search(params *searchParams) ([]*logEntry, time.Time) { return []*logEntry{}, time.Time{} } - // add from file - fileEntries, oldest, total := l.searchFiles(params) + cache := clientCache{} + fileEntries, oldest, total := l.searchFiles(params, cache) + memoryEntries, bufLen := l.searchMemory(params, cache) + total += bufLen - // add from memory buffer - l.bufferLock.Lock() - total += len(l.buffer) - memoryEntries := make([]*logEntry, 0) - - // go through the buffer in the reverse order - // from NEWER to OLDER - for i := len(l.buffer) - 1; i >= 0; i-- { - entry := l.buffer[i] - if !params.match(entry) { - continue - } - memoryEntries = append(memoryEntries, entry) - } - l.bufferLock.Unlock() - - // limits totalLimit := params.offset + params.limit // now let's get a unified collection @@ -74,18 +120,15 @@ func (l *queryLog) search(params *searchParams) ([]*logEntry, time.Time) { return entries, oldest } -// searchFiles reads log entries from all log files and applies the specified search criteria. -// IMPORTANT: this method does not scan more than "maxSearchEntries" so you -// may need to call it many times. -// -// it returns: -// * an array of log entries that we have read -// * time of the oldest processed entry (even if it was discarded) -// * total number of processed entries (including discarded). -func (l *queryLog) searchFiles(params *searchParams) ([]*logEntry, time.Time, int) { - entries := make([]*logEntry, 0) - oldest := time.Time{} - +// searchFiles looks up log records from all log files. It optionally uses the +// client cache, if provided. searchFiles does not scan more than +// maxFileScanEntries so callers may need to call it several times to get all +// results. oldset and total are the time of the oldest processed entry and the +// total number of processed entries, including discarded ones, correspondingly. +func (l *queryLog) searchFiles( + params *searchParams, + cache clientCache, +) (entries []*logEntry, oldest time.Time, total int) { files := []string{ l.logFile + ".1", l.logFile, @@ -104,40 +147,43 @@ func (l *queryLog) searchFiles(params *searchParams) ([]*logEntry, time.Time, in } else { err = r.SeekTS(params.olderThan.UnixNano()) if err == nil { - // Read to the next record right away - // The one that was specified in the "oldest" param is not needed, - // we need only the one next to it + // Read to the next record, because we only need the one + // that goes after it. _, err = r.ReadNext() } } if err != nil { - log.Debug("Cannot SeekTS() to %v: %v", params.olderThan, err) + log.Debug("querylog: cannot seek to %s: %s", params.olderThan, err) + return entries, oldest, 0 } totalLimit := params.offset + params.limit - total := 0 oldestNano := int64(0) - // By default, we do not scan more than "maxFileScanEntries" at once - // The idea is to make search calls faster so that the UI could handle it and show something - // This behavior can be overridden if "maxFileScanEntries" is set to 0 + + // By default, we do not scan more than maxFileScanEntries at once. + // The idea is to make search calls faster so that the UI could handle + // it and show something quicker. This behavior can be overridden if + // maxFileScanEntries is set to 0. for total < params.maxFileScanEntries || params.maxFileScanEntries <= 0 { - var entry *logEntry + var e *logEntry var ts int64 - entry, ts, err = l.readNextEntry(r, params) - if err == io.EOF { - // there's nothing to read anymore - break + e, ts, err = l.readNextEntry(r, params, cache) + if err != nil { + if err == io.EOF { + break + } + + log.Error("querylog: reading next entry: %s", err) } oldestNano = ts total++ - if entry != nil { - entries = append(entries, entry) + if e != nil { + entries = append(entries, e) if len(entries) == totalLimit { - // Do not read more than "totalLimit" records at once break } } @@ -146,36 +192,46 @@ func (l *queryLog) searchFiles(params *searchParams) ([]*logEntry, time.Time, in if oldestNano != 0 { oldest = time.Unix(0, oldestNano) } + return entries, oldest, total } -// readNextEntry - reads the next log entry and checks if it matches the search criteria (getDataParams) -// -// returns: -// * log entry that matches search criteria or null if it was discarded (or if there's nothing to read) -// * timestamp of the processed log entry -// * error if we can't read anymore -func (l *queryLog) readNextEntry(r *QLogReader, params *searchParams) (*logEntry, int64, error) { - line, err := r.ReadNext() +// readNextEntry reads the next log entry and checks if it matches the search +// criteria. It optionally uses the client cache, if provided. e is nil if the +// entry doesn't match the search criteria. ts is the timestamp of the +// processed entry. +func (l *queryLog) readNextEntry( + r *QLogReader, + params *searchParams, + cache clientCache, +) (e *logEntry, ts int64, err error) { + var line string + line, err = r.ReadNext() if err != nil { return nil, 0, err } - // Read the log record timestamp right away - timestamp := readQLogTimestamp(line) + e = &logEntry{} + decodeLogEntry(e, line) - // Quick check without deserializing log entry - if !params.quickMatch(line) { - return nil, timestamp, nil + e.client, err = l.client(e.ClientID, e.IP.String(), cache) + if err != nil { + log.Error( + "querylog: enriching file record at time %s"+ + " for client %q (client id %q): %s", + e.Time, + e.IP, + e.ClientID, + err, + ) + + // Go on and try to match anyway. } - entry := logEntry{} - decodeLogEntry(&entry, line) - - // Full check of the deserialized log entry - if !params.match(&entry) { - return nil, timestamp, nil + ts = e.Time.UnixNano() + if !params.match(e) { + return nil, ts, nil } - return &entry, timestamp, nil + return e, ts, nil } diff --git a/internal/querylog/search_test.go b/internal/querylog/search_test.go new file mode 100644 index 00000000..37420680 --- /dev/null +++ b/internal/querylog/search_test.go @@ -0,0 +1,95 @@ +package querylog + +import ( + "net" + "testing" + "time" + + "github.com/miekg/dns" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestQueryLog_Search_findClient(t *testing.T) { + const knownClientID = "client-1" + const knownClientName = "Known Client 1" + const unknownClientID = "client-2" + + knownClient := &Client{ + IDs: []string{knownClientID}, + Name: knownClientName, + } + + findClientCalls := 0 + findClient := func(ids []string) (c *Client, _ error) { + defer func() { findClientCalls++ }() + + if len(ids) == 0 { + return nil, nil + } + + if ids[0] == knownClientID { + return knownClient, nil + } + + return nil, nil + } + + l := newQueryLog(Config{ + FindClient: findClient, + BaseDir: t.TempDir(), + RotationIvl: 1, + MemSize: 100, + Enabled: true, + FileEnabled: true, + AnonymizeClientIP: false, + }) + t.Cleanup(l.Close) + + q := &dns.Msg{ + Question: []dns.Question{{ + Name: "example.com", + }}, + } + + l.Add(AddParams{ + Question: q, + ClientID: knownClientID, + ClientIP: net.IP{1, 2, 3, 4}, + }) + + // Add the same thing again to test the cache. + l.Add(AddParams{ + Question: q, + ClientID: knownClientID, + ClientIP: net.IP{1, 2, 3, 4}, + }) + + l.Add(AddParams{ + Question: q, + ClientID: unknownClientID, + ClientIP: net.IP{1, 2, 3, 5}, + }) + + sp := &searchParams{ + // Add some time to the "current" one to protect against + // low-resolution timers on some Windows machines. + // + // TODO(a.garipov): Use some kind of timeSource interface + // instead of relying on time.Now() in tests. + olderThan: time.Now().Add(10 * time.Second), + limit: 3, + } + entries, _ := l.search(sp) + assert.Equal(t, 2, findClientCalls) + + require.Len(t, entries, 3) + + assert.Nil(t, entries[0].client) + + gotClient := entries[2].client + require.NotNil(t, gotClient) + + assert.Equal(t, knownClientName, gotClient.Name) + assert.Equal(t, []string{knownClientID}, gotClient.IDs) +} diff --git a/internal/querylog/searchcriteria.go b/internal/querylog/searchcriteria.go index 68672eaf..dd67f6b6 100644 --- a/internal/querylog/searchcriteria.go +++ b/internal/querylog/searchcriteria.go @@ -48,40 +48,6 @@ type searchCriteria struct { strict bool // should we strictly match (equality) or not (indexOf) } -// quickMatch - quickly checks if the log entry matches this search criteria -// the reason is to do it as quickly as possible without de-serializing the entry -func (c *searchCriteria) quickMatch(line string) bool { - // note that we do this only for a limited set of criteria - - switch c.criteriaType { - case ctDomainOrClient: - return c.quickMatchJSONValue(line, "QH") || - c.quickMatchJSONValue(line, "IP") || - c.quickMatchJSONValue(line, "CID") - default: - return true - } -} - -// quickMatchJSONValue - helper used by quickMatch -func (c *searchCriteria) quickMatchJSONValue(line, propertyName string) bool { - val := readJSONValue(line, propertyName) - if len(val) == 0 { - return false - } - val = strings.ToLower(val) - searchVal := strings.ToLower(c.value) - - if c.strict && searchVal == val { - return true - } - if !c.strict && strings.Contains(val, searchVal) { - return true - } - - return false -} - // match - checks if the log entry matches this search criteria func (c *searchCriteria) match(entry *logEntry) bool { switch c.criteriaType { @@ -94,28 +60,41 @@ func (c *searchCriteria) match(entry *logEntry) bool { return false } -func (c *searchCriteria) ctDomainOrClientCase(entry *logEntry) bool { - clientID := strings.ToLower(entry.ClientID) - qhost := strings.ToLower(entry.QHost) - searchVal := strings.ToLower(c.value) - if c.strict && (qhost == searchVal || clientID == searchVal) { - return true +func (c *searchCriteria) ctDomainOrClientCaseStrict(term, clientID, name, host, ip string) bool { + return strings.EqualFold(host, term) || + strings.EqualFold(clientID, term) || + strings.EqualFold(ip, term) || + strings.EqualFold(name, term) +} + +func (c *searchCriteria) ctDomainOrClientCase(e *logEntry) bool { + clientID := e.ClientID + host := e.QHost + + var name string + if e.client != nil { + name = e.client.Name } - if !c.strict && (strings.Contains(qhost, searchVal) || strings.Contains(clientID, searchVal)) { - return true + ip := e.IP.String() + term := strings.ToLower(c.value) + if c.strict { + return c.ctDomainOrClientCaseStrict(term, clientID, name, host, ip) } - ipStr := entry.IP.String() - if c.strict && ipStr == c.value { - return true - } + // TODO(a.garipov): Write a case-insensitive version of strings.Contains + // instead of generating garbage. Or, perhaps in the future, use + // a locale-appropriate matcher from golang.org/x/text. + clientID = strings.ToLower(clientID) + host = strings.ToLower(host) + ip = strings.ToLower(ip) + name = strings.ToLower(name) + term = strings.ToLower(term) - if !c.strict && strings.Contains(ipStr, c.value) { - return true - } - - return false + return strings.Contains(clientID, term) || + strings.Contains(host, term) || + strings.Contains(ip, term) || + strings.Contains(name, term) } func (c *searchCriteria) ctFilteringStatusCase(res dnsfilter.Result) bool { diff --git a/internal/querylog/searchparams.go b/internal/querylog/searchparams.go index da083e45..b4ec6f06 100644 --- a/internal/querylog/searchparams.go +++ b/internal/querylog/searchparams.go @@ -27,19 +27,6 @@ func newSearchParams() *searchParams { } } -// quickMatchesGetDataParams - quickly checks if the line matches the searchParams -// this method does not guarantee anything and the reason is to do a quick check -// without deserializing anything -func (s *searchParams) quickMatch(line string) bool { - for _, c := range s.searchCriteria { - if !c.quickMatch(line) { - return false - } - } - - return true -} - // match - checks if the logEntry matches the searchParams func (s *searchParams) match(entry *logEntry) bool { if !s.olderThan.IsZero() && entry.Time.UnixNano() >= s.olderThan.UnixNano() { diff --git a/internal/tools/go.mod b/internal/tools/go.mod index 45d6e0fa..7bc6d9f9 100644 --- a/internal/tools/go.mod +++ b/internal/tools/go.mod @@ -15,6 +15,6 @@ require ( golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 // indirect golang.org/x/tools v0.1.0 honnef.co/go/tools v0.1.3 - mvdan.cc/gofumpt v0.1.0 + mvdan.cc/gofumpt v0.1.1 mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7 ) diff --git a/internal/tools/go.sum b/internal/tools/go.sum index 266f16e0..1037d8ba 100644 --- a/internal/tools/go.sum +++ b/internal/tools/go.sum @@ -150,7 +150,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -265,7 +264,6 @@ github.com/mwitkow/go-proto-validators v0.2.0/go.mod h1:ZfA1hW+UH/2ZHOWvQ3HnQaU0 github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 h1:4kuARK6Y6FxaNu/BnU2OAaLF86eTVhP2hjTB6iMvItA= github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354/go.mod h1:KSVJerMDfblTH7p5MZaTt+8zaT2iEk3AkVb9PQdZuE8= github.com/nishanths/predeclared v0.0.0-20190419143655-18a43bb90ffc/go.mod h1:62PewwiQTlm/7Rj+cxVYqZvDIUc+JjZq6GHAC1fsObQ= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -335,7 +333,6 @@ github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRci github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -455,7 +452,6 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -529,7 +525,6 @@ golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fq golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -671,7 +666,6 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= @@ -714,8 +708,8 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.3 h1:qTakTkI6ni6LFD5sBwwsdSO+AQqbSIxOauHTTQKZ/7o= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= -mvdan.cc/gofumpt v0.1.0 h1:hsVv+Y9UsZ/mFZTxJZuHVI6shSQCtzZ11h1JEFPAZLw= -mvdan.cc/gofumpt v0.1.0/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= +mvdan.cc/gofumpt v0.1.1 h1:bi/1aS/5W00E2ny5q65w9SnKpWEF/UIOqDYBILpo9rA= +mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7 h1:HT3e4Krq+IE44tiN36RvVEb6tvqeIdtsVSsxmNPqlFU= mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7/go.mod h1:hBpJkZE8H/sb+VRFvw2+rBpHNsTBcvSpk61hr8mzXZE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/internal/tools/tools.go b/internal/tools/tools.go index 9842a5d8..20d2f54f 100644 --- a/internal/tools/tools.go +++ b/internal/tools/tools.go @@ -13,6 +13,6 @@ import ( _ "golang.org/x/tools/go/analysis/passes/nilness/cmd/nilness" _ "golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow" _ "honnef.co/go/tools/cmd/staticcheck" - _ "mvdan.cc/gofumpt/gofumports" + _ "mvdan.cc/gofumpt" _ "mvdan.cc/unparam" ) diff --git a/openapi/CHANGELOG.md b/openapi/CHANGELOG.md index 7cbc626b..cd42f521 100644 --- a/openapi/CHANGELOG.md +++ b/openapi/CHANGELOG.md @@ -2,6 +2,13 @@ +## v0.106: API changes + +### New `"client_info"` field in `GET /querylog` response + +* The new optional field `"client_info"` of `QueryLogItem` objects contains + a more full information about the client. + ## v0.105: API changes ### New `"client_id"` field in `GET /querylog` response diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 5892d704..c0eb9096 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -1815,6 +1815,8 @@ The client ID, if provided in DOH, DOQ, or DOT. 'example': 'cli123' 'type': 'string' + 'client_info': + '$ref': '#/components/schemas/QueryLogItemClient' 'client_proto': 'enum': - 'dot' @@ -1876,6 +1878,58 @@ 'type': 'string' 'description': 'DNS request processing start time' 'example': '2018-11-26T00:02:41+03:00' + 'QueryLogItemClient': + 'description': > + Client information for a query log item. + 'properties': + 'disallowed': + 'type': 'boolean' + 'description': > + Whether the client's IP is blocked or not. + 'disallowed_rule': + 'type': 'string' + 'description': > + The rule due to which the client is disallowed. If disallowed is + set to true, and this string is empty, then the client IP is + disallowed by the "allowed IP list", that is it is not included in + the allowed list. + 'ids': + 'description': > + IP, CIDR, MAC, or client ID. + 'items': + 'type': 'string' + 'type': 'array' + 'name': + 'description': > + Persistent client's name or an empty string if this is a runtime + client. + 'type': 'string' + 'whois': + '$ref': '#/components/schemas/QueryLogItemClientWhois' + 'required': + - 'disallowed' + - 'disallowed_rule' + - 'ids' + - 'name' + - 'whois' + 'type': 'object' + 'QueryLogItemClientWhois': + 'description': > + Client WHOIS information, if any. + 'properties': + 'city': + 'description': > + City, if any. + 'type': 'string' + 'country': + 'description': > + Country, if any. + 'type': 'string' + 'orgname': + 'description': > + Organization name, if any. + 'type': 'string' + 'type': 'object' 'QueryLog': 'type': 'object' 'description': 'Query log' @@ -2205,7 +2259,7 @@ 'use_global_blocked_services': true 'blocked_services': null 'upstreams': null - 'whois_info': null + 'whois_info': {} 'disallowed': false 'disallowed_rule': '' - '1.2.3.4': @@ -2219,7 +2273,7 @@ 'use_global_blocked_services': true 'blocked_services': null 'upstreams': null - 'whois_info': null + 'whois_info': {} 'disallowed': false 'disallowed_rule': '' 'AccessListResponse':