Pull request: querylog: search clients by name, enrich http resp

Updates #1273.

Squashed commit of the following:

commit 55b78153b1b775c855e759011141bbbe6d4b962c
Author: Artem Baskal <a.baskal@adguard.com>
Date:   Fri Apr 2 16:55:39 2021 +0300

    Update client_info in case of null

commit 5c80c1438ed9d961af11617831b704d6ae15cc34
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Apr 2 16:24:14 2021 +0300

    querylog: always set client_info

commit b48efd64d757cc0bcf5b34de22fdd0b0464d98a6
Merge: 4ed7eab5 23c9f528
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Apr 2 16:22:08 2021 +0300

    Merge branch 'master' into 1273-querylog-client-name

commit 4ed7eab52b6b5b0c0ddb5aa5a3225a62d1f9265b
Merge: dbf990eb 70d4c70e
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Apr 2 12:57:17 2021 +0300

    Merge branch 'master' into 1273-querylog-client-name

commit dbf990eb881116754554270e7b691b5db8e9ee34
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Apr 2 12:56:13 2021 +0300

    home: imp names

commit c2cfdef494ca26fff62b9fa008f1b389d9d4d46b
Author: Artem Baskal <a.baskal@adguard.com>
Date:   Thu Apr 1 19:26:04 2021 +0300

    Rename to whois

commit e3cc4a68ee576770b1922680155308e33bed31e8
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Apr 1 19:03:42 2021 +0300

    home: imp whois more

commit 3b8ef8691c298aff35946b35923ef2e5b1f9bbbe
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Apr 1 18:51:14 2021 +0300

    home: imp whois resp

commit fb97e0d74976723a512d6ff4c69e830fe59c8df8
Author: Artem Baskal <a.baskal@adguard.com>
Date:   Thu Apr 1 18:00:03 2021 +0300

    Fix client_info ids prop types

commit 298005189e372651ceff453e88aca19ee925a138
Author: Artem Baskal <a.baskal@adguard.com>
Date:   Thu Apr 1 17:58:14 2021 +0300

    Adapt changes on client

commit aa1769f64197d865478a66271da483babfc5dfd0
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Apr 1 17:18:36 2021 +0300

    all: add more fields to querylog client

commit 4b2a2dbd380ec410f3068d15ea16430912e03e33
Merge: cda92c3f 2e4e2f62
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Apr 1 16:57:26 2021 +0300

    Merge branch 'master' into 1273-querylog-client-name

commit cda92c3f0331cbac252f3163d31457f716bc7f2c
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Mar 29 18:03:51 2021 +0300

    querylog: fix windows tests

commit 5a56f0a32608869ed93a38f18f63ea3a20f7bde2
Merge: 627e4958 e710ce11
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Mar 29 17:45:53 2021 +0300

    Merge branch 'master' into 1273-querylog-client-name

commit 627e495828e82d44cc77aa393536479f23cc68b7
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Mar 29 17:44:49 2021 +0300

    querylog: add tests, imp code, docs

commit 6dec468a2f0c29357875ff99458e0e8f8e580e6d
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Mar 26 16:10:47 2021 +0300

    querylog: search clients by name, enrich http resp
This commit is contained in:
Ainar Garipov 2021-04-02 17:30:39 +03:00
parent 23c9f528db
commit 7c35d208b1
30 changed files with 713 additions and 367 deletions

View File

@ -15,6 +15,7 @@ and this project adheres to
### Added
- Search by clients' names in the query log ([#1273]).
- Verbose version output with `-v --version` ([#2416]).
- The ability to set a custom TLD for known local-network hosts ([#2393]).
- The ability to serve DNS queries on multiple hosts and interfaces ([#1401]).
@ -44,6 +45,7 @@ and this project adheres to
- Go 1.14 support.
[#1273]: https://github.com/AdguardTeam/AdGuardHome/issues/1273
[#1401]: https://github.com/AdguardTeam/AdGuardHome/issues/1401
[#2385]: https://github.com/AdguardTeam/AdGuardHome/issues/2385
[#2393]: https://github.com/AdguardTeam/AdGuardHome/issues/2393

View File

@ -1,23 +1,12 @@
import { createAction } from 'redux-actions';
import apiClient from '../api/Api';
import { normalizeLogs, getParamsForClientsSearch, addClientInfo } from '../helpers/helpers';
import { normalizeLogs } from '../helpers/helpers';
import {
DEFAULT_LOGS_FILTER, FORM_NAME, QUERY_LOGS_PAGE_LIMIT,
} from '../helpers/constants';
import { addErrorToast, addSuccessToast } from './toasts';
const enrichWithClientInfo = async (logs) => {
const clientsParams = getParamsForClientsSearch(logs, 'client', 'client_id');
if (Object.keys(clientsParams).length > 0) {
const clients = await apiClient.findClients(clientsParams);
return addClientInfo(logs, clients, 'client_id', 'client');
}
return logs;
};
const getLogsWithParams = async (config) => {
const { older_than, filter, ...values } = config;
const rawLogs = await apiClient.getQueryLog({
@ -25,11 +14,9 @@ const getLogsWithParams = async (config) => {
older_than,
});
const { data, oldest } = rawLogs;
const normalizedLogs = normalizeLogs(data);
const logs = await enrichWithClientInfo(normalizedLogs);
return {
logs,
logs: normalizeLogs(data),
oldest,
older_than,
filter,
@ -92,10 +79,8 @@ export const updateLogs = () => async (dispatch, getState) => {
try {
const { logs, oldest, older_than } = getState().queryLogs;
const enrichedLogs = await enrichWithClientInfo(logs);
dispatch(getLogsSuccess({
logs: enrichedLogs,
logs,
oldest,
older_than,
}));

View File

@ -17,11 +17,8 @@ import { updateLogs } from '../../../actions/queryLogs';
const ClientCell = ({
client,
client_id,
client_info,
domain,
info,
info: {
name, whois_info, disallowed, disallowed_rule,
},
reason,
}) => {
const { t } = useTranslation();
@ -33,18 +30,22 @@ const ClientCell = ({
const autoClient = autoClients.find((autoClient) => autoClient.name === client);
const source = autoClient?.source;
const whoisAvailable = whois_info && Object.keys(whois_info).length > 0;
const clientName = name || client_id;
const clientInfo = { ...info, name: clientName };
const whoisAvailable = client_info && Object.keys(client_info.whois).length > 0;
const clientName = client_info?.name || client_id;
const clientInfo = client_info && {
...client_info,
whois_info: client_info?.whois,
name: clientName,
};
const id = nanoid();
const data = {
address: client,
name: clientName,
country: whois_info?.country,
city: whois_info?.city,
network: whois_info?.orgname,
country: client_info?.whois?.country,
city: client_info?.whois?.city,
network: client_info?.whois?.orgname,
source_label: source,
};
@ -53,7 +54,7 @@ const ClientCell = ({
const isFiltered = checkFiltered(reason);
const nameClass = classNames('w-90 o-hidden d-flex flex-column', {
'mt-2': isDetailed && !name && !whoisAvailable,
'mt-2': isDetailed && !client_info?.name && !whoisAvailable,
'white-space--nowrap': isDetailed,
});
@ -69,7 +70,11 @@ const ClientCell = ({
confirmMessage,
buttonKey: blockingClientKey,
isNotInAllowedList,
} = getBlockClientInfo(client, disallowed, disallowed_rule);
} = getBlockClientInfo(
client,
client_info?.disallowed || false,
client_info?.disallowed_rule || '',
);
const blockingForClientKey = isFiltered ? 'unblock_for_this_client_only' : 'block_for_this_client_only';
const clientNameBlockingFor = getBlockingClientName(clients, client);
@ -85,7 +90,11 @@ const ClientCell = ({
name: blockingClientKey,
onClick: async () => {
if (window.confirm(confirmMessage)) {
await dispatch(toggleClientBlock(client, disallowed, disallowed_rule));
await dispatch(toggleClientBlock(
client,
client_info?.disallowed || false,
client_info?.disallowed_rule || '',
));
await dispatch(updateLogs());
}
},
@ -199,20 +208,18 @@ const ClientCell = ({
ClientCell.propTypes = {
client: propTypes.string.isRequired,
client_id: propTypes.string,
client_info: propTypes.shape({
ids: propTypes.arrayOf(propTypes.string).isRequired,
name: propTypes.string.isRequired,
whois: propTypes.shape({
country: propTypes.string,
city: propTypes.string,
orgname: propTypes.string,
}).isRequired,
disallowed: propTypes.bool.isRequired,
disallowed_rule: propTypes.string.isRequired,
}),
domain: propTypes.string.isRequired,
info: propTypes.oneOfType([
propTypes.string,
propTypes.shape({
name: propTypes.string.isRequired,
whois_info: propTypes.shape({
country: propTypes.string,
city: propTypes.string,
orgname: propTypes.string,
}),
disallowed: propTypes.bool.isRequired,
disallowed_rule: propTypes.string.isRequired,
}),
]),
reason: propTypes.string.isRequired,
};

View File

@ -29,11 +29,12 @@ import DateCell from './DateCell';
import DomainCell from './DomainCell';
import ResponseCell from './ResponseCell';
import ClientCell from './ClientCell';
import '../Logs.css';
import { toggleClientBlock } from '../../../actions/access';
import { getBlockClientInfo, BUTTON_PREFIX } from './helpers';
import { updateLogs } from '../../../actions/queryLogs';
import '../Logs.css';
const Row = memo(({
style,
rowProps,
@ -61,9 +62,7 @@ const Row = memo(({
client,
domain,
elapsedMs,
info,
info: { disallowed, disallowed_rule },
reason,
client_info,
response,
time,
tracker,
@ -82,11 +81,6 @@ const Row = memo(({
const autoClient = autoClients
.find((autoClient) => autoClient.name === client);
const { whois_info } = info;
const country = whois_info?.country;
const city = whois_info?.city;
const network = whois_info?.orgname;
const source = autoClient?.source;
const formattedElapsedMs = formatElapsedMs(elapsedMs, t);
@ -111,7 +105,11 @@ const Row = memo(({
confirmMessage,
buttonKey: blockingClientKey,
isNotInAllowedList,
} = getBlockClientInfo(client, disallowed, disallowed_rule);
} = getBlockClientInfo(
client,
client_info?.disallowed || false,
client_info?.disallowed_rule || '',
);
const blockingForClientKey = isFiltered ? 'unblock_for_this_client_only' : 'block_for_this_client_only';
const clientNameBlockingFor = getBlockingClientName(clients, client);
@ -122,7 +120,13 @@ const Row = memo(({
const onBlockingClientClick = async () => {
if (window.confirm(confirmMessage)) {
await dispatch(toggleClientBlock(client, disallowed, disallowed_rule));
await dispatch(
toggleClientBlock(
client,
client_info?.disallowed || false,
client_info?.disallowed_rule || '',
),
);
await dispatch(updateLogs());
setModalOpened(false);
}
@ -177,10 +181,10 @@ const Row = memo(({
response_code: status,
client_details: 'title',
ip_address: client,
name: info?.name || client_id,
country,
city,
network,
name: client_info?.name || client_id,
country: client_info?.whois?.country,
city: client_info?.whois?.city,
network: client_info?.whois?.orgname,
source_label: source,
validated_with_dnssec: dnssec_enabled ? Boolean(answer_dnssec) : false,
original_response: originalResponse?.join('\n'),
@ -219,15 +223,6 @@ Row.propTypes = {
client: propTypes.string.isRequired,
domain: propTypes.string.isRequired,
elapsedMs: propTypes.string.isRequired,
info: propTypes.oneOfType([
propTypes.string,
propTypes.shape({
whois_info: propTypes.shape({
country: propTypes.string,
city: propTypes.string,
orgname: propTypes.string,
}),
})]),
response: propTypes.array.isRequired,
time: propTypes.string.isRequired,
tracker: propTypes.object,
@ -235,6 +230,17 @@ Row.propTypes = {
type: propTypes.string.isRequired,
client_proto: propTypes.string.isRequired,
client_id: propTypes.string,
client_info: propTypes.shape({
ids: propTypes.arrayOf(propTypes.string).isRequired,
name: propTypes.string.isRequired,
whois: propTypes.shape({
country: propTypes.string,
city: propTypes.string,
orgname: propTypes.string,
}).isRequired,
disallowed: propTypes.bool.isRequired,
disallowed_rule: propTypes.string.isRequired,
}),
rules: propTypes.arrayOf(propTypes.shape({
text: propTypes.string.isRequired,
filter_list_id: propTypes.number.isRequired,

View File

@ -56,13 +56,13 @@ const InfiniteTable = ({
}, []);
const renderRow = (row, idx) => <Row
key={idx}
rowProps={row}
isSmallScreen={isSmallScreen}
setDetailedDataCurrent={setDetailedDataCurrent}
setButtonType={setButtonType}
setModalOpened={setModalOpened}
/>;
key={idx}
rowProps={row}
isSmallScreen={isSmallScreen}
setDetailedDataCurrent={setDetailedDataCurrent}
setButtonType={setButtonType}
setModalOpened={setModalOpened}
/>;
const isNothingFound = items.length === 0 && !processingGetLogs;

View File

@ -63,6 +63,7 @@ export const normalizeLogs = (logs) => logs.map((log) => {
client,
client_proto,
client_id,
client_info,
elapsedMs,
question,
reason,
@ -101,6 +102,7 @@ export const normalizeLogs = (logs) => logs.map((log) => {
client,
client_proto,
client_id,
client_info,
/* TODO 'filterId' and 'rule' are deprecated, will be removed in 0.106 */
filterId,
rule,

View File

@ -375,5 +375,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// IsBlockedIP - return TRUE if this client should be blocked
func (s *Server) IsBlockedIP(ip net.IP) (bool, string) {
if ip == nil {
return false, ""
}
return s.access.IsBlockedIP(ip)
}

View File

@ -15,6 +15,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/dhcpd"
"github.com/AdguardTeam/AdGuardHome/internal/dnsfilter"
"github.com/AdguardTeam/AdGuardHome/internal/dnsforward"
"github.com/AdguardTeam/AdGuardHome/internal/querylog"
"github.com/AdguardTeam/AdGuardHome/internal/util"
"github.com/AdguardTeam/dnsproxy/proxy"
"github.com/AdguardTeam/dnsproxy/upstream"
@ -60,19 +61,26 @@ const (
ClientSourceHostsFile
)
// ClientHost information
type ClientHost struct {
// RuntimeClient information
type RuntimeClient struct {
Host string
Source clientSource
WhoisInfo [][]string // [[key,value], ...]
WhoisInfo *RuntimeClientWhoisInfo
}
// RuntimeClientWhoisInfo is the filtered WHOIS data for a runtime client.
type RuntimeClientWhoisInfo struct {
City string `json:"city,omitempty"`
Country string `json:"country,omitempty"`
Orgname string `json:"orgname,omitempty"`
}
type clientsContainer struct {
// TODO(a.garipov): Perhaps use a number of separate indices for
// different types (string, net.IP, and so on).
list map[string]*Client // name -> client
idIndex map[string]*Client // ID -> client
ipHost map[string]*ClientHost // IP -> Hostname
list map[string]*Client // name -> client
idIndex map[string]*Client // ID -> client
ipToRC map[string]*RuntimeClient // IP -> runtime client
lock sync.Mutex
allTags map[string]bool
@ -97,7 +105,7 @@ func (clients *clientsContainer) Init(objects []clientObject, dhcpServer *dhcpd.
}
clients.list = make(map[string]*Client)
clients.idIndex = make(map[string]*Client)
clients.ipHost = make(map[string]*ClientHost)
clients.ipToRC = make(map[string]*RuntimeClient)
clients.allTags = make(map[string]bool)
for _, t := range clientTags {
@ -128,7 +136,7 @@ func (clients *clientsContainer) Start() {
}
}
// Reload - reload auto-clients
// Reload reloads runtime clients.
func (clients *clientsContainer) Reload() {
clients.addFromSystemARP()
}
@ -248,21 +256,70 @@ func (clients *clientsContainer) Exists(id string, source clientSource) (ok bool
return true
}
var ch *ClientHost
ch, ok = clients.ipHost[id]
var rc *RuntimeClient
rc, ok = clients.ipToRC[id]
if !ok {
return false
}
// Return false if the new source has higher priority.
return source <= ch.Source
return source <= rc.Source
}
func copyStrings(a []string) (b []string) {
return append(b, a...)
}
// Find searches for a client by its ID.
func toQueryLogWhois(wi *RuntimeClientWhoisInfo) (cw *querylog.ClientWhois) {
if wi == nil {
return &querylog.ClientWhois{}
}
return &querylog.ClientWhois{
City: wi.City,
Country: wi.Country,
Orgname: wi.Orgname,
}
}
// findMultiple is a wrapper around Find to make it a valid client finder for
// the query log. err is always nil.
func (clients *clientsContainer) findMultiple(ids []string) (c *querylog.Client, err error) {
for _, id := range ids {
var name string
var foundIDs []string
whois := &querylog.ClientWhois{}
c, ok := clients.Find(id)
if ok {
name = c.Name
foundIDs = c.IDs
} else {
var rc RuntimeClient
rc, ok = clients.FindRuntimeClient(id)
if !ok {
continue
}
foundIDs = []string{rc.Host}
whois = toQueryLogWhois(rc.WhoisInfo)
}
ip := net.ParseIP(id)
disallowed, disallowedRule := clients.dnsServer.IsBlockedIP(ip)
return &querylog.Client{
Name: name,
DisallowedRule: disallowedRule,
Whois: whois,
IDs: foundIDs,
Disallowed: disallowed,
}, nil
}
return nil, nil
}
func (clients *clientsContainer) Find(id string) (c *Client, ok bool) {
clients.lock.Lock()
defer clients.lock.Unlock()
@ -361,21 +418,22 @@ func (clients *clientsContainer) findLocked(id string) (c *Client, ok bool) {
return nil, false
}
// FindAutoClient - search for an auto-client by IP
func (clients *clientsContainer) FindAutoClient(ip string) (ClientHost, bool) {
// FindRuntimeClient finds a runtime client by their IP.
func (clients *clientsContainer) FindRuntimeClient(ip string) (RuntimeClient, bool) {
ipAddr := net.ParseIP(ip)
if ipAddr == nil {
return ClientHost{}, false
return RuntimeClient{}, false
}
clients.lock.Lock()
defer clients.lock.Unlock()
ch, ok := clients.ipHost[ip]
rc, ok := clients.ipToRC[ip]
if ok {
return *ch, true
return *rc, true
}
return ClientHost{}, false
return RuntimeClient{}, false
}
// check validates the client.
@ -558,9 +616,7 @@ func (clients *clientsContainer) Update(name string, c *Client) (err error) {
}
// SetWhoisInfo sets the WHOIS information for a client.
//
// TODO(a.garipov): Perhaps replace [][]string with map[string]string.
func (clients *clientsContainer) SetWhoisInfo(ip string, info [][]string) {
func (clients *clientsContainer) SetWhoisInfo(ip string, wi *RuntimeClientWhoisInfo) {
clients.lock.Lock()
defer clients.lock.Unlock()
@ -570,21 +626,24 @@ func (clients *clientsContainer) SetWhoisInfo(ip string, info [][]string) {
return
}
ch, ok := clients.ipHost[ip]
rc, ok := clients.ipToRC[ip]
if ok {
ch.WhoisInfo = info
log.Debug("clients: set whois info for auto-client %s: %q", ch.Host, info)
rc.WhoisInfo = wi
log.Debug("clients: set whois info for runtime client %s: %+v", rc.Host, wi)
return
}
// Create a ClientHost implicitly so that we don't do this check again
ch = &ClientHost{
// Create a RuntimeClient implicitly so that we don't do this check
// again.
rc = &RuntimeClient{
Source: ClientSourceWHOIS,
}
ch.WhoisInfo = info
clients.ipHost[ip] = ch
log.Debug("clients: set whois info for auto-client with IP %s: %q", ip, info)
rc.WhoisInfo = wi
clients.ipToRC[ip] = rc
log.Debug("clients: set whois info for runtime client with ip %s: %+v", ip, wi)
}
// AddHost adds a new IP-hostname pairing. The priorities of the sources is
@ -600,24 +659,25 @@ func (clients *clientsContainer) AddHost(ip, host string, src clientSource) (ok
// addHostLocked adds a new IP-hostname pairing. For internal use only.
func (clients *clientsContainer) addHostLocked(ip, host string, src clientSource) (ok bool) {
var ch *ClientHost
ch, ok = clients.ipHost[ip]
var rc *RuntimeClient
rc, ok = clients.ipToRC[ip]
if ok {
if ch.Source > src {
if rc.Source > src {
return false
}
ch.Source = src
rc.Source = src
} else {
ch = &ClientHost{
Host: host,
Source: src,
rc = &RuntimeClient{
Host: host,
Source: src,
WhoisInfo: &RuntimeClientWhoisInfo{},
}
clients.ipHost[ip] = ch
clients.ipToRC[ip] = rc
}
log.Debug("clients: added %q -> %q [%d]", ip, host, len(clients.ipHost))
log.Debug("clients: added %q -> %q [%d]", ip, host, len(clients.ipToRC))
return true
}
@ -625,9 +685,9 @@ func (clients *clientsContainer) addHostLocked(ip, host string, src clientSource
// rmHostsBySrc removes all entries that match the specified source.
func (clients *clientsContainer) rmHostsBySrc(src clientSource) {
n := 0
for k, v := range clients.ipHost {
for k, v := range clients.ipToRC {
if v.Source == src {
delete(clients.ipHost, k)
delete(clients.ipToRC, k)
n++
}
}

View File

@ -163,17 +163,20 @@ func TestClientsWhois(t *testing.T) {
testing: true,
}
clients.Init(nil, nil, nil)
whois := [][]string{{"orgname", "orgname-val"}, {"country", "country-val"}}
whois := &RuntimeClientWhoisInfo{
Country: "AU",
Orgname: "Example Org",
}
t.Run("new_client", func(t *testing.T) {
clients.SetWhoisInfo("1.1.1.255", whois)
require.NotNil(t, clients.ipHost["1.1.1.255"])
h := clients.ipHost["1.1.1.255"]
require.NotNil(t, clients.ipToRC["1.1.1.255"])
require.Len(t, h.WhoisInfo, 2)
require.Len(t, h.WhoisInfo[0], 2)
assert.Equal(t, "orgname-val", h.WhoisInfo[0][1])
h := clients.ipToRC["1.1.1.255"]
require.NotNil(t, h)
assert.Equal(t, h.WhoisInfo, whois)
})
t.Run("existing_auto-client", func(t *testing.T) {
@ -183,12 +186,11 @@ func TestClientsWhois(t *testing.T) {
clients.SetWhoisInfo("1.1.1.1", whois)
require.NotNil(t, clients.ipHost["1.1.1.1"])
h := clients.ipHost["1.1.1.1"]
require.NotNil(t, clients.ipToRC["1.1.1.1"])
h := clients.ipToRC["1.1.1.1"]
require.NotNil(t, h)
require.Len(t, h.WhoisInfo, 2)
require.Len(t, h.WhoisInfo[0], 2)
assert.Equal(t, "orgname-val", h.WhoisInfo[0][1])
assert.Equal(t, h.WhoisInfo, whois)
})
t.Run("can't_set_manually-added", func(t *testing.T) {
@ -200,7 +202,7 @@ func TestClientsWhois(t *testing.T) {
assert.True(t, ok)
clients.SetWhoisInfo("1.1.1.2", whois)
require.Nil(t, clients.ipHost["1.1.1.2"])
require.Nil(t, clients.ipToRC["1.1.1.2"])
assert.True(t, clients.Del("client1"))
})
}

View File

@ -22,7 +22,7 @@ type clientJSON struct {
Upstreams []string `json:"upstreams"`
WhoisInfo map[string]string `json:"whois_info"`
WhoisInfo *RuntimeClientWhoisInfo `json:"whois_info"`
// Disallowed - if true -- client's IP is not disallowed
// Otherwise, it is blocked.
@ -34,18 +34,18 @@ type clientJSON struct {
DisallowedRule string `json:"disallowed_rule"`
}
type clientHostJSON struct {
type runtimeClientJSON struct {
WhoisInfo *RuntimeClientWhoisInfo `json:"whois_info"`
IP string `json:"ip"`
Name string `json:"name"`
Source string `json:"source"`
WhoisInfo map[string]string `json:"whois_info"`
}
type clientListJSON struct {
Clients []clientJSON `json:"clients"`
AutoClients []clientHostJSON `json:"auto_clients"`
Tags []string `json:"supported_tags"`
Clients []clientJSON `json:"clients"`
RuntimeClients []runtimeClientJSON `json:"auto_clients"`
Tags []string `json:"supported_tags"`
}
// respond with information about configured clients
@ -53,18 +53,21 @@ func (clients *clientsContainer) handleGetClients(w http.ResponseWriter, _ *http
data := clientListJSON{}
clients.lock.Lock()
defer clients.lock.Unlock()
for _, c := range clients.list {
cj := clientToJSON(c)
data.Clients = append(data.Clients, cj)
}
for ip, ch := range clients.ipHost {
cj := clientHostJSON{
IP: ip,
Name: ch.Host,
for ip, rc := range clients.ipToRC {
cj := runtimeClientJSON{
IP: ip,
Name: rc.Host,
WhoisInfo: rc.WhoisInfo,
}
cj.Source = "etc/hosts"
switch ch.Source {
switch rc.Source {
case ClientSourceDHCP:
cj.Source = "DHCP"
case ClientSourceRDNS:
@ -75,14 +78,8 @@ func (clients *clientsContainer) handleGetClients(w http.ResponseWriter, _ *http
cj.Source = "WHOIS"
}
cj.WhoisInfo = map[string]string{}
for _, wi := range ch.WhoisInfo {
cj.WhoisInfo[wi[0]] = wi[1]
}
data.AutoClients = append(data.AutoClients, cj)
data.RuntimeClients = append(data.RuntimeClients, cj)
}
clients.lock.Unlock()
data.Tags = clientTags
@ -129,21 +126,21 @@ func clientToJSON(c *Client) clientJSON {
BlockedServices: c.BlockedServices,
Upstreams: c.Upstreams,
WhoisInfo: &RuntimeClientWhoisInfo{},
}
return cj
}
// Convert ClientHost object to JSON
func clientHostToJSON(ip string, ch ClientHost) clientJSON {
cj := clientJSON{
Name: ch.Host,
IDs: []string{ip},
// runtimeClientToJSON converts a RuntimeClient into a JSON struct.
func runtimeClientToJSON(ip string, rc RuntimeClient) (cj clientJSON) {
cj = clientJSON{
Name: rc.Host,
IDs: []string{ip},
WhoisInfo: rc.WhoisInfo,
}
cj.WhoisInfo = map[string]string{}
for _, wi := range ch.WhoisInfo {
cj.WhoisInfo[wi[0]] = wi[1]
}
return cj
}
@ -268,7 +265,7 @@ func (clients *clientsContainer) findTemporary(ip net.IP, idStr string) (cj clie
return cj, false
}
ch, ok := clients.FindAutoClient(idStr)
rc, ok := clients.FindRuntimeClient(idStr)
if !ok {
// It is still possible that the IP used to be in the runtime
// clients list, but then the server was reloaded. So, check
@ -284,12 +281,13 @@ func (clients *clientsContainer) findTemporary(ip net.IP, idStr string) (cj clie
IDs: []string{idStr},
Disallowed: disallowed,
DisallowedRule: rule,
WhoisInfo: &RuntimeClientWhoisInfo{},
}
return cj, true
}
cj = clientHostToJSON(idStr, ch)
cj = runtimeClientToJSON(idStr, rc)
cj.Disallowed, cj.DisallowedRule = clients.dnsServer.IsBlockedIP(ip)
return cj, true

View File

@ -285,7 +285,7 @@ func (c *configuration) write() error {
Context.queryLog.WriteDiskConfig(&dc)
config.DNS.QueryLogEnabled = dc.Enabled
config.DNS.QueryLogFileEnabled = dc.FileEnabled
config.DNS.QueryLogInterval = dc.Interval
config.DNS.QueryLogInterval = dc.RotationIvl
config.DNS.QueryLogMemSize = dc.MemSize
config.DNS.AnonymizeClientIP = dc.AnonymizeClientIP
}

View File

@ -42,15 +42,17 @@ func initDNSServer() error {
if err != nil {
return fmt.Errorf("couldn't initialize statistics module")
}
conf := querylog.Config{
Enabled: config.DNS.QueryLogEnabled,
FileEnabled: config.DNS.QueryLogFileEnabled,
BaseDir: baseDir,
Interval: config.DNS.QueryLogInterval,
MemSize: config.DNS.QueryLogMemSize,
AnonymizeClientIP: config.DNS.AnonymizeClientIP,
ConfigModified: onConfigModified,
HTTPRegister: httpRegister,
FindClient: Context.clients.findMultiple,
BaseDir: baseDir,
RotationIvl: config.DNS.QueryLogInterval,
MemSize: config.DNS.QueryLogMemSize,
Enabled: config.DNS.QueryLogEnabled,
FileEnabled: config.DNS.QueryLogFileEnabled,
AnonymizeClientIP: config.DNS.AnonymizeClientIP,
}
Context.queryLog = querylog.New(conf)

View File

@ -84,7 +84,7 @@ func TestRDNS_Begin(t *testing.T) {
clients: &clientsContainer{
list: map[string]*Client{},
idIndex: tc.cliIDIndex,
ipHost: map[string]*ClientHost{},
ipToRC: map[string]*RuntimeClient{},
allTags: map[string]bool{},
},
}
@ -229,7 +229,7 @@ func TestRDNS_WorkerLoop(t *testing.T) {
cc := &clientsContainer{
list: map[string]*Client{},
idIndex: map[string]*Client{},
ipHost: map[string]*ClientHost{},
ipToRC: map[string]*RuntimeClient{},
allTags: map[string]bool{},
}
ch := make(chan net.IP)

View File

@ -182,29 +182,31 @@ func (w *Whois) queryAll(ctx context.Context, target string) (string, error) {
}
// Request WHOIS information
func (w *Whois) process(ctx context.Context, ip net.IP) [][]string {
data := [][]string{}
func (w *Whois) process(ctx context.Context, ip net.IP) (wi *RuntimeClientWhoisInfo) {
resp, err := w.queryAll(ctx, ip.String())
if err != nil {
log.Debug("Whois: error: %s IP:%s", err, ip)
return data
return nil
}
log.Debug("Whois: IP:%s response: %d bytes", ip, len(resp))
m := whoisParse(resp)
keys := []string{"orgname", "country", "city"}
for _, k := range keys {
v, found := m[k]
if !found {
continue
}
pair := []string{k, v}
data = append(data, pair)
wi = &RuntimeClientWhoisInfo{
City: m["city"],
Country: m["country"],
Orgname: m["orgname"],
}
return data
// Don't return an empty struct so that the frontend doesn't get
// confused.
if *wi == (RuntimeClientWhoisInfo{}) {
return nil
}
return wi
}
// Begin - begin requesting WHOIS info
@ -234,11 +236,9 @@ func (w *Whois) Begin(ip net.IP) {
// workerLoop processes the IP addresses it got from the channel and associates
// the retrieving WHOIS info with a client.
func (w *Whois) workerLoop() {
for {
ip := <-w.ipChan
for ip := range w.ipChan {
info := w.process(context.Background(), ip)
if len(info) == 0 {
if info == nil {
continue
}

View File

@ -0,0 +1,33 @@
package querylog
// Client is the information required by the query log to match against clients
// during searches.
type Client struct {
Name string `json:"name"`
DisallowedRule string `json:"disallowed_rule"`
Whois *ClientWhois `json:"whois,omitempty"`
IDs []string `json:"ids"`
Disallowed bool `json:"disallowed"`
}
// ClientWhois is the filtered WHOIS data for the client.
//
// TODO(a.garipov): Merge with home.RuntimeClientWhoisInfo after the
// refactoring is done.
type ClientWhois struct {
City string `json:"city,omitempty"`
Country string `json:"country,omitempty"`
Orgname string `json:"orgname,omitempty"`
}
// clientCacheKey is the key by which a cached client information is found.
type clientCacheKey struct {
clientID string
ip string
}
// clientCache is the cache of client information found throughout a request to
// the query log API. It is used both to speed up the lookup, as well as to
// make sure that changes in client data between two lookups don't create
// discrepancies in our response.
type clientCache map[clientCacheKey]*Client

View File

@ -68,7 +68,7 @@ func (l *queryLog) handleQueryLogClear(_ http.ResponseWriter, _ *http.Request) {
func (l *queryLog) handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
resp := qlogConfig{}
resp.Enabled = l.conf.Enabled
resp.Interval = l.conf.Interval
resp.Interval = l.conf.RotationIvl
resp.AnonymizeClientIP = l.conf.AnonymizeClientIP
jsonVal, err := json.Marshal(resp)
@ -104,7 +104,7 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request)
conf.Enabled = d.Enabled
}
if req.Exists("interval") {
conf.Interval = d.Interval
conf.RotationIvl = d.Interval
}
if req.Exists("anonymize_client_ip") {
conf.AnonymizeClientIP = d.AnonymizeClientIP

View File

@ -71,6 +71,7 @@ func (l *queryLog) logEntryToJSONEntry(entry *logEntry) (jsonEntry jobject) {
"elapsedMs": strconv.FormatFloat(entry.Elapsed.Seconds()*1000, 'f', -1, 64),
"time": entry.Time.Format(time.RFC3339Nano),
"client": l.getClientIP(entry.IP),
"client_info": entry.client,
"client_proto": entry.ClientProto,
"upstream": entry.Upstream,
"question": jobject{

View File

@ -6,7 +6,6 @@ import (
"fmt"
"net"
"os"
"path/filepath"
"strings"
"sync"
"time"
@ -22,12 +21,17 @@ const (
// queryLog is a structure that writes and reads the DNS query log
type queryLog struct {
findClient func(ids []string) (c *Client, err error)
conf *Config
lock sync.Mutex
logFile string // path to the log file
bufferLock sync.RWMutex
buffer []*logEntry
// bufferLock protects buffer.
bufferLock sync.RWMutex
// buffer contains recent log entries.
buffer []*logEntry
fileFlushLock sync.Mutex // synchronize a file-flushing goroutine and main thread
flushPending bool // don't start another goroutine while the previous one is still running
fileWriteLock sync.Mutex
@ -64,6 +68,9 @@ func NewClientProto(s string) (cp ClientProto, err error) {
// logEntry - represents a single log entry
type logEntry struct {
// client is the found client information, if any.
client *Client
IP net.IP `json:"IP"` // Client IP
Time time.Time `json:"T"`
@ -82,18 +89,6 @@ type logEntry struct {
Upstream string `json:",omitempty"` // if empty, means it was cached
}
// create a new instance of the query log
func newQueryLog(conf Config) *queryLog {
l := queryLog{}
l.logFile = filepath.Join(conf.BaseDir, queryLogFileName)
l.conf = &Config{}
*l.conf = conf
if !checkInterval(l.conf.Interval) {
l.conf.Interval = 1
}
return &l
}
func (l *queryLog) Start() {
if l.conf.HTTPRegister != nil {
l.initWeb()
@ -138,12 +133,16 @@ func (l *queryLog) clear() {
}
func (l *queryLog) Add(params AddParams) {
var err error
if !l.conf.Enabled {
return
}
if params.Question == nil || len(params.Question.Question) != 1 || len(params.Question.Question[0].Name) == 0 ||
params.ClientIP == nil {
err = params.validate()
if err != nil {
log.Error("querylog: adding record: %s, skipping", err)
return
}
@ -168,20 +167,26 @@ func (l *queryLog) Add(params AddParams) {
entry.QClass = dns.Class(q.Qclass).String()
if params.Answer != nil {
a, err := params.Answer.Pack()
var a []byte
a, err = params.Answer.Pack()
if err != nil {
log.Info("Querylog: Answer.Pack(): %s", err)
log.Error("querylog: Answer.Pack(): %s", err)
return
}
entry.Answer = a
}
if params.OrigAnswer != nil {
a, err := params.OrigAnswer.Pack()
var a []byte
a, err = params.OrigAnswer.Pack()
if err != nil {
log.Info("Querylog: OrigAnswer.Pack(): %s", err)
log.Error("querylog: OrigAnswer.Pack(): %s", err)
return
}
entry.OrigAnswer = a
}

View File

@ -26,7 +26,7 @@ func TestQueryLog(t *testing.T) {
l := newQueryLog(Config{
Enabled: true,
FileEnabled: true,
Interval: 1,
RotationIvl: 1,
MemSize: 100,
BaseDir: t.TempDir(),
})
@ -127,10 +127,10 @@ func TestQueryLog(t *testing.T) {
func TestQueryLogOffsetLimit(t *testing.T) {
l := newQueryLog(Config{
Enabled: true,
Interval: 1,
MemSize: 100,
BaseDir: t.TempDir(),
Enabled: true,
RotationIvl: 1,
MemSize: 100,
BaseDir: t.TempDir(),
})
const (
@ -202,7 +202,7 @@ func TestQueryLogMaxFileScanEntries(t *testing.T) {
l := newQueryLog(Config{
Enabled: true,
FileEnabled: true,
Interval: 1,
RotationIvl: 1,
MemSize: 100,
BaseDir: t.TempDir(),
})
@ -230,7 +230,7 @@ func TestQueryLogFileDisabled(t *testing.T) {
l := newQueryLog(Config{
Enabled: true,
FileEnabled: false,
Interval: 1,
RotationIvl: 1,
MemSize: 2,
BaseDir: t.TempDir(),
})

View File

@ -3,9 +3,12 @@ package querylog
import (
"net"
"net/http"
"path/filepath"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/agherr"
"github.com/AdguardTeam/AdGuardHome/internal/dnsfilter"
"github.com/AdguardTeam/golibs/log"
"github.com/miekg/dns"
)
@ -25,18 +28,37 @@ type QueryLog interface {
// Config - configuration object
type Config struct {
Enabled bool // enable the module
FileEnabled bool // write logs to file
BaseDir string // directory where log file is stored
Interval uint32 // interval to rotate logs (in days)
MemSize uint32 // number of entries kept in memory before they are flushed to disk
AnonymizeClientIP bool // anonymize clients' IP addresses
// Called when the configuration is changed by HTTP request
// ConfigModified is called when the configuration is changed, for
// example by HTTP requests.
ConfigModified func()
// Register an HTTP handler
// HTTPRegister registers an HTTP handler.
HTTPRegister func(string, string, func(http.ResponseWriter, *http.Request))
// FindClient returns client information by their IDs.
FindClient func(ids []string) (c *Client, err error)
// BaseDir is the base directory for log files.
BaseDir string
// RotationIvl is the interval for log rotation, in days. After that
// period, the old log file will be renamed, NOT deleted, so the actual
// log retention time is twice the interval.
RotationIvl uint32
// MemSize is the number of entries kept in a memory buffer before they
// are flushed to disk.
MemSize uint32
// Enabled tells if the query log is enabled.
Enabled bool
// FileEnabled tells if the query log writes logs to files.
FileEnabled bool
// AnonymizeClientIP tells if the query log should anonymize clients' IP
// addresses.
AnonymizeClientIP bool
}
// AddParams - parameters for Add()
@ -52,7 +74,52 @@ type AddParams struct {
ClientProto ClientProto
}
// New - create a new instance of the query log
func New(conf Config) QueryLog {
// validate returns an error if the parameters aren't valid.
func (p *AddParams) validate() (err error) {
switch {
case p.Question == nil:
return agherr.Error("question is nil")
case len(p.Question.Question) != 1:
return agherr.Error("more than one question")
case len(p.Question.Question[0].Name) == 0:
return agherr.Error("no host in question")
case p.ClientIP == nil:
return agherr.Error("no client ip")
default:
return nil
}
}
// New creates a new instance of the query log.
func New(conf Config) (ql QueryLog) {
return newQueryLog(conf)
}
// newQueryLog crates a new queryLog.
func newQueryLog(conf Config) (l *queryLog) {
findClient := conf.FindClient
if findClient == nil {
findClient = func(_ []string) (_ *Client, _ error) {
return nil, nil
}
}
l = &queryLog{
findClient: findClient,
logFile: filepath.Join(conf.BaseDir, queryLogFileName),
}
l.conf = &Config{}
*l.conf = conf
if !checkInterval(conf.RotationIvl) {
log.Info(
"querylog: warning: unsupported rotation interval %d, setting to 1 day",
conf.RotationIvl,
)
l.conf.RotationIvl = 1
}
return l
}

View File

@ -129,7 +129,7 @@ func (l *queryLog) readFileFirstTimeValue() int64 {
}
func (l *queryLog) periodicRotate() {
intervalSeconds := uint64(l.conf.Interval) * 24 * 60 * 60
intervalSeconds := uint64(l.conf.RotationIvl) * 24 * 60 * 60
for {
oldest := l.readFileFirstTimeValue()
if uint64(oldest)+intervalSeconds <= uint64(time.Now().Unix()) {

View File

@ -8,6 +8,67 @@ import (
"github.com/AdguardTeam/golibs/log"
)
// client finds the client info, if any, by its client ID and IP address,
// optionally checking the provided cache. It will use the IP address
// regardless of if the IP anonymization is enabled now, because the
// anonymization could have been disabled in the past, and client will try to
// find those records as well.
func (l *queryLog) client(clientID, ip string, cache clientCache) (c *Client, err error) {
cck := clientCacheKey{clientID: clientID, ip: ip}
if c = cache[cck]; c != nil {
return c, nil
}
var ids []string
if clientID != "" {
ids = append(ids, clientID)
}
if ip != "" {
ids = append(ids, ip)
}
c, err = l.findClient(ids)
if err != nil {
return nil, err
}
if cache != nil {
cache[cck] = c
}
return c, nil
}
// searchMemory looks up log records which are currently in the in-memory
// buffer. It optionally uses the client cache, if provided. It also returns
// the total amount of records in the buffer at the moment of searching.
func (l *queryLog) searchMemory(params *searchParams, cache clientCache) (entries []*logEntry, total int) {
l.bufferLock.Lock()
defer l.bufferLock.Unlock()
// Go through the buffer in the reverse order, from newer to older.
var err error
for i := len(l.buffer) - 1; i >= 0; i-- {
e := l.buffer[i]
e.client, err = l.client(e.ClientID, e.IP.String(), cache)
if err != nil {
msg := "querylog: enriching memory record at time %s" +
" for client %q (client id %q): %s"
log.Error(msg, e.Time, e.IP, e.ClientID, err)
// Go on and try to match anyway.
}
if params.match(e) {
entries = append(entries, e)
}
}
return entries, len(l.buffer)
}
// search - searches log entries in the query log using specified parameters
// returns the list of entries found + time of the oldest entry
func (l *queryLog) search(params *searchParams) ([]*logEntry, time.Time) {
@ -17,26 +78,11 @@ func (l *queryLog) search(params *searchParams) ([]*logEntry, time.Time) {
return []*logEntry{}, time.Time{}
}
// add from file
fileEntries, oldest, total := l.searchFiles(params)
cache := clientCache{}
fileEntries, oldest, total := l.searchFiles(params, cache)
memoryEntries, bufLen := l.searchMemory(params, cache)
total += bufLen
// add from memory buffer
l.bufferLock.Lock()
total += len(l.buffer)
memoryEntries := make([]*logEntry, 0)
// go through the buffer in the reverse order
// from NEWER to OLDER
for i := len(l.buffer) - 1; i >= 0; i-- {
entry := l.buffer[i]
if !params.match(entry) {
continue
}
memoryEntries = append(memoryEntries, entry)
}
l.bufferLock.Unlock()
// limits
totalLimit := params.offset + params.limit
// now let's get a unified collection
@ -74,18 +120,15 @@ func (l *queryLog) search(params *searchParams) ([]*logEntry, time.Time) {
return entries, oldest
}
// searchFiles reads log entries from all log files and applies the specified search criteria.
// IMPORTANT: this method does not scan more than "maxSearchEntries" so you
// may need to call it many times.
//
// it returns:
// * an array of log entries that we have read
// * time of the oldest processed entry (even if it was discarded)
// * total number of processed entries (including discarded).
func (l *queryLog) searchFiles(params *searchParams) ([]*logEntry, time.Time, int) {
entries := make([]*logEntry, 0)
oldest := time.Time{}
// searchFiles looks up log records from all log files. It optionally uses the
// client cache, if provided. searchFiles does not scan more than
// maxFileScanEntries so callers may need to call it several times to get all
// results. oldset and total are the time of the oldest processed entry and the
// total number of processed entries, including discarded ones, correspondingly.
func (l *queryLog) searchFiles(
params *searchParams,
cache clientCache,
) (entries []*logEntry, oldest time.Time, total int) {
files := []string{
l.logFile + ".1",
l.logFile,
@ -104,40 +147,43 @@ func (l *queryLog) searchFiles(params *searchParams) ([]*logEntry, time.Time, in
} else {
err = r.SeekTS(params.olderThan.UnixNano())
if err == nil {
// Read to the next record right away
// The one that was specified in the "oldest" param is not needed,
// we need only the one next to it
// Read to the next record, because we only need the one
// that goes after it.
_, err = r.ReadNext()
}
}
if err != nil {
log.Debug("Cannot SeekTS() to %v: %v", params.olderThan, err)
log.Debug("querylog: cannot seek to %s: %s", params.olderThan, err)
return entries, oldest, 0
}
totalLimit := params.offset + params.limit
total := 0
oldestNano := int64(0)
// By default, we do not scan more than "maxFileScanEntries" at once
// The idea is to make search calls faster so that the UI could handle it and show something
// This behavior can be overridden if "maxFileScanEntries" is set to 0
// By default, we do not scan more than maxFileScanEntries at once.
// The idea is to make search calls faster so that the UI could handle
// it and show something quicker. This behavior can be overridden if
// maxFileScanEntries is set to 0.
for total < params.maxFileScanEntries || params.maxFileScanEntries <= 0 {
var entry *logEntry
var e *logEntry
var ts int64
entry, ts, err = l.readNextEntry(r, params)
if err == io.EOF {
// there's nothing to read anymore
break
e, ts, err = l.readNextEntry(r, params, cache)
if err != nil {
if err == io.EOF {
break
}
log.Error("querylog: reading next entry: %s", err)
}
oldestNano = ts
total++
if entry != nil {
entries = append(entries, entry)
if e != nil {
entries = append(entries, e)
if len(entries) == totalLimit {
// Do not read more than "totalLimit" records at once
break
}
}
@ -146,36 +192,46 @@ func (l *queryLog) searchFiles(params *searchParams) ([]*logEntry, time.Time, in
if oldestNano != 0 {
oldest = time.Unix(0, oldestNano)
}
return entries, oldest, total
}
// readNextEntry - reads the next log entry and checks if it matches the search criteria (getDataParams)
//
// returns:
// * log entry that matches search criteria or null if it was discarded (or if there's nothing to read)
// * timestamp of the processed log entry
// * error if we can't read anymore
func (l *queryLog) readNextEntry(r *QLogReader, params *searchParams) (*logEntry, int64, error) {
line, err := r.ReadNext()
// readNextEntry reads the next log entry and checks if it matches the search
// criteria. It optionally uses the client cache, if provided. e is nil if the
// entry doesn't match the search criteria. ts is the timestamp of the
// processed entry.
func (l *queryLog) readNextEntry(
r *QLogReader,
params *searchParams,
cache clientCache,
) (e *logEntry, ts int64, err error) {
var line string
line, err = r.ReadNext()
if err != nil {
return nil, 0, err
}
// Read the log record timestamp right away
timestamp := readQLogTimestamp(line)
e = &logEntry{}
decodeLogEntry(e, line)
// Quick check without deserializing log entry
if !params.quickMatch(line) {
return nil, timestamp, nil
e.client, err = l.client(e.ClientID, e.IP.String(), cache)
if err != nil {
log.Error(
"querylog: enriching file record at time %s"+
" for client %q (client id %q): %s",
e.Time,
e.IP,
e.ClientID,
err,
)
// Go on and try to match anyway.
}
entry := logEntry{}
decodeLogEntry(&entry, line)
// Full check of the deserialized log entry
if !params.match(&entry) {
return nil, timestamp, nil
ts = e.Time.UnixNano()
if !params.match(e) {
return nil, ts, nil
}
return &entry, timestamp, nil
return e, ts, nil
}

View File

@ -0,0 +1,95 @@
package querylog
import (
"net"
"testing"
"time"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestQueryLog_Search_findClient(t *testing.T) {
const knownClientID = "client-1"
const knownClientName = "Known Client 1"
const unknownClientID = "client-2"
knownClient := &Client{
IDs: []string{knownClientID},
Name: knownClientName,
}
findClientCalls := 0
findClient := func(ids []string) (c *Client, _ error) {
defer func() { findClientCalls++ }()
if len(ids) == 0 {
return nil, nil
}
if ids[0] == knownClientID {
return knownClient, nil
}
return nil, nil
}
l := newQueryLog(Config{
FindClient: findClient,
BaseDir: t.TempDir(),
RotationIvl: 1,
MemSize: 100,
Enabled: true,
FileEnabled: true,
AnonymizeClientIP: false,
})
t.Cleanup(l.Close)
q := &dns.Msg{
Question: []dns.Question{{
Name: "example.com",
}},
}
l.Add(AddParams{
Question: q,
ClientID: knownClientID,
ClientIP: net.IP{1, 2, 3, 4},
})
// Add the same thing again to test the cache.
l.Add(AddParams{
Question: q,
ClientID: knownClientID,
ClientIP: net.IP{1, 2, 3, 4},
})
l.Add(AddParams{
Question: q,
ClientID: unknownClientID,
ClientIP: net.IP{1, 2, 3, 5},
})
sp := &searchParams{
// Add some time to the "current" one to protect against
// low-resolution timers on some Windows machines.
//
// TODO(a.garipov): Use some kind of timeSource interface
// instead of relying on time.Now() in tests.
olderThan: time.Now().Add(10 * time.Second),
limit: 3,
}
entries, _ := l.search(sp)
assert.Equal(t, 2, findClientCalls)
require.Len(t, entries, 3)
assert.Nil(t, entries[0].client)
gotClient := entries[2].client
require.NotNil(t, gotClient)
assert.Equal(t, knownClientName, gotClient.Name)
assert.Equal(t, []string{knownClientID}, gotClient.IDs)
}

View File

@ -48,40 +48,6 @@ type searchCriteria struct {
strict bool // should we strictly match (equality) or not (indexOf)
}
// quickMatch - quickly checks if the log entry matches this search criteria
// the reason is to do it as quickly as possible without de-serializing the entry
func (c *searchCriteria) quickMatch(line string) bool {
// note that we do this only for a limited set of criteria
switch c.criteriaType {
case ctDomainOrClient:
return c.quickMatchJSONValue(line, "QH") ||
c.quickMatchJSONValue(line, "IP") ||
c.quickMatchJSONValue(line, "CID")
default:
return true
}
}
// quickMatchJSONValue - helper used by quickMatch
func (c *searchCriteria) quickMatchJSONValue(line, propertyName string) bool {
val := readJSONValue(line, propertyName)
if len(val) == 0 {
return false
}
val = strings.ToLower(val)
searchVal := strings.ToLower(c.value)
if c.strict && searchVal == val {
return true
}
if !c.strict && strings.Contains(val, searchVal) {
return true
}
return false
}
// match - checks if the log entry matches this search criteria
func (c *searchCriteria) match(entry *logEntry) bool {
switch c.criteriaType {
@ -94,28 +60,41 @@ func (c *searchCriteria) match(entry *logEntry) bool {
return false
}
func (c *searchCriteria) ctDomainOrClientCase(entry *logEntry) bool {
clientID := strings.ToLower(entry.ClientID)
qhost := strings.ToLower(entry.QHost)
searchVal := strings.ToLower(c.value)
if c.strict && (qhost == searchVal || clientID == searchVal) {
return true
func (c *searchCriteria) ctDomainOrClientCaseStrict(term, clientID, name, host, ip string) bool {
return strings.EqualFold(host, term) ||
strings.EqualFold(clientID, term) ||
strings.EqualFold(ip, term) ||
strings.EqualFold(name, term)
}
func (c *searchCriteria) ctDomainOrClientCase(e *logEntry) bool {
clientID := e.ClientID
host := e.QHost
var name string
if e.client != nil {
name = e.client.Name
}
if !c.strict && (strings.Contains(qhost, searchVal) || strings.Contains(clientID, searchVal)) {
return true
ip := e.IP.String()
term := strings.ToLower(c.value)
if c.strict {
return c.ctDomainOrClientCaseStrict(term, clientID, name, host, ip)
}
ipStr := entry.IP.String()
if c.strict && ipStr == c.value {
return true
}
// TODO(a.garipov): Write a case-insensitive version of strings.Contains
// instead of generating garbage. Or, perhaps in the future, use
// a locale-appropriate matcher from golang.org/x/text.
clientID = strings.ToLower(clientID)
host = strings.ToLower(host)
ip = strings.ToLower(ip)
name = strings.ToLower(name)
term = strings.ToLower(term)
if !c.strict && strings.Contains(ipStr, c.value) {
return true
}
return false
return strings.Contains(clientID, term) ||
strings.Contains(host, term) ||
strings.Contains(ip, term) ||
strings.Contains(name, term)
}
func (c *searchCriteria) ctFilteringStatusCase(res dnsfilter.Result) bool {

View File

@ -27,19 +27,6 @@ func newSearchParams() *searchParams {
}
}
// quickMatchesGetDataParams - quickly checks if the line matches the searchParams
// this method does not guarantee anything and the reason is to do a quick check
// without deserializing anything
func (s *searchParams) quickMatch(line string) bool {
for _, c := range s.searchCriteria {
if !c.quickMatch(line) {
return false
}
}
return true
}
// match - checks if the logEntry matches the searchParams
func (s *searchParams) match(entry *logEntry) bool {
if !s.olderThan.IsZero() && entry.Time.UnixNano() >= s.olderThan.UnixNano() {

View File

@ -15,6 +15,6 @@ require (
golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 // indirect
golang.org/x/tools v0.1.0
honnef.co/go/tools v0.1.3
mvdan.cc/gofumpt v0.1.0
mvdan.cc/gofumpt v0.1.1
mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7
)

View File

@ -150,7 +150,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@ -265,7 +264,6 @@ github.com/mwitkow/go-proto-validators v0.2.0/go.mod h1:ZfA1hW+UH/2ZHOWvQ3HnQaU0
github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 h1:4kuARK6Y6FxaNu/BnU2OAaLF86eTVhP2hjTB6iMvItA=
github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354/go.mod h1:KSVJerMDfblTH7p5MZaTt+8zaT2iEk3AkVb9PQdZuE8=
github.com/nishanths/predeclared v0.0.0-20190419143655-18a43bb90ffc/go.mod h1:62PewwiQTlm/7Rj+cxVYqZvDIUc+JjZq6GHAC1fsObQ=
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
@ -335,7 +333,6 @@ github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRci
github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@ -455,7 +452,6 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U=
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
@ -529,7 +525,6 @@ golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fq
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@ -671,7 +666,6 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
@ -714,8 +708,8 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.1.3 h1:qTakTkI6ni6LFD5sBwwsdSO+AQqbSIxOauHTTQKZ/7o=
honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
mvdan.cc/gofumpt v0.1.0 h1:hsVv+Y9UsZ/mFZTxJZuHVI6shSQCtzZ11h1JEFPAZLw=
mvdan.cc/gofumpt v0.1.0/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48=
mvdan.cc/gofumpt v0.1.1 h1:bi/1aS/5W00E2ny5q65w9SnKpWEF/UIOqDYBILpo9rA=
mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48=
mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7 h1:HT3e4Krq+IE44tiN36RvVEb6tvqeIdtsVSsxmNPqlFU=
mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7/go.mod h1:hBpJkZE8H/sb+VRFvw2+rBpHNsTBcvSpk61hr8mzXZE=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

View File

@ -13,6 +13,6 @@ import (
_ "golang.org/x/tools/go/analysis/passes/nilness/cmd/nilness"
_ "golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow"
_ "honnef.co/go/tools/cmd/staticcheck"
_ "mvdan.cc/gofumpt/gofumports"
_ "mvdan.cc/gofumpt"
_ "mvdan.cc/unparam"
)

View File

@ -2,6 +2,13 @@
<!-- TODO(a.garipov): Reformat in accordance with the KeepAChangelog spec. -->
## v0.106: API changes
### New `"client_info"` field in `GET /querylog` response
* The new optional field `"client_info"` of `QueryLogItem` objects contains
a more full information about the client.
## v0.105: API changes
### New `"client_id"` field in `GET /querylog` response

View File

@ -1815,6 +1815,8 @@
The client ID, if provided in DOH, DOQ, or DOT.
'example': 'cli123'
'type': 'string'
'client_info':
'$ref': '#/components/schemas/QueryLogItemClient'
'client_proto':
'enum':
- 'dot'
@ -1876,6 +1878,58 @@
'type': 'string'
'description': 'DNS request processing start time'
'example': '2018-11-26T00:02:41+03:00'
'QueryLogItemClient':
'description': >
Client information for a query log item.
'properties':
'disallowed':
'type': 'boolean'
'description': >
Whether the client's IP is blocked or not.
'disallowed_rule':
'type': 'string'
'description': >
The rule due to which the client is disallowed. If disallowed is
set to true, and this string is empty, then the client IP is
disallowed by the "allowed IP list", that is it is not included in
the allowed list.
'ids':
'description': >
IP, CIDR, MAC, or client ID.
'items':
'type': 'string'
'type': 'array'
'name':
'description': >
Persistent client's name or an empty string if this is a runtime
client.
'type': 'string'
'whois':
'$ref': '#/components/schemas/QueryLogItemClientWhois'
'required':
- 'disallowed'
- 'disallowed_rule'
- 'ids'
- 'name'
- 'whois'
'type': 'object'
'QueryLogItemClientWhois':
'description': >
Client WHOIS information, if any.
'properties':
'city':
'description': >
City, if any.
'type': 'string'
'country':
'description': >
Country, if any.
'type': 'string'
'orgname':
'description': >
Organization name, if any.
'type': 'string'
'type': 'object'
'QueryLog':
'type': 'object'
'description': 'Query log'
@ -2205,7 +2259,7 @@
'use_global_blocked_services': true
'blocked_services': null
'upstreams': null
'whois_info': null
'whois_info': {}
'disallowed': false
'disallowed_rule': ''
- '1.2.3.4':
@ -2219,7 +2273,7 @@
'use_global_blocked_services': true
'blocked_services': null
'upstreams': null
'whois_info': null
'whois_info': {}
'disallowed': false
'disallowed_rule': ''
'AccessListResponse':