Pull request: 3978 Query Log ECS

Merge in DNS/adguard-home from 3978-ecs-ip to master

Updates #3978.

Squashed commit of the following:

commit 915b94afa4b6d90169f73d4fa171bc81bcc267a7
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Mar 3 17:46:40 2022 +0300

    all: rm dot

commit 2dd2ed081b199de7e5d8269dae5d08d53b5eea6d
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Mar 3 17:42:45 2022 +0300

    client: imp txt

commit 8d5a23df739f0b650f9f3870141fd83e8fa0c1e0
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Mar 3 14:36:04 2022 +0300

    client: imp text

commit 69c856749a20144822ef3f1f67c5f3e3c24f5374
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Mar 3 14:24:56 2022 +0300

    client: imp description

commit cd0150128ad29d1874492735a5d621c0803ad0bd
Merge: 28181fbc e0b557ed
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Mar 2 21:02:16 2022 +0300

    Merge branch 'master' into 3978-ecs-ip

commit 28181fbc79eb22e7fd13cbd1d5a3c040af9fa2a4
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Mar 2 20:45:50 2022 +0300

    client: show ecs

commit cdc5e7f8c4155b798426d815eed0da547ef6efb7
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Feb 17 20:15:56 2022 +0300

    openapi: fix milestone

commit 404d6d822fa1ba4ed4cd41d92d4c1b805342fe55
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Feb 17 20:08:21 2022 +0300

    all: fix deps, docs

commit 8fb80526f1e251d3b7b193c53a4a6dee0e22c145
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Feb 17 19:39:34 2022 +0300

    all: add querylog ecs backend
This commit is contained in:
Eugene Burkov 2022-03-03 17:52:11 +03:00
parent e0b557eda2
commit 9a764b9b82
14 changed files with 108 additions and 56 deletions

View File

@ -17,6 +17,8 @@ and this project adheres to
### Added ### Added
- EDNS Client-Subnet information in the request details section of a query log
record ([#3978]).
- Support for hostnames for plain UDP upstream servers using the `udp://` scheme - Support for hostnames for plain UDP upstream servers using the `udp://` scheme
([#4166]). ([#4166]).
- Logs are now collected by default on FreeBSD and OpenBSD when AdGuard Home is - Logs are now collected by default on FreeBSD and OpenBSD when AdGuard Home is
@ -84,8 +86,10 @@ In this release, the schema version has changed from 12 to 13.
[#3367]: https://github.com/AdguardTeam/AdGuardHome/issues/3367 [#3367]: https://github.com/AdguardTeam/AdGuardHome/issues/3367
[#3381]: https://github.com/AdguardTeam/AdGuardHome/issues/3381 [#3381]: https://github.com/AdguardTeam/AdGuardHome/issues/3381
[#3503]: https://github.com/AdguardTeam/AdGuardHome/issues/3503 [#3503]: https://github.com/AdguardTeam/AdGuardHome/issues/3503
[#3978]: https://github.com/AdguardTeam/AdGuardHome/issues/3978
[#4166]: https://github.com/AdguardTeam/AdGuardHome/issues/4166 [#4166]: https://github.com/AdguardTeam/AdGuardHome/issues/4166
[#4213]: https://github.com/AdguardTeam/AdGuardHome/issues/4213 [#4213]: https://github.com/AdguardTeam/AdGuardHome/issues/4213
[#4216]: https://github.com/AdguardTeam/AdGuardHome/issues/4216
[#4221]: https://github.com/AdguardTeam/AdGuardHome/issues/4221 [#4221]: https://github.com/AdguardTeam/AdGuardHome/issues/4221
[#4238]: https://github.com/AdguardTeam/AdGuardHome/issues/4238 [#4238]: https://github.com/AdguardTeam/AdGuardHome/issues/4238

View File

@ -287,7 +287,7 @@
"form_enter_rate_limit": "Enter rate limit", "form_enter_rate_limit": "Enter rate limit",
"rate_limit": "Rate limit", "rate_limit": "Rate limit",
"edns_enable": "Enable EDNS client subnet", "edns_enable": "Enable EDNS client subnet",
"edns_cs_desc": "Send clients' subnets to the DNS servers.", "edns_cs_desc": "Add the EDNS Client Subnet option (ECS) to upstream requests and log the values sent by the clients in the query log.",
"rate_limit_desc": "The number of requests per second allowed per client. Setting it to 0 means no limit.", "rate_limit_desc": "The number of requests per second allowed per client. Setting it to 0 means no limit.",
"blocking_ipv4_desc": "IP address to be returned for a blocked A request", "blocking_ipv4_desc": "IP address to be returned for a blocked A request",
"blocking_ipv6_desc": "IP address to be returned for a blocked AAAA request", "blocking_ipv6_desc": "IP address to be returned for a blocked AAAA request",
@ -502,6 +502,7 @@
"interval_days": "{{count}} day", "interval_days": "{{count}} day",
"interval_days_plural": "{{count}} days", "interval_days_plural": "{{count}} days",
"domain": "Domain", "domain": "Domain",
"ecs": "ECS",
"punycode": "Punycode", "punycode": "Punycode",
"answer": "Answer", "answer": "Answer",
"filter_added_successfully": "The list has been successfully added", "filter_added_successfully": "The list has been successfully added",

View File

@ -20,6 +20,7 @@ const DomainCell = ({
time, time,
tracker, tracker,
type, type,
ecs,
}) => { }) => {
const { t } = useTranslation(); const { t } = useTranslation();
const dnssec_enabled = useSelector((state) => state.dnsConfig.dnssec_enabled); const dnssec_enabled = useSelector((state) => state.dnsConfig.dnssec_enabled);
@ -56,6 +57,13 @@ const DomainCell = ({
}; };
} }
if (ecs) {
requestDetailsObj = {
...requestDetailsObj,
ecs,
};
}
requestDetailsObj = { requestDetailsObj = {
...requestDetailsObj, ...requestDetailsObj,
type_table_header: type, type_table_header: type,
@ -168,6 +176,7 @@ DomainCell.propTypes = {
time: propTypes.string.isRequired, time: propTypes.string.isRequired,
type: propTypes.string.isRequired, type: propTypes.string.isRequired,
tracker: propTypes.object, tracker: propTypes.object,
ecs: propTypes.string,
}; };
export default DomainCell; export default DomainCell;

View File

@ -238,6 +238,7 @@ Row.propTypes = {
type: propTypes.string.isRequired, type: propTypes.string.isRequired,
client_proto: propTypes.string.isRequired, client_proto: propTypes.string.isRequired,
client_id: propTypes.string, client_id: propTypes.string,
ecs: propTypes.string,
client_info: propTypes.shape({ client_info: propTypes.shape({
name: propTypes.string.isRequired, name: propTypes.string.isRequired,
whois: propTypes.shape({ whois: propTypes.shape({

View File

@ -76,6 +76,7 @@ export const normalizeLogs = (logs) => logs.map((log) => {
original_answer, original_answer,
upstream, upstream,
cached, cached,
ecs,
} = log; } = log;
const { name: domain, unicode_name: unicodeName, type } = question; const { name: domain, unicode_name: unicodeName, type } = question;
@ -118,6 +119,7 @@ export const normalizeLogs = (logs) => logs.map((log) => {
elapsedMs, elapsedMs,
upstream, upstream,
cached, cached,
ecs,
}; };
}); });

View File

@ -41,55 +41,65 @@ func (s *Server) processQueryLogsAndStats(dctx *dnsContext) (rc resultCode) {
// uninitialized while in use. This can happen after proxy server has been // uninitialized while in use. This can happen after proxy server has been
// stopped, but its workers haven't yet exited. // stopped, but its workers haven't yet exited.
if shouldLog && s.queryLog != nil { if shouldLog && s.queryLog != nil {
p := &querylog.AddParams{ s.logQuery(dctx, pctx, elapsed, ip)
Question: msg,
Answer: pctx.Res,
OrigAnswer: dctx.origResp,
Result: dctx.result,
Elapsed: elapsed,
ClientID: dctx.clientID,
ClientIP: ip,
AuthenticatedData: dctx.responseAD,
}
switch pctx.Proto {
case proxy.ProtoHTTPS:
p.ClientProto = querylog.ClientProtoDoH
case proxy.ProtoQUIC:
p.ClientProto = querylog.ClientProtoDoQ
case proxy.ProtoTLS:
p.ClientProto = querylog.ClientProtoDoT
case proxy.ProtoDNSCrypt:
p.ClientProto = querylog.ClientProtoDNSCrypt
default:
// Consider this a plain DNS-over-UDP or DNS-over-TCP request.
}
if pctx.Upstream != nil {
p.Upstream = pctx.Upstream.Address()
} else if cachedUps := pctx.CachedUpstreamAddr; cachedUps != "" {
p.Upstream = pctx.CachedUpstreamAddr
p.Cached = true
}
s.queryLog.Add(p)
} }
s.updateStats(dctx, elapsed, *dctx.result, ip) if s.stats != nil {
s.updateStats(dctx, elapsed, *dctx.result, ip)
}
return resultCodeSuccess return resultCodeSuccess
} }
// logQuery pushes the request details into the query log.
func (s *Server) logQuery(
dctx *dnsContext,
pctx *proxy.DNSContext,
elapsed time.Duration,
ip net.IP,
) {
p := &querylog.AddParams{
Question: pctx.Req,
ReqECS: pctx.ReqECS,
Answer: pctx.Res,
OrigAnswer: dctx.origResp,
Result: dctx.result,
Elapsed: elapsed,
ClientID: dctx.clientID,
ClientIP: ip,
AuthenticatedData: dctx.responseAD,
}
switch pctx.Proto {
case proxy.ProtoHTTPS:
p.ClientProto = querylog.ClientProtoDoH
case proxy.ProtoQUIC:
p.ClientProto = querylog.ClientProtoDoQ
case proxy.ProtoTLS:
p.ClientProto = querylog.ClientProtoDoT
case proxy.ProtoDNSCrypt:
p.ClientProto = querylog.ClientProtoDNSCrypt
default:
// Consider this a plain DNS-over-UDP or DNS-over-TCP request.
}
if pctx.Upstream != nil {
p.Upstream = pctx.Upstream.Address()
} else if cachedUps := pctx.CachedUpstreamAddr; cachedUps != "" {
p.Upstream = pctx.CachedUpstreamAddr
p.Cached = true
}
s.queryLog.Add(p)
}
// updatesStats writes the request into statistics.
func (s *Server) updateStats( func (s *Server) updateStats(
ctx *dnsContext, ctx *dnsContext,
elapsed time.Duration, elapsed time.Duration,
res filtering.Result, res filtering.Result,
clientIP net.IP, clientIP net.IP,
) { ) {
if s.stats == nil {
return
}
pctx := ctx.proxyCtx pctx := ctx.proxyCtx
e := stats.Entry{} e := stats.Entry{}
e.Domain = strings.ToLower(pctx.Req.Question[0].Name) e.Domain = strings.ToLower(pctx.Req.Question[0].Name)

View File

@ -14,7 +14,7 @@ import (
"github.com/miekg/dns" "github.com/miekg/dns"
) )
type logEntryHandler (func(t json.Token, ent *logEntry) error) type logEntryHandler func(t json.Token, ent *logEntry) error
var logEntryHandlers = map[string]logEntryHandler{ var logEntryHandlers = map[string]logEntryHandler{
"CID": func(t json.Token, ent *logEntry) error { "CID": func(t json.Token, ent *logEntry) error {
@ -109,6 +109,16 @@ var logEntryHandlers = map[string]logEntryHandler{
return err return err
}, },
"ECS": func(t json.Token, ent *logEntry) error {
v, ok := t.(string)
if !ok {
return nil
}
ent.ReqECS = v
return nil
},
"Cached": func(t json.Token, ent *logEntry) error { "Cached": func(t json.Token, ent *logEntry) error {
v, ok := t.(bool) v, ok := t.(bool)
if !ok { if !ok {

View File

@ -32,6 +32,7 @@ func TestDecodeLogEntry(t *testing.T) {
`"QT":"A",` + `"QT":"A",` +
`"QC":"IN",` + `"QC":"IN",` +
`"CP":"",` + `"CP":"",` +
`"ECS":"1.2.3.0/24",` +
`"Answer":"` + ansStr + `",` + `"Answer":"` + ansStr + `",` +
`"Cached":true,` + `"Cached":true,` +
`"AD":true,` + `"AD":true,` +
@ -58,6 +59,7 @@ func TestDecodeLogEntry(t *testing.T) {
QClass: "IN", QClass: "IN",
ClientID: "cli42", ClientID: "cli42",
ClientProto: "", ClientProto: "",
ReqECS: "1.2.3.0/24",
Answer: ans, Answer: ans,
Cached: true, Cached: true,
Result: filtering.Result{ Result: filtering.Result{

View File

@ -78,6 +78,10 @@ func (l *queryLog) entryToJSON(entry *logEntry, anonFunc aghnet.IPMutFunc) (json
jsonEntry["client_id"] = entry.ClientID jsonEntry["client_id"] = entry.ClientID
} }
if entry.ReqECS != "" {
jsonEntry["ecs"] = entry.ReqECS
}
if len(entry.Result.Rules) > 0 { if len(entry.Result.Rules) > 0 {
if r := entry.Result.Rules[0]; len(r.Text) > 0 { if r := entry.Result.Rules[0]; len(r.Text) > 0 {
jsonEntry["rule"] = r.Text jsonEntry["rule"] = r.Text

View File

@ -81,6 +81,8 @@ type logEntry struct {
QType string `json:"QT"` QType string `json:"QT"`
QClass string `json:"QC"` QClass string `json:"QC"`
ReqECS string `json:"ECS,omitempty"`
ClientID string `json:"CID,omitempty"` ClientID string `json:"CID,omitempty"`
ClientProto ClientProto `json:"CP"` ClientProto ClientProto `json:"CP"`
@ -189,6 +191,10 @@ func (l *queryLog) Add(params *AddParams) {
AuthenticatedData: params.AuthenticatedData, AuthenticatedData: params.AuthenticatedData,
} }
if params.ReqECS != nil {
entry.ReqECS = params.ReqECS.String()
}
if params.Answer != nil { if params.Answer != nil {
var a []byte var a []byte
a, err = params.Answer.Pack() a, err = params.Answer.Pack()

View File

@ -77,6 +77,10 @@ type Config struct {
type AddParams struct { type AddParams struct {
Question *dns.Msg Question *dns.Msg
// ReqECS is the IP network extracted from EDNS Client-Subnet option of a
// request.
ReqECS *net.IPNet
// Answer is the response which is sent to the client, if any. // Answer is the response which is sent to the client, if any.
Answer *dns.Msg Answer *dns.Msg

View File

@ -99,24 +99,10 @@ func (c *searchCriterion) quickMatch(line string, findClient quickMatchClientFun
} }
if c.strict { if c.strict {
return ctDomainOrClientCaseStrict( return ctDomainOrClientCaseStrict(c.value, c.asciiVal, clientID, name, host, ip)
c.value,
c.asciiVal,
clientID,
name,
host,
ip,
)
} }
return ctDomainOrClientCaseNonStrict( return ctDomainOrClientCaseNonStrict(c.value, c.asciiVal, clientID, name, host, ip)
c.value,
c.asciiVal,
clientID,
name,
host,
ip,
)
case ctFilteringStatus: case ctFilteringStatus:
// Go on, as we currently don't do quick matches against // Go on, as we currently don't do quick matches against
// filtering statuses. // filtering statuses.

View File

@ -2,7 +2,12 @@
<!-- TODO(a.garipov): Reformat in accordance with the KeepAChangelog spec. --> <!-- TODO(a.garipov): Reformat in accordance with the KeepAChangelog spec. -->
## v0.107.3: API changes ## v0.108.0: API changes
### The new optional field `"ecs"` in `QueryLogItem`
* The new optional field `"ecs"` in `GET /control/querylog` contains the IP
network from an EDNS Client-Subnet option from the request message if any.
### The new possible status code in `/install/configure` response. ### The new possible status code in `/install/configure` response.
@ -10,6 +15,8 @@
`POST /install/configure` which means that the specified password does not `POST /install/configure` which means that the specified password does not
meet the strength requirements. meet the strength requirements.
## v0.107.3: API changes
### The new field `"version"` in `AddressesInfo` ### The new field `"version"` in `AddressesInfo`
* The new field `"version"` in `GET /install/get_addresses` is the version of * The new field `"version"` in `GET /install/get_addresses` is the version of

View File

@ -1905,6 +1905,12 @@
- 'doq' - 'doq'
- 'dnscrypt' - 'dnscrypt'
- '' - ''
'ecs':
'type': 'string'
'example': '192.168.0.0/16'
'description': >
The IP network defined by an EDNS Client-Subnet option in the
request message if any.
'elapsedMs': 'elapsedMs':
'type': 'string' 'type': 'string'
'example': '54.023928' 'example': '54.023928'