AdguardTeam/AdGuardHome
1
1
Fork 0
mirror of https://github.com/AdguardTeam/AdGuardHome.git synced 2024-08-16 22:20:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

Pull request 1983: 5720-wildcard-ignored-domains

Browse Source 
Updates #5720.

Squashed commit of the following:

commit e8093c990f15e2efc496f1a04f87360825e34e96
Merge: df5413eef 28fefaff1
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Sep 5 15:06:33 2023 +0300

    Merge branch 'master' into 5720-wildcard-ignored-domains

commit df5413eefeac2c7e34eb725db9e2908b5b2d08cb
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Sep 5 14:49:05 2023 +0300

    confmigrate: imp docs

commit 1644d99b730cc7f22c9d75b8e990149d3ce5b32a
Merge: 9542ee161 1e4517898
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Sep 5 14:23:42 2023 +0300

    Merge branch 'master' into 5720-wildcard-ignored-domains

commit 9542ee1616c1dd4bdb6ec9a2af79a2af3858a7e3
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Sep 5 12:48:48 2023 +0300

    all: upd chlog

commit 183f84a7f73c7bd33669bd108076f60514ca101e
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Sep 1 17:11:31 2023 +0300

    all: imp chlog

commit a704325352a577a9b6652f011b82180ec3a6e095
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Aug 31 18:59:52 2023 +0300

    all: imp code

commit fe99c3b883500850399b1feb72c914ab878b3107
Merge: 7f11e9460 0182b9ec1
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Aug 31 18:43:09 2023 +0300

    Merge branch 'master' into 5720-wildcard-ignored-domains

commit 7f11e94609027ed821a125d27a1ffde03f37334a
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Aug 30 19:57:51 2023 +0300

    aghnet: add tests

commit f10f9190ce1064a5d03155e8b6bba61db977897b
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Aug 30 18:32:07 2023 +0300

    all: add conf migration

commit a53c14df129765366966c5230dd53aa29bdd25c5
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Aug 30 13:37:30 2023 +0300

    all: add ignore engine
This commit is contained in:
Stanislav Chzhen 2023-09-05 15:13:35 +03:00
parent 28fefaff1a
commit 42291cd547
22 changed files with 577 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
CHANGELOG.md
View File

@ -25,6 +25,8 @@ NOTE: Add new changes BELOW THIS COMMENT.
### Added
- AdBlock-style syntax support for ignored domains in logs and statistics
([#5720]).
- [`Strict-Transport-Security`][hsts] header in the HTTP API and DNS-over-HTTPS
responses when HTTPS is forced ([#2998]). See [RFC 6979][rfc6797].
- UI for the schedule of the service-blocking pause ([#951]).
@ -50,7 +52,11 @@ NOTE: Add new changes BELOW THIS COMMENT.
#### Configuration Changes
In this release, the schema version has changed from 24 to 26.
In this release, the schema version has changed from 24 to 27.
- Ignore rules blocking `.` in `querylog.…` and `stats.…` have been migrated to
AdBlock syntax (`|.^`). To rollback this change, restore the rules and
change the `schema_version` back to `26`.
- Filtering-related settings have been moved from `dns` section of the YAML
configuration file to the new section `filtering`:
@ -156,6 +162,7 @@ In this release, the schema version has changed from 24 to 26.
[#1453]: https://github.com/AdguardTeam/AdGuardHome/issues/1453
[#2998]: https://github.com/AdguardTeam/AdGuardHome/issues/2998
[#3701]: https://github.com/AdguardTeam/AdGuardHome/issues/3701
[#5720]: https://github.com/AdguardTeam/AdGuardHome/issues/5720
[#5793]: https://github.com/AdguardTeam/AdGuardHome/issues/5793
[#5948]: https://github.com/AdguardTeam/AdGuardHome/issues/5948
[#6020]: https://github.com/AdguardTeam/AdGuardHome/issues/6020

4
client/src/__locales/en.json
View File

@ -537,8 +537,8 @@
"statistics_enable": "Enable statistics",
"ignore_domains": "Ignored domains (separated by newline)",
"ignore_domains_title": "Ignored domains",
"ignore_domains_desc_stats": "Queries for these domains are not written to the statistics",
"ignore_domains_desc_query": "Queries for these domains are not written to the query log",
"ignore_domains_desc_stats": "Queries matching these rules are not written to the statistics",
"ignore_domains_desc_query": "Queries matching these rules are not written to the query log",
"interval_hours": "{{count}} hour",
"interval_hours_plural": "{{count}} hours",
"filters_configuration": "Filters configuration",

56
internal/aghnet/ignore.go Normal file
View File

@ -0,0 +1,56 @@
package aghnet
import (
"strings"
"github.com/AdguardTeam/urlfilter"
"github.com/AdguardTeam/urlfilter/filterlist"
"golang.org/x/exp/slices"
)
// IgnoreEngine contains the list of rules for ignoring hostnames and matches
// them.
//
// TODO(s.chzhen): Move all urlfilter stuff to aghfilter.
type IgnoreEngine struct {
// engine is the filtering engine that can match rules for ignoring
// hostnames.
engine *urlfilter.DNSEngine
// ignored is the list of rules for ignoring hostnames.
ignored []string
}
// NewIgnoreEngine creates a new instance of the IgnoreEngine and stores the
// list of rules for ignoring hostnames.
func NewIgnoreEngine(ignored []string) (e *IgnoreEngine, err error) {
ruleList := &filterlist.StringRuleList{
RulesText: strings.ToLower(strings.Join(ignored, "\n")),
IgnoreCosmetic: true,
}
ruleStorage, err := filterlist.NewRuleStorage([]filterlist.RuleList{ruleList})
if err != nil {
return nil, err
}
return &IgnoreEngine{
engine: urlfilter.NewDNSEngine(ruleStorage),
ignored: ignored,
}, nil
}
// Has returns true if IgnoreEngine matches the host.
func (e *IgnoreEngine) Has(host string) (ignore bool) {
if e == nil {
return false
}
_, ignore = e.engine.Match(host)
return ignore
}
// Values returns a copy of list of rules for ignoring hostnames.
func (e *IgnoreEngine) Values() (ignored []string) {
return slices.Clone(e.ignored)
}

46
internal/aghnet/ignore_test.go Normal file
View File

@ -0,0 +1,46 @@
package aghnet_test
import (
"testing"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/stretchr/testify/require"
)
func TestIgnoreEngine_Has(t *testing.T) {
hostnames := []string{
"*.example.com",
"example.com",
"|.^",
}
engine, err := aghnet.NewIgnoreEngine(hostnames)
require.NotNil(t, engine)
require.NoError(t, err)
testCases := []struct {
name string
host string
ignore bool
}{{
name: "basic",
host: "example.com",
ignore: true,
}, {
name: "root",
host: ".",
ignore: true,
}, {
name: "wildcard",
host: "www.example.com",
ignore: true,
}, {
name: "not_ignored",
host: "something.com",
ignore: false,
}}
for _, tc := range testCases {
require.Equal(t, tc.ignore, engine.Has(tc.host))
}
}

83
internal/confmigrate/migrations_test.go
View File

@ -1563,3 +1563,86 @@ func TestUpgradeSchema25to26(t *testing.T) {
})
}
}
func TestUpgradeSchema26to27(t *testing.T) {
const newSchemaVer = 27
testCases := []struct {
in yobj
want yobj
name string
}{{
name: "empty",
in: yobj{},
want: yobj{
"schema_version": newSchemaVer,
},
}, {
name: "single_dot",
in: yobj{
"querylog": yobj{
"ignored": yarr{
".",
},
},
"statistics": yobj{
"ignored": yarr{
".",
},
},
},
want: yobj{
"querylog": yobj{
"ignored": yarr{
"|.^",
},
},
"statistics": yobj{
"ignored": yarr{
"|.^",
},
},
"schema_version": newSchemaVer,
},
}, {
name: "mixed",
in: yobj{
"querylog": yobj{
"ignored": yarr{
".",
"example.com",
},
},
"statistics": yobj{
"ignored": yarr{
".",
"example.org",
},
},
},
want: yobj{
"querylog": yobj{
"ignored": yarr{
"|.^",
"example.com",
},
},
"statistics": yobj{
"ignored": yarr{
"|.^",
"example.org",
},
},
"schema_version": newSchemaVer,
},
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
err := migrateTo27(tc.in)
require.NoError(t, err)
assert.Equal(t, tc.want, tc.in)
})
}
}

3
internal/confmigrate/migrator.go
View File

@ -10,7 +10,7 @@ import (
)
// LastSchemaVersion is the most recent schema version.
const LastSchemaVersion uint = 26
const LastSchemaVersion uint = 27
// Config is a the configuration for initializing a [Migrator].
type Config struct {
@ -122,6 +122,7 @@ func (m *Migrator) upgradeConfigSchema(current, target uint, diskConf yobj) (err
23: migrateTo24,
24: migrateTo25,
25: migrateTo26,
26: migrateTo27,
}
for i, migrate := range upgrades[current:target] {

4
internal/confmigrate/migrator_test.go
View File

@ -181,6 +181,10 @@ func TestMigrateConfig_Migrate(t *testing.T) {
yamlEqFunc: require.YAMLEq,
name: "v26",
targetVersion: 26,
}, {
yamlEqFunc: require.YAMLEq,
name: "v27",
targetVersion: 27,
}}
for _, tc := range testCases {

123
internal/confmigrate/testdata/TestMigrateConfig_Migrate/v27/input.yml vendored Normal file
View File

@ -0,0 +1,123 @@
http:
address: 127.0.0.1:3000
session_ttl: 3h
pprof:
enabled: true
port: 6060
users:
- name: testuser
password: testpassword
dns:
bind_hosts:
- 127.0.0.1
port: 53
parental_sensitivity: 0
upstream_dns:
- tls://1.1.1.1
- tls://1.0.0.1
- quic://8.8.8.8:784
bootstrap_dns:
- 8.8.8.8:53
edns_client_subnet:
enabled: true
use_custom: false
custom_ip: ""
filtering:
filtering_enabled: true
parental_enabled: false
safebrowsing_enabled: false
safe_search:
enabled: false
bing: true
duckduckgo: true
google: true
pixabay: true
yandex: true
youtube: true
protection_enabled: true
blocked_services:
schedule:
time_zone: Local
ids:
- 500px
blocked_response_ttl: 10
filters:
- url: https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt
name: ""
enabled: true
- url: https://adaway.org/hosts.txt
name: AdAway
enabled: false
- url: https://hosts-file.net/ad_servers.txt
name: hpHosts - Ad and Tracking servers only
enabled: false
- url: http://www.malwaredomainlist.com/hostslist/hosts.txt
name: MalwareDomainList.com Hosts List
enabled: false
clients:
persistent:
- name: localhost
ids:
- 127.0.0.1
- aa:aa:aa:aa:aa:aa
use_global_settings: true
use_global_blocked_services: true
filtering_enabled: false
parental_enabled: false
safebrowsing_enabled: false
safe_search:
enabled: true
bing: true
duckduckgo: true
google: true
pixabay: true
yandex: true
youtube: true
blocked_services:
schedule:
time_zone: Local
ids:
- 500px
runtime_sources:
whois: true
arp: true
rdns: true
dhcp: true
hosts: true
dhcp:
enabled: false
interface_name: vboxnet0
local_domain_name: local
dhcpv4:
gateway_ip: 192.168.0.1
subnet_mask: 255.255.255.0
range_start: 192.168.0.10
range_end: 192.168.0.250
lease_duration: 1234
icmp_timeout_msec: 10
schema_version: 26
user_rules: []
querylog:
enabled: true
file_enabled: true
interval: 720h
size_memory: 1000
ignored:
- '.'
statistics:
enabled: true
interval: 240h
ignored:
- '.'
os:
group: ''
rlimit_nofile: 123
user: ''
log:
file: ""
max_backups: 0
max_size: 100
max_age: 3
compress: true
local_time: false
verbose: true

123
internal/confmigrate/testdata/TestMigrateConfig_Migrate/v27/output.yml vendored Normal file
View File

@ -0,0 +1,123 @@
http:
address: 127.0.0.1:3000
session_ttl: 3h
pprof:
enabled: true
port: 6060
users:
- name: testuser
password: testpassword
dns:
bind_hosts:
- 127.0.0.1
port: 53
parental_sensitivity: 0
upstream_dns:
- tls://1.1.1.1
- tls://1.0.0.1
- quic://8.8.8.8:784
bootstrap_dns:
- 8.8.8.8:53
edns_client_subnet:
enabled: true
use_custom: false
custom_ip: ""
filtering:
filtering_enabled: true
parental_enabled: false
safebrowsing_enabled: false
safe_search:
enabled: false
bing: true
duckduckgo: true
google: true
pixabay: true
yandex: true
youtube: true
protection_enabled: true
blocked_services:
schedule:
time_zone: Local
ids:
- 500px
blocked_response_ttl: 10
filters:
- url: https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt
name: ""
enabled: true
- url: https://adaway.org/hosts.txt
name: AdAway
enabled: false
- url: https://hosts-file.net/ad_servers.txt
name: hpHosts - Ad and Tracking servers only
enabled: false
- url: http://www.malwaredomainlist.com/hostslist/hosts.txt
name: MalwareDomainList.com Hosts List
enabled: false
clients:
persistent:
- name: localhost
ids:
- 127.0.0.1
- aa:aa:aa:aa:aa:aa
use_global_settings: true
use_global_blocked_services: true
filtering_enabled: false
parental_enabled: false
safebrowsing_enabled: false
safe_search:
enabled: true
bing: true
duckduckgo: true
google: true
pixabay: true
yandex: true
youtube: true
blocked_services:
schedule:
time_zone: Local
ids:
- 500px
runtime_sources:
whois: true
arp: true
rdns: true
dhcp: true
hosts: true
dhcp:
enabled: false
interface_name: vboxnet0
local_domain_name: local
dhcpv4:
gateway_ip: 192.168.0.1
subnet_mask: 255.255.255.0
range_start: 192.168.0.10
range_end: 192.168.0.250
lease_duration: 1234
icmp_timeout_msec: 10
schema_version: 27
user_rules: []
querylog:
enabled: true
file_enabled: true
interval: 720h
size_memory: 1000
ignored:
- '|.^'
statistics:
enabled: true
interval: 240h
ignored:
- '|.^'
os:
group: ''
rlimit_nofile: 123
user: ''
log:
file: ""
max_backups: 0
max_size: 100
max_age: 3
compress: true
local_time: false
verbose: true

77
internal/confmigrate/v27.go Normal file
View File

@ -0,0 +1,77 @@
package confmigrate
// migrateTo27 performs the following changes:
//
// # BEFORE:
// 'querylog':
// 'ignored':
// - '.'
// - # …
// # …
// 'statistics':
// 'ignored':
// - '.'
// - # …
// # …
// # …
//
// # AFTER:
// 'querylog':
// 'ignored':
// - '|.^'
// - # …
// # …
// 'statistics':
// 'ignored':
// - '|.^'
// - # …
// # …
// # …
func migrateTo27(diskConf yobj) (err error) {
diskConf["schema_version"] = 27
keys := []string{"querylog", "statistics"}
for _, k := range keys {
err = replaceDot(diskConf, k)
if err != nil {
return err
}
}
return nil
}
// replaceDot replaces rules blocking root domain "." with AdBlock style syntax
// "|.^".
func replaceDot(diskConf yobj, key string) (err error) {
var obj yobj
var ok bool
obj, ok, err = fieldVal[yobj](diskConf, key)
if err != nil {
return err
} else if !ok {
return nil
}
var ignored yarr
ignored, ok, err = fieldVal[yarr](obj, "ignored")
if err != nil {
return err
} else if !ok {
return nil
}
for i, hostVal := range ignored {
var host string
host, ok = hostVal.(string)
if !ok {
continue
}
if host == "." {
ignored[i] = "|.^"
}
}
return nil
}

1
internal/dnsforward/access.go
View File

@ -28,6 +28,7 @@ type accessManager struct {
allowedClientIDs *stringutil.Set
blockedClientIDs *stringutil.Set
// TODO(s.chzhen): Use [aghnet.IgnoreEngine].
blockedHostsEng *urlfilter.DNSEngine
// TODO(a.garipov): Create a type for a set of IP networks.

3
internal/home/config.go
View File

@ -22,7 +22,6 @@ import (
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/timeutil"
"github.com/google/renameio/v2/maybe"
"golang.org/x/exp/slices"
yaml "gopkg.in/yaml.v3"
)
@ -594,7 +593,6 @@ func (c *configuration) write() (err error) {
config.Stats.Interval = timeutil.Duration{Duration: statsConf.Limit}
config.Stats.Enabled = statsConf.Enabled
config.Stats.Ignored = statsConf.Ignored.Values()
slices.Sort(config.Stats.Ignored)
}
if Context.queryLog != nil {
@ -606,7 +604,6 @@ func (c *configuration) write() (err error) {
config.QueryLog.Interval = timeutil.Duration{Duration: dc.RotationIvl}
config.QueryLog.MemSize = dc.MemSize
config.QueryLog.Ignored = dc.Ignored.Values()
slices.Sort(config.Stats.Ignored)
}
if Context.filters != nil {

8
internal/home/dns.go
View File

@ -59,12 +59,12 @@ func initDNS() (err error) {
ShouldCountClient: Context.clients.shouldCountClient,
}
set, err := aghnet.NewDomainNameSet(config.Stats.Ignored)
engine, err := aghnet.NewIgnoreEngine(config.Stats.Ignored)
if err != nil {
return fmt.Errorf("statistics: ignored list: %w", err)
}
statsConf.Ignored = set
statsConf.Ignored = engine
Context.stats, err = stats.New(statsConf)
if err != nil {
return fmt.Errorf("init stats: %w", err)
@ -83,12 +83,12 @@ func initDNS() (err error) {
FileEnabled: config.QueryLog.FileEnabled,
}
set, err = aghnet.NewDomainNameSet(config.QueryLog.Ignored)
engine, err = aghnet.NewIgnoreEngine(config.QueryLog.Ignored)
if err != nil {
return fmt.Errorf("querylog: ignored list: %w", err)
}
conf.Ignored = set
conf.Ignored = engine
Context.queryLog, err = querylog.New(conf)
if err != nil {
return fmt.Errorf("init querylog: %w", err)

7
internal/querylog/http.go
View File

@ -17,7 +17,6 @@ import (
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/AdguardTeam/golibs/timeutil"
"golang.org/x/exp/slices"
"golang.org/x/net/idna"
)
@ -141,8 +140,6 @@ func (l *queryLog) handleGetQueryLogConfig(w http.ResponseWriter, r *http.Reques
}
}()
slices.Sort(resp.Ignored)
aghhttp.WriteJSONResponseOK(w, r, resp)
}
@ -224,7 +221,7 @@ func (l *queryLog) handlePutQueryLogConfig(w http.ResponseWriter, r *http.Reques
return
}
set, err := aghnet.NewDomainNameSet(newConf.Ignored)
engine, err := aghnet.NewIgnoreEngine(newConf.Ignored)
if err != nil {
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "ignored: %s", err)
@ -258,7 +255,7 @@ func (l *queryLog) handlePutQueryLogConfig(w http.ResponseWriter, r *http.Reques
conf := *l.conf
conf.Ignored = set
conf.Ignored = engine
conf.RotationIvl = ivl
conf.Enabled = newConf.Enabled == aghalg.NBTrue

1
internal/querylog/qlog.go
View File

@ -127,7 +127,6 @@ func (l *queryLog) WriteDiskConfig(c *Config) {
defer l.confMu.RUnlock()
*c = *l.conf
c.Ignored = l.conf.Ignored.Clone()
}
// Clear memory buffer and remove log files

31
internal/querylog/qlog_test.go
View File

@ -5,8 +5,8 @@ import (
"net"
"testing"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/AdguardTeam/golibs/testutil"
"github.com/AdguardTeam/golibs/timeutil"
"github.com/miekg/dns"
@ -256,10 +256,21 @@ func TestQueryLogFileDisabled(t *testing.T) {
func TestQueryLogShouldLog(t *testing.T) {
const (
ignored1 = "ignor.ed"
ignored2 = "ignored.to"
ignored1 = "ignor.ed"
ignored2 = "ignored.to"
ignoredWildcard = "*.ignored.com"
ignoredRoot = "|.^"
)
set := stringutil.NewSet(ignored1, ignored2)
ignored := []string{
ignored1,
ignored2,
ignoredWildcard,
ignoredRoot,
}
engine, err := aghnet.NewIgnoreEngine(ignored)
require.NoError(t, err)
findClient := func(ids []string) (c *Client, err error) {
log := ids[0] == "no_log"
@ -268,7 +279,7 @@ func TestQueryLogShouldLog(t *testing.T) {
}
l, err := newQueryLog(Config{
Ignored: set,
Ignored: engine,
Enabled: true,
RotationIvl: timeutil.Day,
MemSize: 100,
@ -297,6 +308,16 @@ func TestQueryLogShouldLog(t *testing.T) {
host: ignored2,
ids: []string{"whatever"},
wantLog: false,
}, {
name: "no_log_ignored_wildcard",
host: "www.ignored.com",
ids: []string{"whatever"},
wantLog: false,
}, {
name: "no_log_ignored_root",
host: ".",
ids: []string{"whatever"},
wantLog: false,
}, {
name: "no_log_client_ignore",
host: "example.com",

6
internal/querylog/querylog.go
View File

@ -11,7 +11,6 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/miekg/dns"
)
@ -36,8 +35,9 @@ type QueryLog interface {
//
// Do not alter any fields of this structure after using it.
type Config struct {
// Ignored is the list of host names, which should not be written to log.
Ignored *stringutil.Set
// Ignored contains the list of host names, which should not be written to
// log, and matches them.
Ignored *aghnet.IgnoreEngine
// Anonymizer processes the IP addresses to anonymize those if needed.
Anonymizer *aghnet.IPMut

7
internal/stats/http.go
View File

@ -12,7 +12,6 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/timeutil"
"golang.org/x/exp/slices"
)
// topAddrs is an alias for the types of the TopFoo fields of statsResponse.
@ -140,8 +139,6 @@ func (s *StatsCtx) handleGetStatsConfig(w http.ResponseWriter, r *http.Request)
}
}()
slices.Sort(resp.Ignored)
aghhttp.WriteJSONResponseOK(w, r, resp)
}
@ -184,7 +181,7 @@ func (s *StatsCtx) handlePutStatsConfig(w http.ResponseWriter, r *http.Request)
return
}
set, err := aghnet.NewDomainNameSet(reqData.Ignored)
engine, err := aghnet.NewIgnoreEngine(reqData.Ignored)
if err != nil {
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "ignored: %s", err)
@ -210,7 +207,7 @@ func (s *StatsCtx) handlePutStatsConfig(w http.ResponseWriter, r *http.Request)
s.confMu.Lock()
defer s.confMu.Unlock()
s.ignored = set
s.ignored = engine
s.limit = ivl
s.enabled = reqData.Enabled == aghalg.NBTrue
}

23
internal/stats/http_test.go
View File

@ -75,29 +75,6 @@ func TestHandleStatsConfig(t *testing.T) {
},
wantCode: http.StatusOK,
wantErr: "",
}, {
name: "ignored_duplicate",
body: getConfigResp{
Enabled: aghalg.NBTrue,
Interval: float64(minIvl.Milliseconds()),
Ignored: []string{
"ignor.ed",
"ignor.ed",
},
},
wantCode: http.StatusUnprocessableEntity,
wantErr: "ignored: duplicate hostname \"ignor.ed\" at index 1\n",
}, {
name: "ignored_empty",
body: getConfigResp{
Enabled: aghalg.NBTrue,
Interval: float64(minIvl.Milliseconds()),
Ignored: []string{
"",
},
},
wantCode: http.StatusUnprocessableEntity,
wantErr: "ignored: at index 0: hostname is empty\n",
}, {
name: "enabled_is_null",
body: getConfigResp{

14
internal/stats/stats.go
View File

@ -12,9 +12,9 @@ import (
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/AdguardTeam/golibs/timeutil"
"go.etcd.io/bbolt"
)
@ -58,8 +58,9 @@ type Config struct {
// endpoints.
HTTPRegister aghhttp.RegisterFunc
// Ignored is the list of host names, which should not be counted.
Ignored *stringutil.Set
// Ignored contains the list of host names, which should not be counted,
// and matches them.
Ignored *aghnet.IgnoreEngine
// Filename is the name of the database file.
Filename string
@ -117,8 +118,9 @@ type StatsCtx struct {
// confMu protects ignored, limit, and enabled.
confMu *sync.RWMutex
// ignored is the list of host names, which should not be counted.
ignored *stringutil.Set
// ignored contains the list of host names, which should not be counted,
// and matches them.
ignored *aghnet.IgnoreEngine
// shouldCountClient returns client's ignore setting.
shouldCountClient func([]string) bool
@ -289,7 +291,7 @@ func (s *StatsCtx) WriteDiskConfig(dc *Config) {
s.confMu.RLock()
defer s.confMu.RUnlock()
dc.Ignored = s.ignored.Clone()
dc.Ignored = s.ignored
dc.Limit = s.limit
dc.Enabled = s.enabled
}

8
internal/stats/stats_test.go
View File

@ -11,9 +11,9 @@ import (
"testing"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/AdGuardHome/internal/stats"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/AdguardTeam/golibs/testutil"
"github.com/AdguardTeam/golibs/timeutil"
"github.com/miekg/dns"
@ -215,13 +215,15 @@ func TestShouldCount(t *testing.T) {
ignored1 = "ignor.ed"
ignored2 = "ignored.to"
)
set := stringutil.NewSet(ignored1, ignored2)
ignored := []string{ignored1, ignored2}
engine, err := aghnet.NewIgnoreEngine(ignored)
require.NoError(t, err)
s, err := stats.New(stats.Config{
Enabled: true,
Filename: filepath.Join(t.TempDir(), "stats.db"),
Limit: timeutil.Day,
Ignored: set,
Ignored: engine,
ShouldCountClient: func(ids []string) (a bool) {
return ids[0] != "no_count"
},

4
internal/stats/unit.go
View File

@ -7,9 +7,9 @@ import (
"fmt"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/stringutil"
"go.etcd.io/bbolt"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
@ -370,7 +370,7 @@ type pairsGetter func(u *unitDB) (pairs []countPair)
// topsCollector collects statistics about highest values from the given *unitDB
// slice using pg to retrieve data.
func topsCollector(units []*unitDB, max int, ignored *stringutil.Set, pg pairsGetter) []map[string]uint64 {
func topsCollector(units []*unitDB, max int, ignored *aghnet.IgnoreEngine, pg pairsGetter) []map[string]uint64 {
m := map[string]uint64{}
for _, u := range units {
for _, cp := range pg(u) {