mirror of
https://github.com/AdguardTeam/AdGuardHome.git
synced 2024-08-16 22:20:30 +03:00
Pull request 1983: 5720-wildcard-ignored-domains
Updates #5720. Squashed commit of the following: commit e8093c990f15e2efc496f1a04f87360825e34e96 Merge: df5413eef28fefaff1
Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Tue Sep 5 15:06:33 2023 +0300 Merge branch 'master' into 5720-wildcard-ignored-domains commit df5413eefeac2c7e34eb725db9e2908b5b2d08cb Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Tue Sep 5 14:49:05 2023 +0300 confmigrate: imp docs commit 1644d99b730cc7f22c9d75b8e990149d3ce5b32a Merge: 9542ee1611e4517898
Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Tue Sep 5 14:23:42 2023 +0300 Merge branch 'master' into 5720-wildcard-ignored-domains commit 9542ee1616c1dd4bdb6ec9a2af79a2af3858a7e3 Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Tue Sep 5 12:48:48 2023 +0300 all: upd chlog commit 183f84a7f73c7bd33669bd108076f60514ca101e Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Fri Sep 1 17:11:31 2023 +0300 all: imp chlog commit a704325352a577a9b6652f011b82180ec3a6e095 Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Thu Aug 31 18:59:52 2023 +0300 all: imp code commit fe99c3b883500850399b1feb72c914ab878b3107 Merge: 7f11e94600182b9ec1
Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Thu Aug 31 18:43:09 2023 +0300 Merge branch 'master' into 5720-wildcard-ignored-domains commit 7f11e94609027ed821a125d27a1ffde03f37334a Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Wed Aug 30 19:57:51 2023 +0300 aghnet: add tests commit f10f9190ce1064a5d03155e8b6bba61db977897b Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Wed Aug 30 18:32:07 2023 +0300 all: add conf migration commit a53c14df129765366966c5230dd53aa29bdd25c5 Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Wed Aug 30 13:37:30 2023 +0300 all: add ignore engine
This commit is contained in:
parent
28fefaff1a
commit
42291cd547
@ -25,6 +25,8 @@ NOTE: Add new changes BELOW THIS COMMENT.
|
||||
|
||||
### Added
|
||||
|
||||
- AdBlock-style syntax support for ignored domains in logs and statistics
|
||||
([#5720]).
|
||||
- [`Strict-Transport-Security`][hsts] header in the HTTP API and DNS-over-HTTPS
|
||||
responses when HTTPS is forced ([#2998]). See [RFC 6979][rfc6797].
|
||||
- UI for the schedule of the service-blocking pause ([#951]).
|
||||
@ -50,7 +52,11 @@ NOTE: Add new changes BELOW THIS COMMENT.
|
||||
|
||||
#### Configuration Changes
|
||||
|
||||
In this release, the schema version has changed from 24 to 26.
|
||||
In this release, the schema version has changed from 24 to 27.
|
||||
|
||||
- Ignore rules blocking `.` in `querylog.…` and `stats.…` have been migrated to
|
||||
AdBlock syntax (`|.^`). To rollback this change, restore the rules and
|
||||
change the `schema_version` back to `26`.
|
||||
|
||||
- Filtering-related settings have been moved from `dns` section of the YAML
|
||||
configuration file to the new section `filtering`:
|
||||
@ -156,6 +162,7 @@ In this release, the schema version has changed from 24 to 26.
|
||||
[#1453]: https://github.com/AdguardTeam/AdGuardHome/issues/1453
|
||||
[#2998]: https://github.com/AdguardTeam/AdGuardHome/issues/2998
|
||||
[#3701]: https://github.com/AdguardTeam/AdGuardHome/issues/3701
|
||||
[#5720]: https://github.com/AdguardTeam/AdGuardHome/issues/5720
|
||||
[#5793]: https://github.com/AdguardTeam/AdGuardHome/issues/5793
|
||||
[#5948]: https://github.com/AdguardTeam/AdGuardHome/issues/5948
|
||||
[#6020]: https://github.com/AdguardTeam/AdGuardHome/issues/6020
|
||||
|
@ -537,8 +537,8 @@
|
||||
"statistics_enable": "Enable statistics",
|
||||
"ignore_domains": "Ignored domains (separated by newline)",
|
||||
"ignore_domains_title": "Ignored domains",
|
||||
"ignore_domains_desc_stats": "Queries for these domains are not written to the statistics",
|
||||
"ignore_domains_desc_query": "Queries for these domains are not written to the query log",
|
||||
"ignore_domains_desc_stats": "Queries matching these rules are not written to the statistics",
|
||||
"ignore_domains_desc_query": "Queries matching these rules are not written to the query log",
|
||||
"interval_hours": "{{count}} hour",
|
||||
"interval_hours_plural": "{{count}} hours",
|
||||
"filters_configuration": "Filters configuration",
|
||||
|
56
internal/aghnet/ignore.go
Normal file
56
internal/aghnet/ignore.go
Normal file
@ -0,0 +1,56 @@
|
||||
package aghnet
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/AdguardTeam/urlfilter"
|
||||
"github.com/AdguardTeam/urlfilter/filterlist"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
// IgnoreEngine contains the list of rules for ignoring hostnames and matches
|
||||
// them.
|
||||
//
|
||||
// TODO(s.chzhen): Move all urlfilter stuff to aghfilter.
|
||||
type IgnoreEngine struct {
|
||||
// engine is the filtering engine that can match rules for ignoring
|
||||
// hostnames.
|
||||
engine *urlfilter.DNSEngine
|
||||
|
||||
// ignored is the list of rules for ignoring hostnames.
|
||||
ignored []string
|
||||
}
|
||||
|
||||
// NewIgnoreEngine creates a new instance of the IgnoreEngine and stores the
|
||||
// list of rules for ignoring hostnames.
|
||||
func NewIgnoreEngine(ignored []string) (e *IgnoreEngine, err error) {
|
||||
ruleList := &filterlist.StringRuleList{
|
||||
RulesText: strings.ToLower(strings.Join(ignored, "\n")),
|
||||
IgnoreCosmetic: true,
|
||||
}
|
||||
ruleStorage, err := filterlist.NewRuleStorage([]filterlist.RuleList{ruleList})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &IgnoreEngine{
|
||||
engine: urlfilter.NewDNSEngine(ruleStorage),
|
||||
ignored: ignored,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Has returns true if IgnoreEngine matches the host.
|
||||
func (e *IgnoreEngine) Has(host string) (ignore bool) {
|
||||
if e == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
_, ignore = e.engine.Match(host)
|
||||
|
||||
return ignore
|
||||
}
|
||||
|
||||
// Values returns a copy of list of rules for ignoring hostnames.
|
||||
func (e *IgnoreEngine) Values() (ignored []string) {
|
||||
return slices.Clone(e.ignored)
|
||||
}
|
46
internal/aghnet/ignore_test.go
Normal file
46
internal/aghnet/ignore_test.go
Normal file
@ -0,0 +1,46 @@
|
||||
package aghnet_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestIgnoreEngine_Has(t *testing.T) {
|
||||
hostnames := []string{
|
||||
"*.example.com",
|
||||
"example.com",
|
||||
"|.^",
|
||||
}
|
||||
|
||||
engine, err := aghnet.NewIgnoreEngine(hostnames)
|
||||
require.NotNil(t, engine)
|
||||
require.NoError(t, err)
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
host string
|
||||
ignore bool
|
||||
}{{
|
||||
name: "basic",
|
||||
host: "example.com",
|
||||
ignore: true,
|
||||
}, {
|
||||
name: "root",
|
||||
host: ".",
|
||||
ignore: true,
|
||||
}, {
|
||||
name: "wildcard",
|
||||
host: "www.example.com",
|
||||
ignore: true,
|
||||
}, {
|
||||
name: "not_ignored",
|
||||
host: "something.com",
|
||||
ignore: false,
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
require.Equal(t, tc.ignore, engine.Has(tc.host))
|
||||
}
|
||||
}
|
@ -1563,3 +1563,86 @@ func TestUpgradeSchema25to26(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpgradeSchema26to27(t *testing.T) {
|
||||
const newSchemaVer = 27
|
||||
|
||||
testCases := []struct {
|
||||
in yobj
|
||||
want yobj
|
||||
name string
|
||||
}{{
|
||||
name: "empty",
|
||||
in: yobj{},
|
||||
want: yobj{
|
||||
"schema_version": newSchemaVer,
|
||||
},
|
||||
}, {
|
||||
name: "single_dot",
|
||||
in: yobj{
|
||||
"querylog": yobj{
|
||||
"ignored": yarr{
|
||||
".",
|
||||
},
|
||||
},
|
||||
"statistics": yobj{
|
||||
"ignored": yarr{
|
||||
".",
|
||||
},
|
||||
},
|
||||
},
|
||||
want: yobj{
|
||||
"querylog": yobj{
|
||||
"ignored": yarr{
|
||||
"|.^",
|
||||
},
|
||||
},
|
||||
"statistics": yobj{
|
||||
"ignored": yarr{
|
||||
"|.^",
|
||||
},
|
||||
},
|
||||
"schema_version": newSchemaVer,
|
||||
},
|
||||
}, {
|
||||
name: "mixed",
|
||||
in: yobj{
|
||||
"querylog": yobj{
|
||||
"ignored": yarr{
|
||||
".",
|
||||
"example.com",
|
||||
},
|
||||
},
|
||||
"statistics": yobj{
|
||||
"ignored": yarr{
|
||||
".",
|
||||
"example.org",
|
||||
},
|
||||
},
|
||||
},
|
||||
want: yobj{
|
||||
"querylog": yobj{
|
||||
"ignored": yarr{
|
||||
"|.^",
|
||||
"example.com",
|
||||
},
|
||||
},
|
||||
"statistics": yobj{
|
||||
"ignored": yarr{
|
||||
"|.^",
|
||||
"example.org",
|
||||
},
|
||||
},
|
||||
"schema_version": newSchemaVer,
|
||||
},
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := migrateTo27(tc.in)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, tc.want, tc.in)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
@ -10,7 +10,7 @@ import (
|
||||
)
|
||||
|
||||
// LastSchemaVersion is the most recent schema version.
|
||||
const LastSchemaVersion uint = 26
|
||||
const LastSchemaVersion uint = 27
|
||||
|
||||
// Config is a the configuration for initializing a [Migrator].
|
||||
type Config struct {
|
||||
@ -122,6 +122,7 @@ func (m *Migrator) upgradeConfigSchema(current, target uint, diskConf yobj) (err
|
||||
23: migrateTo24,
|
||||
24: migrateTo25,
|
||||
25: migrateTo26,
|
||||
26: migrateTo27,
|
||||
}
|
||||
|
||||
for i, migrate := range upgrades[current:target] {
|
||||
|
@ -181,6 +181,10 @@ func TestMigrateConfig_Migrate(t *testing.T) {
|
||||
yamlEqFunc: require.YAMLEq,
|
||||
name: "v26",
|
||||
targetVersion: 26,
|
||||
}, {
|
||||
yamlEqFunc: require.YAMLEq,
|
||||
name: "v27",
|
||||
targetVersion: 27,
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
|
123
internal/confmigrate/testdata/TestMigrateConfig_Migrate/v27/input.yml
vendored
Normal file
123
internal/confmigrate/testdata/TestMigrateConfig_Migrate/v27/input.yml
vendored
Normal file
@ -0,0 +1,123 @@
|
||||
http:
|
||||
address: 127.0.0.1:3000
|
||||
session_ttl: 3h
|
||||
pprof:
|
||||
enabled: true
|
||||
port: 6060
|
||||
users:
|
||||
- name: testuser
|
||||
password: testpassword
|
||||
dns:
|
||||
bind_hosts:
|
||||
- 127.0.0.1
|
||||
port: 53
|
||||
parental_sensitivity: 0
|
||||
upstream_dns:
|
||||
- tls://1.1.1.1
|
||||
- tls://1.0.0.1
|
||||
- quic://8.8.8.8:784
|
||||
bootstrap_dns:
|
||||
- 8.8.8.8:53
|
||||
edns_client_subnet:
|
||||
enabled: true
|
||||
use_custom: false
|
||||
custom_ip: ""
|
||||
filtering:
|
||||
filtering_enabled: true
|
||||
parental_enabled: false
|
||||
safebrowsing_enabled: false
|
||||
safe_search:
|
||||
enabled: false
|
||||
bing: true
|
||||
duckduckgo: true
|
||||
google: true
|
||||
pixabay: true
|
||||
yandex: true
|
||||
youtube: true
|
||||
protection_enabled: true
|
||||
blocked_services:
|
||||
schedule:
|
||||
time_zone: Local
|
||||
ids:
|
||||
- 500px
|
||||
blocked_response_ttl: 10
|
||||
filters:
|
||||
- url: https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt
|
||||
name: ""
|
||||
enabled: true
|
||||
- url: https://adaway.org/hosts.txt
|
||||
name: AdAway
|
||||
enabled: false
|
||||
- url: https://hosts-file.net/ad_servers.txt
|
||||
name: hpHosts - Ad and Tracking servers only
|
||||
enabled: false
|
||||
- url: http://www.malwaredomainlist.com/hostslist/hosts.txt
|
||||
name: MalwareDomainList.com Hosts List
|
||||
enabled: false
|
||||
clients:
|
||||
persistent:
|
||||
- name: localhost
|
||||
ids:
|
||||
- 127.0.0.1
|
||||
- aa:aa:aa:aa:aa:aa
|
||||
use_global_settings: true
|
||||
use_global_blocked_services: true
|
||||
filtering_enabled: false
|
||||
parental_enabled: false
|
||||
safebrowsing_enabled: false
|
||||
safe_search:
|
||||
enabled: true
|
||||
bing: true
|
||||
duckduckgo: true
|
||||
google: true
|
||||
pixabay: true
|
||||
yandex: true
|
||||
youtube: true
|
||||
blocked_services:
|
||||
schedule:
|
||||
time_zone: Local
|
||||
ids:
|
||||
- 500px
|
||||
runtime_sources:
|
||||
whois: true
|
||||
arp: true
|
||||
rdns: true
|
||||
dhcp: true
|
||||
hosts: true
|
||||
dhcp:
|
||||
enabled: false
|
||||
interface_name: vboxnet0
|
||||
local_domain_name: local
|
||||
dhcpv4:
|
||||
gateway_ip: 192.168.0.1
|
||||
subnet_mask: 255.255.255.0
|
||||
range_start: 192.168.0.10
|
||||
range_end: 192.168.0.250
|
||||
lease_duration: 1234
|
||||
icmp_timeout_msec: 10
|
||||
schema_version: 26
|
||||
user_rules: []
|
||||
querylog:
|
||||
enabled: true
|
||||
file_enabled: true
|
||||
interval: 720h
|
||||
size_memory: 1000
|
||||
ignored:
|
||||
- '.'
|
||||
statistics:
|
||||
enabled: true
|
||||
interval: 240h
|
||||
ignored:
|
||||
- '.'
|
||||
os:
|
||||
group: ''
|
||||
rlimit_nofile: 123
|
||||
user: ''
|
||||
log:
|
||||
file: ""
|
||||
max_backups: 0
|
||||
max_size: 100
|
||||
max_age: 3
|
||||
compress: true
|
||||
local_time: false
|
||||
verbose: true
|
123
internal/confmigrate/testdata/TestMigrateConfig_Migrate/v27/output.yml
vendored
Normal file
123
internal/confmigrate/testdata/TestMigrateConfig_Migrate/v27/output.yml
vendored
Normal file
@ -0,0 +1,123 @@
|
||||
http:
|
||||
address: 127.0.0.1:3000
|
||||
session_ttl: 3h
|
||||
pprof:
|
||||
enabled: true
|
||||
port: 6060
|
||||
users:
|
||||
- name: testuser
|
||||
password: testpassword
|
||||
dns:
|
||||
bind_hosts:
|
||||
- 127.0.0.1
|
||||
port: 53
|
||||
parental_sensitivity: 0
|
||||
upstream_dns:
|
||||
- tls://1.1.1.1
|
||||
- tls://1.0.0.1
|
||||
- quic://8.8.8.8:784
|
||||
bootstrap_dns:
|
||||
- 8.8.8.8:53
|
||||
edns_client_subnet:
|
||||
enabled: true
|
||||
use_custom: false
|
||||
custom_ip: ""
|
||||
filtering:
|
||||
filtering_enabled: true
|
||||
parental_enabled: false
|
||||
safebrowsing_enabled: false
|
||||
safe_search:
|
||||
enabled: false
|
||||
bing: true
|
||||
duckduckgo: true
|
||||
google: true
|
||||
pixabay: true
|
||||
yandex: true
|
||||
youtube: true
|
||||
protection_enabled: true
|
||||
blocked_services:
|
||||
schedule:
|
||||
time_zone: Local
|
||||
ids:
|
||||
- 500px
|
||||
blocked_response_ttl: 10
|
||||
filters:
|
||||
- url: https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt
|
||||
name: ""
|
||||
enabled: true
|
||||
- url: https://adaway.org/hosts.txt
|
||||
name: AdAway
|
||||
enabled: false
|
||||
- url: https://hosts-file.net/ad_servers.txt
|
||||
name: hpHosts - Ad and Tracking servers only
|
||||
enabled: false
|
||||
- url: http://www.malwaredomainlist.com/hostslist/hosts.txt
|
||||
name: MalwareDomainList.com Hosts List
|
||||
enabled: false
|
||||
clients:
|
||||
persistent:
|
||||
- name: localhost
|
||||
ids:
|
||||
- 127.0.0.1
|
||||
- aa:aa:aa:aa:aa:aa
|
||||
use_global_settings: true
|
||||
use_global_blocked_services: true
|
||||
filtering_enabled: false
|
||||
parental_enabled: false
|
||||
safebrowsing_enabled: false
|
||||
safe_search:
|
||||
enabled: true
|
||||
bing: true
|
||||
duckduckgo: true
|
||||
google: true
|
||||
pixabay: true
|
||||
yandex: true
|
||||
youtube: true
|
||||
blocked_services:
|
||||
schedule:
|
||||
time_zone: Local
|
||||
ids:
|
||||
- 500px
|
||||
runtime_sources:
|
||||
whois: true
|
||||
arp: true
|
||||
rdns: true
|
||||
dhcp: true
|
||||
hosts: true
|
||||
dhcp:
|
||||
enabled: false
|
||||
interface_name: vboxnet0
|
||||
local_domain_name: local
|
||||
dhcpv4:
|
||||
gateway_ip: 192.168.0.1
|
||||
subnet_mask: 255.255.255.0
|
||||
range_start: 192.168.0.10
|
||||
range_end: 192.168.0.250
|
||||
lease_duration: 1234
|
||||
icmp_timeout_msec: 10
|
||||
schema_version: 27
|
||||
user_rules: []
|
||||
querylog:
|
||||
enabled: true
|
||||
file_enabled: true
|
||||
interval: 720h
|
||||
size_memory: 1000
|
||||
ignored:
|
||||
- '|.^'
|
||||
statistics:
|
||||
enabled: true
|
||||
interval: 240h
|
||||
ignored:
|
||||
- '|.^'
|
||||
os:
|
||||
group: ''
|
||||
rlimit_nofile: 123
|
||||
user: ''
|
||||
log:
|
||||
file: ""
|
||||
max_backups: 0
|
||||
max_size: 100
|
||||
max_age: 3
|
||||
compress: true
|
||||
local_time: false
|
||||
verbose: true
|
77
internal/confmigrate/v27.go
Normal file
77
internal/confmigrate/v27.go
Normal file
@ -0,0 +1,77 @@
|
||||
package confmigrate
|
||||
|
||||
// migrateTo27 performs the following changes:
|
||||
//
|
||||
// # BEFORE:
|
||||
// 'querylog':
|
||||
// 'ignored':
|
||||
// - '.'
|
||||
// - # …
|
||||
// # …
|
||||
// 'statistics':
|
||||
// 'ignored':
|
||||
// - '.'
|
||||
// - # …
|
||||
// # …
|
||||
// # …
|
||||
//
|
||||
// # AFTER:
|
||||
// 'querylog':
|
||||
// 'ignored':
|
||||
// - '|.^'
|
||||
// - # …
|
||||
// # …
|
||||
// 'statistics':
|
||||
// 'ignored':
|
||||
// - '|.^'
|
||||
// - # …
|
||||
// # …
|
||||
// # …
|
||||
func migrateTo27(diskConf yobj) (err error) {
|
||||
diskConf["schema_version"] = 27
|
||||
|
||||
keys := []string{"querylog", "statistics"}
|
||||
for _, k := range keys {
|
||||
err = replaceDot(diskConf, k)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// replaceDot replaces rules blocking root domain "." with AdBlock style syntax
|
||||
// "|.^".
|
||||
func replaceDot(diskConf yobj, key string) (err error) {
|
||||
var obj yobj
|
||||
var ok bool
|
||||
obj, ok, err = fieldVal[yobj](diskConf, key)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
var ignored yarr
|
||||
ignored, ok, err = fieldVal[yarr](obj, "ignored")
|
||||
if err != nil {
|
||||
return err
|
||||
} else if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
for i, hostVal := range ignored {
|
||||
var host string
|
||||
host, ok = hostVal.(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
if host == "." {
|
||||
ignored[i] = "|.^"
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
@ -28,6 +28,7 @@ type accessManager struct {
|
||||
allowedClientIDs *stringutil.Set
|
||||
blockedClientIDs *stringutil.Set
|
||||
|
||||
// TODO(s.chzhen): Use [aghnet.IgnoreEngine].
|
||||
blockedHostsEng *urlfilter.DNSEngine
|
||||
|
||||
// TODO(a.garipov): Create a type for a set of IP networks.
|
||||
|
@ -22,7 +22,6 @@ import (
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/timeutil"
|
||||
"github.com/google/renameio/v2/maybe"
|
||||
"golang.org/x/exp/slices"
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
@ -594,7 +593,6 @@ func (c *configuration) write() (err error) {
|
||||
config.Stats.Interval = timeutil.Duration{Duration: statsConf.Limit}
|
||||
config.Stats.Enabled = statsConf.Enabled
|
||||
config.Stats.Ignored = statsConf.Ignored.Values()
|
||||
slices.Sort(config.Stats.Ignored)
|
||||
}
|
||||
|
||||
if Context.queryLog != nil {
|
||||
@ -606,7 +604,6 @@ func (c *configuration) write() (err error) {
|
||||
config.QueryLog.Interval = timeutil.Duration{Duration: dc.RotationIvl}
|
||||
config.QueryLog.MemSize = dc.MemSize
|
||||
config.QueryLog.Ignored = dc.Ignored.Values()
|
||||
slices.Sort(config.Stats.Ignored)
|
||||
}
|
||||
|
||||
if Context.filters != nil {
|
||||
|
@ -59,12 +59,12 @@ func initDNS() (err error) {
|
||||
ShouldCountClient: Context.clients.shouldCountClient,
|
||||
}
|
||||
|
||||
set, err := aghnet.NewDomainNameSet(config.Stats.Ignored)
|
||||
engine, err := aghnet.NewIgnoreEngine(config.Stats.Ignored)
|
||||
if err != nil {
|
||||
return fmt.Errorf("statistics: ignored list: %w", err)
|
||||
}
|
||||
|
||||
statsConf.Ignored = set
|
||||
statsConf.Ignored = engine
|
||||
Context.stats, err = stats.New(statsConf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init stats: %w", err)
|
||||
@ -83,12 +83,12 @@ func initDNS() (err error) {
|
||||
FileEnabled: config.QueryLog.FileEnabled,
|
||||
}
|
||||
|
||||
set, err = aghnet.NewDomainNameSet(config.QueryLog.Ignored)
|
||||
engine, err = aghnet.NewIgnoreEngine(config.QueryLog.Ignored)
|
||||
if err != nil {
|
||||
return fmt.Errorf("querylog: ignored list: %w", err)
|
||||
}
|
||||
|
||||
conf.Ignored = set
|
||||
conf.Ignored = engine
|
||||
Context.queryLog, err = querylog.New(conf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init querylog: %w", err)
|
||||
|
@ -17,7 +17,6 @@ import (
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/stringutil"
|
||||
"github.com/AdguardTeam/golibs/timeutil"
|
||||
"golang.org/x/exp/slices"
|
||||
"golang.org/x/net/idna"
|
||||
)
|
||||
|
||||
@ -141,8 +140,6 @@ func (l *queryLog) handleGetQueryLogConfig(w http.ResponseWriter, r *http.Reques
|
||||
}
|
||||
}()
|
||||
|
||||
slices.Sort(resp.Ignored)
|
||||
|
||||
aghhttp.WriteJSONResponseOK(w, r, resp)
|
||||
}
|
||||
|
||||
@ -224,7 +221,7 @@ func (l *queryLog) handlePutQueryLogConfig(w http.ResponseWriter, r *http.Reques
|
||||
return
|
||||
}
|
||||
|
||||
set, err := aghnet.NewDomainNameSet(newConf.Ignored)
|
||||
engine, err := aghnet.NewIgnoreEngine(newConf.Ignored)
|
||||
if err != nil {
|
||||
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "ignored: %s", err)
|
||||
|
||||
@ -258,7 +255,7 @@ func (l *queryLog) handlePutQueryLogConfig(w http.ResponseWriter, r *http.Reques
|
||||
|
||||
conf := *l.conf
|
||||
|
||||
conf.Ignored = set
|
||||
conf.Ignored = engine
|
||||
conf.RotationIvl = ivl
|
||||
conf.Enabled = newConf.Enabled == aghalg.NBTrue
|
||||
|
||||
|
@ -127,7 +127,6 @@ func (l *queryLog) WriteDiskConfig(c *Config) {
|
||||
defer l.confMu.RUnlock()
|
||||
|
||||
*c = *l.conf
|
||||
c.Ignored = l.conf.Ignored.Clone()
|
||||
}
|
||||
|
||||
// Clear memory buffer and remove log files
|
||||
|
@ -5,8 +5,8 @@ import (
|
||||
"net"
|
||||
"testing"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
|
||||
"github.com/AdguardTeam/golibs/stringutil"
|
||||
"github.com/AdguardTeam/golibs/testutil"
|
||||
"github.com/AdguardTeam/golibs/timeutil"
|
||||
"github.com/miekg/dns"
|
||||
@ -256,10 +256,21 @@ func TestQueryLogFileDisabled(t *testing.T) {
|
||||
|
||||
func TestQueryLogShouldLog(t *testing.T) {
|
||||
const (
|
||||
ignored1 = "ignor.ed"
|
||||
ignored2 = "ignored.to"
|
||||
ignored1 = "ignor.ed"
|
||||
ignored2 = "ignored.to"
|
||||
ignoredWildcard = "*.ignored.com"
|
||||
ignoredRoot = "|.^"
|
||||
)
|
||||
set := stringutil.NewSet(ignored1, ignored2)
|
||||
|
||||
ignored := []string{
|
||||
ignored1,
|
||||
ignored2,
|
||||
ignoredWildcard,
|
||||
ignoredRoot,
|
||||
}
|
||||
|
||||
engine, err := aghnet.NewIgnoreEngine(ignored)
|
||||
require.NoError(t, err)
|
||||
|
||||
findClient := func(ids []string) (c *Client, err error) {
|
||||
log := ids[0] == "no_log"
|
||||
@ -268,7 +279,7 @@ func TestQueryLogShouldLog(t *testing.T) {
|
||||
}
|
||||
|
||||
l, err := newQueryLog(Config{
|
||||
Ignored: set,
|
||||
Ignored: engine,
|
||||
Enabled: true,
|
||||
RotationIvl: timeutil.Day,
|
||||
MemSize: 100,
|
||||
@ -297,6 +308,16 @@ func TestQueryLogShouldLog(t *testing.T) {
|
||||
host: ignored2,
|
||||
ids: []string{"whatever"},
|
||||
wantLog: false,
|
||||
}, {
|
||||
name: "no_log_ignored_wildcard",
|
||||
host: "www.ignored.com",
|
||||
ids: []string{"whatever"},
|
||||
wantLog: false,
|
||||
}, {
|
||||
name: "no_log_ignored_root",
|
||||
host: ".",
|
||||
ids: []string{"whatever"},
|
||||
wantLog: false,
|
||||
}, {
|
||||
name: "no_log_client_ignore",
|
||||
host: "example.com",
|
||||
|
@ -11,7 +11,6 @@ import (
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
"github.com/AdguardTeam/golibs/stringutil"
|
||||
"github.com/miekg/dns"
|
||||
)
|
||||
|
||||
@ -36,8 +35,9 @@ type QueryLog interface {
|
||||
//
|
||||
// Do not alter any fields of this structure after using it.
|
||||
type Config struct {
|
||||
// Ignored is the list of host names, which should not be written to log.
|
||||
Ignored *stringutil.Set
|
||||
// Ignored contains the list of host names, which should not be written to
|
||||
// log, and matches them.
|
||||
Ignored *aghnet.IgnoreEngine
|
||||
|
||||
// Anonymizer processes the IP addresses to anonymize those if needed.
|
||||
Anonymizer *aghnet.IPMut
|
||||
|
@ -12,7 +12,6 @@ import (
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/timeutil"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
// topAddrs is an alias for the types of the TopFoo fields of statsResponse.
|
||||
@ -140,8 +139,6 @@ func (s *StatsCtx) handleGetStatsConfig(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
}()
|
||||
|
||||
slices.Sort(resp.Ignored)
|
||||
|
||||
aghhttp.WriteJSONResponseOK(w, r, resp)
|
||||
}
|
||||
|
||||
@ -184,7 +181,7 @@ func (s *StatsCtx) handlePutStatsConfig(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
set, err := aghnet.NewDomainNameSet(reqData.Ignored)
|
||||
engine, err := aghnet.NewIgnoreEngine(reqData.Ignored)
|
||||
if err != nil {
|
||||
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "ignored: %s", err)
|
||||
|
||||
@ -210,7 +207,7 @@ func (s *StatsCtx) handlePutStatsConfig(w http.ResponseWriter, r *http.Request)
|
||||
s.confMu.Lock()
|
||||
defer s.confMu.Unlock()
|
||||
|
||||
s.ignored = set
|
||||
s.ignored = engine
|
||||
s.limit = ivl
|
||||
s.enabled = reqData.Enabled == aghalg.NBTrue
|
||||
}
|
||||
|
@ -75,29 +75,6 @@ func TestHandleStatsConfig(t *testing.T) {
|
||||
},
|
||||
wantCode: http.StatusOK,
|
||||
wantErr: "",
|
||||
}, {
|
||||
name: "ignored_duplicate",
|
||||
body: getConfigResp{
|
||||
Enabled: aghalg.NBTrue,
|
||||
Interval: float64(minIvl.Milliseconds()),
|
||||
Ignored: []string{
|
||||
"ignor.ed",
|
||||
"ignor.ed",
|
||||
},
|
||||
},
|
||||
wantCode: http.StatusUnprocessableEntity,
|
||||
wantErr: "ignored: duplicate hostname \"ignor.ed\" at index 1\n",
|
||||
}, {
|
||||
name: "ignored_empty",
|
||||
body: getConfigResp{
|
||||
Enabled: aghalg.NBTrue,
|
||||
Interval: float64(minIvl.Milliseconds()),
|
||||
Ignored: []string{
|
||||
"",
|
||||
},
|
||||
},
|
||||
wantCode: http.StatusUnprocessableEntity,
|
||||
wantErr: "ignored: at index 0: hostname is empty\n",
|
||||
}, {
|
||||
name: "enabled_is_null",
|
||||
body: getConfigResp{
|
||||
|
@ -12,9 +12,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/stringutil"
|
||||
"github.com/AdguardTeam/golibs/timeutil"
|
||||
"go.etcd.io/bbolt"
|
||||
)
|
||||
@ -58,8 +58,9 @@ type Config struct {
|
||||
// endpoints.
|
||||
HTTPRegister aghhttp.RegisterFunc
|
||||
|
||||
// Ignored is the list of host names, which should not be counted.
|
||||
Ignored *stringutil.Set
|
||||
// Ignored contains the list of host names, which should not be counted,
|
||||
// and matches them.
|
||||
Ignored *aghnet.IgnoreEngine
|
||||
|
||||
// Filename is the name of the database file.
|
||||
Filename string
|
||||
@ -117,8 +118,9 @@ type StatsCtx struct {
|
||||
// confMu protects ignored, limit, and enabled.
|
||||
confMu *sync.RWMutex
|
||||
|
||||
// ignored is the list of host names, which should not be counted.
|
||||
ignored *stringutil.Set
|
||||
// ignored contains the list of host names, which should not be counted,
|
||||
// and matches them.
|
||||
ignored *aghnet.IgnoreEngine
|
||||
|
||||
// shouldCountClient returns client's ignore setting.
|
||||
shouldCountClient func([]string) bool
|
||||
@ -289,7 +291,7 @@ func (s *StatsCtx) WriteDiskConfig(dc *Config) {
|
||||
s.confMu.RLock()
|
||||
defer s.confMu.RUnlock()
|
||||
|
||||
dc.Ignored = s.ignored.Clone()
|
||||
dc.Ignored = s.ignored
|
||||
dc.Limit = s.limit
|
||||
dc.Enabled = s.enabled
|
||||
}
|
||||
|
@ -11,9 +11,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/stats"
|
||||
"github.com/AdguardTeam/golibs/netutil"
|
||||
"github.com/AdguardTeam/golibs/stringutil"
|
||||
"github.com/AdguardTeam/golibs/testutil"
|
||||
"github.com/AdguardTeam/golibs/timeutil"
|
||||
"github.com/miekg/dns"
|
||||
@ -215,13 +215,15 @@ func TestShouldCount(t *testing.T) {
|
||||
ignored1 = "ignor.ed"
|
||||
ignored2 = "ignored.to"
|
||||
)
|
||||
set := stringutil.NewSet(ignored1, ignored2)
|
||||
ignored := []string{ignored1, ignored2}
|
||||
engine, err := aghnet.NewIgnoreEngine(ignored)
|
||||
require.NoError(t, err)
|
||||
|
||||
s, err := stats.New(stats.Config{
|
||||
Enabled: true,
|
||||
Filename: filepath.Join(t.TempDir(), "stats.db"),
|
||||
Limit: timeutil.Day,
|
||||
Ignored: set,
|
||||
Ignored: engine,
|
||||
ShouldCountClient: func(ids []string) (a bool) {
|
||||
return ids[0] != "no_count"
|
||||
},
|
||||
|
@ -7,9 +7,9 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/stringutil"
|
||||
"go.etcd.io/bbolt"
|
||||
"golang.org/x/exp/maps"
|
||||
"golang.org/x/exp/slices"
|
||||
@ -370,7 +370,7 @@ type pairsGetter func(u *unitDB) (pairs []countPair)
|
||||
|
||||
// topsCollector collects statistics about highest values from the given *unitDB
|
||||
// slice using pg to retrieve data.
|
||||
func topsCollector(units []*unitDB, max int, ignored *stringutil.Set, pg pairsGetter) []map[string]uint64 {
|
||||
func topsCollector(units []*unitDB, max int, ignored *aghnet.IgnoreEngine, pg pairsGetter) []map[string]uint64 {
|
||||
m := map[string]uint64{}
|
||||
for _, u := range units {
|
||||
for _, cp := range pg(u) {
|
||||
|
Loading…
Reference in New Issue
Block a user