mirror of
https://github.com/AdguardTeam/AdGuardHome.git
synced 2024-08-16 22:20:30 +03:00
Pull request 1731: 4299-stats-ignore
Merge in DNS/adguard-home from 4299-stats-ignore to master Updates #1717. Updates #4299. Squashed commit of the following: commit 1d1212d088c944e995deae2fd599eccb0a075033 Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Mon Feb 13 17:53:36 2023 +0300 fix changelog commit 5f56852c21d794bd87c13192d3857757be10f9b2 Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Mon Feb 13 17:39:02 2023 +0300 add todo; fix data race commit 89b8b16ddf5a43ebf68174cbaf9e8a53365f8cbe Merge: e0a6bb49ec19a85e
Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Fri Feb 10 17:21:38 2023 +0300 Merge branch 'master' into 4299-stats-ignore commit e0a6bb490b651d1cf31589a7f17095fff4cb4dbb Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Fri Feb 10 17:21:06 2023 +0300 interval under mutex commit c569c7bc237f11b23fe47c98a20a1c5cb36751cb Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Fri Feb 10 16:19:35 2023 +0300 fix mutex commit 9374cf0c54dccc2fbfc38765b52c64e1c479137c Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Fri Feb 10 16:03:17 2023 +0300 fix typo commit 1f4fd1e7ab1b3c2f8e9c3d32ef7e4958f99abb47 Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Fri Feb 10 15:55:44 2023 +0300 add mutex commit 2148048ce9ad228381cbb51a806c9b9cc21458fd Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Fri Feb 10 12:27:36 2023 +0300 add key check commit a19350977c463f888aea70d0dace26dff0173a65 Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Thu Feb 9 18:34:36 2023 +0300 fix changelog commit 23c3b6da162dfd513884b460c265ba4cafeb9727 Merge: 8fccc0b8b89105e3
Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Thu Feb 9 13:28:59 2023 +0300 Merge branch 'master' into 4299-stats-ignore commit 8fccc0b8ec670a37e5209d795f35c43dd64afeb3 Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Thu Feb 9 13:27:42 2023 +0300 add changelog commit 0416c71742795b2fb8adb0173dcd6a99d9d9c676 Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Wed Feb 8 14:31:55 2023 +0300 all: stats ignore
This commit is contained in:
parent
ec19a85ed0
commit
ff04b2a7d3
30
CHANGELOG.md
30
CHANGELOG.md
@ -25,16 +25,38 @@ NOTE: Add new changes BELOW THIS COMMENT.
|
||||
|
||||
### Added
|
||||
|
||||
- The ability to exclude domain names from the query log by using the new
|
||||
`querylog.ignored` field ([#1717], [#4299]).
|
||||
- The ability to disable statistics by using the new `statistics.enabled`
|
||||
field. Previously it was necessary to set the `statistics_interval` to 0,
|
||||
losing the previous value ([#1717], [#4299]).
|
||||
- The ability to exclude domain names from the query log or statistics by using
|
||||
the new `querylog.ignored` or `statistics.ignored` fields ([#1717], [#4299]).
|
||||
|
||||
### Changed
|
||||
|
||||
#### Configuration Changes
|
||||
|
||||
In this release, the schema version has changed from 14 to 15.
|
||||
In this release, the schema version has changed from 14 to 16.
|
||||
|
||||
- The fields `dns.…` have been moved to the new `querylog` object.
|
||||
- Property `statistics_interval`, which in schema versions 15 and earlier used
|
||||
to be a part of the `dns` object, is now a part of the `statistics` object:
|
||||
|
||||
```yaml
|
||||
# BEFORE:
|
||||
'dns':
|
||||
# …
|
||||
'statistics_interval': 1
|
||||
|
||||
# AFTER:
|
||||
'statistics':
|
||||
# …
|
||||
'interval': 1
|
||||
```
|
||||
|
||||
To rollback this change, move the property back into the `dns` object and
|
||||
change the `schema_version` back to `15`.
|
||||
- The fields `dns.querylog_enabled`, `dns.querylog_file_enabled`,
|
||||
`dns.querylog_interval`, `dns.querylog_size_memory` have been moved to the
|
||||
new `querylog` object.
|
||||
|
||||
```yaml
|
||||
# BEFORE:
|
||||
|
@ -48,10 +48,15 @@ func (s *Server) processQueryLogsAndStats(dctx *dnsContext) (rc resultCode) {
|
||||
s.queryLog.ShouldLog(host, q.Qtype, q.Qclass) {
|
||||
s.logQuery(dctx, pctx, elapsed, ip)
|
||||
} else {
|
||||
log.Debug("request for %s from %s ignored; not logging", host, ip)
|
||||
log.Debug(
|
||||
"dnsforward: request %s %s from %s ignored; not logging",
|
||||
dns.Type(q.Qtype),
|
||||
host,
|
||||
ip,
|
||||
)
|
||||
}
|
||||
|
||||
if s.stats != nil {
|
||||
if s.stats != nil && s.stats.ShouldCount(host, q.Qtype, q.Qclass) {
|
||||
s.updateStats(dctx, elapsed, *dctx.result, ip)
|
||||
}
|
||||
|
||||
|
@ -35,20 +35,25 @@ func (l *testQueryLog) ShouldLog(string, uint16, uint16) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// testStats is a simple stats.Stats implementation for tests.
|
||||
// testStats is a simple [stats.Interface] implementation for tests.
|
||||
type testStats struct {
|
||||
// Stats is embedded here simply to make testStats a stats.Stats without
|
||||
// actually implementing all methods.
|
||||
// Stats is embedded here simply to make testStats a [stats.Interface]
|
||||
// without actually implementing all methods.
|
||||
stats.Interface
|
||||
|
||||
lastEntry stats.Entry
|
||||
}
|
||||
|
||||
// Update implements the stats.Stats interface for *testStats.
|
||||
// Update implements the [stats.Interface] interface for *testStats.
|
||||
func (l *testStats) Update(e stats.Entry) {
|
||||
l.lastEntry = e
|
||||
}
|
||||
|
||||
// ShouldCount implements the [stats.Interface] interface for *testStats.
|
||||
func (l *testStats) ShouldCount(string, uint16, uint16) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func TestProcessQueryLogsAndStats(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
|
@ -116,6 +116,7 @@ type configuration struct {
|
||||
DNS dnsConfig `yaml:"dns"`
|
||||
TLS tlsConfigSettings `yaml:"tls"`
|
||||
QueryLog queryLogConfig `yaml:"querylog"`
|
||||
Stats statsConfig `yaml:"statistics"`
|
||||
|
||||
// Filters reflects the filters from [filtering.Config]. It's cloned to the
|
||||
// config used in the filtering module at the startup. Afterwards it's
|
||||
@ -149,10 +150,6 @@ type dnsConfig struct {
|
||||
BindHosts []netip.Addr `yaml:"bind_hosts"`
|
||||
Port int `yaml:"port"`
|
||||
|
||||
// StatsInterval is the time interval for flushing statistics to the disk in
|
||||
// days.
|
||||
StatsInterval uint32 `yaml:"statistics_interval"`
|
||||
|
||||
// AnonymizeClientIP defines if clients' IP addresses should be anonymized
|
||||
// in query log and statistics.
|
||||
AnonymizeClientIP bool `yaml:"anonymize_client_ip"`
|
||||
@ -234,8 +231,20 @@ type queryLogConfig struct {
|
||||
// flushed to disk.
|
||||
MemSize uint32 `yaml:"size_memory"`
|
||||
|
||||
// Ignored is the list of host names, which are should not be written
|
||||
// to log.
|
||||
// Ignored is the list of host names, which should not be written to
|
||||
// log.
|
||||
Ignored []string `yaml:"ignored"`
|
||||
}
|
||||
|
||||
type statsConfig struct {
|
||||
// Enabled defines if the statistics are enabled.
|
||||
Enabled bool `yaml:"enabled"`
|
||||
|
||||
// Interval is the time interval for flushing statistics to the disk in
|
||||
// days.
|
||||
Interval uint32 `yaml:"interval"`
|
||||
|
||||
// Ignored is the list of host names, which should not be counted.
|
||||
Ignored []string `yaml:"ignored"`
|
||||
}
|
||||
|
||||
@ -249,9 +258,8 @@ var config = &configuration{
|
||||
AuthBlockMin: 15,
|
||||
WebSessionTTLHours: 30 * 24,
|
||||
DNS: dnsConfig{
|
||||
BindHosts: []netip.Addr{netip.IPv4Unspecified()},
|
||||
Port: defaultPortDNS,
|
||||
StatsInterval: 1,
|
||||
BindHosts: []netip.Addr{netip.IPv4Unspecified()},
|
||||
Port: defaultPortDNS,
|
||||
FilteringConfig: dnsforward.FilteringConfig{
|
||||
ProtectionEnabled: true, // whether or not use any of filtering features
|
||||
BlockingMode: dnsforward.BlockingModeDefault,
|
||||
@ -296,6 +304,11 @@ var config = &configuration{
|
||||
MemSize: 1000,
|
||||
Ignored: []string{},
|
||||
},
|
||||
Stats: statsConfig{
|
||||
Enabled: true,
|
||||
Interval: 1,
|
||||
Ignored: []string{},
|
||||
},
|
||||
// NOTE: Keep these parameters in sync with the one put into
|
||||
// client/src/helpers/filters/filters.js by scripts/vetted-filters.
|
||||
//
|
||||
@ -472,9 +485,12 @@ func (c *configuration) write() (err error) {
|
||||
}
|
||||
|
||||
if Context.stats != nil {
|
||||
sdc := stats.DiskConfig{}
|
||||
Context.stats.WriteDiskConfig(&sdc)
|
||||
config.DNS.StatsInterval = sdc.Interval
|
||||
statsConf := stats.Config{}
|
||||
Context.stats.WriteDiskConfig(&statsConf)
|
||||
config.Stats.Interval = statsConf.LimitDays
|
||||
config.Stats.Enabled = statsConf.Enabled
|
||||
config.Stats.Ignored = statsConf.Ignored.Values()
|
||||
sort.Strings(config.Stats.Ignored)
|
||||
}
|
||||
|
||||
if Context.queryLog != nil {
|
||||
|
@ -53,10 +53,18 @@ func initDNS() (err error) {
|
||||
|
||||
statsConf := stats.Config{
|
||||
Filename: filepath.Join(baseDir, "stats.db"),
|
||||
LimitDays: config.DNS.StatsInterval,
|
||||
LimitDays: config.Stats.Interval,
|
||||
ConfigModified: onConfigModified,
|
||||
HTTPRegister: httpRegister,
|
||||
Enabled: config.Stats.Enabled,
|
||||
}
|
||||
|
||||
set, err := nonDupEmptyHostNames(config.Stats.Ignored)
|
||||
if err != nil {
|
||||
return fmt.Errorf("statistics: ignored list: %w", err)
|
||||
}
|
||||
|
||||
statsConf.Ignored = set
|
||||
Context.stats, err = stats.New(statsConf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init stats: %w", err)
|
||||
@ -73,16 +81,14 @@ func initDNS() (err error) {
|
||||
MemSize: config.QueryLog.MemSize,
|
||||
Enabled: config.QueryLog.Enabled,
|
||||
FileEnabled: config.QueryLog.FileEnabled,
|
||||
Ignored: stringutil.NewSet(),
|
||||
}
|
||||
for _, v := range config.QueryLog.Ignored {
|
||||
host := strings.ToLower(strings.TrimSuffix(v, "."))
|
||||
if conf.Ignored.Has(host) {
|
||||
return fmt.Errorf("duplicate ignored host %s", host)
|
||||
}
|
||||
|
||||
conf.Ignored.Add(host)
|
||||
set, err = nonDupEmptyHostNames(config.QueryLog.Ignored)
|
||||
if err != nil {
|
||||
return fmt.Errorf("querylog: ignored list: %w", err)
|
||||
}
|
||||
|
||||
conf.Ignored = set
|
||||
Context.queryLog = querylog.New(conf)
|
||||
|
||||
Context.filters, err = filtering.New(config.DNS.DnsfilterConf, nil)
|
||||
@ -526,3 +532,27 @@ func closeDNSServer() {
|
||||
|
||||
log.Debug("all dns modules are closed")
|
||||
}
|
||||
|
||||
// nonDupEmptyHostNames returns nil and error, if list has duplicate or empty
|
||||
// host name. Otherwise returns a set, which contains lowercase host names
|
||||
// without dot at the end, and nil error.
|
||||
func nonDupEmptyHostNames(list []string) (set *stringutil.Set, err error) {
|
||||
set = stringutil.NewSet()
|
||||
|
||||
for _, v := range list {
|
||||
host := strings.ToLower(strings.TrimSuffix(v, "."))
|
||||
// TODO(a.garipov): Think about ignoring empty (".") names in
|
||||
// the future.
|
||||
if host == "" {
|
||||
return nil, errors.Error("host name is empty")
|
||||
}
|
||||
|
||||
if set.Has(host) {
|
||||
return nil, fmt.Errorf("duplicate host name %q", host)
|
||||
}
|
||||
|
||||
set.Add(host)
|
||||
}
|
||||
|
||||
return set, nil
|
||||
}
|
||||
|
@ -22,7 +22,7 @@ import (
|
||||
)
|
||||
|
||||
// currentSchemaVersion is the current schema version.
|
||||
const currentSchemaVersion = 15
|
||||
const currentSchemaVersion = 16
|
||||
|
||||
// These aliases are provided for convenience.
|
||||
type (
|
||||
@ -88,6 +88,7 @@ func upgradeConfigSchema(oldVersion int, diskConf yobj) (err error) {
|
||||
upgradeSchema12to13,
|
||||
upgradeSchema13to14,
|
||||
upgradeSchema14to15,
|
||||
upgradeSchema15to16,
|
||||
}
|
||||
|
||||
n := 0
|
||||
@ -823,8 +824,12 @@ func upgradeSchema14to15(diskConf yobj) (err error) {
|
||||
log.Printf("Upgrade yaml: 14 to 15")
|
||||
diskConf["schema_version"] = 15
|
||||
|
||||
dnsVal := diskConf["dns"]
|
||||
dns, ok := dnsVal.(map[string]any)
|
||||
dnsVal, ok := diskConf["dns"]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
dns, ok := dnsVal.(yobj)
|
||||
if !ok {
|
||||
return fmt.Errorf("unexpected type of dns: %T", dnsVal)
|
||||
}
|
||||
@ -856,6 +861,50 @@ func upgradeSchema14to15(diskConf yobj) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// upgradeSchema15to16 performs the following changes:
|
||||
//
|
||||
// # BEFORE:
|
||||
// 'dns':
|
||||
// 'statistics_interval': 1
|
||||
//
|
||||
// # AFTER:
|
||||
// 'statistics':
|
||||
// 'enabled': true
|
||||
// 'interval': 1
|
||||
// 'ignored': []
|
||||
func upgradeSchema15to16(diskConf yobj) (err error) {
|
||||
log.Printf("Upgrade yaml: 15 to 16")
|
||||
diskConf["schema_version"] = 16
|
||||
|
||||
dnsVal, ok := diskConf["dns"]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
dns, ok := dnsVal.(yobj)
|
||||
if !ok {
|
||||
return fmt.Errorf("unexpected type of dns: %T", dnsVal)
|
||||
}
|
||||
|
||||
stats := map[string]any{
|
||||
"enabled": true,
|
||||
"interval": 1,
|
||||
"ignored": []any{},
|
||||
}
|
||||
|
||||
k := "statistics_interval"
|
||||
v, has := dns[k]
|
||||
if has {
|
||||
stats["enabled"] = v != 0
|
||||
stats["interval"] = v
|
||||
}
|
||||
delete(dns, k)
|
||||
|
||||
diskConf["statistics"] = stats
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// TODO(a.garipov): Replace with log.Output when we port it to our logging
|
||||
// package.
|
||||
func funcName() string {
|
||||
|
@ -688,3 +688,62 @@ func TestUpgradeSchema14to15(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpgradeSchema15to16(t *testing.T) {
|
||||
const newSchemaVer = 16
|
||||
|
||||
defaultWantObj := yobj{
|
||||
"statistics": map[string]any{
|
||||
"enabled": true,
|
||||
"interval": 1,
|
||||
"ignored": []any{},
|
||||
},
|
||||
"dns": map[string]any{},
|
||||
"schema_version": newSchemaVer,
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
in yobj
|
||||
want yobj
|
||||
name string
|
||||
}{{
|
||||
in: yobj{
|
||||
"dns": map[string]any{
|
||||
"statistics_interval": 1,
|
||||
},
|
||||
},
|
||||
want: defaultWantObj,
|
||||
name: "basic",
|
||||
}, {
|
||||
in: yobj{
|
||||
"dns": map[string]any{},
|
||||
},
|
||||
want: defaultWantObj,
|
||||
name: "default_values",
|
||||
}, {
|
||||
in: yobj{
|
||||
"dns": map[string]any{
|
||||
"statistics_interval": 0,
|
||||
},
|
||||
},
|
||||
want: yobj{
|
||||
"statistics": map[string]any{
|
||||
"enabled": false,
|
||||
"interval": 0,
|
||||
"ignored": []any{},
|
||||
},
|
||||
"dns": map[string]any{},
|
||||
"schema_version": newSchemaVer,
|
||||
},
|
||||
name: "stats_disabled",
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := upgradeSchema15to16(tc.in)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, tc.want, tc.in)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
@ -44,6 +44,9 @@ func (l *queryLog) initWeb() {
|
||||
}
|
||||
|
||||
func (l *queryLog) handleQueryLog(w http.ResponseWriter, r *http.Request) {
|
||||
l.lock.Lock()
|
||||
defer l.lock.Unlock()
|
||||
|
||||
params, err := l.parseSearchParams(r)
|
||||
if err != nil {
|
||||
aghhttp.Error(r, w, http.StatusBadRequest, "failed to parse params: %s", err)
|
||||
|
@ -76,8 +76,8 @@ type Config struct {
|
||||
// addresses.
|
||||
AnonymizeClientIP bool
|
||||
|
||||
// Ignored is the list of host names, which are should not be written
|
||||
// to log.
|
||||
// Ignored is the list of host names, which should not be written to
|
||||
// log.
|
||||
Ignored *stringutil.Set
|
||||
}
|
||||
|
||||
|
@ -155,6 +155,9 @@ func (l *queryLog) periodicRotate() {
|
||||
// checkAndRotate rotates log files if those are older than the specified
|
||||
// rotation interval.
|
||||
func (l *queryLog) checkAndRotate() {
|
||||
l.lock.Lock()
|
||||
defer l.lock.Unlock()
|
||||
|
||||
oldest, err := l.readFileFirstTimeValue()
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
log.Error("querylog: reading oldest record for rotation: %s", err)
|
||||
|
@ -5,7 +5,6 @@ package stats
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
|
||||
@ -41,10 +40,11 @@ type StatsResp struct {
|
||||
|
||||
// handleStats handles requests to the GET /control/stats endpoint.
|
||||
func (s *StatsCtx) handleStats(w http.ResponseWriter, r *http.Request) {
|
||||
limit := atomic.LoadUint32(&s.limitHours)
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
start := time.Now()
|
||||
resp, ok := s.getData(limit)
|
||||
resp, ok := s.getData(s.limitHours)
|
||||
log.Debug("stats: prepared data in %v", time.Since(start))
|
||||
|
||||
if !ok {
|
||||
@ -65,7 +65,13 @@ type configResp struct {
|
||||
|
||||
// handleStatsInfo handles requests to the GET /control/stats_info endpoint.
|
||||
func (s *StatsCtx) handleStatsInfo(w http.ResponseWriter, r *http.Request) {
|
||||
resp := configResp{IntervalDays: atomic.LoadUint32(&s.limitHours) / 24}
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
resp := configResp{IntervalDays: s.limitHours / 24}
|
||||
if !s.enabled {
|
||||
resp.IntervalDays = 0
|
||||
}
|
||||
_ = aghhttp.WriteJSONResponse(w, r, resp)
|
||||
}
|
||||
|
||||
|
@ -15,16 +15,10 @@ import (
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/stringutil"
|
||||
"go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
// DiskConfig is the configuration structure that is stored in file.
|
||||
type DiskConfig struct {
|
||||
// Interval is the number of days for which the statistics are collected
|
||||
// before flushing to the database.
|
||||
Interval uint32 `yaml:"statistics_interval"`
|
||||
}
|
||||
|
||||
// checkInterval returns true if days is valid to be used as statistics
|
||||
// retention interval. The valid values are 0, 1, 7, 30 and 90.
|
||||
func checkInterval(days uint32) (ok bool) {
|
||||
@ -51,6 +45,12 @@ type Config struct {
|
||||
// LimitDays is the maximum number of days to collect statistics into the
|
||||
// current unit.
|
||||
LimitDays uint32
|
||||
|
||||
// Enabled tells if the statistics are enabled.
|
||||
Enabled bool
|
||||
|
||||
// Ignored is the list of host names, which should not be counted.
|
||||
Ignored *stringutil.Set
|
||||
}
|
||||
|
||||
// Interface is the statistics interface to be used by other packages.
|
||||
@ -68,19 +68,15 @@ type Interface interface {
|
||||
TopClientsIP(limit uint) []netip.Addr
|
||||
|
||||
// WriteDiskConfig puts the Interface's configuration to the dc.
|
||||
WriteDiskConfig(dc *DiskConfig)
|
||||
WriteDiskConfig(dc *Config)
|
||||
|
||||
// ShouldCount returns true if request for the host should be counted.
|
||||
ShouldCount(host string, qType, qClass uint16) bool
|
||||
}
|
||||
|
||||
// StatsCtx collects the statistics and flushes it to the database. Its default
|
||||
// flushing interval is one hour.
|
||||
type StatsCtx struct {
|
||||
// limitHours is the maximum number of hours to collect statistics into the
|
||||
// current unit.
|
||||
//
|
||||
// It is of type uint32 to be accessed by atomic. It's arranged at the
|
||||
// beginning of the structure to keep 64-bit alignment.
|
||||
limitHours uint32
|
||||
|
||||
// currMu protects curr.
|
||||
currMu *sync.RWMutex
|
||||
// curr is the actual statistics collection result.
|
||||
@ -102,6 +98,21 @@ type StatsCtx struct {
|
||||
|
||||
// filename is the name of database file.
|
||||
filename string
|
||||
|
||||
// lock protects all the fields below.
|
||||
lock sync.Mutex
|
||||
|
||||
// enabled tells if the statistics are enabled.
|
||||
enabled bool
|
||||
|
||||
// limitHours is the maximum number of hours to collect statistics into the
|
||||
// current unit.
|
||||
//
|
||||
// TODO(s.chzhen): Rewrite to use time.Duration.
|
||||
limitHours uint32
|
||||
|
||||
// ignored is the list of host names, which should not be counted.
|
||||
ignored *stringutil.Set
|
||||
}
|
||||
|
||||
// New creates s from conf and properly initializes it. Don't use s before
|
||||
@ -110,10 +121,12 @@ func New(conf Config) (s *StatsCtx, err error) {
|
||||
defer withRecovered(&err)
|
||||
|
||||
s = &StatsCtx{
|
||||
enabled: conf.Enabled,
|
||||
currMu: &sync.RWMutex{},
|
||||
filename: conf.Filename,
|
||||
configModified: conf.ConfigModified,
|
||||
httpRegister: conf.HTTPRegister,
|
||||
ignored: conf.Ignored,
|
||||
}
|
||||
if s.limitHours = conf.LimitDays * 24; !checkInterval(conf.LimitDays) {
|
||||
s.limitHours = 24
|
||||
@ -215,7 +228,10 @@ func (s *StatsCtx) Close() (err error) {
|
||||
|
||||
// Update implements the Interface interface for *StatsCtx.
|
||||
func (s *StatsCtx) Update(e Entry) {
|
||||
if atomic.LoadUint32(&s.limitHours) == 0 {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
if !s.enabled || s.limitHours == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
@ -243,14 +259,22 @@ func (s *StatsCtx) Update(e Entry) {
|
||||
}
|
||||
|
||||
// WriteDiskConfig implements the Interface interface for *StatsCtx.
|
||||
func (s *StatsCtx) WriteDiskConfig(dc *DiskConfig) {
|
||||
dc.Interval = atomic.LoadUint32(&s.limitHours) / 24
|
||||
func (s *StatsCtx) WriteDiskConfig(dc *Config) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
dc.LimitDays = s.limitHours / 24
|
||||
dc.Enabled = s.enabled
|
||||
dc.Ignored = s.ignored
|
||||
}
|
||||
|
||||
// TopClientsIP implements the [Interface] interface for *StatsCtx.
|
||||
func (s *StatsCtx) TopClientsIP(maxCount uint) (ips []netip.Addr) {
|
||||
limit := atomic.LoadUint32(&s.limitHours)
|
||||
if limit == 0 {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
limit := s.limitHours
|
||||
if !s.enabled || limit == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -342,6 +366,9 @@ func (s *StatsCtx) openDB() (err error) {
|
||||
func (s *StatsCtx) flush() (cont bool, sleepFor time.Duration) {
|
||||
id := s.unitIDGen()
|
||||
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
s.currMu.Lock()
|
||||
defer s.currMu.Unlock()
|
||||
|
||||
@ -350,7 +377,7 @@ func (s *StatsCtx) flush() (cont bool, sleepFor time.Duration) {
|
||||
return false, 0
|
||||
}
|
||||
|
||||
limit := atomic.LoadUint32(&s.limitHours)
|
||||
limit := s.limitHours
|
||||
if limit == 0 || ptr.id == id {
|
||||
return true, time.Second
|
||||
}
|
||||
@ -410,14 +437,23 @@ func (s *StatsCtx) periodicFlush() {
|
||||
}
|
||||
|
||||
func (s *StatsCtx) setLimit(limitDays int) {
|
||||
atomic.StoreUint32(&s.limitHours, uint32(24*limitDays))
|
||||
if limitDays == 0 {
|
||||
if err := s.clear(); err != nil {
|
||||
log.Error("stats: %s", err)
|
||||
}
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
if limitDays != 0 {
|
||||
s.enabled = true
|
||||
s.limitHours = uint32(24 * limitDays)
|
||||
log.Debug("stats: set limit: %d days", limitDays)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
log.Debug("stats: set limit: %d days", limitDays)
|
||||
s.enabled = false
|
||||
log.Debug("stats: disabled")
|
||||
|
||||
if err := s.clear(); err != nil {
|
||||
log.Error("stats: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Reset counters and clear database
|
||||
@ -520,3 +556,13 @@ func (s *StatsCtx) loadUnits(limit uint32) (units []*unitDB, firstID uint32) {
|
||||
|
||||
return units, firstID
|
||||
}
|
||||
|
||||
// ShouldCount returns true if request for the host should be counted.
|
||||
func (s *StatsCtx) ShouldCount(host string, _, _ uint16) bool {
|
||||
return !s.isIgnored(host)
|
||||
}
|
||||
|
||||
// isIgnored returns true if the host is in the Ignored list.
|
||||
func (s *StatsCtx) isIgnored(host string) bool {
|
||||
return s.ignored.Has(host)
|
||||
}
|
||||
|
@ -53,6 +53,7 @@ func TestStats(t *testing.T) {
|
||||
conf := stats.Config{
|
||||
Filename: filepath.Join(t.TempDir(), "stats.db"),
|
||||
LimitDays: 1,
|
||||
Enabled: true,
|
||||
UnitID: constUnitID,
|
||||
HTTPRegister: func(_, url string, handler http.HandlerFunc) {
|
||||
handlers[url] = handler
|
||||
@ -158,6 +159,7 @@ func TestLargeNumbers(t *testing.T) {
|
||||
conf := stats.Config{
|
||||
Filename: filepath.Join(t.TempDir(), "stats.db"),
|
||||
LimitDays: 1,
|
||||
Enabled: true,
|
||||
UnitID: func() (id uint32) { return atomic.LoadUint32(&curHour) },
|
||||
HTTPRegister: func(_, url string, handler http.HandlerFunc) { handlers[url] = handler },
|
||||
}
|
||||
|
@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/stringutil"
|
||||
"go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
@ -341,11 +342,13 @@ type pairsGetter func(u *unitDB) (pairs []countPair)
|
||||
|
||||
// topsCollector collects statistics about highest values from the given *unitDB
|
||||
// slice using pg to retrieve data.
|
||||
func topsCollector(units []*unitDB, max int, pg pairsGetter) []map[string]uint64 {
|
||||
func topsCollector(units []*unitDB, max int, ignored *stringutil.Set, pg pairsGetter) []map[string]uint64 {
|
||||
m := map[string]uint64{}
|
||||
for _, u := range units {
|
||||
for _, cp := range pg(u) {
|
||||
m[cp.Name] += cp.Count
|
||||
if !ignored.Has(cp.Name) {
|
||||
m[cp.Name] += cp.Count
|
||||
}
|
||||
}
|
||||
}
|
||||
a2 := convertMapToSlice(m, max)
|
||||
@ -408,9 +411,9 @@ func (s *StatsCtx) getData(limit uint32) (StatsResp, bool) {
|
||||
BlockedFiltering: statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RFiltered] }),
|
||||
ReplacedSafebrowsing: statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RSafeBrowsing] }),
|
||||
ReplacedParental: statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RParental] }),
|
||||
TopQueried: topsCollector(units, maxDomains, func(u *unitDB) (pairs []countPair) { return u.Domains }),
|
||||
TopBlocked: topsCollector(units, maxDomains, func(u *unitDB) (pairs []countPair) { return u.BlockedDomains }),
|
||||
TopClients: topsCollector(units, maxClients, func(u *unitDB) (pairs []countPair) { return u.Clients }),
|
||||
TopQueried: topsCollector(units, maxDomains, s.ignored, func(u *unitDB) (pairs []countPair) { return u.Domains }),
|
||||
TopBlocked: topsCollector(units, maxDomains, s.ignored, func(u *unitDB) (pairs []countPair) { return u.BlockedDomains }),
|
||||
TopClients: topsCollector(units, maxClients, nil, func(u *unitDB) (pairs []countPair) { return u.Clients }),
|
||||
}
|
||||
|
||||
// Total counters:
|
||||
|
Loading…
Reference in New Issue
Block a user