From 388583cefefe1a3a479ed0200757145538f6ae42 Mon Sep 17 00:00:00 2001 From: Dimitry Kolyshev Date: Fri, 17 Nov 2023 15:51:51 +0300 Subject: [PATCH] Pull request: custom upstreams cache Merge in DNS/adguard-home from custom-ups-cache to master Squashed commit of the following: commit 98428a87520f70cb522701d8eccfe4c529be1e40 Author: Dimitry Kolyshev Date: Fri Nov 17 10:53:32 2023 +0200 all: upd dep commit 775a639af4a2a45220b17e8b0037edc126ff62e4 Author: Dimitry Kolyshev Date: Fri Nov 17 09:52:31 2023 +0200 dnsforward: imp test commit e9e2a58b48e8588dfcb28df319d4651e1fe77af5 Author: Dimitry Kolyshev Date: Fri Nov 17 09:44:46 2023 +0200 docs: changelog commit a6d67218f037c8fec29e5fa2967476d63c3cfc32 Author: Dimitry Kolyshev Date: Fri Nov 17 09:37:17 2023 +0200 all: upd dep commit b101ff6e0cf393dacdee6fb68d33ba8f11c36280 Merge: d61f4eb88 8bb1aad73 Author: Dimitry Kolyshev Date: Thu Nov 16 15:54:05 2023 +0200 Merge remote-tracking branch 'origin/master' into custom-ups-cache commit d61f4eb8871f8ae8504259998bf9015b29001cfb Merge: 567a8a4af fdf60eeed Author: Dimitry Kolyshev Date: Mon Nov 13 10:32:22 2023 +0200 Merge remote-tracking branch 'origin/master' into custom-ups-cache commit 567a8a4af34ad001d0e6d7d2efdc123205569e8c Author: Dimitry Kolyshev Date: Mon Nov 13 10:30:24 2023 +0200 home: imp code commit a3c16facbebc166e5c0c731c1e892b61c0950d9e Author: Dimitry Kolyshev Date: Fri Nov 10 14:34:04 2023 +0200 all: imp code commit 84160eafee1d0f2d0cd3f025f2d5070e4f597ad6 Author: Dimitry Kolyshev Date: Fri Nov 10 14:31:26 2023 +0200 all: conf custom ups cache commit b7f6581901ebad96c87e765a305a1fa5b336efbb Author: Dimitry Kolyshev Date: Fri Nov 10 14:29:47 2023 +0200 all: conf custom ups cache commit d07df945d4e7614a679ef5dc77756096abf1e66c Author: Dimitry Kolyshev Date: Fri Nov 10 09:26:29 2023 +0200 all: docs commit 998124bac08889c7d354dd1a099929726725bccc Merge: f665e2f85 53170d871 Author: Dimitry Kolyshev Date: Fri Nov 10 09:24:28 2023 +0200 Merge remote-tracking branch 'origin/master' into custom-ups-cache commit f665e2f85bce12d95f80aba6614b6bfd4874b122 Author: Dimitry Kolyshev Date: Fri Nov 10 09:22:46 2023 +0200 all: conf custom ups cache commit a4b26973bef4f3b339198ffbe52a50baca303daf Author: Dimitry Kolyshev Date: Thu Nov 9 12:46:39 2023 +0200 all: conf custom ups cache --- CHANGELOG.md | 6 +++++ go.mod | 3 ++- go.sum | 8 ++++++ internal/aghtest/interface.go | 4 +-- internal/dnsforward/config.go | 5 +++- internal/dnsforward/dnsforward_test.go | 36 +++++++++++++++++++++----- internal/home/client.go | 16 +++++++----- internal/home/clients.go | 25 +++++++++++++++--- internal/home/clients_internal_test.go | 10 +++---- 9 files changed, 85 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 439afa36..1078b296 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,12 @@ NOTE: Add new changes BELOW THIS COMMENT. - Ability to specify rate limiting settings in the Web UI ([#6369]). +#### Configuration changes + +- Added new properties `clients.persistent.upstreams_cache_enabled` and + `clients.persistent.upstreams_cache_size` which describe cache configuration + for each client custom upstream configuration. + ### Fixed - Pre-filling the New static lease window with data ([#6402]). diff --git a/go.mod b/go.mod index 375bed97..a969dc57 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/AdguardTeam/AdGuardHome go 1.20 require ( - github.com/AdguardTeam/dnsproxy v0.58.0 + github.com/AdguardTeam/dnsproxy v0.59.0 github.com/AdguardTeam/golibs v0.17.2 github.com/AdguardTeam/urlfilter v0.17.3 github.com/NYTimes/gziphandler v1.1.1 @@ -48,6 +48,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a // indirect + github.com/jessevdk/go-flags v1.5.0 // indirect github.com/mdlayher/socket v0.5.0 // indirect github.com/onsi/ginkgo/v2 v2.13.1 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect diff --git a/go.sum b/go.sum index dcb90678..d0568ebc 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ github.com/AdguardTeam/dnsproxy v0.58.0 h1:1zPmDYWIc60D5Mn2idt3TcH+CQzKBvkWzJ5/u49wraw= github.com/AdguardTeam/dnsproxy v0.58.0/go.mod h1:ZvkbM71HwpilgkCnTubDiR4Ba6x5Qvnhy2iasMWaTDM= +github.com/AdguardTeam/dnsproxy v0.58.1-0.20231114082217-26ee425d7cb9 h1:8JLgm92qlmd9LHh9maxo1Ao+vGLvLRSOClYH2/IvJ2g= +github.com/AdguardTeam/dnsproxy v0.58.1-0.20231114082217-26ee425d7cb9/go.mod h1:ZvkbM71HwpilgkCnTubDiR4Ba6x5Qvnhy2iasMWaTDM= +github.com/AdguardTeam/dnsproxy v0.59.0 h1:J0Iw4+GZ7U44scgkrNCM5W2G+tNS0r9nfQOuSLBqapE= +github.com/AdguardTeam/dnsproxy v0.59.0/go.mod h1:ZvkbM71HwpilgkCnTubDiR4Ba6x5Qvnhy2iasMWaTDM= github.com/AdguardTeam/golibs v0.17.2 h1:vg6wHMjUKscnyPGRvxS5kAt7Uw4YxcJiITZliZ476W8= github.com/AdguardTeam/golibs v0.17.2/go.mod h1:DKhCIXHcUYtBhU8ibTLKh1paUL96n5zhQBlx763sj+U= github.com/AdguardTeam/urlfilter v0.17.3 h1:fg/ObbnO0Cv6aw0tW6N/ETDMhhNvmcUUOZ7HlmKC3rw= @@ -52,6 +56,8 @@ github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8 github.com/insomniacslk/dhcp v0.0.0-20231016090811-6a2c8fbdcc1c h1:PgxFEySCI41sH0mB7/2XswdXbUykQsRUGod8Rn+NubM= github.com/insomniacslk/dhcp v0.0.0-20231016090811-6a2c8fbdcc1c/go.mod h1:3A9PQ1cunSDF/1rbTq99Ts4pVnycWg+vlPkfeD2NLFI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.0.1-0.20221213033349-c1e37c09b531/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86 h1:elKwZS1OcdQ0WwEDBeqxKwb7WB62QX8bvZ/FJnVXIfk= @@ -100,6 +106,7 @@ github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0 github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= @@ -145,6 +152,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/aghtest/interface.go b/internal/aghtest/interface.go index 3d88f0c1..f21f6e57 100644 --- a/internal/aghtest/interface.go +++ b/internal/aghtest/interface.go @@ -125,7 +125,7 @@ type ClientsContainer struct { OnUpstreamConfigByID func( id string, boot upstream.Resolver, - ) (conf *proxy.UpstreamConfig, err error) + ) (conf *proxy.CustomUpstreamConfig, err error) } // UpstreamConfigByID implements the [dnsforward.ClientsContainer] interface @@ -133,7 +133,7 @@ type ClientsContainer struct { func (c *ClientsContainer) UpstreamConfigByID( id string, boot upstream.Resolver, -) (conf *proxy.UpstreamConfig, err error) { +) (conf *proxy.CustomUpstreamConfig, err error) { return c.OnUpstreamConfigByID(id, boot) } diff --git a/internal/dnsforward/config.go b/internal/dnsforward/config.go index 0947bf8f..d3d58e9d 100644 --- a/internal/dnsforward/config.go +++ b/internal/dnsforward/config.go @@ -34,7 +34,10 @@ type ClientsContainer interface { // returns nil if there is no custom upstream configuration for the client. // The id is expected to be either a string representation of an IP address // or the ClientID. - UpstreamConfigByID(id string, boot upstream.Resolver) (conf *proxy.UpstreamConfig, err error) + UpstreamConfigByID( + id string, + boot upstream.Resolver, + ) (conf *proxy.CustomUpstreamConfig, err error) } // Config represents the DNS filtering configuration of AdGuard Home. The zero diff --git a/internal/dnsforward/dnsforward_test.go b/internal/dnsforward/dnsforward_test.go index 2b52aba0..c74d97f7 100644 --- a/internal/dnsforward/dnsforward_test.go +++ b/internal/dnsforward/dnsforward_test.go @@ -644,10 +644,15 @@ func TestBlockedRequest(t *testing.T) { } func TestServerCustomClientUpstream(t *testing.T) { + const defaultCacheSize = 1024 * 1024 + + var upsCalledCounter uint32 + forwardConf := ServerConfig{ UDPListenAddrs: []*net.UDPAddr{{}}, TCPListenAddrs: []*net.TCPAddr{{}}, Config: Config{ + CacheSize: defaultCacheSize, EDNSClientSubnet: &EDNSClientSubnet{ Enabled: false, }, @@ -658,34 +663,51 @@ func TestServerCustomClientUpstream(t *testing.T) { }, forwardConf, nil) ups := aghtest.NewUpstreamMock(func(req *dns.Msg) (resp *dns.Msg, err error) { + atomic.AddUint32(&upsCalledCounter, 1) + return aghalg.Coalesce( aghtest.MatchedResponse(req, dns.TypeA, "host", "192.168.0.1"), new(dns.Msg).SetRcode(req, dns.RcodeNameError), ), nil }) + + customUpsConf := proxy.NewCustomUpstreamConfig( + &proxy.UpstreamConfig{ + Upstreams: []upstream.Upstream{ups}, + }, + true, + defaultCacheSize, + forwardConf.EDNSClientSubnet.Enabled, + ) + s.conf.ClientsContainer = &aghtest.ClientsContainer{ OnUpstreamConfigByID: func( _ string, _ upstream.Resolver, - ) (conf *proxy.UpstreamConfig, err error) { - return &proxy.UpstreamConfig{Upstreams: []upstream.Upstream{ups}}, nil + ) (conf *proxy.CustomUpstreamConfig, err error) { + return customUpsConf, nil }, } + startDeferStop(t, s) - addr := s.dnsProxy.Addr(proxy.ProtoUDP) + addr := s.dnsProxy.Addr(proxy.ProtoUDP).String() // Send test request. req := createTestMessage("host.") - reply, err := dns.Exchange(req, addr.String()) + reply, err := dns.Exchange(req, addr) require.NoError(t, err) + require.NotEmpty(t, reply.Answer) + require.Len(t, reply.Answer, 1) assert.Equal(t, dns.RcodeSuccess, reply.Rcode) - require.NotEmpty(t, reply.Answer) - - require.Len(t, reply.Answer, 1) assert.Equal(t, net.IP{192, 168, 0, 1}, reply.Answer[0].(*dns.A).A) + assert.Equal(t, uint32(1), atomic.LoadUint32(&upsCalledCounter)) + + _, err = dns.Exchange(req, addr) + require.NoError(t, err) + assert.Equal(t, uint32(1), atomic.LoadUint32(&upsCalledCounter)) } // testCNAMEs is a map of names and CNAMEs necessary for the TestUpstream work. diff --git a/internal/home/client.go b/internal/home/client.go index 751bde6d..49260ed9 100644 --- a/internal/home/client.go +++ b/internal/home/client.go @@ -14,11 +14,11 @@ import ( // Client contains information about persistent clients. type Client struct { - // upstreamConfig is the custom upstream config for this client. If - // it's nil, it has not been initialized yet. If it's non-nil and - // empty, there are no valid upstreams. If it's non-nil and non-empty, - // these upstream must be used. - upstreamConfig *proxy.UpstreamConfig + // upstreamConfig is the custom upstream configuration for this client. If + // it's nil, it has not been initialized yet. If it's non-nil and empty, + // there are no valid upstreams. If it's non-nil and non-empty, these + // upstream must be used. + upstreamConfig *proxy.CustomUpstreamConfig safeSearchConf filtering.SafeSearchConfig SafeSearch filtering.SafeSearch @@ -32,6 +32,9 @@ type Client struct { Tags []string Upstreams []string + UpstreamsCacheSize uint32 + UpstreamsCacheEnabled bool + UseOwnSettings bool FilteringEnabled bool SafeBrowsingEnabled bool @@ -57,8 +60,7 @@ func (c *Client) ShallowClone() (sh *Client) { // closeUpstreams closes the client-specific upstream config of c if any. func (c *Client) closeUpstreams() (err error) { if c.upstreamConfig != nil { - err = c.upstreamConfig.Close() - if err != nil { + if err = c.upstreamConfig.Close(); err != nil { return fmt.Errorf("closing upstreams of client %q: %w", c.Name, err) } } diff --git a/internal/home/clients.go b/internal/home/clients.go index d9f5dcd4..d99bd632 100644 --- a/internal/home/clients.go +++ b/internal/home/clients.go @@ -185,6 +185,14 @@ type clientObject struct { Tags []string `yaml:"tags"` Upstreams []string `yaml:"upstreams"` + // UpstreamsCacheSize is the DNS cache size (in bytes). + // + // TODO(d.kolyshev): Use [datasize.Bytesize]. + UpstreamsCacheSize uint32 `yaml:"upstreams_cache_size"` + + // UpstreamsCacheEnabled indicates if the DNS cache is enabled. + UpstreamsCacheEnabled bool `yaml:"upstreams_cache_enabled"` + UseGlobalSettings bool `yaml:"use_global_settings"` FilteringEnabled bool `yaml:"filtering_enabled"` ParentalEnabled bool `yaml:"parental_enabled"` @@ -216,6 +224,8 @@ func (clients *clientsContainer) addFromConfig( UseOwnBlockedServices: !o.UseGlobalBlockedServices, IgnoreQueryLog: o.IgnoreQueryLog, IgnoreStatistics: o.IgnoreStatistics, + UpstreamsCacheEnabled: o.UpstreamsCacheEnabled, + UpstreamsCacheSize: o.UpstreamsCacheSize, } if o.SafeSearchConf.Enabled { @@ -429,11 +439,12 @@ func (clients *clientsContainer) shouldCountClient(ids []string) (y bool) { var _ dnsforward.ClientsContainer = (*clientsContainer)(nil) // UpstreamConfigByID implements the [dnsforward.ClientsContainer] interface for -// *clientsContainer. +// *clientsContainer. upsConf is nil if the client isn't found or if the client +// has no custom upstreams. func (clients *clientsContainer) UpstreamConfigByID( id string, bootstrap upstream.Resolver, -) (upsConf *proxy.UpstreamConfig, err error) { +) (conf *proxy.CustomUpstreamConfig, err error) { clients.lock.Lock() defer clients.lock.Unlock() @@ -449,8 +460,8 @@ func (clients *clientsContainer) UpstreamConfigByID( return nil, nil } - var conf *proxy.UpstreamConfig - conf, err = proxy.ParseUpstreamsConfig( + var upsConf *proxy.UpstreamConfig + upsConf, err = proxy.ParseUpstreamsConfig( upstreams, &upstream.Options{ Bootstrap: bootstrap, @@ -464,6 +475,12 @@ func (clients *clientsContainer) UpstreamConfigByID( return nil, err } + conf = proxy.NewCustomUpstreamConfig( + upsConf, + c.UpstreamsCacheEnabled, + int(c.UpstreamsCacheSize), + config.DNS.EDNSClientSubnet.Enabled, + ) c.upstreamConfig = conf return conf, nil diff --git a/internal/home/clients_internal_test.go b/internal/home/clients_internal_test.go index 45e89cf3..30d735bb 100644 --- a/internal/home/clients_internal_test.go +++ b/internal/home/clients_internal_test.go @@ -355,13 +355,11 @@ func TestClientsCustomUpstream(t *testing.T) { require.NoError(t, err) assert.True(t, ok) - config, err := clients.UpstreamConfigByID("1.2.3.4", net.DefaultResolver) - assert.Nil(t, config) + upsConf, err := clients.UpstreamConfigByID("1.2.3.4", net.DefaultResolver) + assert.Nil(t, upsConf) assert.NoError(t, err) - config, err = clients.UpstreamConfigByID("1.1.1.1", net.DefaultResolver) - require.NotNil(t, config) + upsConf, err = clients.UpstreamConfigByID("1.1.1.1", net.DefaultResolver) + require.NotNil(t, upsConf) assert.NoError(t, err) - assert.Len(t, config.Upstreams, 1) - assert.Len(t, config.DomainReservedUpstreams, 1) }