+ DNS: add "ipset" configuration setting

Close #1191

Squashed commit of the following:

commit ba14b53f9e3d98ad8127aa3af1def0da4269e8c4
Merge: 362f4c44 6b614295
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Wed Sep 2 14:03:19 2020 +0300

    Merge remote-tracking branch 'origin/master' into 1191-ipset

commit 362f4c44915cb8946db2e80f9a3f5afd74fe5de1
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Wed Sep 2 12:50:56 2020 +0300

    minor

commit 28e12459166fe3d13fb0dbe59ac11b7d86adb9b4
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Wed Sep 2 12:43:25 2020 +0300

    minor

commit bdbd7324501f6111bea1e91eda7d730c7ea57b11
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Tue Sep 1 18:40:04 2020 +0300

    move code, ipset-v6

commit 77f4d943e74b70b5bc5aea279875ab1e2fab2192
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Tue Sep 1 15:53:27 2020 +0300

    comment

commit 16401325bbefeba08e447257b12a8424b78c9475
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Mon Aug 31 17:43:23 2020 +0300

    minor

commit c8410e9a519b87911bc50f504e8b4aaf8dce6e02
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Mon Aug 31 15:30:52 2020 +0300

    + DNS: add "ipset" configuration setting

AGHTechDoc.md

@@ -69,6 +69,7 @@ Contents:
 	* API: Log out
 	* API: Get current user info
 * Safe services
+* ipset
 
 
 ## Relations between subsystems
@@ -1882,3 +1883,25 @@ Check if host name is blocked by SB/PC service:
 		sha256(host.com)[0..1] -> hashes[0],hashes[1],...
 		sha256(sub.host.com)[0..1] -> hashes[2],...
 		...
+
+
+## ipset
+
+AGH can add IP addresses of the specified in configuration domain names to an ipset list.
+
+Prepare: user creates an ipset list and configures AGH for using it.
+
+	1. User --( ipset create my_ipset hash:ip ) -> OS
+	2. User --( ipset: host.com,host2.com/my_ipset )-> AGH
+
+		Syntax:
+
+			ipset: "DOMAIN[,DOMAIN].../IPSET1_NAME[,IPSET2_NAME]..."
+
+		IPv4 addresses are added to an ipset list with `ipv4` family, IPv6 addresses - to `ipv6` ipset list.
+
+Run-time: AGH adds IP addresses of a domain name to a corresponding ipset list.
+
+	1. AGH --( resolve host.com )-> upstream
+	2. AGH <-( host.com:[1.1.1.1,2.2.2.2] )-- upstream
+	3. AGH --( ipset.add(my_ipset, [1.1.1.1,2.2.2.2] ))-> OS

dnsforward/config.go

@@ -82,6 +82,11 @@ type FilteringConfig struct {
 	EnableDNSSEC           bool     `yaml:"enable_dnssec"`      // Set DNSSEC flag in outcoming DNS request
 	EnableEDNSClientSubnet bool     `yaml:"edns_client_subnet"` // Enable EDNS Client Subnet option
 	MaxGoroutines          uint32   `yaml:"max_goroutines"`     // Max. number of parallel goroutines for processing incoming requests
+
+	// IPSET configuration - add IP addresses of the specified domain names to an ipset list
+	// Syntax:
+	// "DOMAIN[,DOMAIN].../IPSET_NAME"
+	IPSETList []string `yaml:"ipset"`
 }
 
 // TLSConfig is the TLS configuration for HTTPS, DNS-over-HTTPS, and DNS-over-TLS

dnsforward/dnsforward.go

@@ -51,6 +51,8 @@ type Server struct {
 	stats      stats.Stats
 	access     *accessCtx
 
+	ipset ipsetCtx
+
 	tableHostToIP     map[string]net.IP // "hostname -> IP" table for internal addresses (DHCP)
 	tableHostToIPLock sync.Mutex
 
@@ -168,7 +170,7 @@ func (s *Server) startInternal() error {
 
 // Prepare the object
 func (s *Server) Prepare(config *ServerConfig) error {
-	// 1. Initialize the server configuration
+	// Initialize the server configuration
 	// --
 	if config != nil {
 		s.conf = *config
@@ -184,18 +186,22 @@ func (s *Server) Prepare(config *ServerConfig) error {
 		}
 	}
 
-	// 2. Set default values in the case if nothing is configured
+	// Set default values in the case if nothing is configured
 	// --
 	s.initDefaultSettings()
 
-	// 3. Prepare DNS servers settings
+	// Initialize IPSET configuration
+	// --
+	s.ipset.init(s.conf.IPSETList)
+
+	// Prepare DNS servers settings
 	// --
 	err := s.prepareUpstreamSettings()
 	if err != nil {
 		return err
 	}
 
-	// 3. Create DNS proxy configuration
+	// Create DNS proxy configuration
 	// --
 	var proxyConfig proxy.Config
 	proxyConfig, err = s.createProxyConfig()
@@ -203,11 +209,11 @@ func (s *Server) Prepare(config *ServerConfig) error {
 		return err
 	}
 
-	// 4. Prepare a DNS proxy instance that we use for internal DNS queries
+	// Prepare a DNS proxy instance that we use for internal DNS queries
 	// --
 	s.prepareIntlProxy()
 
-	// 5. Initialize DNS access module
+	// Initialize DNS access module
 	// --
 	s.access = &accessCtx{}
 	err = s.access.Init(s.conf.AllowedClients, s.conf.DisallowedClients, s.conf.BlockedHosts)
@@ -215,14 +221,14 @@ func (s *Server) Prepare(config *ServerConfig) error {
 		return err
 	}
 
-	// 6. Register web handlers if necessary
+	// Register web handlers if necessary
 	// --
 	if !webRegistered && s.conf.HTTPRegister != nil {
 		webRegistered = true
 		s.registerHandlers()
 	}
 
-	// 7. Create the main DNS proxy instance
+	// Create the main DNS proxy instance
 	// --
 	s.dnsProxy = &proxy.Proxy{Config: proxyConfig}
 	return nil

dnsforward/handle_dns.go

@@ -49,6 +49,7 @@ func (s *Server) handleDNSRequest(_ *proxy.Proxy, d *proxy.DNSContext) error {
 		processUpstream,
 		processDNSSECAfterResponse,
 		processFilteringAfterResponse,
+		s.ipset.process,
 		processQueryLogsAndStats,
 	}
 	for _, process := range mods {

dnsforward/ipset.go

@@ -0,0 +1,133 @@
+package dnsforward
+
+import (
+	"net"
+	"strings"
+
+	"github.com/AdguardTeam/AdGuardHome/util"
+	"github.com/AdguardTeam/golibs/log"
+	"github.com/miekg/dns"
+)
+
+type ipsetCtx struct {
+	ipsetList   map[string][]string // domain -> []ipset_name
+	ipsetCache  map[[4]byte]bool    // cache for IP[] to prevent duplicate calls to ipset program
+	ipset6Cache map[[16]byte]bool   // cache for IP[] to prevent duplicate calls to ipset program
+}
+
+// Convert configuration settings to an internal map
+// DOMAIN[,DOMAIN].../IPSET1_NAME[,IPSET2_NAME]...
+func (c *ipsetCtx) init(ipsetConfig []string) {
+	c.ipsetList = make(map[string][]string)
+	c.ipsetCache = make(map[[4]byte]bool)
+	c.ipset6Cache = make(map[[16]byte]bool)
+
+	for _, it := range ipsetConfig {
+		it = strings.TrimSpace(it)
+		hostsAndNames := strings.Split(it, "/")
+		if len(hostsAndNames) != 2 {
+			log.Debug("IPSET: invalid value '%s'", it)
+			continue
+		}
+
+		ipsetNames := strings.Split(hostsAndNames[1], ",")
+		if len(ipsetNames) == 0 {
+			log.Debug("IPSET: invalid value '%s'", it)
+			continue
+		}
+		bad := false
+		for i := range ipsetNames {
+			ipsetNames[i] = strings.TrimSpace(ipsetNames[i])
+			if len(ipsetNames[i]) == 0 {
+				bad = true
+				break
+			}
+		}
+		if bad {
+			log.Debug("IPSET: invalid value '%s'", it)
+			continue
+		}
+
+		hosts := strings.Split(hostsAndNames[0], ",")
+		for _, host := range hosts {
+			host = strings.TrimSpace(host)
+			host = strings.ToLower(host)
+			if len(host) == 0 {
+				log.Debug("IPSET: invalid value '%s'", it)
+				continue
+			}
+			c.ipsetList[host] = ipsetNames
+		}
+	}
+	log.Debug("IPSET: added %d hosts", len(c.ipsetList))
+}
+
+func (c *ipsetCtx) getIP(rr dns.RR) net.IP {
+	switch a := rr.(type) {
+	case *dns.A:
+		var ip4 [4]byte
+		copy(ip4[:], a.A.To4())
+		_, found := c.ipsetCache[ip4]
+		if found {
+			return nil // this IP was added before
+		}
+		c.ipsetCache[ip4] = false
+		return a.A
+
+	case *dns.AAAA:
+		var ip6 [16]byte
+		copy(ip6[:], a.AAAA)
+		_, found := c.ipset6Cache[ip6]
+		if found {
+			return nil // this IP was added before
+		}
+		c.ipset6Cache[ip6] = false
+		return a.AAAA
+
+	default:
+		return nil
+	}
+}
+
+// Add IP addresses of the specified in configuration domain names to an ipset list
+func (c *ipsetCtx) process(ctx *dnsContext) int {
+	req := ctx.proxyCtx.Req
+	if !(req.Question[0].Qtype == dns.TypeA ||
+		req.Question[0].Qtype == dns.TypeAAAA) ||
+		!ctx.responseFromUpstream {
+		return resultDone
+	}
+
+	host := req.Question[0].Name
+	host = strings.TrimSuffix(host, ".")
+	host = strings.ToLower(host)
+	ipsetNames, found := c.ipsetList[host]
+	if !found {
+		return resultDone
+	}
+
+	log.Debug("IPSET: found ipsets %v for host %s", ipsetNames, host)
+
+	for _, it := range ctx.proxyCtx.Res.Answer {
+		ip := c.getIP(it)
+		if ip == nil {
+			continue
+		}
+
+		ipStr := ip.String()
+		for _, name := range ipsetNames {
+			code, out, err := util.RunCommand("ipset", "add", name, ipStr)
+			if err != nil {
+				log.Info("IPSET: %s(%s) -> %s: %s", host, ipStr, name, err)
+				continue
+			}
+			if code != 0 {
+				log.Info("IPSET: ipset add:  code:%d  output:'%s'", code, out)
+				continue
+			}
+			log.Debug("IPSET: added %s(%s) -> %s", host, ipStr, name)
+		}
+	}
+
+	return resultDone
+}

dnsforward/ipset_test.go

@@ -0,0 +1,41 @@
+package dnsforward
+
+import (
+	"testing"
+
+	"github.com/AdguardTeam/dnsproxy/proxy"
+	"github.com/miekg/dns"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIPSET(t *testing.T) {
+	s := Server{}
+	s.conf.IPSETList = append(s.conf.IPSETList, "HOST.com/name")
+	s.conf.IPSETList = append(s.conf.IPSETList, "host2.com,host3.com/name23")
+	s.conf.IPSETList = append(s.conf.IPSETList, "host4.com/name4,name41")
+	c := ipsetCtx{}
+	c.init(s.conf.IPSETList)
+
+	assert.Equal(t, "name", c.ipsetList["host.com"][0])
+	assert.Equal(t, "name23", c.ipsetList["host2.com"][0])
+	assert.Equal(t, "name23", c.ipsetList["host3.com"][0])
+	assert.Equal(t, "name4", c.ipsetList["host4.com"][0])
+	assert.Equal(t, "name41", c.ipsetList["host4.com"][1])
+
+	_, ok := c.ipsetList["host0.com"]
+	assert.False(t, ok)
+
+	ctx := &dnsContext{
+		srv: &s,
+	}
+	ctx.proxyCtx = &proxy.DNSContext{}
+	ctx.proxyCtx.Req = &dns.Msg{
+		Question: []dns.Question{
+			{
+				Name:  "host.com.",
+				Qtype: dns.TypeA,
+			},
+		},
+	}
+	assert.Equal(t, resultDone, c.process(ctx))
+}