AdGuardHome/dnsfilter/dnsfilter_test.go

1026 lines
28 KiB
Go
Raw Normal View History

2018-08-30 17:25:33 +03:00
package dnsfilter
import (
"archive/zip"
"bytes"
"io/ioutil"
2018-08-30 17:25:33 +03:00
"net/http"
"net/http/httptest"
"path"
"runtime/pprof"
"strings"
2018-08-30 17:25:33 +03:00
"testing"
"time"
"bufio"
"fmt"
"os"
"runtime"
"github.com/shirou/gopsutil/process"
2018-08-30 17:25:33 +03:00
"go.uber.org/goleak"
)
// first in file because it must be run first
func TestLotsOfRulesMemoryUsage(t *testing.T) {
start := getRSS()
trace("RSS before loading rules - %d kB\n", start/1024)
dumpMemProfile(_Func() + "1.pprof")
d := NewForTest()
defer d.Destroy()
err := loadTestRules(d)
if err != nil {
t.Error(err)
}
afterLoad := getRSS()
trace("RSS after loading rules - %d kB (%d kB diff)\n", afterLoad/1024, (afterLoad-start)/1024)
dumpMemProfile(_Func() + "2.pprof")
tests := []struct {
host string
match bool
}{
{"asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.thisistesthost.com", false},
{"asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.ad.doubleclick.net", true},
}
for _, testcase := range tests {
ret, err := d.CheckHost(testcase.host)
if err != nil {
t.Errorf("Error while matching host %s: %s", testcase.host, err)
}
if !ret.IsFiltered && ret.IsFiltered != testcase.match {
t.Errorf("Expected hostname %s to not match", testcase.host)
}
if ret.IsFiltered && ret.IsFiltered != testcase.match {
t.Errorf("Expected hostname %s to match", testcase.host)
}
}
afterMatch := getRSS()
trace("RSS after matching - %d kB (%d kB diff)\n", afterMatch/1024, (afterMatch-afterLoad)/1024)
dumpMemProfile(_Func() + "3.pprof")
}
func getRSS() uint64 {
proc, err := process.NewProcess(int32(os.Getpid()))
if err != nil {
panic(err)
}
minfo, err := proc.MemoryInfo()
return minfo.RSS
}
func dumpMemProfile(name string) {
runtime.GC()
f, err := os.Create(name)
if err != nil {
panic(err)
}
defer f.Close()
runtime.GC() // update the stats before writing them
err = pprof.WriteHeapProfile(f)
if err != nil {
panic(err)
}
}
const topHostsFilename = "../tests/top-1m.csv"
func fetchTopHostsFromNet() {
trace("Fetching top hosts from network")
resp, err := http.Get("http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip")
if err != nil {
panic(err)
}
defer resp.Body.Close()
trace("Reading zipfile body")
zipfile, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
trace("Opening zipfile")
r, err := zip.NewReader(bytes.NewReader(zipfile), int64(len(zipfile)))
if err != nil {
panic(err)
}
if len(r.File) != 1 {
panic(fmt.Errorf("zipfile must have only one entry: %+v", r))
}
f := r.File[0]
trace("Unpacking file %s from zipfile", f.Name)
rc, err := f.Open()
if err != nil {
panic(err)
}
trace("Reading file %s contents", f.Name)
body, err := ioutil.ReadAll(rc)
if err != nil {
panic(err)
}
rc.Close()
trace("Writing file %s contents to disk", f.Name)
err = ioutil.WriteFile(topHostsFilename+".tmp", body, 0644)
if err != nil {
panic(err)
}
err = os.Rename(topHostsFilename+".tmp", topHostsFilename)
if err != nil {
panic(err)
}
}
func getTopHosts() {
// if file doesn't exist, fetch it
if _, err := os.Stat(topHostsFilename); os.IsNotExist(err) {
// file does not exist, fetch it
fetchTopHostsFromNet()
}
}
func TestLotsOfRulesLotsOfHostsMemoryUsage(t *testing.T) {
start := getRSS()
trace("RSS before loading rules - %d kB\n", start/1024)
dumpMemProfile(_Func() + "1.pprof")
d := NewForTest()
defer d.Destroy()
mustLoadTestRules(d)
trace("Have %d rules", d.Count())
afterLoad := getRSS()
trace("RSS after loading rules - %d kB (%d kB diff)\n", afterLoad/1024, (afterLoad-start)/1024)
dumpMemProfile(_Func() + "2.pprof")
getTopHosts()
hostnames, err := os.Open(topHostsFilename)
if err != nil {
t.Fatal(err)
}
defer hostnames.Close()
afterHosts := getRSS()
trace("RSS after loading hosts - %d kB (%d kB diff)\n", afterHosts/1024, (afterHosts-afterLoad)/1024)
dumpMemProfile(_Func() + "2.pprof")
{
scanner := bufio.NewScanner(hostnames)
for scanner.Scan() {
line := scanner.Text()
records := strings.Split(line, ",")
ret, err := d.CheckHost(records[1] + "." + records[1])
if err != nil {
t.Error(err)
}
if ret.Reason.Matched() {
// log.Printf("host \"%s\" mathed. Rule \"%s\", reason: %v", host, ret.Rule, ret.Reason)
}
}
}
afterMatch := getRSS()
trace("RSS after matching - %d kB (%d kB diff)\n", afterMatch/1024, (afterMatch-afterLoad)/1024)
dumpMemProfile(_Func() + "3.pprof")
}
2018-08-30 17:25:33 +03:00
func TestRuleToRegexp(t *testing.T) {
tests := []struct {
rule string
result string
err error
}{
{"/doubleclick/", "doubleclick", nil},
{"/", "", ErrInvalidSyntax},
{`|double*?.+[]|(){}#$\|`, `^double.*\?\.\+\[\]\|\(\)\{\}\#\$\\$`, nil},
{`||doubleclick.net^`, `(?:^|\.)doubleclick\.net$`, nil},
2018-08-30 17:25:33 +03:00
}
for _, testcase := range tests {
converted, err := ruleToRegexp(testcase.rule)
if err != testcase.err {
t.Error("Errors do not match, got ", err, " expected ", testcase.err)
}
if converted != testcase.result {
t.Error("Results do not match, got ", converted, " expected ", testcase.result)
}
}
}
func TestSuffixRule(t *testing.T) {
for _, testcase := range []struct {
rule string
isSuffix bool
suffix string
}{
{`||doubleclick.net^`, true, `doubleclick.net`}, // entire string or subdomain match
{`||doubleclick.net|`, true, `doubleclick.net`}, // entire string or subdomain match
{`|doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
{`*doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
{`doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
{`|*doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
{`||*doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
{`||*doubleclick.net|`, false, ``}, // TODO: ends with doubleclick.net
{`||*doublec*lick.net^`, false, ``}, // has a wildcard inside, has to be regexp
{`||*doublec|lick.net^`, false, ``}, // has a special symbol inside, has to be regexp
{`/abracadabra/`, false, ``}, // regexp, not anchored
{`/abracadabra$/`, false, ``}, // TODO: simplify simple suffix regexes
} {
isSuffix, suffix := getSuffix(testcase.rule)
if testcase.isSuffix != isSuffix {
t.Errorf("Results do not match for \"%s\": got %v expected %v", testcase.rule, isSuffix, testcase.isSuffix)
continue
}
if testcase.isSuffix && testcase.suffix != suffix {
t.Errorf("Result suffix does not match for \"%s\": got \"%s\" expected \"%s\"", testcase.rule, suffix, testcase.suffix)
continue
}
// trace("\"%s\": %v: %s", testcase.rule, isSuffix, suffix)
}
}
2018-08-30 17:25:33 +03:00
//
// helper functions
//
func (d *Dnsfilter) checkAddRule(t *testing.T, rule string) {
t.Helper()
err := d.AddRule(rule, 0)
if err == nil {
// nothing to report
return
}
if err == ErrInvalidSyntax {
t.Errorf("This rule has invalid syntax: %s", rule)
}
if err != nil {
t.Errorf("Error while adding rule %s: %s", rule, err)
}
}
func (d *Dnsfilter) checkAddRuleFail(t *testing.T, rule string) {
t.Helper()
err := d.AddRule(rule, 0)
2018-10-29 16:17:18 +03:00
if err == ErrInvalidSyntax || err == ErrAlreadyExists {
2018-08-30 17:25:33 +03:00
return
}
if err != nil {
t.Errorf("Error while adding rule %s: %s", rule, err)
}
t.Errorf("Adding this rule should have failed: %s", rule)
}
func (d *Dnsfilter) checkMatch(t *testing.T, hostname string) {
t.Helper()
ret, err := d.CheckHost(hostname)
if err != nil {
t.Errorf("Error while matching host %s: %s", hostname, err)
}
if !ret.IsFiltered {
t.Errorf("Expected hostname %s to match", hostname)
}
}
func (d *Dnsfilter) checkMatchIp(t *testing.T, hostname string, ip string) {
t.Helper()
ret, err := d.CheckHost(hostname)
if err != nil {
t.Errorf("Error while matching host %s: %s", hostname, err)
}
if !ret.IsFiltered {
t.Errorf("Expected hostname %s to match", hostname)
}
if ret.Ip == nil || ret.Ip.String() != ip {
t.Errorf("Expected ip %s to match, actual: %v", ip, ret.Ip)
}
}
2018-08-30 17:25:33 +03:00
func (d *Dnsfilter) checkMatchEmpty(t *testing.T, hostname string) {
t.Helper()
ret, err := d.CheckHost(hostname)
if err != nil {
t.Errorf("Error while matching host %s: %s", hostname, err)
}
if ret.IsFiltered {
t.Errorf("Expected hostname %s to not match", hostname)
}
}
func loadTestRules(d *Dnsfilter) error {
filterFileName := "../tests/dns.txt"
file, err := os.Open(filterFileName)
if err != nil {
return err
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
rule := scanner.Text()
err = d.AddRule(rule, 0)
2018-10-29 16:17:18 +03:00
if err == ErrInvalidSyntax || err == ErrAlreadyExists {
2018-08-30 17:25:33 +03:00
continue
}
if err != nil {
return err
}
}
err = scanner.Err()
return err
}
func mustLoadTestRules(d *Dnsfilter) {
err := loadTestRules(d)
if err != nil {
panic(err)
}
}
2018-08-30 17:25:33 +03:00
func NewForTest() *Dnsfilter {
d := New()
purgeCaches()
return d
}
//
// tests
//
func TestSanityCheck(t *testing.T) {
d := NewForTest()
defer d.Destroy()
2018-08-30 17:25:33 +03:00
d.checkAddRule(t, "||doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
2018-08-30 17:25:33 +03:00
d.checkMatch(t, "www.doubleclick.net")
d.checkMatchEmpty(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
d.checkMatchEmpty(t, "wmconvirus.narod.ru")
d.checkAddRuleFail(t, "lkfaojewhoawehfwacoefawr$@#$@3413841384")
}
func TestEtcHostsMatching(t *testing.T) {
d := NewForTest()
defer d.Destroy()
addr := "216.239.38.120"
text := fmt.Sprintf(" %s google.com www.google.com # enforce google's safesearch ", addr)
d.checkAddRule(t, text)
d.checkMatchIp(t, "google.com", addr)
d.checkMatchIp(t, "www.google.com", addr)
d.checkMatchEmpty(t, "subdomain.google.com")
d.checkMatchEmpty(t, "example.org")
}
func TestSuffixMatching1(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "||doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatch(t, "www.doubleclick.net")
d.checkMatchEmpty(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
}
func TestSuffixMatching2(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "|doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatchEmpty(t, "www.doubleclick.net")
d.checkMatchEmpty(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
}
func TestSuffixMatching3(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatch(t, "www.doubleclick.net")
d.checkMatch(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
}
func TestSuffixMatching4(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "*doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatch(t, "www.doubleclick.net")
d.checkMatch(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
}
func TestSuffixMatching5(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "|*doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatch(t, "www.doubleclick.net")
d.checkMatch(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
}
func TestSuffixMatching6(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "||*doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatch(t, "www.doubleclick.net")
d.checkMatch(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
}
2018-08-30 17:25:33 +03:00
func TestCount(t *testing.T) {
d := NewForTest()
defer d.Destroy()
err := loadTestRules(d)
if err != nil {
t.Fatal(err)
}
count := d.Count()
expected := 12747
if count != expected {
t.Fatalf("Number of rules parsed should be %d, but it is %d\n", expected, count)
}
}
func TestDnsFilterBlocking(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "||example.org^")
d.checkMatch(t, "example.org")
d.checkMatch(t, "test.example.org")
d.checkMatch(t, "test.test.example.org")
d.checkMatchEmpty(t, "testexample.org")
d.checkMatchEmpty(t, "onemoreexample.org")
}
func TestDnsFilterWhitelist(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "||example.org^")
d.checkAddRule(t, "@@||test.example.org")
d.checkMatch(t, "example.org")
d.checkMatchEmpty(t, "test.example.org")
d.checkMatchEmpty(t, "test.test.example.org")
2018-11-26 14:09:43 +03:00
d.checkAddRule(t, "||googleadapis.l.google.com^|")
d.checkMatch(t, "googleadapis.l.google.com")
d.checkMatch(t, "test.googleadapis.l.google.com")
d.checkAddRule(t, "@@||googleadapis.l.google.com|")
d.checkMatchEmpty(t, "googleadapis.l.google.com")
d.checkMatchEmpty(t, "test.googleadapis.l.google.com")
2018-08-30 17:25:33 +03:00
}
func TestDnsFilterImportant(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "@@||example.org^")
d.checkAddRule(t, "||test.example.org^$important")
d.checkMatchEmpty(t, "example.org")
d.checkMatch(t, "test.example.org")
d.checkMatch(t, "test.test.example.org")
d.checkMatchEmpty(t, "testexample.org")
d.checkMatchEmpty(t, "onemoreexample.org")
}
func TestDnsFilterRegexrule(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "/example\\.org/")
d.checkAddRule(t, "@@||test.example.org^")
d.checkMatch(t, "example.org")
d.checkMatchEmpty(t, "test.example.org")
d.checkMatchEmpty(t, "test.test.example.org")
d.checkMatch(t, "testexample.org")
d.checkMatch(t, "onemoreexample.org")
}
func TestDomainMask(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "test*.example.org^")
d.checkAddRule(t, "exam*.com")
d.checkMatch(t, "test.example.org")
d.checkMatch(t, "test2.example.org")
d.checkMatch(t, "example.com")
d.checkMatch(t, "exampleeee.com")
d.checkMatchEmpty(t, "example.org")
d.checkMatchEmpty(t, "testexample.org")
d.checkMatchEmpty(t, "example.co.uk")
}
func TestAddRuleFail(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRuleFail(t, "lkfaojewhoawehfwacoefawr$@#$@3413841384")
}
func TestSafeBrowsing(t *testing.T) {
testCases := []string{
"",
"sb.adtidy.org",
}
for _, tc := range testCases {
t.Run(fmt.Sprintf("%s in %s", tc, _Func()), func(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.EnableSafeBrowsing()
stats.Safebrowsing.Requests = 0
d.checkMatch(t, "wmconvirus.narod.ru")
d.checkMatch(t, "wmconvirus.narod.ru")
if stats.Safebrowsing.Requests != 1 {
t.Errorf("Safebrowsing lookup positive cache is not working: %v", stats.Safebrowsing.Requests)
}
d.checkMatch(t, "WMconvirus.narod.ru")
if stats.Safebrowsing.Requests != 1 {
t.Errorf("Safebrowsing lookup positive cache is not working: %v", stats.Safebrowsing.Requests)
}
d.checkMatch(t, "wmconvirus.narod.ru.")
d.checkMatch(t, "test.wmconvirus.narod.ru")
d.checkMatch(t, "test.wmconvirus.narod.ru.")
d.checkMatchEmpty(t, "yandex.ru")
d.checkMatchEmpty(t, "pornhub.com")
l := stats.Safebrowsing.Requests
d.checkMatchEmpty(t, "pornhub.com")
if stats.Safebrowsing.Requests != l {
t.Errorf("Safebrowsing lookup negative cache is not working: %v", stats.Safebrowsing.Requests)
}
})
}
}
func TestParallelSB(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.EnableSafeBrowsing()
t.Run("group", func(t *testing.T) {
for i := 0; i < 100; i++ {
t.Run(fmt.Sprintf("aaa%d", i), func(t *testing.T) {
t.Parallel()
d.checkMatch(t, "wmconvirus.narod.ru")
d.checkMatch(t, "wmconvirus.narod.ru.")
d.checkMatch(t, "test.wmconvirus.narod.ru")
d.checkMatch(t, "test.wmconvirus.narod.ru.")
d.checkMatchEmpty(t, "yandex.ru")
d.checkMatchEmpty(t, "pornhub.com")
})
}
})
}
// the only way to verify that custom server option is working is to point it at a server that does serve safebrowsing
func TestSafeBrowsingCustomServerFail(t *testing.T) {
d := NewForTest()
defer d.Destroy()
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// w.Write("Hello, client")
fmt.Fprintln(w, "Hello, client")
}))
defer ts.Close()
address := ts.Listener.Addr().String()
d.EnableSafeBrowsing()
d.SetHTTPTimeout(time.Second * 5)
d.SetSafeBrowsingServer(address) // this will ensure that test fails
d.checkMatchEmpty(t, "wmconvirus.narod.ru")
}
func TestParentalControl(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.EnableParental(3)
d.checkMatch(t, "pornhub.com")
d.checkMatch(t, "pornhub.com")
if stats.Parental.Requests != 1 {
t.Errorf("Parental lookup positive cache is not working")
}
d.checkMatch(t, "PORNhub.com")
if stats.Parental.Requests != 1 {
t.Errorf("Parental lookup positive cache is not working")
}
d.checkMatch(t, "www.pornhub.com")
d.checkMatch(t, "pornhub.com.")
d.checkMatch(t, "www.pornhub.com.")
d.checkMatchEmpty(t, "www.yandex.ru")
d.checkMatchEmpty(t, "yandex.ru")
l := stats.Parental.Requests
d.checkMatchEmpty(t, "yandex.ru")
if stats.Parental.Requests != l {
t.Errorf("Parental lookup negative cache is not working")
}
d.checkMatchEmpty(t, "api.jquery.com")
2018-08-30 17:25:33 +03:00
}
func TestSafeSearch(t *testing.T) {
d := NewForTest()
defer d.Destroy()
_, ok := d.SafeSearchDomain("www.google.com")
if ok {
t.Errorf("Expected safesearch to error when disabled")
}
d.EnableSafeSearch()
val, ok := d.SafeSearchDomain("www.google.com")
if !ok {
t.Errorf("Expected safesearch to find result for www.google.com")
}
if val != "forcesafesearch.google.com" {
t.Errorf("Expected safesearch for google.com to be forcesafesearch.google.com")
}
}
//
// parametrized testing
//
var blockingRules = []string{"||example.org^"}
var whitelistRules = []string{"||example.org^", "@@||test.example.org"}
var importantRules = []string{"@@||example.org^", "||test.example.org^$important"}
var regexRules = []string{"/example\\.org/", "@@||test.example.org^"}
var maskRules = []string{"test*.example.org^", "exam*.com"}
var tests = []struct {
testname string
rules []string
hostname string
isFiltered bool
reason Reason
2018-08-30 17:25:33 +03:00
}{
{"sanity", []string{"||doubleclick.net^"}, "www.doubleclick.net", true, FilteredBlackList},
{"sanity", []string{"||doubleclick.net^"}, "nodoubleclick.net", false, NotFilteredNotFound},
{"sanity", []string{"||doubleclick.net^"}, "doubleclick.net.ru", false, NotFilteredNotFound},
{"sanity", []string{"||doubleclick.net^"}, "wmconvirus.narod.ru", false, NotFilteredNotFound},
{"blocking", blockingRules, "example.org", true, FilteredBlackList},
{"blocking", blockingRules, "test.example.org", true, FilteredBlackList},
{"blocking", blockingRules, "test.test.example.org", true, FilteredBlackList},
{"blocking", blockingRules, "testexample.org", false, NotFilteredNotFound},
{"blocking", blockingRules, "onemoreexample.org", false, NotFilteredNotFound},
{"whitelist", whitelistRules, "example.org", true, FilteredBlackList},
{"whitelist", whitelistRules, "test.example.org", false, NotFilteredWhiteList},
{"whitelist", whitelistRules, "test.test.example.org", false, NotFilteredWhiteList},
{"whitelist", whitelistRules, "testexample.org", false, NotFilteredNotFound},
{"whitelist", whitelistRules, "onemoreexample.org", false, NotFilteredNotFound},
{"important", importantRules, "example.org", false, NotFilteredWhiteList},
{"important", importantRules, "test.example.org", true, FilteredBlackList},
{"important", importantRules, "test.test.example.org", true, FilteredBlackList},
{"important", importantRules, "testexample.org", false, NotFilteredNotFound},
{"important", importantRules, "onemoreexample.org", false, NotFilteredNotFound},
{"regex", regexRules, "example.org", true, FilteredBlackList},
{"regex", regexRules, "test.example.org", false, NotFilteredWhiteList},
{"regex", regexRules, "test.test.example.org", false, NotFilteredWhiteList},
{"regex", regexRules, "testexample.org", true, FilteredBlackList},
{"regex", regexRules, "onemoreexample.org", true, FilteredBlackList},
{"mask", maskRules, "test.example.org", true, FilteredBlackList},
{"mask", maskRules, "test2.example.org", true, FilteredBlackList},
{"mask", maskRules, "example.com", true, FilteredBlackList},
{"mask", maskRules, "exampleeee.com", true, FilteredBlackList},
{"mask", maskRules, "onemoreexamsite.com", true, FilteredBlackList},
{"mask", maskRules, "example.org", false, NotFilteredNotFound},
{"mask", maskRules, "testexample.org", false, NotFilteredNotFound},
{"mask", maskRules, "example.co.uk", false, NotFilteredNotFound},
2018-08-30 17:25:33 +03:00
}
func TestMatching(t *testing.T) {
for _, test := range tests {
t.Run(fmt.Sprintf("%s-%s", test.testname, test.hostname), func(t *testing.T) {
d := NewForTest()
defer d.Destroy()
for _, rule := range test.rules {
err := d.AddRule(rule, 0)
if err != nil {
t.Fatal(err)
}
}
ret, err := d.CheckHost(test.hostname)
if err != nil {
t.Errorf("Error while matching host %s: %s", test.hostname, err)
}
if ret.IsFiltered != test.isFiltered {
t.Errorf("Hostname %s has wrong result (%v must be %v)", test.hostname, ret.IsFiltered, test.isFiltered)
}
if ret.Reason != test.reason {
t.Errorf("Hostname %s has wrong reason (%v must be %v)", test.hostname, ret.Reason.String(), test.reason.String())
2018-08-30 17:25:33 +03:00
}
})
}
}
//
// benchmarks
//
func BenchmarkAddRule(b *testing.B) {
d := NewForTest()
defer d.Destroy()
for n := 0; n < b.N; n++ {
rule := "||doubleclick.net^"
err := d.AddRule(rule, 0)
switch err {
case nil:
2018-10-29 16:17:18 +03:00
case ErrAlreadyExists: // ignore rules which were already added
2018-08-30 17:25:33 +03:00
case ErrInvalidSyntax: // ignore invalid syntax
default:
b.Fatalf("Error while adding rule %s: %s", rule, err)
}
}
}
func BenchmarkAddRuleParallel(b *testing.B) {
d := NewForTest()
defer d.Destroy()
rule := "||doubleclick.net^"
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
var err error
for pb.Next() {
err = d.AddRule(rule, 0)
}
switch err {
case nil:
2018-10-29 16:17:18 +03:00
case ErrAlreadyExists: // ignore rules which were already added
2018-08-30 17:25:33 +03:00
case ErrInvalidSyntax: // ignore invalid syntax
default:
b.Fatalf("Error while adding rule %s: %s", rule, err)
}
})
}
func BenchmarkLotsOfRulesNoMatch(b *testing.B) {
d := NewForTest()
defer d.Destroy()
err := loadTestRules(d)
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for n := 0; n < b.N; n++ {
hostname := "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.thisistesthost.com"
ret, err := d.CheckHost(hostname)
if err != nil {
b.Errorf("Error while matching host %s: %s", hostname, err)
}
if ret.IsFiltered {
b.Errorf("Expected hostname %s to not match", hostname)
}
}
}
func BenchmarkLotsOfRulesNoMatchParallel(b *testing.B) {
d := NewForTest()
defer d.Destroy()
err := loadTestRules(d)
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
hostname := "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.thisistesthost.com"
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
ret, err := d.CheckHost(hostname)
if err != nil {
b.Errorf("Error while matching host %s: %s", hostname, err)
}
if ret.IsFiltered {
b.Errorf("Expected hostname %s to not match", hostname)
}
}
})
}
func BenchmarkLotsOfRulesMatch(b *testing.B) {
d := NewForTest()
defer d.Destroy()
err := loadTestRules(d)
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for n := 0; n < b.N; n++ {
const hostname = "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.ad.doubleclick.net"
ret, err := d.CheckHost(hostname)
if err != nil {
b.Errorf("Error while matching host %s: %s", hostname, err)
}
if !ret.IsFiltered {
b.Errorf("Expected hostname %s to match", hostname)
}
}
}
func BenchmarkLotsOfRulesMatchParallel(b *testing.B) {
d := NewForTest()
defer d.Destroy()
err := loadTestRules(d)
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
const hostname = "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.ad.doubleclick.net"
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
ret, err := d.CheckHost(hostname)
if err != nil {
b.Errorf("Error while matching host %s: %s", hostname, err)
}
if !ret.IsFiltered {
b.Errorf("Expected hostname %s to match", hostname)
}
}
})
}
func BenchmarkLotsOfRulesLotsOfHosts(b *testing.B) {
d := NewForTest()
defer d.Destroy()
mustLoadTestRules(d)
getTopHosts()
hostnames, err := os.Open(topHostsFilename)
if err != nil {
b.Fatal(err)
}
defer hostnames.Close()
scanner := bufio.NewScanner(hostnames)
b.ResetTimer()
for n := 0; n < b.N; n++ {
havedata := scanner.Scan()
if !havedata {
hostnames.Seek(0, 0)
scanner = bufio.NewScanner(hostnames)
havedata = scanner.Scan()
}
if !havedata {
b.Fatal(scanner.Err())
}
line := scanner.Text()
records := strings.Split(line, ",")
ret, err := d.CheckHost(records[1] + "." + records[1])
if err != nil {
b.Error(err)
}
if ret.Reason.Matched() {
// log.Printf("host \"%s\" mathed. Rule \"%s\", reason: %v", host, ret.Rule, ret.Reason)
}
}
}
func BenchmarkLotsOfRulesLotsOfHostsParallel(b *testing.B) {
d := NewForTest()
defer d.Destroy()
mustLoadTestRules(d)
getTopHosts()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
hostnames, err := os.Open(topHostsFilename)
if err != nil {
b.Fatal(err)
}
defer hostnames.Close()
scanner := bufio.NewScanner(hostnames)
for pb.Next() {
havedata := scanner.Scan()
if !havedata {
hostnames.Seek(0, 0)
scanner = bufio.NewScanner(hostnames)
havedata = scanner.Scan()
}
if !havedata {
b.Fatal(scanner.Err())
}
line := scanner.Text()
records := strings.Split(line, ",")
ret, err := d.CheckHost(records[1] + "." + records[1])
if err != nil {
b.Error(err)
}
if ret.Reason.Matched() {
// log.Printf("host \"%s\" mathed. Rule \"%s\", reason: %v", host, ret.Rule, ret.Reason)
}
}
})
}
2018-08-30 17:25:33 +03:00
func BenchmarkSafeBrowsing(b *testing.B) {
d := NewForTest()
defer d.Destroy()
d.EnableSafeBrowsing()
for n := 0; n < b.N; n++ {
hostname := "wmconvirus.narod.ru"
ret, err := d.CheckHost(hostname)
if err != nil {
b.Errorf("Error while matching host %s: %s", hostname, err)
}
if !ret.IsFiltered {
b.Errorf("Expected hostname %s to match", hostname)
}
}
}
func BenchmarkSafeBrowsingParallel(b *testing.B) {
d := NewForTest()
defer d.Destroy()
d.EnableSafeBrowsing()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
hostname := "wmconvirus.narod.ru"
ret, err := d.CheckHost(hostname)
if err != nil {
b.Errorf("Error while matching host %s: %s", hostname, err)
}
if !ret.IsFiltered {
b.Errorf("Expected hostname %s to match", hostname)
}
}
})
}
func BenchmarkSafeSearch(b *testing.B) {
d := NewForTest()
defer d.Destroy()
d.EnableSafeSearch()
for n := 0; n < b.N; n++ {
val, ok := d.SafeSearchDomain("www.google.com")
if !ok {
b.Errorf("Expected safesearch to find result for www.google.com")
}
if val != "forcesafesearch.google.com" {
b.Errorf("Expected safesearch for google.com to be forcesafesearch.google.com")
}
}
}
func BenchmarkSafeSearchParallel(b *testing.B) {
d := NewForTest()
defer d.Destroy()
d.EnableSafeSearch()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
val, ok := d.SafeSearchDomain("www.google.com")
if !ok {
b.Errorf("Expected safesearch to find result for www.google.com")
}
if val != "forcesafesearch.google.com" {
b.Errorf("Expected safesearch for google.com to be forcesafesearch.google.com")
}
}
})
}
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}
//
// helper functions for debugging and testing
//
func purgeCaches() {
if safebrowsingCache != nil {
safebrowsingCache.Purge()
}
if parentalCache != nil {
parentalCache.Purge()
}
}
func _Func() string {
pc := make([]uintptr, 10) // at least 1 entry needed
runtime.Callers(2, pc)
f := runtime.FuncForPC(pc[0])
return path.Base(f.Name())
}
func trace(format string, args ...interface{}) {
pc := make([]uintptr, 10) // at least 1 entry needed
runtime.Callers(2, pc)
f := runtime.FuncForPC(pc[0])
var buf strings.Builder
buf.WriteString(fmt.Sprintf("%s(): ", path.Base(f.Name())))
text := fmt.Sprintf(format, args...)
buf.WriteString(text)
if len(text) == 0 || text[len(text)-1] != '\n' {
buf.WriteRune('\n')
}
fmt.Print(buf.String())
}