mirror of
https://github.com/AdguardTeam/AdGuardHome.git
synced 2024-12-15 19:31:45 +03:00
* dnsfilter: adapt tests to new interface
This commit is contained in:
Simon Zolin
parent
829415da5b
commit
5ec747b30b
@ -1,278 +1,51 @@
|
||||
package dnsfilter
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path"
|
||||
"runtime/pprof"
|
||||
"strings"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/shirou/gopsutil/process"
|
||||
"go.uber.org/goleak"
|
||||
)
|
||||
|
||||
// first in file because it must be run first
|
||||
func TestLotsOfRulesMemoryUsage(t *testing.T) {
|
||||
start := getRSS()
|
||||
log.Tracef("RSS before loading rules - %d kB\n", start/1024)
|
||||
dumpMemProfile("tests/" + _Func() + "1.pprof")
|
||||
// HELPERS
|
||||
// SAFE BROWSING
|
||||
// SAFE SEARCH
|
||||
// PARENTAL
|
||||
// FILTERING
|
||||
// BENCHMARKS
|
||||
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
err := loadTestRules(d)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
// HELPERS
|
||||
|
||||
afterLoad := getRSS()
|
||||
log.Tracef("RSS after loading rules - %d kB (%d kB diff)\n", afterLoad/1024, (afterLoad-start)/1024)
|
||||
dumpMemProfile("tests/" + _Func() + "2.pprof")
|
||||
|
||||
tests := []struct {
|
||||
host string
|
||||
match bool
|
||||
}{
|
||||
{"asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.thisistesthost.com", false},
|
||||
{"asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.ad.doubleclick.net", true},
|
||||
func purgeCaches() {
|
||||
if safebrowsingCache != nil {
|
||||
safebrowsingCache.Purge()
|
||||
}
|
||||
for _, testcase := range tests {
|
||||
ret, err := d.CheckHost(testcase.host)
|
||||
if err != nil {
|
||||
t.Errorf("Error while matching host %s: %s", testcase.host, err)
|
||||
}
|
||||
if !ret.IsFiltered && ret.IsFiltered != testcase.match {
|
||||
t.Errorf("Expected hostname %s to not match", testcase.host)
|
||||
}
|
||||
if ret.IsFiltered && ret.IsFiltered != testcase.match {
|
||||
t.Errorf("Expected hostname %s to match", testcase.host)
|
||||
}
|
||||
}
|
||||
afterMatch := getRSS()
|
||||
log.Tracef("RSS after matching - %d kB (%d kB diff)\n", afterMatch/1024, (afterMatch-afterLoad)/1024)
|
||||
dumpMemProfile("tests/" + _Func() + "3.pprof")
|
||||
}
|
||||
|
||||
func getRSS() uint64 {
|
||||
proc, err := process.NewProcess(int32(os.Getpid()))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
minfo, err := proc.MemoryInfo()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return minfo.RSS
|
||||
}
|
||||
|
||||
func dumpMemProfile(name string) {
|
||||
runtime.GC()
|
||||
f, err := os.Create(name)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer f.Close()
|
||||
runtime.GC() // update the stats before writing them
|
||||
err = pprof.WriteHeapProfile(f)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
if parentalCache != nil {
|
||||
parentalCache.Purge()
|
||||
}
|
||||
}
|
||||
|
||||
const topHostsFilename = "tests/top-1m.csv"
|
||||
|
||||
func fetchTopHostsFromNet() {
|
||||
log.Tracef("Fetching top hosts from network")
|
||||
resp, err := http.Get("http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
log.Tracef("Reading zipfile body")
|
||||
zipfile, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
log.Tracef("Opening zipfile")
|
||||
r, err := zip.NewReader(bytes.NewReader(zipfile), int64(len(zipfile)))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if len(r.File) != 1 {
|
||||
panic(fmt.Errorf("zipfile must have only one entry: %+v", r))
|
||||
}
|
||||
f := r.File[0]
|
||||
log.Tracef("Unpacking file %s from zipfile", f.Name)
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
log.Tracef("Reading file %s contents", f.Name)
|
||||
body, err := ioutil.ReadAll(rc)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
rc.Close()
|
||||
|
||||
log.Tracef("Writing file %s contents to disk", f.Name)
|
||||
err = ioutil.WriteFile(topHostsFilename+".tmp", body, 0644)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = os.Rename(topHostsFilename+".tmp", topHostsFilename)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
func _Func() string {
|
||||
pc := make([]uintptr, 10) // at least 1 entry needed
|
||||
runtime.Callers(2, pc)
|
||||
f := runtime.FuncForPC(pc[0])
|
||||
return path.Base(f.Name())
|
||||
}
|
||||
|
||||
func getTopHosts() {
|
||||
// if file doesn't exist, fetch it
|
||||
if _, err := os.Stat(topHostsFilename); os.IsNotExist(err) {
|
||||
// file does not exist, fetch it
|
||||
fetchTopHostsFromNet()
|
||||
}
|
||||
func NewForTest() *Dnsfilter {
|
||||
d := New(nil, nil)
|
||||
purgeCaches()
|
||||
return d
|
||||
}
|
||||
|
||||
func TestLotsOfRulesLotsOfHostsMemoryUsage(t *testing.T) {
|
||||
start := getRSS()
|
||||
log.Tracef("RSS before loading rules - %d kB\n", start/1024)
|
||||
dumpMemProfile("tests/" + _Func() + "1.pprof")
|
||||
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
mustLoadTestRules(d)
|
||||
log.Tracef("Have %d rules", d.Count())
|
||||
|
||||
afterLoad := getRSS()
|
||||
log.Tracef("RSS after loading rules - %d kB (%d kB diff)\n", afterLoad/1024, (afterLoad-start)/1024)
|
||||
dumpMemProfile("tests/" + _Func() + "2.pprof")
|
||||
|
||||
getTopHosts()
|
||||
hostnames, err := os.Open(topHostsFilename)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer hostnames.Close()
|
||||
afterHosts := getRSS()
|
||||
log.Tracef("RSS after loading hosts - %d kB (%d kB diff)\n", afterHosts/1024, (afterHosts-afterLoad)/1024)
|
||||
dumpMemProfile("tests/" + _Func() + "2.pprof")
|
||||
|
||||
{
|
||||
scanner := bufio.NewScanner(hostnames)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
records := strings.Split(line, ",")
|
||||
ret, err := d.CheckHost(records[1] + "." + records[1])
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if ret.Reason.Matched() {
|
||||
// log.Printf("host \"%s\" mathed. Rule \"%s\", reason: %v", host, ret.Rule, ret.Reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
afterMatch := getRSS()
|
||||
log.Tracef("RSS after matching - %d kB (%d kB diff)\n", afterMatch/1024, (afterMatch-afterLoad)/1024)
|
||||
dumpMemProfile("tests/" + _Func() + "3.pprof")
|
||||
}
|
||||
|
||||
func TestRuleToRegexp(t *testing.T) {
|
||||
tests := []struct {
|
||||
rule string
|
||||
result string
|
||||
err error
|
||||
}{
|
||||
{"/doubleclick/", "doubleclick", nil},
|
||||
{"/", "", ErrInvalidSyntax},
|
||||
{`|double*?.+[]|(){}#$\|`, `^double.*\?\.\+\[\]\|\(\)\{\}\#\$\\$`, nil},
|
||||
{`||doubleclick.net^`, `(?:^|\.)doubleclick\.net$`, nil},
|
||||
}
|
||||
for _, testcase := range tests {
|
||||
converted, err := ruleToRegexp(testcase.rule)
|
||||
if err != testcase.err {
|
||||
t.Error("Errors do not match, got ", err, " expected ", testcase.err)
|
||||
}
|
||||
if converted != testcase.result {
|
||||
t.Error("Results do not match, got ", converted, " expected ", testcase.result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuffixRule(t *testing.T) {
|
||||
for _, testcase := range []struct {
|
||||
rule string
|
||||
isSuffix bool
|
||||
suffix string
|
||||
}{
|
||||
{`||doubleclick.net^`, true, `doubleclick.net`}, // entire string or subdomain match
|
||||
{`||doubleclick.net|`, true, `doubleclick.net`}, // entire string or subdomain match
|
||||
{`|doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
|
||||
{`*doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
|
||||
{`doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
|
||||
{`|*doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
|
||||
{`||*doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
|
||||
{`||*doubleclick.net|`, false, ``}, // TODO: ends with doubleclick.net
|
||||
{`||*doublec*lick.net^`, false, ``}, // has a wildcard inside, has to be regexp
|
||||
{`||*doublec|lick.net^`, false, ``}, // has a special symbol inside, has to be regexp
|
||||
{`/abracadabra/`, false, ``}, // regexp, not anchored
|
||||
{`/abracadabra$/`, false, ``}, // TODO: simplify simple suffix regexes
|
||||
} {
|
||||
isSuffix, suffix := getSuffix(testcase.rule)
|
||||
if testcase.isSuffix != isSuffix {
|
||||
t.Errorf("Results do not match for \"%s\": got %v expected %v", testcase.rule, isSuffix, testcase.isSuffix)
|
||||
continue
|
||||
}
|
||||
if testcase.isSuffix && testcase.suffix != suffix {
|
||||
t.Errorf("Result suffix does not match for \"%s\": got \"%s\" expected \"%s\"", testcase.rule, suffix, testcase.suffix)
|
||||
continue
|
||||
}
|
||||
// log.Tracef("\"%s\": %v: %s", testcase.rule, isSuffix, suffix)
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// helper functions
|
||||
//
|
||||
func (d *Dnsfilter) checkAddRule(t *testing.T, rule string) {
|
||||
t.Helper()
|
||||
err := d.AddRule(rule, 0)
|
||||
if err == nil {
|
||||
// nothing to report
|
||||
return
|
||||
}
|
||||
if err == ErrInvalidSyntax {
|
||||
t.Errorf("This rule has invalid syntax: %s", rule)
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("Error while adding rule %s: %s", rule, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dnsfilter) checkAddRuleFail(t *testing.T, rule string) {
|
||||
t.Helper()
|
||||
err := d.AddRule(rule, 0)
|
||||
if err == ErrInvalidSyntax || err == ErrAlreadyExists {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("Error while adding rule %s: %s", rule, err)
|
||||
}
|
||||
t.Errorf("Adding this rule should have failed: %s", rule)
|
||||
func NewForTestFilters(filters map[int]string) *Dnsfilter {
|
||||
d := New(nil, filters)
|
||||
purgeCaches()
|
||||
return d
|
||||
}
|
||||
|
||||
func (d *Dnsfilter) checkMatch(t *testing.T, hostname string) {
|
||||
@ -311,232 +84,21 @@ func (d *Dnsfilter) checkMatchEmpty(t *testing.T, hostname string) {
|
||||
}
|
||||
}
|
||||
|
||||
func loadTestRules(d *Dnsfilter) error {
|
||||
filterFileName := "tests/dns.txt"
|
||||
file, err := os.Open(filterFileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
rule := scanner.Text()
|
||||
err = d.AddRule(rule, 0)
|
||||
if err == ErrInvalidSyntax || err == ErrAlreadyExists {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = scanner.Err()
|
||||
return err
|
||||
}
|
||||
|
||||
func mustLoadTestRules(d *Dnsfilter) {
|
||||
err := loadTestRules(d)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func NewForTest() *Dnsfilter {
|
||||
d := New(nil)
|
||||
purgeCaches()
|
||||
return d
|
||||
}
|
||||
|
||||
//
|
||||
// tests
|
||||
//
|
||||
func TestSanityCheck(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
|
||||
d.checkAddRule(t, "||doubleclick.net^")
|
||||
d.checkMatch(t, "doubleclick.net")
|
||||
d.checkMatch(t, "www.doubleclick.net")
|
||||
d.checkMatchEmpty(t, "nodoubleclick.net")
|
||||
d.checkMatchEmpty(t, "doubleclick.net.ru")
|
||||
d.checkMatchEmpty(t, "wmconvirus.narod.ru")
|
||||
d.checkAddRuleFail(t, "lkfaojewhoawehfwacoefawr$@#$@3413841384")
|
||||
}
|
||||
|
||||
func TestEtcHostsMatching(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
|
||||
addr := "216.239.38.120"
|
||||
text := fmt.Sprintf(" %s google.com www.google.com # enforce google's safesearch ", addr)
|
||||
filters := make(map[int]string)
|
||||
filters[0] = text
|
||||
d := NewForTestFilters(filters)
|
||||
defer d.Destroy()
|
||||
|
||||
d.checkAddRule(t, text)
|
||||
d.checkMatchIP(t, "google.com", addr)
|
||||
d.checkMatchIP(t, "www.google.com", addr)
|
||||
d.checkMatchEmpty(t, "subdomain.google.com")
|
||||
d.checkMatchEmpty(t, "example.org")
|
||||
}
|
||||
|
||||
func TestSuffixMatching1(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
|
||||
d.checkAddRule(t, "||doubleclick.net^")
|
||||
d.checkMatch(t, "doubleclick.net")
|
||||
d.checkMatch(t, "www.doubleclick.net")
|
||||
d.checkMatchEmpty(t, "nodoubleclick.net")
|
||||
d.checkMatchEmpty(t, "doubleclick.net.ru")
|
||||
}
|
||||
|
||||
func TestSuffixMatching2(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
|
||||
d.checkAddRule(t, "|doubleclick.net^")
|
||||
d.checkMatch(t, "doubleclick.net")
|
||||
d.checkMatchEmpty(t, "www.doubleclick.net")
|
||||
d.checkMatchEmpty(t, "nodoubleclick.net")
|
||||
d.checkMatchEmpty(t, "doubleclick.net.ru")
|
||||
}
|
||||
|
||||
func TestSuffixMatching3(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
|
||||
d.checkAddRule(t, "doubleclick.net^")
|
||||
d.checkMatch(t, "doubleclick.net")
|
||||
d.checkMatch(t, "www.doubleclick.net")
|
||||
d.checkMatch(t, "nodoubleclick.net")
|
||||
d.checkMatchEmpty(t, "doubleclick.net.ru")
|
||||
}
|
||||
|
||||
func TestSuffixMatching4(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
|
||||
d.checkAddRule(t, "*doubleclick.net^")
|
||||
d.checkMatch(t, "doubleclick.net")
|
||||
d.checkMatch(t, "www.doubleclick.net")
|
||||
d.checkMatch(t, "nodoubleclick.net")
|
||||
d.checkMatchEmpty(t, "doubleclick.net.ru")
|
||||
}
|
||||
|
||||
func TestSuffixMatching5(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
|
||||
d.checkAddRule(t, "|*doubleclick.net^")
|
||||
d.checkMatch(t, "doubleclick.net")
|
||||
d.checkMatch(t, "www.doubleclick.net")
|
||||
d.checkMatch(t, "nodoubleclick.net")
|
||||
d.checkMatchEmpty(t, "doubleclick.net.ru")
|
||||
}
|
||||
|
||||
func TestSuffixMatching6(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
|
||||
d.checkAddRule(t, "||*doubleclick.net^")
|
||||
d.checkMatch(t, "doubleclick.net")
|
||||
d.checkMatch(t, "www.doubleclick.net")
|
||||
d.checkMatch(t, "nodoubleclick.net")
|
||||
d.checkMatchEmpty(t, "doubleclick.net.ru")
|
||||
}
|
||||
|
||||
func TestCount(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
err := loadTestRules(d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
count := d.Count()
|
||||
expected := 12747
|
||||
if count != expected {
|
||||
t.Fatalf("Number of rules parsed should be %d, but it is %d\n", expected, count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDnsFilterBlocking(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
d.checkAddRule(t, "||example.org^")
|
||||
|
||||
d.checkMatch(t, "example.org")
|
||||
d.checkMatch(t, "test.example.org")
|
||||
d.checkMatch(t, "test.test.example.org")
|
||||
d.checkMatchEmpty(t, "testexample.org")
|
||||
d.checkMatchEmpty(t, "onemoreexample.org")
|
||||
}
|
||||
|
||||
func TestDnsFilterWhitelist(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
d.checkAddRule(t, "||example.org^")
|
||||
d.checkAddRule(t, "@@||test.example.org")
|
||||
|
||||
d.checkMatch(t, "example.org")
|
||||
d.checkMatchEmpty(t, "test.example.org")
|
||||
d.checkMatchEmpty(t, "test.test.example.org")
|
||||
|
||||
d.checkAddRule(t, "||googleadapis.l.google.com^|")
|
||||
d.checkMatch(t, "googleadapis.l.google.com")
|
||||
d.checkMatch(t, "test.googleadapis.l.google.com")
|
||||
|
||||
d.checkAddRule(t, "@@||googleadapis.l.google.com|")
|
||||
d.checkMatchEmpty(t, "googleadapis.l.google.com")
|
||||
d.checkMatchEmpty(t, "test.googleadapis.l.google.com")
|
||||
|
||||
}
|
||||
|
||||
func TestDnsFilterImportant(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
d.checkAddRule(t, "@@||example.org^")
|
||||
d.checkAddRule(t, "||test.example.org^$important")
|
||||
|
||||
d.checkMatchEmpty(t, "example.org")
|
||||
d.checkMatch(t, "test.example.org")
|
||||
d.checkMatch(t, "test.test.example.org")
|
||||
d.checkMatchEmpty(t, "testexample.org")
|
||||
d.checkMatchEmpty(t, "onemoreexample.org")
|
||||
}
|
||||
|
||||
func TestDnsFilterRegexrule(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
d.checkAddRule(t, "/example\\.org/")
|
||||
d.checkAddRule(t, "@@||test.example.org^")
|
||||
|
||||
d.checkMatch(t, "example.org")
|
||||
d.checkMatchEmpty(t, "test.example.org")
|
||||
d.checkMatchEmpty(t, "test.test.example.org")
|
||||
d.checkMatch(t, "testexample.org")
|
||||
d.checkMatch(t, "onemoreexample.org")
|
||||
}
|
||||
|
||||
func TestDomainMask(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
d.checkAddRule(t, "test*.example.org^")
|
||||
d.checkAddRule(t, "exam*.com")
|
||||
|
||||
d.checkMatch(t, "test.example.org")
|
||||
d.checkMatch(t, "test2.example.org")
|
||||
d.checkMatch(t, "example.com")
|
||||
d.checkMatch(t, "exampleeee.com")
|
||||
|
||||
d.checkMatchEmpty(t, "example.org")
|
||||
d.checkMatchEmpty(t, "testexample.org")
|
||||
d.checkMatchEmpty(t, "example.co.uk")
|
||||
}
|
||||
|
||||
func TestAddRuleFail(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
d.checkAddRuleFail(t, "lkfaojewhoawehfwacoefawr$@#$@3413841384")
|
||||
}
|
||||
// SAFE BROWSING
|
||||
|
||||
func TestSafeBrowsing(t *testing.T) {
|
||||
testCases := []string{
|
||||
@ -608,6 +170,25 @@ func TestSafeBrowsingCustomServerFail(t *testing.T) {
|
||||
d.checkMatchEmpty(t, "wmconvirus.narod.ru")
|
||||
}
|
||||
|
||||
// SAFE SEARCH
|
||||
|
||||
func TestSafeSearch(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
_, ok := d.SafeSearchDomain("www.google.com")
|
||||
if ok {
|
||||
t.Errorf("Expected safesearch to error when disabled")
|
||||
}
|
||||
d.SafeSearchEnabled = true
|
||||
val, ok := d.SafeSearchDomain("www.google.com")
|
||||
if !ok {
|
||||
t.Errorf("Expected safesearch to find result for www.google.com")
|
||||
}
|
||||
if val != "forcesafesearch.google.com" {
|
||||
t.Errorf("Expected safesearch for google.com to be forcesafesearch.google.com")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckHostSafeSearchYandex(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
@ -757,6 +338,8 @@ func TestSafeSearchCacheGoogle(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// PARENTAL
|
||||
|
||||
func TestParentalControl(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
@ -785,63 +368,50 @@ func TestParentalControl(t *testing.T) {
|
||||
d.checkMatchEmpty(t, "api.jquery.com")
|
||||
}
|
||||
|
||||
func TestSafeSearch(t *testing.T) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
_, ok := d.SafeSearchDomain("www.google.com")
|
||||
if ok {
|
||||
t.Errorf("Expected safesearch to error when disabled")
|
||||
}
|
||||
d.SafeSearchEnabled = true
|
||||
val, ok := d.SafeSearchDomain("www.google.com")
|
||||
if !ok {
|
||||
t.Errorf("Expected safesearch to find result for www.google.com")
|
||||
}
|
||||
if val != "forcesafesearch.google.com" {
|
||||
t.Errorf("Expected safesearch for google.com to be forcesafesearch.google.com")
|
||||
}
|
||||
}
|
||||
// FILTERING
|
||||
|
||||
//
|
||||
// parametrized testing
|
||||
//
|
||||
var blockingRules = []string{"||example.org^"}
|
||||
var whitelistRules = []string{"||example.org^", "@@||test.example.org"}
|
||||
var importantRules = []string{"@@||example.org^", "||test.example.org^$important"}
|
||||
var regexRules = []string{"/example\\.org/", "@@||test.example.org^"}
|
||||
var maskRules = []string{"test*.example.org^", "exam*.com"}
|
||||
var blockingRules = "||example.org^\n"
|
||||
var whitelistRules = "||example.org^\n@@||test.example.org\n"
|
||||
var importantRules = "@@||example.org^\n||test.example.org^$important\n"
|
||||
var regexRules = "/example\\.org/\n@@||test.example.org^\n"
|
||||
var maskRules = "test*.example.org^\nexam*.com\n"
|
||||
|
||||
var tests = []struct {
|
||||
testname string
|
||||
rules []string
|
||||
rules string
|
||||
hostname string
|
||||
isFiltered bool
|
||||
reason Reason
|
||||
}{
|
||||
{"sanity", []string{"||doubleclick.net^"}, "www.doubleclick.net", true, FilteredBlackList},
|
||||
{"sanity", []string{"||doubleclick.net^"}, "nodoubleclick.net", false, NotFilteredNotFound},
|
||||
{"sanity", []string{"||doubleclick.net^"}, "doubleclick.net.ru", false, NotFilteredNotFound},
|
||||
{"sanity", []string{"||doubleclick.net^"}, "wmconvirus.narod.ru", false, NotFilteredNotFound},
|
||||
{"sanity", "||doubleclick.net^", "www.doubleclick.net", true, FilteredBlackList},
|
||||
{"sanity", "||doubleclick.net^", "nodoubleclick.net", false, NotFilteredNotFound},
|
||||
{"sanity", "||doubleclick.net^", "doubleclick.net.ru", false, NotFilteredNotFound},
|
||||
{"sanity", "||doubleclick.net^", "wmconvirus.narod.ru", false, NotFilteredNotFound},
|
||||
|
||||
{"blocking", blockingRules, "example.org", true, FilteredBlackList},
|
||||
{"blocking", blockingRules, "test.example.org", true, FilteredBlackList},
|
||||
{"blocking", blockingRules, "test.test.example.org", true, FilteredBlackList},
|
||||
{"blocking", blockingRules, "testexample.org", false, NotFilteredNotFound},
|
||||
{"blocking", blockingRules, "onemoreexample.org", false, NotFilteredNotFound},
|
||||
|
||||
{"whitelist", whitelistRules, "example.org", true, FilteredBlackList},
|
||||
{"whitelist", whitelistRules, "test.example.org", false, NotFilteredWhiteList},
|
||||
{"whitelist", whitelistRules, "test.test.example.org", false, NotFilteredWhiteList},
|
||||
{"whitelist", whitelistRules, "testexample.org", false, NotFilteredNotFound},
|
||||
{"whitelist", whitelistRules, "onemoreexample.org", false, NotFilteredNotFound},
|
||||
|
||||
{"important", importantRules, "example.org", false, NotFilteredWhiteList},
|
||||
{"important", importantRules, "test.example.org", true, FilteredBlackList},
|
||||
{"important", importantRules, "test.test.example.org", true, FilteredBlackList},
|
||||
{"important", importantRules, "testexample.org", false, NotFilteredNotFound},
|
||||
{"important", importantRules, "onemoreexample.org", false, NotFilteredNotFound},
|
||||
|
||||
{"regex", regexRules, "example.org", true, FilteredBlackList},
|
||||
{"regex", regexRules, "test.example.org", false, NotFilteredWhiteList},
|
||||
{"regex", regexRules, "test.test.example.org", false, NotFilteredWhiteList},
|
||||
{"regex", regexRules, "testexample.org", true, FilteredBlackList},
|
||||
{"regex", regexRules, "onemoreexample.org", true, FilteredBlackList},
|
||||
|
||||
{"mask", maskRules, "test.example.org", true, FilteredBlackList},
|
||||
{"mask", maskRules, "test2.example.org", true, FilteredBlackList},
|
||||
{"mask", maskRules, "example.com", true, FilteredBlackList},
|
||||
@ -855,14 +425,11 @@ var tests = []struct {
|
||||
func TestMatching(t *testing.T) {
|
||||
for _, test := range tests {
|
||||
t.Run(fmt.Sprintf("%s-%s", test.testname, test.hostname), func(t *testing.T) {
|
||||
d := NewForTest()
|
||||
filters := make(map[int]string)
|
||||
filters[0] = test.rules
|
||||
d := NewForTestFilters(filters)
|
||||
defer d.Destroy()
|
||||
for _, rule := range test.rules {
|
||||
err := d.AddRule(rule, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
ret, err := d.CheckHost(test.hostname)
|
||||
if err != nil {
|
||||
t.Errorf("Error while matching host %s: %s", test.hostname, err)
|
||||
@ -877,204 +444,7 @@ func TestMatching(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// benchmarks
|
||||
//
|
||||
func BenchmarkAddRule(b *testing.B) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
for n := 0; n < b.N; n++ {
|
||||
rule := "||doubleclick.net^"
|
||||
err := d.AddRule(rule, 0)
|
||||
switch err {
|
||||
case nil:
|
||||
case ErrAlreadyExists: // ignore rules which were already added
|
||||
case ErrInvalidSyntax: // ignore invalid syntax
|
||||
default:
|
||||
b.Fatalf("Error while adding rule %s: %s", rule, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAddRuleParallel(b *testing.B) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
rule := "||doubleclick.net^"
|
||||
b.ResetTimer()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
var err error
|
||||
for pb.Next() {
|
||||
err = d.AddRule(rule, 0)
|
||||
}
|
||||
switch err {
|
||||
case nil:
|
||||
case ErrAlreadyExists: // ignore rules which were already added
|
||||
case ErrInvalidSyntax: // ignore invalid syntax
|
||||
default:
|
||||
b.Fatalf("Error while adding rule %s: %s", rule, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkLotsOfRulesNoMatch(b *testing.B) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
err := loadTestRules(d)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
hostname := "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.thisistesthost.com"
|
||||
ret, err := d.CheckHost(hostname)
|
||||
if err != nil {
|
||||
b.Errorf("Error while matching host %s: %s", hostname, err)
|
||||
}
|
||||
if ret.IsFiltered {
|
||||
b.Errorf("Expected hostname %s to not match", hostname)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkLotsOfRulesNoMatchParallel(b *testing.B) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
err := loadTestRules(d)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.ResetTimer()
|
||||
hostname := "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.thisistesthost.com"
|
||||
b.ResetTimer()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
ret, err := d.CheckHost(hostname)
|
||||
if err != nil {
|
||||
b.Errorf("Error while matching host %s: %s", hostname, err)
|
||||
}
|
||||
if ret.IsFiltered {
|
||||
b.Errorf("Expected hostname %s to not match", hostname)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkLotsOfRulesMatch(b *testing.B) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
err := loadTestRules(d)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
const hostname = "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.ad.doubleclick.net"
|
||||
ret, err := d.CheckHost(hostname)
|
||||
if err != nil {
|
||||
b.Errorf("Error while matching host %s: %s", hostname, err)
|
||||
}
|
||||
if !ret.IsFiltered {
|
||||
b.Errorf("Expected hostname %s to match", hostname)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkLotsOfRulesMatchParallel(b *testing.B) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
err := loadTestRules(d)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.ResetTimer()
|
||||
const hostname = "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.ad.doubleclick.net"
|
||||
b.ResetTimer()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
ret, err := d.CheckHost(hostname)
|
||||
if err != nil {
|
||||
b.Errorf("Error while matching host %s: %s", hostname, err)
|
||||
}
|
||||
if !ret.IsFiltered {
|
||||
b.Errorf("Expected hostname %s to match", hostname)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkLotsOfRulesLotsOfHosts(b *testing.B) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
mustLoadTestRules(d)
|
||||
|
||||
getTopHosts()
|
||||
hostnames, err := os.Open(topHostsFilename)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
defer hostnames.Close()
|
||||
|
||||
scanner := bufio.NewScanner(hostnames)
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
havedata := scanner.Scan()
|
||||
if !havedata {
|
||||
_, _ = hostnames.Seek(0, 0)
|
||||
scanner = bufio.NewScanner(hostnames)
|
||||
havedata = scanner.Scan()
|
||||
}
|
||||
if !havedata {
|
||||
b.Fatal(scanner.Err())
|
||||
}
|
||||
line := scanner.Text()
|
||||
records := strings.Split(line, ",")
|
||||
ret, err := d.CheckHost(records[1] + "." + records[1])
|
||||
if err != nil {
|
||||
b.Error(err)
|
||||
}
|
||||
if ret.Reason.Matched() {
|
||||
// log.Printf("host \"%s\" mathed. Rule \"%s\", reason: %v", host, ret.Rule, ret.Reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkLotsOfRulesLotsOfHostsParallel(b *testing.B) {
|
||||
d := NewForTest()
|
||||
defer d.Destroy()
|
||||
mustLoadTestRules(d)
|
||||
|
||||
getTopHosts()
|
||||
|
||||
b.ResetTimer()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
hostnames, err := os.Open(topHostsFilename)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
defer hostnames.Close()
|
||||
scanner := bufio.NewScanner(hostnames)
|
||||
for pb.Next() {
|
||||
havedata := scanner.Scan()
|
||||
if !havedata {
|
||||
_, _ = hostnames.Seek(0, 0)
|
||||
scanner = bufio.NewScanner(hostnames)
|
||||
havedata = scanner.Scan()
|
||||
}
|
||||
if !havedata {
|
||||
b.Fatal(scanner.Err())
|
||||
}
|
||||
line := scanner.Text()
|
||||
records := strings.Split(line, ",")
|
||||
ret, err := d.CheckHost(records[1] + "." + records[1])
|
||||
if err != nil {
|
||||
b.Error(err)
|
||||
}
|
||||
if ret.Reason.Matched() {
|
||||
// log.Printf("host \"%s\" mathed. Rule \"%s\", reason: %v", host, ret.Rule, ret.Reason)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
// BENCHMARKS
|
||||
|
||||
func BenchmarkSafeBrowsing(b *testing.B) {
|
||||
d := NewForTest()
|
||||
@ -1141,26 +511,3 @@ func BenchmarkSafeSearchParallel(b *testing.B) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
goleak.VerifyTestMain(m)
|
||||
}
|
||||
|
||||
//
|
||||
// helper functions for debugging and testing
|
||||
//
|
||||
func purgeCaches() {
|
||||
if safebrowsingCache != nil {
|
||||
safebrowsingCache.Purge()
|
||||
}
|
||||
if parentalCache != nil {
|
||||
parentalCache.Purge()
|
||||
}
|
||||
}
|
||||
|
||||
func _Func() string {
|
||||
pc := make([]uintptr, 10) // at least 1 entry needed
|
||||
runtime.Callers(2, pc)
|
||||
f := runtime.FuncForPC(pc[0])
|
||||
return path.Base(f.Name())
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user