AdguardTeam/AdGuardHome
1
1
Fork 0
mirror of https://github.com/AdguardTeam/AdGuardHome.git synced 2024-09-11 10:45:42 +03:00
Code Issues Packages Projects Releases Wiki Activity

Pull request 1743: upd-sorting

Browse Source 
Merge in DNS/adguard-home from upd-sorting to master

Squashed commit of the following:

commit 7bd21de65c50168d5ad83ff46e63f4cbca365d23
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Tue Feb 21 10:57:17 2023 +0300

    all: upd sorting, go-lint
This commit is contained in:
Ainar Garipov 2023-02-21 16:38:22 +03:00
parent a556ce8fb8
commit 91dee0986b
13 changed files with 185 additions and 296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
internal/dnsforward/config.go
View File

@ -7,7 +7,6 @@ import (
"net"
"net/netip"
"os"
"sort"
"strings"
"time"
@ -23,6 +22,7 @@ import (
"github.com/AdguardTeam/golibs/stringutil"
"github.com/AdguardTeam/golibs/timeutil"
"github.com/ameshkov/dnscrypt/v2"
"golang.org/x/exp/slices"
)
// BlockingMode is an enum of all allowed blocking modes.
@ -510,7 +510,7 @@ func (s *Server) prepareTLS(proxyConfig *proxy.Config) (err error) {
if len(cert.DNSNames) != 0 {
s.conf.dnsNames = cert.DNSNames
log.Debug("dnsforward: using certificate's SAN as DNS names: %v", cert.DNSNames)
sort.Strings(s.conf.dnsNames)
slices.Sort(s.conf.dnsNames)
} else {
s.conf.dnsNames = append(s.conf.dnsNames, cert.Subject.CommonName)
log.Debug("dnsforward: using certificate's CN as DNS name: %s", cert.Subject.CommonName)
@ -526,16 +526,6 @@ func (s *Server) prepareTLS(proxyConfig *proxy.Config) (err error) {
return nil
}
// isInSorted returns true if s is in the sorted slice strs.
func isInSorted(strs []string, s string) (ok bool) {
i := sort.SearchStrings(strs, s)
if i == len(strs) || strs[i] != s {
return false
}
return true
}
// isWildcard returns true if host is a wildcard hostname.
func isWildcard(host string) (ok bool) {
return len(host) >= 2 && host[0] == '*' && host[1] == '.'
@ -554,7 +544,7 @@ func anyNameMatches(dnsNames []string, sni string) (ok bool) {
return false
}
if isInSorted(dnsNames, sni) {
if _, ok = slices.BinarySearch(dnsNames, sni); ok {
return true
}

4
internal/dnsforward/config_test.go
View File

@ -1,15 +1,15 @@
package dnsforward
import (
"sort"
"testing"
"github.com/stretchr/testify/assert"
"golang.org/x/exp/slices"
)
func TestAnyNameMatches(t *testing.T) {
dnsNames := []string{"host1", "*.host2", "1.2.3.4"}
sort.Strings(dnsNames)
slices.Sort(dnsNames)
testCases := []struct {
name string

38
internal/filtering/blocked_test.go
View File

@ -1,38 +0,0 @@
//go:build ignore
// +build ignore
package filtering
import (
"fmt"
"sort"
"testing"
)
// This is a simple tool that takes a list of services and prints them to the output.
// It is supposed to be used to update:
// client/src/helpers/constants.js
// client/src/components/ui/Icons.js
//
// Usage:
// 1. go run ./internal/filtering/blocked_test.go
// 2. Use the output to replace `SERVICES` array in "client/src/helpers/constants.js".
// 3. You'll need to enter services names manually.
// 4. Don't forget to add missing icons to "client/src/components/ui/Icons.js".
//
// TODO(ameshkov): Rework generator: have a JSON file with all the metadata we need
// then use this JSON file to generate JS and Go code
func TestGenServicesArray(t *testing.T) {
services := make([]svc, len(serviceRulesArray))
copy(services, serviceRulesArray)
sort.Slice(services, func(i, j int) bool {
return services[i].name < services[j].name
})
fmt.Println("export const SERVICES = [")
for _, s := range services {
fmt.Printf(" {\n id: '%s',\n name: '%s',\n },\n", s.name, s.name)
}
fmt.Println("];")
}

43
internal/filtering/rewrites.go
View File

@ -1,11 +1,8 @@
// DNS Rewrites
package filtering
import (
"fmt"
"net"
"sort"
"strings"
"github.com/AdguardTeam/golibs/errors"
@ -14,6 +11,8 @@ import (
"golang.org/x/exp/slices"
)
// Legacy DNS rewrites
// LegacyRewrite is a single legacy DNS rewrite record.
//
// Instances of *LegacyRewrite must never be nil.
@ -123,38 +122,24 @@ func matchDomainWildcard(host, wildcard string) (ok bool) {
return isWildcard(wildcard) && strings.HasSuffix(host, wildcard[1:])
}
// rewritesSorted is a slice of legacy rewrites for sorting.
// legacyRewriteSortsBefore sorts rewirtes according to the following priority:
//
// The sorting priority:
//
// 1. A and AAAA > CNAME
// 2. wildcard > exact
// 3. lower level wildcard > higher level wildcard
//
// TODO(a.garipov): Replace with slices.Sort.
type rewritesSorted []*LegacyRewrite
// Len implements the sort.Interface interface for rewritesSorted.
func (a rewritesSorted) Len() (l int) { return len(a) }
// Swap implements the sort.Interface interface for rewritesSorted.
func (a rewritesSorted) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
// Less implements the sort.Interface interface for rewritesSorted.
func (a rewritesSorted) Less(i, j int) (less bool) {
ith, jth := a[i], a[j]
if ith.Type == dns.TypeCNAME && jth.Type != dns.TypeCNAME {
// 1. A and AAAA > CNAME;
// 2. wildcard > exact;
// 3. lower level wildcard > higher level wildcard;
func legacyRewriteSortsBefore(a, b *LegacyRewrite) (sortsBefore bool) {
if a.Type == dns.TypeCNAME && b.Type != dns.TypeCNAME {
return true
} else if ith.Type != dns.TypeCNAME && jth.Type == dns.TypeCNAME {
} else if a.Type != dns.TypeCNAME && b.Type == dns.TypeCNAME {
return false
}
if iw, jw := isWildcard(ith.Domain), isWildcard(jth.Domain); iw != jw {
return jw
if aIsWld, bIsWld := isWildcard(a.Domain), isWildcard(b.Domain); aIsWld != bIsWld {
return bIsWld
}
// Both are either wildcards or not.
return len(ith.Domain) > len(jth.Domain)
// Both are either wildcards or both aren't.
return len(a.Domain) > len(b.Domain)
}
// prepareRewrites normalizes and validates all legacy DNS rewrites.
@ -196,7 +181,7 @@ func findRewrites(
return nil, matched
}
sort.Sort(rewritesSorted(rewrites))
slices.SortFunc(rewrites, legacyRewriteSortsBefore)
for i, r := range rewrites {
if isWildcard(r.Domain) {

6
internal/filtering/safebrowsing.go
View File

@ -8,7 +8,6 @@ import (
"fmt"
"net"
"net/http"
"sort"
"strings"
"sync"
"time"
@ -19,6 +18,7 @@ import (
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/miekg/dns"
"golang.org/x/exp/slices"
"golang.org/x/net/publicsuffix"
)
@ -241,8 +241,8 @@ func (c *sbCtx) processTXT(resp *dns.Msg) (bool, [][]byte) {
}
func (c *sbCtx) storeCache(hashes [][]byte) {
sort.Slice(hashes, func(a, b int) bool {
return bytes.Compare(hashes[a], hashes[b]) == -1
slices.SortFunc(hashes, func(a, b []byte) (sortsBefore bool) {
return bytes.Compare(a, b) == -1
})
var curData []byte

9
internal/home/clients.go
View File

@ -6,7 +6,6 @@ import (
"fmt"
"net"
"net/netip"
"sort"
"strings"
"sync"
"time"
@ -271,7 +270,7 @@ func (clients *clientsContainer) addFromConfig(objects []*clientObject) {
}
}
sort.Strings(cli.Tags)
slices.Sort(cli.Tags)
_, err := clients.Add(cli)
if err != nil {
@ -311,7 +310,9 @@ func (clients *clientsContainer) forConfig() (objs []*clientObject) {
// above loop can generate different orderings when writing to the config
// file: this produces lots of diffs in config files, so sort objects by
// name before writing.
sort.Slice(objs, func(i, j int) bool { return objs[i].Name < objs[j].Name })
slices.SortStableFunc(objs, func(a, b *clientObject) (sortsBefore bool) {
return a.Name < b.Name
})
return objs
}
@ -590,7 +591,7 @@ func (clients *clientsContainer) check(c *Client) (err error) {
}
}
sort.Strings(c.Tags)
slices.Sort(c.Tags)
err = dnsforward.ValidateUpstreams(c.Upstreams)
if err != nil {

6
internal/home/config.go
View File

@ -6,7 +6,6 @@ import (
"net/netip"
"os"
"path/filepath"
"sort"
"sync"
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
@ -21,6 +20,7 @@ import (
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/timeutil"
"github.com/google/renameio/maybe"
"golang.org/x/exp/slices"
yaml "gopkg.in/yaml.v3"
)
@ -490,7 +490,7 @@ func (c *configuration) write() (err error) {
config.Stats.Interval = statsConf.LimitDays
config.Stats.Enabled = statsConf.Enabled
config.Stats.Ignored = statsConf.Ignored.Values()
sort.Strings(config.Stats.Ignored)
slices.Sort(config.Stats.Ignored)
}
if Context.queryLog != nil {
@ -502,7 +502,7 @@ func (c *configuration) write() (err error) {
config.QueryLog.Interval = timeutil.Duration{Duration: dc.RotationIvl}
config.QueryLog.MemSize = dc.MemSize
config.QueryLog.Ignored = dc.Ignored.Values()
sort.Strings(config.QueryLog.Ignored)
slices.Sort(config.Stats.Ignored)
}
if Context.filters != nil {

72
internal/querylog/qlog_test.go
View File

@ -2,11 +2,8 @@ package querylog
import (
"fmt"
"math/rand"
"net"
"sort"
"testing"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/dnsproxy/proxyutil"
@ -352,72 +349,3 @@ func assertLogEntry(t *testing.T, entry *logEntry, host string, answer, client n
ip := proxyutil.IPFromRR(msg.Answer[0]).To16()
assert.Equal(t, answer, ip)
}
func testEntries() (entries []*logEntry) {
rsrc := rand.NewSource(time.Now().UnixNano())
rgen := rand.New(rsrc)
entries = make([]*logEntry, 1000)
for i := range entries {
min := rgen.Intn(60)
sec := rgen.Intn(60)
entries[i] = &logEntry{
Time: time.Date(2020, 1, 1, 0, min, sec, 0, time.UTC),
}
}
return entries
}
// logEntriesByTimeDesc is a wrapper over []*logEntry for sorting.
//
// NOTE(a.garipov): Weirdly enough, on my machine this gets consistently
// outperformed by sort.Slice, see the benchmark below. I'm leaving this
// implementation here, in tests, in case we want to make sure it outperforms on
// most machines, but for now this is unused in the actual code.
type logEntriesByTimeDesc []*logEntry
// Len implements the sort.Interface interface for logEntriesByTimeDesc.
func (les logEntriesByTimeDesc) Len() (n int) { return len(les) }
// Less implements the sort.Interface interface for logEntriesByTimeDesc.
func (les logEntriesByTimeDesc) Less(i, j int) (less bool) {
return les[i].Time.After(les[j].Time)
}
// Swap implements the sort.Interface interface for logEntriesByTimeDesc.
func (les logEntriesByTimeDesc) Swap(i, j int) { les[i], les[j] = les[j], les[i] }
func BenchmarkLogEntry_sort(b *testing.B) {
b.Run("methods", func(b *testing.B) {
for i := 0; i < b.N; i++ {
b.StopTimer()
entries := testEntries()
b.StartTimer()
sort.Stable(logEntriesByTimeDesc(entries))
}
})
b.Run("reflect", func(b *testing.B) {
for i := 0; i < b.N; i++ {
b.StopTimer()
entries := testEntries()
b.StartTimer()
sort.SliceStable(entries, func(i, j int) (less bool) {
return entries[i].Time.After(entries[j].Time)
})
}
})
}
func TestLogEntriesByTime_sort(t *testing.T) {
entries := testEntries()
sort.Sort(logEntriesByTimeDesc(entries))
for i := range entries[1:] {
assert.False(t, entries[i+1].Time.After(entries[i].Time),
"%s %s", entries[i+1].Time, entries[i].Time)
}
}

6
internal/querylog/search.go
View File

@ -2,10 +2,10 @@ package querylog
import (
"io"
"sort"
"time"
"github.com/AdguardTeam/golibs/log"
"golang.org/x/exp/slices"
)
// client finds the client info, if any, by its ClientID and IP address,
@ -98,8 +98,8 @@ func (l *queryLog) search(params *searchParams) (entries []*logEntry, oldest tim
// weird on the frontend.
//
// See https://github.com/AdguardTeam/AdGuardHome/issues/2293.
sort.SliceStable(entries, func(i, j int) (less bool) {
return entries[i].Time.After(entries[j].Time)
slices.SortStableFunc(entries, func(a, b *logEntry) (sortsBefore bool) {
return a.Time.After(b.Time)
})
if params.offset > 0 {

6
internal/stats/unit.go
View File

@ -5,13 +5,13 @@ import (
"encoding/binary"
"encoding/gob"
"fmt"
"sort"
"time"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/stringutil"
"go.etcd.io/bbolt"
"golang.org/x/exp/slices"
)
// TODO(a.garipov): Rewrite all of this. Add proper error handling and
@ -180,8 +180,8 @@ func convertMapToSlice(m map[string]uint64, max int) (s []countPair) {
s = append(s, countPair{Name: k, Count: v})
}
sort.Slice(s, func(i, j int) bool {
return s[j].Count < s[i].Count
slices.SortFunc(s, func(a, b countPair) (sortsBefore bool) {
return a.Count < b.Count
})
if max > len(s) {
max = len(s)

155
scripts/make/go-lint.sh
View File

@ -1,8 +1,13 @@
#!/bin/sh
verbose="${VERBOSE:-0}"
# This comment is used to simplify checking local copies of the script. Bump
# this number every time a significant change is made to this script.
#
# AdGuard-Project-Version: 3
verbose="${VERBOSE:-0}"
readonly verbose
# Set verbosity.
if [ "$verbose" -gt '0' ]
then
set -x
@ -16,34 +21,12 @@ else
set -e
fi
# We don't need glob expansions and we want to see errors about unset variables.
set -f -u
# Deferred Helpers
not_found_msg='
looks like a binary not found error.
make sure you have installed the linter binaries using:
$ make go-tools
'
readonly not_found_msg
# TODO(a.garipov): Put it into a separate script and source it both here and in
# txt-lint.sh?
not_found() {
if [ "$?" -eq '127' ]
then
# Code 127 is the exit status a shell uses when a command or
# a file is not found, according to the Bash Hackers wiki.
#
# See https://wiki.bash-hackers.org/dict/terms/exit_status.
echo "$not_found_msg" 1>&2
fi
}
trap not_found EXIT
# Source the common helpers, including not_found and run_linter.
. ./scripts/make/helper.sh
@ -52,7 +35,7 @@ trap not_found EXIT
go_version="$( "${GO:-go}" version )"
readonly go_version
go_min_version='go1.19'
go_min_version='go1.19.6'
go_version_msg="
warning: your go version (${go_version}) is different from the recommended minimal one (${go_min_version}).
if you have the version installed, please set the GO environment variable.
@ -74,7 +57,7 @@ esac
# Simple Analyzers
# Simple analyzers
# blocklist_imports is a simple check against unwanted packages. The following
# packages are banned:
@ -91,6 +74,8 @@ esac
#
# See https://github.com/golang/go/issues/45200.
#
# * Package sort is replaced by golang.org/x/exp/slices.
#
# * Package unsafe is… unsafe.
#
# * Package golang.org/x/net/context has been moved into stdlib.
@ -101,6 +86,7 @@ blocklist_imports() {
-e '[[:space:]]"io/ioutil"$'\
-e '[[:space:]]"log"$'\
-e '[[:space:]]"reflect"$'\
-e '[[:space:]]"sort"$'\
-e '[[:space:]]"unsafe"$'\
-e '[[:space:]]"golang.org/x/net/context"$'\
-n\
@ -158,96 +144,67 @@ underscores() {
# Helpers
# exit_on_output exits with a nonzero exit code if there is anything in the
# command's combined output.
exit_on_output() (
set +e
if [ "$VERBOSE" -lt '2' ]
then
set +x
fi
cmd="$1"
shift
output="$( "$cmd" "$@" 2>&1 )"
exitcode="$?"
if [ "$exitcode" -ne '0' ]
then
echo "'$cmd' failed with code $exitcode"
fi
if [ "$output" != '' ]
then
if [ "$*" != '' ]
then
echo "combined output of linter '$cmd $*':"
else
echo "combined output of linter '$cmd':"
fi
echo "$output"
if [ "$exitcode" -eq '0' ]
then
exitcode='1'
fi
fi
return "$exitcode"
)
# Checks
exit_on_output blocklist_imports
run_linter -e blocklist_imports
exit_on_output method_const
run_linter -e method_const
exit_on_output underscores
run_linter -e underscores
exit_on_output gofumpt --extra -e -l .
run_linter -e gofumpt --extra -e -l .
# TODO(a.garipov): golint is deprecated, and seems to cause more and more
# issues with each release. Find a suitable replacement.
#
# golint --set_exit_status ./...
# TODO(a.garipov): golint is deprecated, find a suitable replacement.
"$GO" vet ./...
run_linter "$GO" vet ./...
govulncheck ./...
run_linter govulncheck ./...
# Apply more lax standards to the code we haven't properly refactored yet.
gocyclo --over 17 ./internal/querylog/
gocyclo --over 13 ./internal/dhcpd ./internal/filtering/ ./internal/home/
run_linter gocyclo --over 17 ./internal/querylog/
run_linter gocyclo --over 13\
./internal/dhcpd\
./internal/filtering/\
./internal/home/\
;
# Apply stricter standards to new or somewhat refactored code.
gocyclo --over 10 ./internal/aghio/ ./internal/aghnet/ ./internal/aghos/\
./internal/aghtest/ ./internal/dnsforward/ ./internal/filtering/rewrite/\
./internal/stats/ ./internal/tools/ ./internal/updater/ ./internal/next/\
./internal/version/ ./scripts/blocked-services/ ./scripts/vetted-filters/\
./main.go
# Apply the normal standards to new or somewhat refactored code.
run_linter gocyclo --over 10\
./internal/aghio/\
./internal/aghnet/\
./internal/aghos/\
./internal/aghtest/\
./internal/dnsforward/\
./internal/filtering/rewrite/\
./internal/stats/\
./internal/tools/\
./internal/updater/\
./internal/next/\
./internal/version/\
./scripts/blocked-services/\
./scripts/vetted-filters/\
./main.go\
;
ineffassign ./...
run_linter ineffassign ./...
unparam ./...
run_linter unparam ./...
git ls-files -- '*.go' '*.mod' '*.sh' 'Makefile' | xargs misspell --error
git ls-files -- 'Makefile' '*.go' '*.mod' '*.sh' '*.yaml' '*.yml'\
| xargs misspell --error
looppointer ./...
run_linter looppointer ./...
nilness ./...
run_linter nilness ./...
exit_on_output shadow --strict ./...
# TODO(a.garipov): Add fieldalignment?
run_linter -e shadow --strict ./...
# TODO(a.garipov): Enable in v0.108.0.
# gosec --quiet ./...
# run_linter gosec --quiet ./...
# TODO(a.garipov): Enable --blank?
errcheck --asserts ./...
run_linter errcheck --asserts ./...
staticcheck ./...
run_linter staticcheck ./...

84
scripts/make/helper.sh Normal file
View File

@ -0,0 +1,84 @@
#!/bin/sh
# Common script helpers
#
# This file contains common script helpers. It should be sourced in scripts
# right after the initial environment processing.
# This comment is used to simplify checking local copies of the script. Bump
# this number every time a remarkable change is made to this script.
#
# AdGuard-Project-Version: 2
# Deferred helpers
not_found_msg='
looks like a binary not found error.
make sure you have installed the linter binaries using:
$ make go-tools
'
readonly not_found_msg
not_found() {
if [ "$?" -eq '127' ]
then
# Code 127 is the exit status a shell uses when a command or a file is
# not found, according to the Bash Hackers wiki.
#
# See https://wiki.bash-hackers.org/dict/terms/exit_status.
echo "$not_found_msg" 1>&2
fi
}
trap not_found EXIT
# Helpers
# run_linter runs the given linter with two additions:
#
# 1. If the first argument is "-e", run_linter exits with a nonzero exit code
# if there is anything in the command's combined output.
#
# 2. In any case, run_linter adds the program's name to its combined output.
run_linter() (
set +e
if [ "$VERBOSE" -lt '2' ]
then
set +x
fi
cmd="${1:?run_linter: provide a command}"
shift
exit_on_output='0'
if [ "$cmd" = '-e' ]
then
exit_on_output='1'
cmd="${1:?run_linter: provide a command}"
shift
fi
readonly cmd
output="$( "$cmd" "$@" )"
exitcode="$?"
readonly output
if [ "$output" != '' ]
then
echo "$output" | sed -e "s/^/${cmd}: /"
if [ "$exitcode" -eq '0' ] && [ "$exit_on_output" -eq '1' ]
then
exitcode='1'
fi
fi
return "$exitcode"
)

36
scripts/make/txt-lint.sh
View File

@ -1,9 +1,13 @@
#!/bin/sh
# This comment is used to simplify checking local copies of the script. Bump
# this number every time a remarkable change is made to this script.
#
# AdGuard-Project-Version: 2
verbose="${VERBOSE:-0}"
readonly verbose
# Set verbosity.
if [ "$verbose" -gt '0' ]
then
set -x
@ -20,31 +24,9 @@ fi
# We don't need glob expansions and we want to see errors about unset variables.
set -f -u
# Deferred Helpers
not_found_msg='
looks like a binary not found error.
make sure you have installed the linter binaries using:
$ make go-tools
'
readonly not_found_msg
# TODO(a.garipov): Put it into a separate script and source it both here and in
# go-lint.sh?
not_found() {
if [ "$?" -eq '127' ]
then
# Code 127 is the exit status a shell uses when a command or
# a file is not found, according to the Bash Hackers wiki.
#
# See https://wiki.bash-hackers.org/dict/terms/exit_status.
echo "$not_found_msg" 1>&2
fi
}
trap not_found EXIT
# Source the common helpers, including not_found.
. ./scripts/make/helper.sh
git ls-files -- '*.md' '*.yaml' '*.yml' 'client/src/__locales/en.json'\
| xargs misspell --error
| xargs misspell --error\
| sed -e 's/^/misspell: /'