From 7b92d53b848cdd2a42b1c6ddb748bf585f8a0ada Mon Sep 17 00:00:00 2001 From: Ainar Garipov Date: Fri, 7 Jul 2023 18:27:33 +0300 Subject: [PATCH] Pull request 1910: new-rulelist-parser Squashed commit of the following: commit bac0da6818388d67840b8fe9b633ce0804964ed9 Merge: cb6759b63 f7dd83251 Author: Ainar Garipov Date: Fri Jul 7 18:22:40 2023 +0300 Merge branch 'master' into new-rulelist-parser commit cb6759b63546b35074ec0ae04769ddb5e83ebac1 Author: Ainar Garipov Date: Fri Jul 7 12:18:44 2023 +0300 all: upd tools commit d28bf4cb42057b84e0c1325389db121a91f7c989 Author: Ainar Garipov Date: Thu Jul 6 19:35:48 2023 +0300 all: upd chlog commit 7df637b00331dff5810c3a76f4a7d2cee24148f1 Author: Ainar Garipov Date: Thu Jul 6 19:30:42 2023 +0300 rulelist: fix tabs commit 0598d0d43504b246570e9ee76d79dff0d86413c5 Author: Ainar Garipov Date: Thu Jul 6 19:06:18 2023 +0300 all: add go-bench, go-fuzz; imp docs commit a9ab7726048e216b13876a85991f3e3e8696a029 Author: Ainar Garipov Date: Thu Jul 6 18:18:14 2023 +0300 filtering: add new parser --- .gitignore | 1 + CHANGELOG.md | 2 + Makefile | 4 +- internal/aghtest/interface.go | 71 ++--- internal/filtering/filter.go | 211 ++++----------- internal/filtering/filtering.go | 56 ++-- internal/filtering/filtering_test.go | 2 +- internal/filtering/http.go | 2 +- internal/filtering/rewrites.go | 2 +- internal/filtering/rulelist/error.go | 9 + internal/filtering/rulelist/parser.go | 184 +++++++++++++ internal/filtering/rulelist/parser_test.go | 247 ++++++++++++++++++ internal/filtering/rulelist/rulelist.go | 11 + internal/filtering/rulelist/rulelist_test.go | 14 + .../next/websvc/waitlistener_internal_test.go | 4 +- internal/tools/go.mod | 9 +- internal/tools/go.sum | 18 +- scripts/README.md | 51 +++- scripts/make/go-bench.sh | 55 ++++ scripts/make/go-fuzz.sh | 58 ++++ scripts/make/go-lint.sh | 7 +- 21 files changed, 759 insertions(+), 259 deletions(-) create mode 100644 internal/filtering/rulelist/error.go create mode 100644 internal/filtering/rulelist/parser.go create mode 100644 internal/filtering/rulelist/parser_test.go create mode 100644 internal/filtering/rulelist/rulelist.go create mode 100644 internal/filtering/rulelist/rulelist_test.go create mode 100644 scripts/make/go-bench.sh create mode 100644 scripts/make/go-fuzz.sh diff --git a/.gitignore b/.gitignore index e5124973..9e19558a 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ *.db *.log *.snap +*.test /agh-backup/ /bin/ /build/* diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e88d5b7..83342942 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,8 @@ NOTE: Add new changes BELOW THIS COMMENT. ### Changed +- Improved CPU and RAM consumption during updates of filtering-rule lists. + #### Configuration Changes In this release, the schema version has changed from 23 to 24. diff --git a/Makefile b/Makefile index 864734d4..61a3ed0f 100644 --- a/Makefile +++ b/Makefile @@ -78,7 +78,7 @@ build: deps quick-build quick-build: js-build go-build -ci: deps test +ci: deps test go-bench go-fuzz deps: js-deps go-deps lint: js-lint go-lint @@ -104,8 +104,10 @@ js-deps: js-lint: ; $(NPM) $(NPM_FLAGS) run lint js-test: ; $(NPM) $(NPM_FLAGS) run test +go-bench: ; $(ENV) "$(SHELL)" ./scripts/make/go-bench.sh go-build: ; $(ENV) "$(SHELL)" ./scripts/make/go-build.sh go-deps: ; $(ENV) "$(SHELL)" ./scripts/make/go-deps.sh +go-fuzz: ; $(ENV) "$(SHELL)" ./scripts/make/go-fuzz.sh go-lint: ; $(ENV) "$(SHELL)" ./scripts/make/go-lint.sh go-tools: ; $(ENV) "$(SHELL)" ./scripts/make/go-tools.sh diff --git a/internal/aghtest/interface.go b/internal/aghtest/interface.go index 66f0211b..cce49776 100644 --- a/internal/aghtest/interface.go +++ b/internal/aghtest/interface.go @@ -2,8 +2,8 @@ package aghtest import ( "context" + "io" "io/fs" - "net" "github.com/AdguardTeam/AdGuardHome/internal/aghos" "github.com/AdguardTeam/AdGuardHome/internal/next/agh" @@ -19,23 +19,23 @@ import ( // Package fs -// type check -var _ fs.FS = &FS{} - -// FS is a mock [fs.FS] implementation for tests. +// FS is a fake [fs.FS] implementation for tests. type FS struct { OnOpen func(name string) (fs.File, error) } +// type check +var _ fs.FS = (*FS)(nil) + // Open implements the [fs.FS] interface for *FS. func (fsys *FS) Open(name string) (fs.File, error) { return fsys.OnOpen(name) } // type check -var _ fs.GlobFS = &GlobFS{} +var _ fs.GlobFS = (*GlobFS)(nil) -// GlobFS is a mock [fs.GlobFS] implementation for tests. +// GlobFS is a fake [fs.GlobFS] implementation for tests. type GlobFS struct { // FS is embedded here to avoid implementing all it's methods. FS @@ -48,9 +48,9 @@ func (fsys *GlobFS) Glob(pattern string) ([]string, error) { } // type check -var _ fs.StatFS = &StatFS{} +var _ fs.StatFS = (*StatFS)(nil) -// StatFS is a mock [fs.StatFS] implementation for tests. +// StatFS is a fake [fs.StatFS] implementation for tests. type StatFS struct { // FS is embedded here to avoid implementing all it's methods. FS @@ -62,47 +62,34 @@ func (fsys *StatFS) Stat(name string) (fs.FileInfo, error) { return fsys.OnStat(name) } -// Package net +// Package io -// type check -var _ net.Listener = (*Listener)(nil) - -// Listener is a mock [net.Listener] implementation for tests. -type Listener struct { - OnAccept func() (conn net.Conn, err error) - OnAddr func() (addr net.Addr) - OnClose func() (err error) +// Writer is a fake [io.Writer] implementation for tests. +type Writer struct { + OnWrite func(b []byte) (n int, err error) } -// Accept implements the [net.Listener] interface for *Listener. -func (l *Listener) Accept() (conn net.Conn, err error) { - return l.OnAccept() -} +var _ io.Writer = (*Writer)(nil) -// Addr implements the [net.Listener] interface for *Listener. -func (l *Listener) Addr() (addr net.Addr) { - return l.OnAddr() -} - -// Close implements the [net.Listener] interface for *Listener. -func (l *Listener) Close() (err error) { - return l.OnClose() +// Write implements the [io.Writer] interface for *Writer. +func (w *Writer) Write(b []byte) (n int, err error) { + return w.OnWrite(b) } // Module adguard-home // Package aghos -// type check -var _ aghos.FSWatcher = (*FSWatcher)(nil) - -// FSWatcher is a mock [aghos.FSWatcher] implementation for tests. +// FSWatcher is a fake [aghos.FSWatcher] implementation for tests. type FSWatcher struct { OnEvents func() (e <-chan struct{}) OnAdd func(name string) (err error) OnClose func() (err error) } +// type check +var _ aghos.FSWatcher = (*FSWatcher)(nil) + // Events implements the [aghos.FSWatcher] interface for *FSWatcher. func (w *FSWatcher) Events() (e <-chan struct{}) { return w.OnEvents() @@ -120,16 +107,16 @@ func (w *FSWatcher) Close() (err error) { // Package agh -// type check -var _ agh.ServiceWithConfig[struct{}] = (*ServiceWithConfig[struct{}])(nil) - -// ServiceWithConfig is a mock [agh.ServiceWithConfig] implementation for tests. +// ServiceWithConfig is a fake [agh.ServiceWithConfig] implementation for tests. type ServiceWithConfig[ConfigType any] struct { OnStart func() (err error) OnShutdown func(ctx context.Context) (err error) OnConfig func() (c ConfigType) } +// type check +var _ agh.ServiceWithConfig[struct{}] = (*ServiceWithConfig[struct{}])(nil) + // Start implements the [agh.ServiceWithConfig] interface for // *ServiceWithConfig. func (s *ServiceWithConfig[_]) Start() (err error) { @@ -152,10 +139,7 @@ func (s *ServiceWithConfig[ConfigType]) Config() (c ConfigType) { // Package upstream -// type check -var _ upstream.Upstream = (*UpstreamMock)(nil) - -// UpstreamMock is a mock [upstream.Upstream] implementation for tests. +// UpstreamMock is a fake [upstream.Upstream] implementation for tests. // // TODO(a.garipov): Replace with all uses of Upstream with UpstreamMock and // rename it to just Upstream. @@ -165,6 +149,9 @@ type UpstreamMock struct { OnClose func() (err error) } +// type check +var _ upstream.Upstream = (*UpstreamMock)(nil) + // Address implements the [upstream.Upstream] interface for *UpstreamMock. func (u *UpstreamMock) Address() (addr string) { return u.OnAddress() diff --git a/internal/filtering/filter.go b/internal/filtering/filter.go index 44dc7f76..fa512c29 100644 --- a/internal/filtering/filter.go +++ b/internal/filtering/filter.go @@ -1,10 +1,7 @@ package filtering import ( - "bufio" - "bytes" "fmt" - "hash/crc32" "io" "net/http" "os" @@ -14,6 +11,7 @@ import ( "time" "github.com/AdguardTeam/AdGuardHome/internal/aghalg" + "github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/stringutil" @@ -29,9 +27,9 @@ const filterDir = "filters" // TODO(e.burkov): Use more deterministic approach. var nextFilterID = time.Now().Unix() -// FilterYAML respresents a filter list in the configuration file. +// FilterYAML represents a filter list in the configuration file. // -// TODO(e.burkov): Investigate if the field oredering is important. +// TODO(e.burkov): Investigate if the field ordering is important. type FilterYAML struct { Enabled bool URL string // URL or a file path @@ -213,7 +211,7 @@ func (d *DNSFilter) loadFilters(array []FilterYAML) { err := d.load(filter) if err != nil { - log.Error("Couldn't load filter %d contents due to %s", filter.ID, err) + log.Error("filtering: loading filter %d: %s", filter.ID, err) } } } @@ -338,7 +336,8 @@ func (d *DNSFilter) refreshFiltersArray(filters *[]FilterYAML, force bool) (int, updateFlags = append(updateFlags, updated) if err != nil { nfail++ - log.Printf("Failed to update filter %s: %s\n", uf.URL, err) + log.Info("filtering: updating filter from url %q: %s\n", uf.URL, err) + continue } } @@ -367,7 +366,13 @@ func (d *DNSFilter) refreshFiltersArray(filters *[]FilterYAML, force bool) (int, continue } - log.Info("Updated filter #%d. Rules: %d -> %d", f.ID, f.RulesCount, uf.RulesCount) + log.Info( + "filtering: updated filter %d; rule count: %d (was %d)", + f.ID, + uf.RulesCount, + f.RulesCount, + ) + f.Name = uf.Name f.RulesCount = uf.RulesCount f.checksum = uf.checksum @@ -397,9 +402,10 @@ func (d *DNSFilter) refreshFiltersArray(filters *[]FilterYAML, force bool) (int, // // TODO(a.garipov, e.burkov): What the hell? func (d *DNSFilter) refreshFiltersIntl(block, allow, force bool) (int, bool) { - log.Debug("filtering: updating...") - updNum := 0 + log.Debug("filtering: starting updating") + defer func() { log.Debug("filtering: finished updating, %d updated", updNum) }() + var lists []FilterYAML var toUpd []bool isNetErr := false @@ -437,131 +443,9 @@ func (d *DNSFilter) refreshFiltersIntl(block, allow, force bool) (int, bool) { } } - log.Debug("filtering: update finished: %d lists updated", updNum) - return updNum, false } -// isPrintableText returns true if data is printable UTF-8 text with CR, LF, TAB -// characters. -// -// TODO(e.burkov): Investigate the purpose of this and improve the -// implementation. Perhaps, use something from the unicode package. -func isPrintableText(data string) (ok bool) { - for _, c := range []byte(data) { - if (c >= ' ' && c != 0x7f) || c == '\n' || c == '\r' || c == '\t' { - continue - } - - return false - } - - return true -} - -// scanLinesWithBreak is essentially a [bufio.ScanLines] which keeps trailing -// line breaks. -func scanLinesWithBreak(data []byte, atEOF bool) (advance int, token []byte, err error) { - if atEOF && len(data) == 0 { - return 0, nil, nil - } - - if i := bytes.IndexByte(data, '\n'); i >= 0 { - return i + 1, data[0 : i+1], nil - } - - if atEOF { - return len(data), data, nil - } - - // Request more data. - return 0, nil, nil -} - -// parseFilter copies filter's content from src to dst and returns the number of -// rules, number of bytes written, checksum, and title of the parsed list. dst -// must not be nil. -func (d *DNSFilter) parseFilter( - src io.Reader, - dst io.Writer, -) (rulesNum, written int, checksum uint32, title string, err error) { - scanner := bufio.NewScanner(src) - scanner.Split(scanLinesWithBreak) - - titleFound := false - for n := 0; scanner.Scan(); written += n { - line := scanner.Text() - var isRule bool - var likelyTitle string - isRule, likelyTitle, err = d.parseFilterLine(line, !titleFound, written == 0) - if err != nil { - return 0, written, 0, "", err - } - - if isRule { - rulesNum++ - } else if likelyTitle != "" { - title, titleFound = likelyTitle, true - } - - checksum = crc32.Update(checksum, crc32.IEEETable, []byte(line)) - - n, err = dst.Write([]byte(line)) - if err != nil { - return 0, written, 0, "", fmt.Errorf("writing filter line: %w", err) - } - } - - if err = scanner.Err(); err != nil { - return 0, written, 0, "", fmt.Errorf("scanning filter contents: %w", err) - } - - return rulesNum, written, checksum, title, nil -} - -// parseFilterLine returns true if the passed line is a rule. line is -// considered a rule if it's not a comment and contains no title. -func (d *DNSFilter) parseFilterLine( - line string, - lookForTitle bool, - testHTML bool, -) (isRule bool, title string, err error) { - if !isPrintableText(line) { - return false, "", errors.Error("filter contains non-printable characters") - } - - line = strings.TrimSpace(line) - if line == "" || line[0] == '#' { - return false, "", nil - } - - if testHTML && isHTML(line) { - return false, "", errors.Error("data is HTML, not plain text") - } - - if line[0] == '!' && lookForTitle { - match := d.filterTitleRegexp.FindStringSubmatch(line) - if len(match) > 1 { - title = match[1] - } - - return false, title, nil - } - - return true, "", nil -} - -// isHTML returns true if the line contains HTML tags instead of plain text. -// line shouldn have no leading space symbols. -// -// TODO(ameshkov): It actually gives too much false-positives. Perhaps, just -// check if trimmed string begins with angle bracket. -func isHTML(line string) (ok bool) { - line = strings.ToLower(line) - - return strings.HasPrefix(line, "= log.DEBUG { timer := log.StartTimer() - defer timer.LogElapsed("safebrowsing lookup for %q", host) + defer timer.LogElapsed("filtering: safebrowsing lookup for %q", host) } res = Result{ @@ -1079,7 +1077,7 @@ func (d *DNSFilter) checkParental( if log.GetLevel() >= log.DEBUG { timer := log.StartTimer() - defer timer.LogElapsed("parental lookup for %q", host) + defer timer.LogElapsed("filtering: parental lookup for %q", host) } res = Result{ diff --git a/internal/filtering/filtering_test.go b/internal/filtering/filtering_test.go index 8636606b..e7b55d6f 100644 --- a/internal/filtering/filtering_test.go +++ b/internal/filtering/filtering_test.go @@ -547,7 +547,7 @@ func TestWhitelist(t *testing.T) { }} d, setts := newForTest(t, nil, filters) - err := d.SetFilters(filters, whiteFilters, false) + err := d.setFilters(filters, whiteFilters, false) require.NoError(t, err) t.Cleanup(d.Close) diff --git a/internal/filtering/http.go b/internal/filtering/http.go index 4aef0409..8d3f202f 100644 --- a/internal/filtering/http.go +++ b/internal/filtering/http.go @@ -95,7 +95,7 @@ func (d *DNSFilter) handleFilteringAddURL(w http.ResponseWriter, r *http.Request r, w, http.StatusBadRequest, - "Couldn't fetch filter from url %s: %s", + "Couldn't fetch filter from URL %q: %s", filt.URL, err, ) diff --git a/internal/filtering/rewrites.go b/internal/filtering/rewrites.go index 3e10da55..e3625c84 100644 --- a/internal/filtering/rewrites.go +++ b/internal/filtering/rewrites.go @@ -122,7 +122,7 @@ func matchDomainWildcard(host, wildcard string) (ok bool) { return isWildcard(wildcard) && strings.HasSuffix(host, wildcard[1:]) } -// legacyRewriteSortsBefore sorts rewirtes according to the following priority: +// legacyRewriteSortsBefore sorts rewrites according to the following priority: // // 1. A and AAAA > CNAME; // 2. wildcard > exact; diff --git a/internal/filtering/rulelist/error.go b/internal/filtering/rulelist/error.go new file mode 100644 index 00000000..54322da5 --- /dev/null +++ b/internal/filtering/rulelist/error.go @@ -0,0 +1,9 @@ +package rulelist + +import "github.com/AdguardTeam/golibs/errors" + +// ErrHTML is returned by [Parser.Parse] if the data is likely to be HTML. +// +// TODO(a.garipov): This error is currently returned to the UI. Stop that and +// make it all-lowercase. +const ErrHTML errors.Error = "data is HTML, not plain text" diff --git a/internal/filtering/rulelist/parser.go b/internal/filtering/rulelist/parser.go new file mode 100644 index 00000000..0bf5dba8 --- /dev/null +++ b/internal/filtering/rulelist/parser.go @@ -0,0 +1,184 @@ +package rulelist + +import ( + "bufio" + "bytes" + "fmt" + "hash/crc32" + "io" + "unicode" + + "github.com/AdguardTeam/golibs/errors" +) + +// Parser is a filtering-rule parser that collects data, such as the checksum +// and the title, as well as counts rules and removes comments. +type Parser struct { + title string + rulesCount int + written int + checksum uint32 + titleFound bool +} + +// NewParser returns a new filtering-rule parser. +func NewParser() (p *Parser) { + return &Parser{} +} + +// ParseResult contains information about the results of parsing a +// filtering-rule list by [Parser.Parse]. +type ParseResult struct { + // Title is the title contained within the filtering-rule list, if any. + Title string + + // RulesCount is the number of rules in the list. It excludes empty lines + // and comments. + RulesCount int + + // BytesWritten is the number of bytes written to dst. + BytesWritten int + + // Checksum is the CRC-32 checksum of the rules content. That is, excluding + // empty lines and comments. + Checksum uint32 +} + +// Parse parses data from src into dst using buf during parsing. r is never +// nil. +func (p *Parser) Parse(dst io.Writer, src io.Reader, buf []byte) (r *ParseResult, err error) { + s := bufio.NewScanner(src) + s.Buffer(buf, MaxRuleLen) + + lineIdx := 0 + for s.Scan() { + var n int + n, err = p.processLine(dst, s.Bytes(), lineIdx) + p.written += n + if err != nil { + // Don't wrap the error, because it's informative enough as is. + return p.result(), err + } + + lineIdx++ + } + + r = p.result() + err = s.Err() + + return r, errors.Annotate(err, "scanning filter contents: %w") +} + +// result returns the current parsing result. +func (p *Parser) result() (r *ParseResult) { + return &ParseResult{ + Title: p.title, + RulesCount: p.rulesCount, + BytesWritten: p.written, + Checksum: p.checksum, + } +} + +// processLine processes a single line. It may write to dst, and if it does, n +// is the number of bytes written. +func (p *Parser) processLine(dst io.Writer, line []byte, lineIdx int) (n int, err error) { + trimmed := bytes.TrimSpace(line) + if p.written == 0 && isHTMLLine(trimmed) { + return 0, ErrHTML + } + + badIdx, isRule := 0, false + if p.titleFound { + badIdx, isRule = parseLine(trimmed) + } else { + badIdx, isRule = p.parseLineTitle(trimmed) + } + if badIdx != -1 { + return 0, fmt.Errorf( + "line at index %d: character at index %d: non-printable character", + lineIdx, + badIdx+bytes.Index(line, trimmed), + ) + } + + if !isRule { + return 0, nil + } + + p.rulesCount++ + p.checksum = crc32.Update(p.checksum, crc32.IEEETable, trimmed) + + // Assume that there is generally enough space in the buffer to add a + // newline. + n, err = dst.Write(append(trimmed, '\n')) + + return n, errors.Annotate(err, "writing rule line: %w") +} + +// isHTMLLine returns true if line is likely an HTML line. line is assumed to +// be trimmed of whitespace characters. +func isHTMLLine(line []byte) (isHTML bool) { + return hasPrefixFold(line, []byte("= l && bytes.EqualFold(b[:l], prefix) +} + +// parseLine returns true if the parsed line is a filtering rule. line is +// assumed to be trimmed of whitespace characters. nonPrintIdx is the index of +// the first non-printable character, if any; if there are none, nonPrintIdx is +// -1. +// +// A line is considered a rule if it's not empty, not a comment, and contains +// only printable characters. +func parseLine(line []byte) (nonPrintIdx int, isRule bool) { + if len(line) == 0 || line[0] == '#' || line[0] == '!' { + return -1, false + } + + nonPrintIdx = bytes.IndexFunc(line, isNotPrintable) + + return nonPrintIdx, nonPrintIdx == -1 +} + +// isNotPrintable returns true if r is not a printable character that can be +// contained in a filtering rule. +func isNotPrintable(r rune) (ok bool) { + // Tab isn't included into Unicode's graphic symbols, so include it here + // explicitly. + return r != '\t' && !unicode.IsGraphic(r) +} + +// parseLineTitle is like [parseLine] but additionally looks for a title. line +// is assumed to be trimmed of whitespace characters. +func (p *Parser) parseLineTitle(line []byte) (nonPrintIdx int, isRule bool) { + if len(line) == 0 || line[0] == '#' { + return -1, false + } + + if line[0] != '!' { + nonPrintIdx = bytes.IndexFunc(line, isNotPrintable) + + return nonPrintIdx, nonPrintIdx == -1 + } + + const titlePattern = "! Title: " + if !bytes.HasPrefix(line, []byte(titlePattern)) { + return -1, false + } + + title := bytes.TrimSpace(line[len(titlePattern):]) + if title != nil { + // Note that title can be a non-nil empty slice. Consider that normal + // and just stop looking for other titles. + p.title = string(title) + p.titleFound = true + } + + return -1, false +} diff --git a/internal/filtering/rulelist/parser_test.go b/internal/filtering/rulelist/parser_test.go new file mode 100644 index 00000000..5e912988 --- /dev/null +++ b/internal/filtering/rulelist/parser_test.go @@ -0,0 +1,247 @@ +package rulelist_test + +import ( + "bufio" + "bytes" + "strings" + "testing" + + "github.com/AdguardTeam/AdGuardHome/internal/aghtest" + "github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist" + "github.com/AdguardTeam/golibs/errors" + "github.com/AdguardTeam/golibs/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParser_Parse(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + in string + wantDst string + wantErrMsg string + wantTitle string + wantRulesNum int + wantWritten int + }{{ + name: "empty", + in: "", + wantDst: "", + wantErrMsg: "", + wantTitle: "", + wantRulesNum: 0, + wantWritten: 0, + }, { + name: "html", + in: testRuleTextHTML, + wantErrMsg: rulelist.ErrHTML.Error(), + wantTitle: "", + wantRulesNum: 0, + wantWritten: 0, + }, { + name: "comments", + in: "# Comment 1\n" + + "! Comment 2\n", + wantErrMsg: "", + wantTitle: "", + wantRulesNum: 0, + wantWritten: 0, + }, {}, { + name: "rule", + in: testRuleTextBlocked, + wantDst: testRuleTextBlocked, + wantErrMsg: "", + wantRulesNum: 1, + wantTitle: "", + wantWritten: len(testRuleTextBlocked), + }, { + name: "html_in_rule", + in: testRuleTextBlocked + testRuleTextHTML, + wantDst: testRuleTextBlocked + testRuleTextHTML, + wantErrMsg: "", + wantTitle: "", + wantRulesNum: 2, + wantWritten: len(testRuleTextBlocked) + len(testRuleTextHTML), + }, { + name: "title", + in: "! Title: Test Title \n" + + "! Title: Bad, Ignored Title\n" + + testRuleTextBlocked, + wantDst: testRuleTextBlocked, + wantErrMsg: "", + wantTitle: "Test Title", + wantRulesNum: 1, + wantWritten: len(testRuleTextBlocked), + }, { + name: "bad_char", + in: "! Title: Test Title \n" + + testRuleTextBlocked + + ">>>\x7F<<<", + wantDst: testRuleTextBlocked, + wantErrMsg: "line at index 2: " + + "character at index 3: " + + "non-printable character", + wantTitle: "Test Title", + wantRulesNum: 1, + wantWritten: len(testRuleTextBlocked), + }, { + name: "too_long", + in: strings.Repeat("a", rulelist.MaxRuleLen+1), + wantDst: "", + wantErrMsg: "scanning filter contents: " + bufio.ErrTooLong.Error(), + wantTitle: "", + wantRulesNum: 0, + wantWritten: 0, + }, { + name: "bad_tab_and_comment", + in: testRuleTextBadTab, + wantDst: testRuleTextBadTab, + wantErrMsg: "", + wantTitle: "", + wantRulesNum: 1, + wantWritten: len(testRuleTextBadTab), + }, { + name: "etc_hosts_tab_and_comment", + in: testRuleTextEtcHostsTab, + wantDst: testRuleTextEtcHostsTab, + wantErrMsg: "", + wantTitle: "", + wantRulesNum: 1, + wantWritten: len(testRuleTextEtcHostsTab), + }} + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + dst := &bytes.Buffer{} + buf := make([]byte, rulelist.MaxRuleLen) + + p := rulelist.NewParser() + r, err := p.Parse(dst, strings.NewReader(tc.in), buf) + require.NotNil(t, r) + + testutil.AssertErrorMsg(t, tc.wantErrMsg, err) + assert.Equal(t, tc.wantDst, dst.String()) + assert.Equal(t, tc.wantTitle, r.Title) + assert.Equal(t, tc.wantRulesNum, r.RulesCount) + assert.Equal(t, tc.wantWritten, r.BytesWritten) + + if tc.wantWritten > 0 { + assert.NotZero(t, r.Checksum) + } + }) + } +} + +func TestParser_Parse_writeError(t *testing.T) { + t.Parallel() + + dst := &aghtest.Writer{ + OnWrite: func(b []byte) (n int, err error) { + return 1, errors.Error("test error") + }, + } + buf := make([]byte, rulelist.MaxRuleLen) + + p := rulelist.NewParser() + r, err := p.Parse(dst, strings.NewReader(testRuleTextBlocked), buf) + require.NotNil(t, r) + + testutil.AssertErrorMsg(t, "writing rule line: test error", err) + assert.Equal(t, 1, r.BytesWritten) +} + +func TestParser_Parse_checksums(t *testing.T) { + t.Parallel() + + const ( + withoutComments = testRuleTextBlocked + withComments = "! Some comment.\n" + + " " + testRuleTextBlocked + + "# Another comment.\n" + ) + + buf := make([]byte, rulelist.MaxRuleLen) + + p := rulelist.NewParser() + r, err := p.Parse(&bytes.Buffer{}, strings.NewReader(withoutComments), buf) + require.NotNil(t, r) + require.NoError(t, err) + + gotWithoutComments := r.Checksum + + p = rulelist.NewParser() + + r, err = p.Parse(&bytes.Buffer{}, strings.NewReader(withComments), buf) + require.NotNil(t, r) + require.NoError(t, err) + + gotWithComments := r.Checksum + assert.Equal(t, gotWithoutComments, gotWithComments) +} + +var ( + resSink *rulelist.ParseResult + errSink error +) + +func BenchmarkParser_Parse(b *testing.B) { + dst := &bytes.Buffer{} + src := strings.NewReader(strings.Repeat(testRuleTextBlocked, 1000)) + buf := make([]byte, rulelist.MaxRuleLen) + p := rulelist.NewParser() + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + resSink, errSink = p.Parse(dst, src, buf) + dst.Reset() + } + + require.NoError(b, errSink) + require.NotNil(b, resSink) +} + +func FuzzParser_Parse(f *testing.F) { + const n = 64 + + testCases := []string{ + "", + "# Comment", + "! Comment", + "! Title ", + "! Title XXX", + testRuleTextEtcHostsTab, + testRuleTextHTML, + testRuleTextBlocked, + testRuleTextBadTab, + "1.2.3.4", + "1.2.3.4 etc-hosts.example", + ">>>\x00<<<", + ">>>\x7F<<<", + strings.Repeat("a", n+1), + } + + for _, tc := range testCases { + f.Add(tc) + } + + buf := make([]byte, n) + + f.Fuzz(func(t *testing.T, input string) { + require.Eventually(t, func() (ok bool) { + dst := &bytes.Buffer{} + src := strings.NewReader(input) + + p := rulelist.NewParser() + r, _ := p.Parse(dst, src, buf) + require.NotNil(t, r) + + return true + }, testTimeout, testTimeout/100) + }) +} diff --git a/internal/filtering/rulelist/rulelist.go b/internal/filtering/rulelist/rulelist.go new file mode 100644 index 00000000..1a6236c5 --- /dev/null +++ b/internal/filtering/rulelist/rulelist.go @@ -0,0 +1,11 @@ +// Package rulelist contains the implementation of the standard rule-list +// filter that wraps an urlfilter filtering-engine. +// +// TODO(a.garipov): Expand. +package rulelist + +// MaxRuleLen is the maximum length of a line with a filtering rule, in bytes. +// +// TODO(a.garipov): Consider changing this to a rune length, like AdGuardDNS +// does. +const MaxRuleLen = 1024 diff --git a/internal/filtering/rulelist/rulelist_test.go b/internal/filtering/rulelist/rulelist_test.go new file mode 100644 index 00000000..0c3a3b84 --- /dev/null +++ b/internal/filtering/rulelist/rulelist_test.go @@ -0,0 +1,14 @@ +package rulelist_test + +import "time" + +// testTimeout is the common timeout for tests. +const testTimeout = 1 * time.Second + +// Common texts for tests. +const ( + testRuleTextHTML = "\n" + testRuleTextBlocked = "||blocked.example^\n" + testRuleTextBadTab = "||bad-tab-and-comment.example^\t# A comment.\n" + testRuleTextEtcHostsTab = "0.0.0.0 tab..example^\t# A comment.\n" +) diff --git a/internal/next/websvc/waitlistener_internal_test.go b/internal/next/websvc/waitlistener_internal_test.go index 6911d137..089c0531 100644 --- a/internal/next/websvc/waitlistener_internal_test.go +++ b/internal/next/websvc/waitlistener_internal_test.go @@ -6,13 +6,13 @@ import ( "sync/atomic" "testing" - "github.com/AdguardTeam/AdGuardHome/internal/aghtest" + "github.com/AdguardTeam/golibs/testutil/fakenet" "github.com/stretchr/testify/assert" ) func TestWaitListener_Accept(t *testing.T) { var accepted atomic.Bool - var l net.Listener = &aghtest.Listener{ + var l net.Listener = &fakenet.Listener{ OnAccept: func() (conn net.Conn, err error) { accepted.Store(true) diff --git a/internal/tools/go.mod b/internal/tools/go.mod index 540064a7..4e00c92b 100644 --- a/internal/tools/go.mod +++ b/internal/tools/go.mod @@ -10,9 +10,10 @@ require ( github.com/kyoh86/looppointer v0.2.1 github.com/securego/gosec/v2 v2.16.0 github.com/uudashr/gocognit v1.0.6 - golang.org/x/tools v0.10.0 + golang.org/x/tools v0.11.0 golang.org/x/vuln v0.2.0 - honnef.co/go/tools v0.4.3 + // TODO(a.garipov): Return to tagged releases once a new one appears. + honnef.co/go/tools v0.5.0-0.dev.0.20230706211743-ddee6bbaa341 mvdan.cc/gofumpt v0.5.0 mvdan.cc/unparam v0.0.0-20230610194454-9ea02bef9868 ) @@ -27,8 +28,8 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect golang.org/x/exp/typeparams v0.0.0-20230626212559-97b1e661b5df // indirect - golang.org/x/mod v0.11.0 // indirect + golang.org/x/mod v0.12.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.9.0 // indirect + golang.org/x/sys v0.10.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/internal/tools/go.sum b/internal/tools/go.sum index 5f85cfa4..c27ed5b9 100644 --- a/internal/tools/go.sum +++ b/internal/tools/go.sum @@ -58,15 +58,15 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -82,8 +82,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220702020025-31831981b65f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -96,8 +96,8 @@ golang.org/x/tools v0.0.0-20201007032633-0806396f153e/go.mod h1:z6u4i615ZeAfBE4X golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= -golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= -golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= +golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= +golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= golang.org/x/vuln v0.2.0 h1:Dlz47lW0pvPHU7tnb10S8vbMn9GnV2B6eyT7Tem5XBI= golang.org/x/vuln v0.2.0/go.mod h1:V0eyhHwaAaHrt42J9bgrN6rd12f6GU4T0Lu0ex2wDg4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -107,8 +107,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.4.3 h1:o/n5/K5gXqk8Gozvs2cnL0F2S1/g1vcGCAx2vETjITw= -honnef.co/go/tools v0.4.3/go.mod h1:36ZgoUOrqOk1GxwHhyryEkq8FQWkUO2xGuSMhUCcdvA= +honnef.co/go/tools v0.5.0-0.dev.0.20230706211743-ddee6bbaa341 h1:jNlTAPEjbDiN9qda/1wple0GSpewFnWhvc1GO7bZX1U= +honnef.co/go/tools v0.5.0-0.dev.0.20230706211743-ddee6bbaa341/go.mod h1:GUV+uIBCLpdf0/v6UhHHG/yzI/z6qPskBeQCjcNB96k= mvdan.cc/gofumpt v0.5.0 h1:0EQ+Z56k8tXjj/6TQD25BFNKQXpCvT0rnansIc7Ug5E= mvdan.cc/gofumpt v0.5.0/go.mod h1:HBeVDtMKRZpXyxFciAirzdKklDlGu8aAy1wEbH5Y9js= mvdan.cc/unparam v0.0.0-20230610194454-9ea02bef9868 h1:F4Q7pXcrU9UiU1fq0ZWqSOxKjNAteRuDr7JDk7uVLRQ= diff --git a/scripts/README.md b/scripts/README.md index 579ee08a..d9ef6851 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -18,7 +18,7 @@ Run `make init` from the project root. -## `make/`: Makefile Scripts +## `make/`: Makefile scripts The release channels are: `development` (the default), `edge`, `beta`, and `release`. If verbosity levels aren't documented here, there are only two: `0`, @@ -26,7 +26,7 @@ don't print anything, and `1`, be verbose. - ### `build-docker.sh`: Build A Multi-Architecture Docker Image + ### `build-docker.sh`: Build a multi-architecture Docker image Required environment: @@ -51,7 +51,7 @@ Optional environment: - ### `build-release.sh`: Build A Release For All Platforms + ### `build-release.sh`: Build a release for all platforms Required environment: @@ -101,7 +101,22 @@ Required environment: - ### `go-build.sh`: Build The Backend + ### `go-bench.sh`: Run backend benchmarks + +Optional environment: + + * `GO`: set an alternative name for the Go compiler. + + * `TIMEOUT_FLAGS`: set timeout flags for tests. The default value is + `--timeout=30s`. + + * `VERBOSE`: verbosity level. `1` shows every command that is run and every + Go package that is processed. `2` also shows subcommands and environment. + The default value is `0`, don't be verbose. + + + + ### `go-build.sh`: Build the backend Optional environment: @@ -135,7 +150,7 @@ Required environment: - ### `go-deps.sh`: Install Backend Dependencies + ### `go-deps.sh`: Install backend dependencies Optional environment: @@ -147,7 +162,25 @@ Optional environment: - ### `go-lint.sh`: Run Backend Static Analyzers + ### `go-fuzz.sh`: Run backend fuzz tests + +Optional environment: + + * `GO`: set an alternative name for the Go compiler. + + * `FUZZTIME_FLAGS`: set fuss flags for tests. The default value is + `--fuzztime=20s`. + + * `TIMEOUT_FLAGS`: set timeout flags for tests. The default value is + `--timeout=30s`. + + * `VERBOSE`: verbosity level. `1` shows every command that is run and every + Go package that is processed. `2` also shows subcommands and environment. + The default value is `0`, don't be verbose. + + + + ### `go-lint.sh`: Run backend static analyzers Don't forget to run `make go-tools` once first! @@ -163,7 +196,7 @@ Optional environment: - ### `go-test.sh`: Run Backend Tests + ### `go-test.sh`: Run backend tests Optional environment: @@ -173,7 +206,7 @@ Optional environment: `1`, use the race detector. * `TIMEOUT_FLAGS`: set timeout flags for tests. The default value is - `--timeout 30s`. + `--timeout=30s`. * `VERBOSE`: verbosity level. `1` shows every command that is run and every Go package that is processed. `2` also shows subcommands. The default @@ -181,7 +214,7 @@ Optional environment: - ### `go-tools.sh`: Install Backend Tooling + ### `go-tools.sh`: Install backend tooling Installs the Go static analysis and other tools into `${PWD}/bin`. Either add `${PWD}/bin` to your `$PATH` before all other entries, or use the commands diff --git a/scripts/make/go-bench.sh b/scripts/make/go-bench.sh new file mode 100644 index 00000000..5ddf5d03 --- /dev/null +++ b/scripts/make/go-bench.sh @@ -0,0 +1,55 @@ +#!/bin/sh + +verbose="${VERBOSE:-0}" +readonly verbose + +# Verbosity levels: +# 0 = Don't print anything except for errors. +# 1 = Print commands, but not nested commands. +# 2 = Print everything. +if [ "$verbose" -gt '1' ] +then + set -x + v_flags='-v=1' + x_flags='-x=1' +elif [ "$verbose" -gt '0' ] +then + set -x + v_flags='-v=1' + x_flags='-x=0' +else + set +x + v_flags='-v=0' + x_flags='-x=0' +fi +readonly v_flags x_flags + +set -e -f -u + +if [ "${RACE:-1}" -eq '0' ] +then + race_flags='--race=0' +else + race_flags='--race=1' +fi +readonly race_flags + +go="${GO:-go}" + +count_flags='--count=1' +shuffle_flags='--shuffle=on' +timeout_flags="${TIMEOUT_FLAGS:---timeout=30s}" +readonly go count_flags shuffle_flags timeout_flags + +"$go" test\ + "$count_flags"\ + "$shuffle_flags"\ + "$race_flags"\ + "$timeout_flags"\ + "$x_flags"\ + "$v_flags"\ + --bench='.'\ + --benchmem\ + --benchtime=1s\ + --run='^$'\ + ./... diff --git a/scripts/make/go-fuzz.sh b/scripts/make/go-fuzz.sh new file mode 100644 index 00000000..8f10b4f1 --- /dev/null +++ b/scripts/make/go-fuzz.sh @@ -0,0 +1,58 @@ +#!/bin/sh + +verbose="${VERBOSE:-0}" +readonly verbose + +# Verbosity levels: +# 0 = Don't print anything except for errors. +# 1 = Print commands, but not nested commands. +# 2 = Print everything. +if [ "$verbose" -gt '1' ] +then + set -x + v_flags='-v=1' + x_flags='-x=1' +elif [ "$verbose" -gt '0' ] +then + set -x + v_flags='-v=1' + x_flags='-x=0' +else + set +x + v_flags='-v=0' + x_flags='-x=0' +fi +readonly v_flags x_flags + +set -e -f -u + +if [ "${RACE:-1}" -eq '0' ] +then + race_flags='--race=0' +else + race_flags='--race=1' +fi +readonly race_flags + +go="${GO:-go}" + +count_flags='--count=1' +shuffle_flags='--shuffle=on' +timeout_flags="${TIMEOUT_FLAGS:---timeout=30s}" +fuzztime_flags="${FUZZTIME_FLAGS:---fuzztime=20s}" + +readonly go count_flags shuffle_flags timeout_flags fuzztime_flags + +# TODO(a.garipov): File an issue about using --fuzz with multiple packages. +"$go" test\ + "$count_flags"\ + "$shuffle_flags"\ + "$race_flags"\ + "$timeout_flags"\ + "$x_flags"\ + "$v_flags"\ + "$fuzztime_flags"\ + --fuzz='.'\ + --run='^$'\ + ./internal/filtering/rulelist/\ + ; diff --git a/scripts/make/go-lint.sh b/scripts/make/go-lint.sh index 16e589b3..a0a9569a 100644 --- a/scripts/make/go-lint.sh +++ b/scripts/make/go-lint.sh @@ -176,6 +176,8 @@ run_linter gocognit --over 10\ ./internal/aghchan/\ ./internal/aghhttp/\ ./internal/aghio/\ + ./internal/filtering/hashprefix/\ + ./internal/filtering/rulelist/\ ./internal/next/\ ./internal/rdns/\ ./internal/tools/\ @@ -211,6 +213,8 @@ run_linter gosec --quiet\ ./internal/dhcpd\ ./internal/dhcpsvc\ ./internal/dnsforward\ + ./internal/filtering/hashprefix/\ + ./internal/filtering/rulelist/\ ./internal/next\ ./internal/schedule\ ./internal/stats\ @@ -219,8 +223,7 @@ run_linter gosec --quiet\ ./internal/whois\ ; -# TODO(a.garipov): Enable --blank? -run_linter errcheck --asserts ./... +run_linter errcheck ./... staticcheck_matrix=' darwin: GOOS=darwin