umputun/reproxy
1
1
Fork 0
mirror of https://github.com/umputun/reproxy.git synced 2024-10-26 10:10:12 +03:00
Code Issues Projects Releases Wiki Activity

Merge pull request #97

Browse Source 
* revendor with latest rest lib

* simplify with passThroughHandler

* add deps for throttling
This commit is contained in:
Umputun 2021-07-03 01:23:50 -05:00 committed by GitHub
parent a9c7db27b6
commit 71039681e2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 2437 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
README.md
View File

@ -236,6 +236,12 @@ _see also [examples/metrics](https://github.com/umputun/reproxy/tree/master/exam
Reproxy returns 502 (Bad Gateway) error in case if request doesn't match to any provided routes and assets. In case if some unexpected, internal error happened it returns 500. By default reproxy renders the simplest text version of the error - "Server error". Setting `--error.enabled` turns on the default html error message and with `--error.template` user may set any custom html template file for the error rendering. The template has two vars: `{{.ErrCode}}` and `{{.ErrMessage}}`. For example this template `oh my! {{.ErrCode}} - {{.ErrMessage}}` will be rendered to `oh my! 502 - Bad Gateway`
## Throttling
Reproxy allows to define system level max req/sec value for the overall system activity as well as per user. 0 values (default) treated as unlimited.
User activity limited for both matched and unmatched routes. All unmatched routes considered as a "single destination group" and get a common limiter which is `rate*3`. It means if 10 (req/sec) defined with `--throttle.user=10` the end user will be able to perform up to 30 request pers second for either static assets or unmatched routes. For matched routes this limiter maintained per destination (route), i.e. request proxied to s1.example.com/api will allow 10 r/s and the request proxied to s2.example.com will allow another 10 r/s.
## Plugins support
The core functionality of reproxy can be extended with external plugins. Each plugin is an independent process/container implementing [rpc server](https://golang.org/pkg/net/rpc/). Plugins registered with reproxy conductor and added to the chain of the middlewares. Each plugin receives request with the original url, headers and all matching route info and responds with the headers and the status code. Any status code >= 400 treated as an error response and terminates flow immediately with the proxy error. There are two types of headers plugins can set:
@ -349,6 +355,10 @@ health-check:
--health-check.enabled enable automatic health-check [$HEALTH_CHECK_ENABLED]
--health-check.interval= automatic health-check interval (default: 300s) [$HEALTH_CHECK_INTERVAL]
throttle:
--throttle.system= throttle overall activity' (default: 0) [$THROTTLE_SYSTEM]
--throttle.user= limit req/sec per user and per proxy destination (default: 0) [$THROTTLE_USER]
Help Options:
-h, --help Show this help message

7
app/main.go
View File

@ -115,6 +115,11 @@ var opts struct {
Interval time.Duration `long:"interval" env:"INTERVAL" default:"300s" description:"automatic health-check interval"`
} `group:"health-check" namespace:"health-check" env-namespace:"HEALTH_CHECK"`
Throttle struct {
System int `long:"system" env:"SYSTEM" default:"0" description:"throttle overall activity'"`
User int `long:"user" env:"USER" default:"0" description:"limit req/sec per user and per proxy destination"`
} `group:"throttle" namespace:"throttle" env-namespace:"THROTTLE"`
Plugin struct {
Enabled bool `long:"enabled" env:"ENABLED" description:"enable plugin support"`
Listen string `long:"listen" env:"LISTEN" default:"127.0.0.1:8081" description:"registration listen on host:port"`
@ -246,6 +251,8 @@ func run() error {
Metrics: makeMetrics(ctx, svc),
Reporter: errReporter,
PluginConductor: makePluginConductor(ctx),
ThrottleSystem: opts.Throttle.System * 3,
ThottleUser: opts.Throttle.User,
}
err = px.Run(ctx)

115
app/proxy/handlers.go
View File

@ -5,9 +5,13 @@ import (
"net/http"
"strings"
"github.com/didip/tollbooth/v6"
"github.com/didip/tollbooth/v6/libstring"
log "github.com/go-pkgz/lgr"
R "github.com/go-pkgz/rest"
"github.com/gorilla/handlers"
"github.com/umputun/reproxy/app/discovery"
)
func headersHandler(headers []string) func(next http.Handler) http.Handler {
@ -32,11 +36,7 @@ func headersHandler(headers []string) func(next http.Handler) http.Handler {
func maxReqSizeHandler(maxSize int64) func(next http.Handler) http.Handler {
if maxSize <= 0 {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
next.ServeHTTP(w, r)
})
}
return passThroughHandler
}
log.Printf("[DEBUG] request size limited to %d", maxSize)
@ -65,49 +65,92 @@ func accessLogHandler(wr io.Writer) func(next http.Handler) http.Handler {
func stdoutLogHandler(enable bool, lh func(next http.Handler) http.Handler) func(next http.Handler) http.Handler {
if enable {
log.Printf("[DEBUG] stdout logging enabled")
return func(next http.Handler) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
// don't log to stdout GET ~/(.*)/ping$ requests
if r.Method == "GET" && strings.HasSuffix(r.URL.Path, "/ping") {
next.ServeHTTP(w, r)
return
}
lh(next).ServeHTTP(w, r)
}
return http.HandlerFunc(fn)
}
if !enable {
return passThroughHandler
}
log.Printf("[DEBUG] stdout logging enabled")
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
next.ServeHTTP(w, r)
})
fn := func(w http.ResponseWriter, r *http.Request) {
// don't log to stdout GET ~/(.*)/ping$ requests
if r.Method == "GET" && strings.HasSuffix(r.URL.Path, "/ping") {
next.ServeHTTP(w, r)
return
}
lh(next).ServeHTTP(w, r)
}
return http.HandlerFunc(fn)
}
}
func gzipHandler(enabled bool) func(next http.Handler) http.Handler {
if enabled {
log.Printf("[DEBUG] gzip enabled")
return handlers.CompressHandler
if !enabled {
return passThroughHandler
}
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
next.ServeHTTP(w, r)
})
}
log.Printf("[DEBUG] gzip enabled")
return handlers.CompressHandler
}
func signatureHandler(enabled bool, version string) func(next http.Handler) http.Handler {
if enabled {
log.Printf("[DEBUG] signature headers enabled")
return R.AppInfo("reproxy", "umputun", version)
if !enabled {
return passThroughHandler
}
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
next.ServeHTTP(w, r)
})
log.Printf("[DEBUG] signature headers enabled")
return R.AppInfo("reproxy", "umputun", version)
}
// limiterSystemHandler throttles overall activity of reproxy server, 0 means disabled
func limiterSystemHandler(reqSec int) func(next http.Handler) http.Handler {
if reqSec <= 0 {
return passThroughHandler
}
return func(h http.Handler) http.Handler {
lmt := tollbooth.NewLimiter(float64(reqSec), nil)
fn := func(w http.ResponseWriter, r *http.Request) {
if httpError := tollbooth.LimitByKeys(lmt, []string{"system"}); httpError != nil {
http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests)
return
}
h.ServeHTTP(w, r)
}
return http.HandlerFunc(fn)
}
}
// limiterUserHandler throttles per user activity. In case if match found the limit is per destination
// otherwise global (per user in any case). 0 means disabled
func limiterUserHandler(reqSec int) func(next http.Handler) http.Handler {
if reqSec <= 0 {
return passThroughHandler
}
return func(h http.Handler) http.Handler {
lmt := tollbooth.NewLimiter(float64(reqSec), nil)
fn := func(w http.ResponseWriter, r *http.Request) {
keys := []string{libstring.RemoteIP(lmt.GetIPLookups(), lmt.GetForwardedForIndexFromBehind(), r)}
// add dst proxy if matched
if r.Context().Value(ctxMatch) != nil { // route match detected by matchHandler
match := r.Context().Value(ctxMatch).(discovery.MatchedRoute)
matchType := r.Context().Value(ctxMatchType).(discovery.MatchType)
if matchType == discovery.MTProxy {
keys = append(keys, match.Mapper.Dst)
}
}
if httpError := tollbooth.LimitByKeys(lmt, keys); httpError != nil {
http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests)
return
}
h.ServeHTTP(w, r)
}
return http.HandlerFunc(fn)
}
}
func passThroughHandler(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
next.ServeHTTP(w, r)
})
}

97
app/proxy/handlers_test.go
View File

@ -2,12 +2,18 @@ package proxy
import (
"bytes"
"context"
"net/http"
"net/http/httptest"
"strconv"
"sync"
"sync/atomic"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/reproxy/app/discovery"
)
func Test_headersHandler(t *testing.T) {
@ -83,3 +89,94 @@ func Test_signatureHandler(t *testing.T) {
assert.Equal(t, "", wr.Result().Header.Get("App-Version"), wr.Result().Header)
}
}
func Test_limiterSystemHandler(t *testing.T) {
var passed int32
handler := limiterSystemHandler(10)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&passed, 1)
}))
ts := httptest.NewServer(handler)
var wg sync.WaitGroup
wg.Add(100)
for i := 0; i < 100; i++ {
go func() {
defer wg.Done()
req, err := http.NewRequest("GET", ts.URL, nil)
require.NoError(t, err)
client := http.Client{}
resp, err := client.Do(req)
require.NoError(t, err)
resp.Body.Close()
}()
}
wg.Wait()
assert.Equal(t, int32(10), atomic.LoadInt32(&passed))
}
func Test_limiterClientHandlerNoMatches(t *testing.T) {
var passed int32
handler := limiterUserHandler(10)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&passed, 1)
}))
ts := httptest.NewServer(handler)
var wg sync.WaitGroup
wg.Add(100)
for i := 0; i < 100; i++ {
go func() {
defer wg.Done()
req, err := http.NewRequest("GET", ts.URL, nil)
require.NoError(t, err)
client := http.Client{}
resp, err := client.Do(req)
require.NoError(t, err)
resp.Body.Close()
}()
}
wg.Wait()
assert.Equal(t, int32(10), atomic.LoadInt32(&passed))
}
func Test_limiterClientHandlerWithMatches(t *testing.T) {
var passed int32
handler := limiterUserHandler(10)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&passed, 1)
}))
wrapWithContext := func(next http.Handler) http.Handler {
var id int32
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
n := int(atomic.AddInt32(&id, 1))
m := discovery.MatchedRoute{Mapper: discovery.URLMapper{Dst: strconv.Itoa(n % 2)}}
ctx := context.WithValue(context.Background(), ctxMatchType, discovery.MTProxy)
ctx = context.WithValue(ctx, ctxMatch, m)
next.ServeHTTP(w, r.WithContext(ctx))
})
}
ts := httptest.NewServer(wrapWithContext(handler))
var wg sync.WaitGroup
wg.Add(100)
for i := 0; i < 100; i++ {
go func(id int) {
defer wg.Done()
req, err := http.NewRequest("POST", ts.URL, bytes.NewBufferString("123456"))
require.NoError(t, err)
m := discovery.MatchedRoute{Mapper: discovery.URLMapper{Dst: strconv.Itoa(id % 2)}}
ctx := context.WithValue(context.Background(), ctxMatchType, discovery.MTProxy)
ctx = context.WithValue(ctx, ctxMatch, m)
req = req.WithContext(ctx)
client := http.Client{}
resp, err := client.Do(req)
require.NoError(t, err)
resp.Body.Close()
}(i)
}
wg.Wait()
assert.Equal(t, int32(20), atomic.LoadInt32(&passed))
}

51
app/proxy/proxy.go
View File

@ -43,6 +43,9 @@ type Http struct { // nolint golint
PluginConductor MiddlewareProvider
Reporter Reporter
LBSelector func(len int) int
ThrottleSystem int
ThottleUser int
}
// Matcher source info (server and route) to the destination url
@ -107,18 +110,20 @@ func (h *Http) Run(ctx context.Context) error {
}()
handler := R.Wrap(h.proxyHandler(),
R.Recoverer(log.Default()),
signatureHandler(h.Signature, h.Version),
h.pingHandler,
h.healthMiddleware,
h.matchHandler,
h.mgmtHandler(),
h.pluginHandler(),
headersHandler(h.ProxyHeaders),
accessLogHandler(h.AccessLog),
R.Recoverer(log.Default()), // recover on errors
signatureHandler(h.Signature, h.Version), // send app signature
h.pingHandler, // respond to /ping
h.healthMiddleware, // respond to /health
h.matchHandler, // set matched routes to context
limiterSystemHandler(h.ThrottleSystem), // limit total requests/sec
limiterUserHandler(h.ThottleUser), // req/seq per user/route match
h.mgmtHandler(), // handles /metrics and /routes for prometheus
h.pluginHandler(), // prc to external plugins
headersHandler(h.ProxyHeaders), // set response headers
accessLogHandler(h.AccessLog), // apache-format log file
stdoutLogHandler(h.StdOutEnabled, logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]")).Handler),
maxReqSizeHandler(h.MaxBodySize),
gzipHandler(h.GzEnabled),
maxReqSizeHandler(h.MaxBodySize), // limit request max size
gzipHandler(h.GzEnabled), // gzip response
)
if len(h.SSLConfig.FQDNs) == 0 && h.SSLConfig.SSLMode == SSLAuto {
@ -347,27 +352,19 @@ func (h *Http) toHTTP(address string, httpPort int) string {
}
func (h *Http) pluginHandler() func(next http.Handler) http.Handler {
if h.PluginConductor != nil {
log.Printf("[INFO] plugin support enabled")
return h.PluginConductor.Middleware
}
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
next.ServeHTTP(w, r)
})
if h.PluginConductor == nil {
return passThroughHandler
}
log.Printf("[INFO] plugin support enabled")
return h.PluginConductor.Middleware
}
func (h *Http) mgmtHandler() func(next http.Handler) http.Handler {
if h.Metrics != nil {
log.Printf("[DEBUG] metrics enabled")
return h.Metrics.Middleware
}
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
next.ServeHTTP(w, r)
})
if h.Metrics == nil {
return passThroughHandler
}
log.Printf("[DEBUG] metrics enabled")
return h.Metrics.Middleware
}
func (h *Http) makeHTTPServer(addr string, router http.Handler) *http.Server {

1
go.mod
View File

@ -3,6 +3,7 @@ module github.com/umputun/reproxy
go 1.16
require (
github.com/didip/tollbooth/v6 v6.1.0
github.com/go-pkgz/lgr v0.10.4
github.com/go-pkgz/repeater v1.1.3
github.com/go-pkgz/rest v1.11.0

7
go.sum
View File

@ -44,6 +44,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/didip/tollbooth/v6 v6.1.0 h1:ZS2fNa9JhFdRSJCj3+V12VfuUifYrGB4Z0jSwXmKMeE=
github.com/didip/tollbooth/v6 v6.1.0/go.mod h1:xjcse6CTHCLuOkzsWrEgdy9WPJFv+p/x6v+MyfP+O9s=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
@ -65,6 +67,8 @@ github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgO
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-pkgz/expirable-cache v0.0.3 h1:rTh6qNPp78z0bQE6HDhXBHUwqnV9i09Vm6dksJLXQDc=
github.com/go-pkgz/expirable-cache v0.0.3/go.mod h1:+IauqN00R2FqNRLCLA+X5YljQJrwB179PfiAoMPlTlQ=
github.com/go-pkgz/lgr v0.10.4 h1:l7qyFjqEZgwRgaQQSEp6tve4A3OU80VrfzpvtEX8ngw=
github.com/go-pkgz/lgr v0.10.4/go.mod h1:CD0s1z6EFpIUplV067gitF77tn25JItzwHNKAPqeCF0=
github.com/go-pkgz/repeater v1.1.3 h1:q6+JQF14ESSy28Dd7F+wRelY4F+41HJ0LEy/szNnMiE=
@ -269,6 +273,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@ -365,6 +370,8 @@ golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI=
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

2
vendor/github.com/didip/tollbooth/v6/.gitignore generated vendored Normal file
View File

@ -0,0 +1,2 @@
/debug
/.vscode

37
vendor/github.com/didip/tollbooth/v6/.golangci.yml generated vendored Normal file
View File

@ -0,0 +1,37 @@
linters:
enable:
- megacheck
- golint
- govet
- unconvert
- megacheck
- structcheck
- gas
- gocyclo
- dupl
- misspell
- unparam
- varcheck
- deadcode
- typecheck
- ineffassign
- varcheck
- stylecheck
- gochecknoinits
- scopelint
- gocritic
- nakedret
- gosimple
- prealloc
fast: false
disable-all: true
issues:
exclude-rules:
- path: _test\.go
linters:
- dupl
- text: "Errors unhandled"
linters:
- gosec
exclude-use-default: false

21
vendor/github.com/didip/tollbooth/v6/LICENSE generated vendored Normal file
View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2015 Didip Kerabat
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

175
vendor/github.com/didip/tollbooth/v6/README.md generated vendored Normal file
View File

@ -0,0 +1,175 @@
[![GoDoc](https://godoc.org/github.com/didip/tollbooth?status.svg)](http://godoc.org/github.com/didip/tollbooth)
[![license](http://img.shields.io/badge/license-MIT-red.svg?style=flat)](https://raw.githubusercontent.com/didip/tollbooth/master/LICENSE)
## Tollbooth
This is a generic middleware to rate-limit HTTP requests.
**NOTE 1:** This library is considered finished.
**NOTE 2:** Major version changes are backward-incompatible. `v2.0.0` streamlines the ugliness of the old API.
## Versions
**v1.0.0:** This version maintains the old API but all the thirdparty modules are moved to their own repo.
**v2.x.x:** Brand-new API for the sake of code cleanup, thread safety, & auto-expiring data structures.
**v3.x.x:** Apparently we have been using golang.org/x/time/rate incorrectly. See issue #48. It always limits X number per 1 second. The time duration is not changeable, so it does not make sense to pass TTL to tollbooth.
**v4.x.x:** Float64 for max requests per second
**v5.x.x:** go.mod and go.sum
**v6.x.x:** Replaced `go-cache` with `github.com/go-pkgz/expirable-cache` because `go-cache` leaks goroutines.
## Five Minute Tutorial
```go
package main
import (
"github.com/didip/tollbooth"
"net/http"
)
func HelloHandler(w http.ResponseWriter, req *http.Request) {
w.Write([]byte("Hello, World!"))
}
func main() {
// Create a request limiter per handler.
http.Handle("/", tollbooth.LimitFuncHandler(tollbooth.NewLimiter(1, nil), HelloHandler))
http.ListenAndServe(":12345", nil)
}
```
## Features
1. Rate-limit by request's remote IP, path, methods, custom headers, & basic auth usernames.
```go
import (
"time"
"github.com/didip/tollbooth/limiter"
)
lmt := tollbooth.NewLimiter(1, nil)
// or create a limiter with expirable token buckets
// This setting means:
// create a 1 request/second limiter and
// every token bucket in it will expire 1 hour after it was initially set.
lmt = tollbooth.NewLimiter(1, &limiter.ExpirableOptions{DefaultExpirationTTL: time.Hour})
// Configure list of places to look for IP address.
// By default it's: "RemoteAddr", "X-Forwarded-For", "X-Real-IP"
// If your application is behind a proxy, set "X-Forwarded-For" first.
lmt.SetIPLookups([]string{"RemoteAddr", "X-Forwarded-For", "X-Real-IP"})
// Limit only GET and POST requests.
lmt.SetMethods([]string{"GET", "POST"})
// Limit based on basic auth usernames.
// You add them on-load, or later as you handle requests.
lmt.SetBasicAuthUsers([]string{"bob", "jane", "didip", "vip"})
// You can remove them later as well.
lmt.RemoveBasicAuthUsers([]string{"vip"})
// Limit request headers containing certain values.
// You add them on-load, or later as you handle requests.
lmt.SetHeader("X-Access-Token", []string{"abc123", "xyz098"})
// You can remove all entries at once.
lmt.RemoveHeader("X-Access-Token")
// Or remove specific ones.
lmt.RemoveHeaderEntries("X-Access-Token", []string{"limitless-token"})
// By the way, the setters are chainable. Example:
lmt.SetIPLookups([]string{"RemoteAddr", "X-Forwarded-For", "X-Real-IP"}).
SetMethods([]string{"GET", "POST"}).
SetBasicAuthUsers([]string{"sansa"}).
SetBasicAuthUsers([]string{"tyrion"})
```
2. Compose your own middleware by using `LimitByKeys()`.
3. Header entries and basic auth users can expire over time (to conserve memory).
```go
import "time"
lmt := tollbooth.NewLimiter(1, nil)
// Set a custom expiration TTL for token bucket.
lmt.SetTokenBucketExpirationTTL(time.Hour)
// Set a custom expiration TTL for basic auth users.
lmt.SetBasicAuthExpirationTTL(time.Hour)
// Set a custom expiration TTL for header entries.
lmt.SetHeaderEntryExpirationTTL(time.Hour)
```
4. Upon rejection, the following HTTP response headers are available to users:
* `X-Rate-Limit-Limit` The maximum request limit.
* `X-Rate-Limit-Duration` The rate-limiter duration.
* `X-Rate-Limit-Request-Forwarded-For` The rejected request `X-Forwarded-For`.
* `X-Rate-Limit-Request-Remote-Addr` The rejected request `RemoteAddr`.
5. Customize your own message or function when limit is reached.
```go
lmt := tollbooth.NewLimiter(1, nil)
// Set a custom message.
lmt.SetMessage("You have reached maximum request limit.")
// Set a custom content-type.
lmt.SetMessageContentType("text/plain; charset=utf-8")
// Set a custom function for rejection.
lmt.SetOnLimitReached(func(w http.ResponseWriter, r *http.Request) { fmt.Println("A request was rejected") })
```
6. Tollbooth does not require external storage since it uses an algorithm called [Token Bucket](http://en.wikipedia.org/wiki/Token_bucket) [(Go library: golang.org/x/time/rate)](https://godoc.org/golang.org/x/time/rate).
## Other Web Frameworks
Sometimes, other frameworks require a little bit of shim to use Tollbooth. These shims below are contributed by the community, so I make no promises on how well they work. The one I am familiar with are: Chi, Gin, and Negroni.
* [Chi](https://github.com/didip/tollbooth_chi)
* [Echo](https://github.com/didip/tollbooth_echo)
* [FastHTTP](https://github.com/didip/tollbooth_fasthttp)
* [Gin](https://github.com/didip/tollbooth_gin)
* [GoRestful](https://github.com/didip/tollbooth_gorestful)
* [HTTPRouter](https://github.com/didip/tollbooth_httprouter)
* [Iris](https://github.com/didip/tollbooth_iris)
* [Negroni](https://github.com/didip/tollbooth_negroni)
## My other Go libraries
* [Stopwatch](https://github.com/didip/stopwatch): A small library to measure latency of things. Useful if you want to report latency data to Graphite.
* [LaborUnion](https://github.com/didip/laborunion): A dynamic worker pool library.
* [Gomet](https://github.com/didip/gomet): Simple HTTP client & server long poll library for Go. Useful for receiving live updates without needing Websocket.
## Contributions
Before sending a PR with code changes, please make sure altered code is covered with tests which are passing, and that golangci-lint shows no errors.
To check the linter output, [install it](https://golangci-lint.run/usage/install/#local-installation) and then run `golangci-lint run` in the root directory of the repository.

15
vendor/github.com/didip/tollbooth/v6/errors/errors.go generated vendored Normal file
View File

@ -0,0 +1,15 @@
// Package errors provide data structure for errors.
package errors
import "fmt"
// HTTPError is an error struct that returns both message and status code.
type HTTPError struct {
Message string
StatusCode int
}
// Error returns error message.
func (httperror *HTTPError) Error() string {
return fmt.Sprintf("%v: %v", httperror.StatusCode, httperror.Message)
}

10
vendor/github.com/didip/tollbooth/v6/go.mod generated vendored Normal file
View File

@ -0,0 +1,10 @@
module github.com/didip/tollbooth/v6
go 1.12
require (
github.com/go-pkgz/expirable-cache v0.0.3
github.com/kr/pretty v0.1.0 // indirect
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
)

24
vendor/github.com/didip/tollbooth/v6/go.sum generated vendored Normal file
View File

@ -0,0 +1,24 @@
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-pkgz/expirable-cache v0.0.3 h1:rTh6qNPp78z0bQE6HDhXBHUwqnV9i09Vm6dksJLXQDc=
github.com/go-pkgz/expirable-cache v0.0.3/go.mod h1:+IauqN00R2FqNRLCLA+X5YljQJrwB179PfiAoMPlTlQ=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI=
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

55
vendor/github.com/didip/tollbooth/v6/libstring/libstring.go generated vendored Normal file
View File

@ -0,0 +1,55 @@
// Package libstring provides various string related functions.
package libstring
import (
"net"
"net/http"
"strings"
)
// StringInSlice finds needle in a slice of strings.
func StringInSlice(sliceString []string, needle string) bool {
for _, b := range sliceString {
if b == needle {
return true
}
}
return false
}
// RemoteIP finds IP Address given http.Request struct.
func RemoteIP(ipLookups []string, forwardedForIndexFromBehind int, r *http.Request) string {
realIP := r.Header.Get("X-Real-IP")
forwardedFor := r.Header.Get("X-Forwarded-For")
for _, lookup := range ipLookups {
if lookup == "RemoteAddr" {
// 1. Cover the basic use cases for both ipv4 and ipv6
ip, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
// 2. Upon error, just return the remote addr.
return r.RemoteAddr
}
return ip
}
if lookup == "X-Forwarded-For" && forwardedFor != "" {
// X-Forwarded-For is potentially a list of addresses separated with ","
parts := strings.Split(forwardedFor, ",")
for i, p := range parts {
parts[i] = strings.TrimSpace(p)
}
partIndex := len(parts) - 1 - forwardedForIndexFromBehind
if partIndex < 0 {
partIndex = 0
}
return parts[partIndex]
}
if lookup == "X-Real-IP" && realIP != "" {
return realIP
}
}
return ""
}

574
vendor/github.com/didip/tollbooth/v6/limiter/limiter.go generated vendored Normal file
View File

@ -0,0 +1,574 @@
// Package limiter provides data structure to configure rate-limiter.
package limiter
import (
"net/http"
"sync"
"time"
"github.com/go-pkgz/expirable-cache"
"golang.org/x/time/rate"
)
// New is a constructor for Limiter.
func New(generalExpirableOptions *ExpirableOptions) *Limiter {
lmt := &Limiter{}
lmt.SetMessageContentType("text/plain; charset=utf-8").
SetMessage("You have reached maximum request limit.").
SetStatusCode(429).
SetOnLimitReached(nil).
SetIPLookups([]string{"RemoteAddr", "X-Forwarded-For", "X-Real-IP"}).
SetForwardedForIndexFromBehind(0).
SetHeaders(make(map[string][]string)).
SetContextValues(make(map[string][]string))
if generalExpirableOptions != nil {
lmt.generalExpirableOptions = generalExpirableOptions
} else {
lmt.generalExpirableOptions = &ExpirableOptions{}
}
// Default for DefaultExpirationTTL is 10 years.
if lmt.generalExpirableOptions.DefaultExpirationTTL <= 0 {
lmt.generalExpirableOptions.DefaultExpirationTTL = 87600 * time.Hour
}
lmt.tokenBuckets, _ = cache.NewCache(cache.TTL(lmt.generalExpirableOptions.DefaultExpirationTTL))
lmt.basicAuthUsers, _ = cache.NewCache(cache.TTL(lmt.generalExpirableOptions.DefaultExpirationTTL))
return lmt
}
// Limiter is a config struct to limit a particular request handler.
type Limiter struct {
// Maximum number of requests to limit per second.
max float64
// Limiter burst size
burst int
// HTTP message when limit is reached.
message string
// Content-Type for Message
messageContentType string
// HTTP status code when limit is reached.
statusCode int
// A function to call when a request is rejected.
onLimitReached func(w http.ResponseWriter, r *http.Request)
// An option to write back what you want upon reaching a limit.
overrideDefaultResponseWriter bool
// List of places to look up IP address.
// Default is "RemoteAddr", "X-Forwarded-For", "X-Real-IP".
// You can rearrange the order as you like.
ipLookups []string
forwardedForIndex int
// List of HTTP Methods to limit (GET, POST, PUT, etc.).
// Empty means limit all methods.
methods []string
// Able to configure token bucket expirations.
generalExpirableOptions *ExpirableOptions
// List of basic auth usernames to limit.
basicAuthUsers cache.Cache
// Map of HTTP headers to limit.
// Empty means skip headers checking.
headers map[string]cache.Cache
// Map of Context values to limit.
contextValues map[string]cache.Cache
// Map of limiters with TTL
tokenBuckets cache.Cache
tokenBucketExpirationTTL time.Duration
basicAuthExpirationTTL time.Duration
headerEntryExpirationTTL time.Duration
contextEntryExpirationTTL time.Duration
sync.RWMutex
}
// SetTokenBucketExpirationTTL is thread-safe way of setting custom token bucket expiration TTL.
func (l *Limiter) SetTokenBucketExpirationTTL(ttl time.Duration) *Limiter {
l.Lock()
l.tokenBucketExpirationTTL = ttl
l.Unlock()
return l
}
// GetTokenBucketExpirationTTL is thread-safe way of getting custom token bucket expiration TTL.
func (l *Limiter) GetTokenBucketExpirationTTL() time.Duration {
l.RLock()
defer l.RUnlock()
return l.tokenBucketExpirationTTL
}
// SetBasicAuthExpirationTTL is thread-safe way of setting custom basic auth expiration TTL.
func (l *Limiter) SetBasicAuthExpirationTTL(ttl time.Duration) *Limiter {
l.Lock()
l.basicAuthExpirationTTL = ttl
l.Unlock()
return l
}
// GetBasicAuthExpirationTTL is thread-safe way of getting custom basic auth expiration TTL.
func (l *Limiter) GetBasicAuthExpirationTTL() time.Duration {
l.RLock()
defer l.RUnlock()
return l.basicAuthExpirationTTL
}
// SetHeaderEntryExpirationTTL is thread-safe way of setting custom basic auth expiration TTL.
func (l *Limiter) SetHeaderEntryExpirationTTL(ttl time.Duration) *Limiter {
l.Lock()
l.headerEntryExpirationTTL = ttl
l.Unlock()
return l
}
// GetHeaderEntryExpirationTTL is thread-safe way of getting custom basic auth expiration TTL.
func (l *Limiter) GetHeaderEntryExpirationTTL() time.Duration {
l.RLock()
defer l.RUnlock()
return l.headerEntryExpirationTTL
}
// SetContextValueEntryExpirationTTL is thread-safe way of setting custom Context value expiration TTL.
func (l *Limiter) SetContextValueEntryExpirationTTL(ttl time.Duration) *Limiter {
l.Lock()
l.contextEntryExpirationTTL = ttl
l.Unlock()
return l
}
// GetContextValueEntryExpirationTTL is thread-safe way of getting custom Context value expiration TTL.
func (l *Limiter) GetContextValueEntryExpirationTTL() time.Duration {
l.RLock()
defer l.RUnlock()
return l.contextEntryExpirationTTL
}
// SetMax is thread-safe way of setting maximum number of requests to limit per second.
func (l *Limiter) SetMax(max float64) *Limiter {
l.Lock()
l.max = max
l.Unlock()
return l
}
// GetMax is thread-safe way of getting maximum number of requests to limit per second.
func (l *Limiter) GetMax() float64 {
l.RLock()
defer l.RUnlock()
return l.max
}
// SetBurst is thread-safe way of setting maximum burst size.
func (l *Limiter) SetBurst(burst int) *Limiter {
l.Lock()
l.burst = burst
l.Unlock()
return l
}
// GetBurst is thread-safe way of setting maximum burst size.
func (l *Limiter) GetBurst() int {
l.RLock()
defer l.RUnlock()
return l.burst
}
// SetMessage is thread-safe way of setting HTTP message when limit is reached.
func (l *Limiter) SetMessage(msg string) *Limiter {
l.Lock()
l.message = msg
l.Unlock()
return l
}
// GetMessage is thread-safe way of getting HTTP message when limit is reached.
func (l *Limiter) GetMessage() string {
l.RLock()
defer l.RUnlock()
return l.message
}
// SetMessageContentType is thread-safe way of setting HTTP message Content-Type when limit is reached.
func (l *Limiter) SetMessageContentType(contentType string) *Limiter {
l.Lock()
l.messageContentType = contentType
l.Unlock()
return l
}
// GetMessageContentType is thread-safe way of getting HTTP message Content-Type when limit is reached.
func (l *Limiter) GetMessageContentType() string {
l.RLock()
defer l.RUnlock()
return l.messageContentType
}
// SetStatusCode is thread-safe way of setting HTTP status code when limit is reached.
func (l *Limiter) SetStatusCode(statusCode int) *Limiter {
l.Lock()
l.statusCode = statusCode
l.Unlock()
return l
}
// GetStatusCode is thread-safe way of getting HTTP status code when limit is reached.
func (l *Limiter) GetStatusCode() int {
l.RLock()
defer l.RUnlock()
return l.statusCode
}
// SetOnLimitReached is thread-safe way of setting after-rejection function when limit is reached.
func (l *Limiter) SetOnLimitReached(fn func(w http.ResponseWriter, r *http.Request)) *Limiter {
l.Lock()
l.onLimitReached = fn
l.Unlock()
return l
}
// ExecOnLimitReached is thread-safe way of executing after-rejection function when limit is reached.
func (l *Limiter) ExecOnLimitReached(w http.ResponseWriter, r *http.Request) {
l.RLock()
defer l.RUnlock()
fn := l.onLimitReached
if fn != nil {
fn(w, r)
}
}
// SetOverrideDefaultResponseWriter is a thread-safe way of setting the response writer override variable.
func (l *Limiter) SetOverrideDefaultResponseWriter(override bool) {
l.Lock()
l.overrideDefaultResponseWriter = override
l.Unlock()
}
// GetOverrideDefaultResponseWriter is a thread-safe way of getting the response writer override variable.
func (l *Limiter) GetOverrideDefaultResponseWriter() bool {
l.RLock()
defer l.RUnlock()
return l.overrideDefaultResponseWriter
}
// SetIPLookups is thread-safe way of setting list of places to look up IP address.
func (l *Limiter) SetIPLookups(ipLookups []string) *Limiter {
l.Lock()
l.ipLookups = ipLookups
l.Unlock()
return l
}
// GetIPLookups is thread-safe way of getting list of places to look up IP address.
func (l *Limiter) GetIPLookups() []string {
l.RLock()
defer l.RUnlock()
return l.ipLookups
}
// SetForwardedForIndexFromBehind is thread-safe way of setting which X-Forwarded-For index to choose.
func (l *Limiter) SetForwardedForIndexFromBehind(forwardedForIndex int) *Limiter {
l.Lock()
l.forwardedForIndex = forwardedForIndex
l.Unlock()
return l
}
// GetForwardedForIndexFromBehind is thread-safe way of getting which X-Forwarded-For index to choose.
func (l *Limiter) GetForwardedForIndexFromBehind() int {
l.RLock()
defer l.RUnlock()
return l.forwardedForIndex
}
// SetMethods is thread-safe way of setting list of HTTP Methods to limit (GET, POST, PUT, etc.).
func (l *Limiter) SetMethods(methods []string) *Limiter {
l.Lock()
l.methods = methods
l.Unlock()
return l
}
// GetMethods is thread-safe way of getting list of HTTP Methods to limit (GET, POST, PUT, etc.).
func (l *Limiter) GetMethods() []string {
l.RLock()
defer l.RUnlock()
return l.methods
}
// SetBasicAuthUsers is thread-safe way of setting list of basic auth usernames to limit.
func (l *Limiter) SetBasicAuthUsers(basicAuthUsers []string) *Limiter {
ttl := l.GetBasicAuthExpirationTTL()
if ttl <= 0 {
ttl = l.generalExpirableOptions.DefaultExpirationTTL
}
for _, basicAuthUser := range basicAuthUsers {
l.basicAuthUsers.Set(basicAuthUser, true, ttl)
}
return l
}
// GetBasicAuthUsers is thread-safe way of getting list of basic auth usernames to limit.
func (l *Limiter) GetBasicAuthUsers() []string {
return l.basicAuthUsers.Keys()
}
// RemoveBasicAuthUsers is thread-safe way of removing basic auth usernames from existing list.
func (l *Limiter) RemoveBasicAuthUsers(basicAuthUsers []string) *Limiter {
for _, toBeRemoved := range basicAuthUsers {
l.basicAuthUsers.Invalidate(toBeRemoved)
}
return l
}
// SetHeaders is thread-safe way of setting map of HTTP headers to limit.
func (l *Limiter) SetHeaders(headers map[string][]string) *Limiter {
if l.headers == nil {
l.headers = make(map[string]cache.Cache)
}
for header, entries := range headers {
l.SetHeader(header, entries)
}
return l
}
// GetHeaders is thread-safe way of getting map of HTTP headers to limit.
func (l *Limiter) GetHeaders() map[string][]string {
results := make(map[string][]string)
l.RLock()
defer l.RUnlock()
for header, entriesAsGoCache := range l.headers {
results[header] = entriesAsGoCache.Keys()
}
return results
}
// SetHeader is thread-safe way of setting entries of 1 HTTP header.
func (l *Limiter) SetHeader(header string, entries []string) *Limiter {
l.RLock()
existing, found := l.headers[header]
l.RUnlock()
ttl := l.GetHeaderEntryExpirationTTL()
if ttl <= 0 {
ttl = l.generalExpirableOptions.DefaultExpirationTTL
}
if !found {
existing, _ = cache.NewCache(cache.TTL(ttl))
}
for _, entry := range entries {
existing.Set(entry, true, ttl)
}
l.Lock()
l.headers[header] = existing
l.Unlock()
return l
}
// GetHeader is thread-safe way of getting entries of 1 HTTP header.
func (l *Limiter) GetHeader(header string) []string {
l.RLock()
entriesAsGoCache := l.headers[header]
l.RUnlock()
return entriesAsGoCache.Keys()
}
// RemoveHeader is thread-safe way of removing entries of 1 HTTP header.
func (l *Limiter) RemoveHeader(header string) *Limiter {
ttl := l.GetHeaderEntryExpirationTTL()
if ttl <= 0 {
ttl = l.generalExpirableOptions.DefaultExpirationTTL
}
l.Lock()
l.headers[header], _ = cache.NewCache(cache.TTL(ttl))
l.Unlock()
return l
}
// RemoveHeaderEntries is thread-safe way of removing new entries to 1 HTTP header rule.
func (l *Limiter) RemoveHeaderEntries(header string, entriesForRemoval []string) *Limiter {
l.RLock()
entries, found := l.headers[header]
l.RUnlock()
if !found {
return l
}
for _, toBeRemoved := range entriesForRemoval {
entries.Invalidate(toBeRemoved)
}
return l
}
// SetContextValues is thread-safe way of setting map of HTTP headers to limit.
func (l *Limiter) SetContextValues(contextValues map[string][]string) *Limiter {
if l.contextValues == nil {
l.contextValues = make(map[string]cache.Cache)
}
for contextValue, entries := range contextValues {
l.SetContextValue(contextValue, entries)
}
return l
}
// GetContextValues is thread-safe way of getting a map of Context values to limit.
func (l *Limiter) GetContextValues() map[string][]string {
results := make(map[string][]string)
l.RLock()
defer l.RUnlock()
for contextValue, entriesAsGoCache := range l.contextValues {
results[contextValue] = entriesAsGoCache.Keys()
}
return results
}
// SetContextValue is thread-safe way of setting entries of 1 Context value.
func (l *Limiter) SetContextValue(contextValue string, entries []string) *Limiter {
l.RLock()
existing, found := l.contextValues[contextValue]
l.RUnlock()
ttl := l.GetContextValueEntryExpirationTTL()
if ttl <= 0 {
ttl = l.generalExpirableOptions.DefaultExpirationTTL
}
if !found {
existing, _ = cache.NewCache(cache.TTL(ttl))
}
for _, entry := range entries {
existing.Set(entry, true, ttl)
}
l.Lock()
l.contextValues[contextValue] = existing
l.Unlock()
return l
}
// GetContextValue is thread-safe way of getting 1 Context value entry.
func (l *Limiter) GetContextValue(contextValue string) []string {
l.RLock()
entriesAsGoCache := l.contextValues[contextValue]
l.RUnlock()
return entriesAsGoCache.Keys()
}
// RemoveContextValue is thread-safe way of removing entries of 1 Context value.
func (l *Limiter) RemoveContextValue(contextValue string) *Limiter {
ttl := l.GetContextValueEntryExpirationTTL()
if ttl <= 0 {
ttl = l.generalExpirableOptions.DefaultExpirationTTL
}
l.Lock()
l.contextValues[contextValue], _ = cache.NewCache(cache.TTL(ttl))
l.Unlock()
return l
}
// RemoveContextValuesEntries is thread-safe way of removing entries to a ContextValue.
func (l *Limiter) RemoveContextValuesEntries(contextValue string, entriesForRemoval []string) *Limiter {
l.RLock()
entries, found := l.contextValues[contextValue]
l.RUnlock()
if !found {
return l
}
for _, toBeRemoved := range entriesForRemoval {
entries.Invalidate(toBeRemoved)
}
return l
}
func (l *Limiter) limitReachedWithTokenBucketTTL(key string, tokenBucketTTL time.Duration) bool {
lmtMax := l.GetMax()
lmtBurst := l.GetBurst()
l.Lock()
defer l.Unlock()
if _, found := l.tokenBuckets.Get(key); !found {
l.tokenBuckets.Set(
key,
rate.NewLimiter(rate.Limit(lmtMax), lmtBurst),
tokenBucketTTL,
)
}
expiringMap, found := l.tokenBuckets.Get(key)
if !found {
return false
}
return !expiringMap.(*rate.Limiter).Allow()
}
// LimitReached returns a bool indicating if the Bucket identified by key ran out of tokens.
func (l *Limiter) LimitReached(key string) bool {
ttl := l.GetTokenBucketExpirationTTL()
if ttl <= 0 {
ttl = l.generalExpirableOptions.DefaultExpirationTTL
}
return l.limitReachedWithTokenBucketTTL(key, ttl)
}

14
vendor/github.com/didip/tollbooth/v6/limiter/limiter_options.go generated vendored Normal file
View File

@ -0,0 +1,14 @@
package limiter
import (
"time"
)
// ExpirableOptions are options used for new limiter creation
type ExpirableOptions struct {
DefaultExpirationTTL time.Duration
// How frequently expire job triggers
// Deprecated: not used anymore
ExpireJobInterval time.Duration
}

314
vendor/github.com/didip/tollbooth/v6/tollbooth.go generated vendored Normal file
View File

@ -0,0 +1,314 @@
// Package tollbooth provides rate-limiting logic to HTTP request handler.
package tollbooth
import (
"fmt"
"math"
"net/http"
"strings"
"github.com/didip/tollbooth/v6/errors"
"github.com/didip/tollbooth/v6/libstring"
"github.com/didip/tollbooth/v6/limiter"
)
// setResponseHeaders configures X-Rate-Limit-Limit and X-Rate-Limit-Duration
func setResponseHeaders(lmt *limiter.Limiter, w http.ResponseWriter, r *http.Request) {
w.Header().Add("X-Rate-Limit-Limit", fmt.Sprintf("%.2f", lmt.GetMax()))
w.Header().Add("X-Rate-Limit-Duration", "1")
xForwardedFor := r.Header.Get("X-Forwarded-For")
if strings.TrimSpace(xForwardedFor) != "" {
w.Header().Add("X-Rate-Limit-Request-Forwarded-For", xForwardedFor)
}
w.Header().Add("X-Rate-Limit-Request-Remote-Addr", r.RemoteAddr)
}
// NewLimiter is a convenience function to limiter.New.
func NewLimiter(max float64, tbOptions *limiter.ExpirableOptions) *limiter.Limiter {
return limiter.New(tbOptions).
SetMax(max).
SetBurst(int(math.Max(1, max))).
SetIPLookups([]string{"X-Forwarded-For", "X-Real-IP", "RemoteAddr"})
}
// LimitByKeys keeps track number of request made by keys separated by pipe.
// It returns HTTPError when limit is exceeded.
func LimitByKeys(lmt *limiter.Limiter, keys []string) *errors.HTTPError {
if lmt.LimitReached(strings.Join(keys, "|")) {
return &errors.HTTPError{Message: lmt.GetMessage(), StatusCode: lmt.GetStatusCode()}
}
return nil
}
// ShouldSkipLimiter is a series of filter that decides if request should be limited or not.
func ShouldSkipLimiter(lmt *limiter.Limiter, r *http.Request) bool {
// ---------------------------------
// Filter by remote ip
// If we are unable to find remoteIP, skip limiter
remoteIP := libstring.RemoteIP(lmt.GetIPLookups(), lmt.GetForwardedForIndexFromBehind(), r)
if remoteIP == "" {
return true
}
// ---------------------------------
// Filter by request method
lmtMethods := lmt.GetMethods()
lmtMethodsIsSet := len(lmtMethods) > 0
if lmtMethodsIsSet {
// If request does not contain all of the methods in limiter,
// skip limiter
requestMethodDefinedInLimiter := libstring.StringInSlice(lmtMethods, r.Method)
if !requestMethodDefinedInLimiter {
return true
}
}
// ---------------------------------
// Filter by request headers
lmtHeaders := lmt.GetHeaders()
lmtHeadersIsSet := len(lmtHeaders) > 0
if lmtHeadersIsSet {
// If request does not contain all of the headers in limiter,
// skip limiter
requestHeadersDefinedInLimiter := false
for headerKey := range lmtHeaders {
reqHeaderValue := r.Header.Get(headerKey)
if reqHeaderValue != "" {
requestHeadersDefinedInLimiter = true
break
}
}
if !requestHeadersDefinedInLimiter {
return true
}
// ------------------------------
// If request contains the header key but not the values,
// skip limiter
requestHeadersDefinedInLimiter = false
for headerKey, headerValues := range lmtHeaders {
for _, headerValue := range headerValues {
if r.Header.Get(headerKey) == headerValue {
requestHeadersDefinedInLimiter = true
break
}
}
}
if !requestHeadersDefinedInLimiter {
return true
}
}
// ---------------------------------
// Filter by context values
lmtContextValues := lmt.GetContextValues()
lmtContextValuesIsSet := len(lmtContextValues) > 0
if lmtContextValuesIsSet {
// If request does not contain all of the contexts in limiter,
// skip limiter
requestContextValuesDefinedInLimiter := false
for contextKey := range lmtContextValues {
reqContextValue := fmt.Sprintf("%v", r.Context().Value(contextKey))
if reqContextValue != "" {
requestContextValuesDefinedInLimiter = true
break
}
}
if !requestContextValuesDefinedInLimiter {
return true
}
// ------------------------------
// If request contains the context key but not the values,
// skip limiter
requestContextValuesDefinedInLimiter = false
for contextKey, contextValues := range lmtContextValues {
for _, contextValue := range contextValues {
if r.Header.Get(contextKey) == contextValue {
requestContextValuesDefinedInLimiter = true
break
}
}
}
if !requestContextValuesDefinedInLimiter {
return true
}
}
// ---------------------------------
// Filter by basic auth usernames
lmtBasicAuthUsers := lmt.GetBasicAuthUsers()
lmtBasicAuthUsersIsSet := len(lmtBasicAuthUsers) > 0
if lmtBasicAuthUsersIsSet {
// If request does not contain all of the basic auth users in limiter,
// skip limiter
requestAuthUsernameDefinedInLimiter := false
username, _, ok := r.BasicAuth()
if ok && libstring.StringInSlice(lmtBasicAuthUsers, username) {
requestAuthUsernameDefinedInLimiter = true
}
if !requestAuthUsernameDefinedInLimiter {
return true
}
}
return false
}
// BuildKeys generates a slice of keys to rate-limit by given limiter and request structs.
func BuildKeys(lmt *limiter.Limiter, r *http.Request) [][]string {
remoteIP := libstring.RemoteIP(lmt.GetIPLookups(), lmt.GetForwardedForIndexFromBehind(), r)
path := r.URL.Path
sliceKeys := make([][]string, 0)
lmtMethods := lmt.GetMethods()
lmtHeaders := lmt.GetHeaders()
lmtContextValues := lmt.GetContextValues()
lmtBasicAuthUsers := lmt.GetBasicAuthUsers()
lmtHeadersIsSet := len(lmtHeaders) > 0
lmtContextValuesIsSet := len(lmtContextValues) > 0
lmtBasicAuthUsersIsSet := len(lmtBasicAuthUsers) > 0
usernameToLimit := ""
if lmtBasicAuthUsersIsSet {
username, _, ok := r.BasicAuth()
if ok && libstring.StringInSlice(lmtBasicAuthUsers, username) {
usernameToLimit = username
}
}
headerValuesToLimit := [][]string{}
if lmtHeadersIsSet {
for headerKey, headerValues := range lmtHeaders {
reqHeaderValue := r.Header.Get(headerKey)
if reqHeaderValue == "" {
continue
}
if len(headerValues) == 0 {
// If header values are empty, rate-limit all request containing headerKey.
headerValuesToLimit = append(headerValuesToLimit, []string{headerKey, reqHeaderValue})
} else {
// If header values are not empty, rate-limit all request with headerKey and headerValues.
for _, headerValue := range headerValues {
if r.Header.Get(headerKey) == headerValue {
headerValuesToLimit = append(headerValuesToLimit, []string{headerKey, headerValue})
break
}
}
}
}
}
contextValuesToLimit := [][]string{}
if lmtContextValuesIsSet {
for contextKey, contextValues := range lmtContextValues {
reqContextValue := fmt.Sprintf("%v", r.Context().Value(contextKey))
if reqContextValue == "" {
continue
}
if len(contextValues) == 0 {
// If context values are empty, rate-limit all request containing contextKey.
contextValuesToLimit = append(contextValuesToLimit, []string{contextKey, reqContextValue})
} else {
// If context values are not empty, rate-limit all request with contextKey and contextValues.
for _, contextValue := range contextValues {
if reqContextValue == contextValue {
contextValuesToLimit = append(contextValuesToLimit, []string{contextKey, contextValue})
break
}
}
}
}
}
sliceKey := []string{remoteIP, path}
sliceKey = append(sliceKey, lmtMethods...)
for _, header := range headerValuesToLimit {
sliceKey = append(sliceKey, header[0], header[1])
}
for _, contextValue := range contextValuesToLimit {
sliceKey = append(sliceKey, contextValue[0], contextValue[1])
}
sliceKey = append(sliceKey, usernameToLimit)
sliceKeys = append(sliceKeys, sliceKey)
return sliceKeys
}
// LimitByRequest builds keys based on http.Request struct,
// loops through all the keys, and check if any one of them returns HTTPError.
func LimitByRequest(lmt *limiter.Limiter, w http.ResponseWriter, r *http.Request) *errors.HTTPError {
setResponseHeaders(lmt, w, r)
shouldSkip := ShouldSkipLimiter(lmt, r)
if shouldSkip {
return nil
}
sliceKeys := BuildKeys(lmt, r)
// Loop sliceKeys and check if one of them has error.
for _, keys := range sliceKeys {
httpError := LimitByKeys(lmt, keys)
if httpError != nil {
return httpError
}
}
return nil
}
// LimitHandler is a middleware that performs rate-limiting given http.Handler struct.
func LimitHandler(lmt *limiter.Limiter, next http.Handler) http.Handler {
middle := func(w http.ResponseWriter, r *http.Request) {
httpError := LimitByRequest(lmt, w, r)
if httpError != nil {
lmt.ExecOnLimitReached(w, r)
if lmt.GetOverrideDefaultResponseWriter() {
return
}
w.Header().Add("Content-Type", lmt.GetMessageContentType())
w.WriteHeader(httpError.StatusCode)
w.Write([]byte(httpError.Message))
return
}
// There's no rate-limit error, serve the next handler.
next.ServeHTTP(w, r)
}
return http.HandlerFunc(middle)
}
// LimitFuncHandler is a middleware that performs rate-limiting given request handler function.
func LimitFuncHandler(lmt *limiter.Limiter, nextFunc func(http.ResponseWriter, *http.Request)) http.Handler {
return LimitHandler(lmt, http.HandlerFunc(nextFunc))
}

15
vendor/github.com/go-pkgz/expirable-cache/.gitignore generated vendored Normal file
View File

@ -0,0 +1,15 @@
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Dependency directories (remove the comment below to include it)
# vendor/

64
vendor/github.com/go-pkgz/expirable-cache/.golangci.yml generated vendored Normal file
View File

@ -0,0 +1,64 @@
linters-settings:
govet:
check-shadowing: true
golint:
min-confidence: 0
gocyclo:
min-complexity: 15
maligned:
suggest-new: true
goconst:
min-len: 2
min-occurrences: 2
misspell:
locale: US
lll:
line-length: 140
gocritic:
enabled-tags:
- performance
- style
- experimental
disabled-checks:
- wrapperFunc
linters:
enable:
- megacheck
- golint
- govet
- unconvert
- megacheck
- structcheck
- gas
- gocyclo
- dupl
- misspell
- unparam
- varcheck
- deadcode
- typecheck
- ineffassign
- varcheck
- stylecheck
- gochecknoinits
- scopelint
- gocritic
- nakedret
- gosimple
- prealloc
fast: false
disable-all: true
run:
output:
format: tab
skip-dirs:
- vendor
issues:
exclude-rules:
- text: "should have a package comment, unless it's in another file for this package"
linters:
- golint
exclude-use-default: false

22
vendor/github.com/go-pkgz/expirable-cache/LICENSE generated vendored Normal file
View File

@ -0,0 +1,22 @@
MIT License
Copyright (c) 2020 Umputun
Copyright (c) 2020 Dmitry Verhoturov
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

70
vendor/github.com/go-pkgz/expirable-cache/README.md generated vendored Normal file
View File

@ -0,0 +1,70 @@
# expirable-cache
[![Build Status](https://github.com/go-pkgz/expirable-cache/workflows/build/badge.svg)](https://github.com/go-pkgz/expirable-cache/actions)
[![Coverage Status](https://coveralls.io/repos/github/go-pkgz/expirable-cache/badge.svg?branch=master)](https://coveralls.io/github/go-pkgz/expirable-cache?branch=master)
[![godoc](https://godoc.org/github.com/go-pkgz/expirable-cache?status.svg)](https://pkg.go.dev/github.com/go-pkgz/expirable-cache?tab=doc)
Package cache implements expirable cache.
- Support LRC, LRU and TTL-based eviction.
- Package is thread-safe and doesn't spawn any goroutines.
- On every Set() call, cache deletes single oldest entry in case it's expired.
- In case MaxSize is set, cache deletes the oldest entry disregarding its expiration date to maintain the size,
either using LRC or LRU eviction.
- In case of default TTL (10 years) and default MaxSize (0, unlimited) the cache will be truly unlimited
and will never delete entries from itself automatically.
**Important**: only reliable way of not having expired entries stuck in a cache is to
run cache.DeleteExpired periodically using [time.Ticker](https://golang.org/pkg/time/#Ticker),
advisable period is 1/2 of TTL.
This cache is heavily inspired by [hashicorp/golang-lru](https://github.com/hashicorp/golang-lru) _simplelru_ implementation.
### Usage example
```go
package main
import (
"fmt"
"time"
"github.com/go-pkgz/expirable-cache"
)
func main() {
// make cache with short TTL and 3 max keys
c, _ := cache.NewCache(cache.MaxKeys(3), cache.TTL(time.Millisecond*10))
// set value under key1.
// with 0 ttl (last parameter) will use cache-wide setting instead (10ms).
c.Set("key1", "val1", 0)
// get value under key1
r, ok := c.Get("key1")
// check for OK value, because otherwise return would be nil and
// type conversion will panic
if ok {
rstr := r.(string) // convert cached value from interface{} to real type
fmt.Printf("value before expiration is found: %v, value: %v\n", ok, rstr)
}
time.Sleep(time.Millisecond * 11)
// get value under key1 after key expiration
r, ok = c.Get("key1")
// don't convert to string as with ok == false value would be nil
fmt.Printf("value after expiration is found: %v, value: %v\n", ok, r)
// set value under key2, would evict old entry because it is already expired.
// ttl (last parameter) overrides cache-wide ttl.
c.Set("key2", "val2", time.Minute*5)
fmt.Printf("%+v\n", c)
// Output:
// value before expiration is found: true, value: val1
// value after expiration is found: false, value: <nil>
// Size: 1, Stats: {Hits:1 Misses:1 Added:2 Evicted:1} (50.0%)
}
```

274
vendor/github.com/go-pkgz/expirable-cache/cache.go generated vendored Normal file
View File

@ -0,0 +1,274 @@
// Package cache implements Cache similar to hashicorp/golang-lru
//
// Support LRC, LRU and TTL-based eviction.
// Package is thread-safe and doesn't spawn any goroutines.
// On every Set() call, cache deletes single oldest entry in case it's expired.
// In case MaxSize is set, cache deletes the oldest entry disregarding its expiration date to maintain the size,
// either using LRC or LRU eviction.
// In case of default TTL (10 years) and default MaxSize (0, unlimited) the cache will be truly unlimited
// and will never delete entries from itself automatically.
//
// Important: only reliable way of not having expired entries stuck in a cache is to
// run cache.DeleteExpired periodically using time.Ticker, advisable period is 1/2 of TTL.
package cache
import (
"container/list"
"fmt"
"sync"
"time"
"github.com/pkg/errors"
)
// Cache defines cache interface
type Cache interface {
fmt.Stringer
Set(key string, value interface{}, ttl time.Duration)
Get(key string) (interface{}, bool)
Peek(key string) (interface{}, bool)
Keys() []string
Len() int
Invalidate(key string)
InvalidateFn(fn func(key string) bool)
RemoveOldest()
DeleteExpired()
Purge()
Stat() Stats
}
// Stats provides statistics for cache
type Stats struct {
Hits, Misses int // cache effectiveness
Added, Evicted int // number of added and evicted records
}
// cacheImpl provides Cache interface implementation.
type cacheImpl struct {
ttl time.Duration
maxKeys int
isLRU bool
onEvicted func(key string, value interface{})
sync.Mutex
stat Stats
items map[string]*list.Element
evictList *list.List
}
// noEvictionTTL - very long ttl to prevent eviction
const noEvictionTTL = time.Hour * 24 * 365 * 10
// NewCache returns a new Cache.
// Default MaxKeys is unlimited (0).
// Default TTL is 10 years, sane value for expirable cache is 5 minutes.
// Default eviction mode is LRC, appropriate option allow to change it to LRU.
func NewCache(options ...Option) (Cache, error) {
res := cacheImpl{
items: map[string]*list.Element{},
evictList: list.New(),
ttl: noEvictionTTL,
maxKeys: 0,
}
for _, opt := range options {
if err := opt(&res); err != nil {
return nil, errors.Wrap(err, "failed to set cache option")
}
}
return &res, nil
}
// Set key, ttl of 0 would use cache-wide TTL
func (c *cacheImpl) Set(key string, value interface{}, ttl time.Duration) {
c.Lock()
defer c.Unlock()
now := time.Now()
if ttl == 0 {
ttl = c.ttl
}
// Check for existing item
if ent, ok := c.items[key]; ok {
c.evictList.MoveToFront(ent)
ent.Value.(*cacheItem).value = value
ent.Value.(*cacheItem).expiresAt = now.Add(ttl)
return
}
// Add new item
ent := &cacheItem{key: key, value: value, expiresAt: now.Add(ttl)}
entry := c.evictList.PushFront(ent)
c.items[key] = entry
c.stat.Added++
// Remove oldest entry if it is expired, only in case of non-default TTL.
if c.ttl != noEvictionTTL || ttl != noEvictionTTL {
c.removeOldestIfExpired()
}
// Verify size not exceeded
if c.maxKeys > 0 && len(c.items) > c.maxKeys {
c.removeOldest()
}
}
// Get returns the key value if it's not expired
func (c *cacheImpl) Get(key string) (interface{}, bool) {
c.Lock()
defer c.Unlock()
if ent, ok := c.items[key]; ok {
// Expired item check
if time.Now().After(ent.Value.(*cacheItem).expiresAt) {
c.stat.Misses++
return nil, false
}
if c.isLRU {
c.evictList.MoveToFront(ent)
}
c.stat.Hits++
return ent.Value.(*cacheItem).value, true
}
c.stat.Misses++
return nil, false
}
// Peek returns the key value (or undefined if not found) without updating the "recently used"-ness of the key.
// Works exactly the same as Get in case of LRC mode (default one).
func (c *cacheImpl) Peek(key string) (interface{}, bool) {
c.Lock()
defer c.Unlock()
if ent, ok := c.items[key]; ok {
// Expired item check
if time.Now().After(ent.Value.(*cacheItem).expiresAt) {
c.stat.Misses++
return nil, false
}
c.stat.Hits++
return ent.Value.(*cacheItem).value, true
}
c.stat.Misses++
return nil, false
}
// Keys returns a slice of the keys in the cache, from oldest to newest.
func (c *cacheImpl) Keys() []string {
c.Lock()
defer c.Unlock()
return c.keys()
}
// Len return count of items in cache, including expired
func (c *cacheImpl) Len() int {
c.Lock()
defer c.Unlock()
return c.evictList.Len()
}
// Invalidate key (item) from the cache
func (c *cacheImpl) Invalidate(key string) {
c.Lock()
defer c.Unlock()
if ent, ok := c.items[key]; ok {
c.removeElement(ent)
}
}
// InvalidateFn deletes multiple keys if predicate is true
func (c *cacheImpl) InvalidateFn(fn func(key string) bool) {
c.Lock()
defer c.Unlock()
for key, ent := range c.items {
if fn(key) {
c.removeElement(ent)
}
}
}
// RemoveOldest remove oldest element in the cache
func (c *cacheImpl) RemoveOldest() {
c.Lock()
defer c.Unlock()
c.removeOldest()
}
// DeleteExpired clears cache of expired items
func (c *cacheImpl) DeleteExpired() {
c.Lock()
defer c.Unlock()
for _, key := range c.keys() {
if time.Now().After(c.items[key].Value.(*cacheItem).expiresAt) {
c.removeElement(c.items[key])
}
}
}
// Purge clears the cache completely.
func (c *cacheImpl) Purge() {
c.Lock()
defer c.Unlock()
for k, v := range c.items {
delete(c.items, k)
c.stat.Evicted++
if c.onEvicted != nil {
c.onEvicted(k, v.Value.(*cacheItem).value)
}
}
c.evictList.Init()
}
// Stat gets the current stats for cache
func (c *cacheImpl) Stat() Stats {
c.Lock()
defer c.Unlock()
return c.stat
}
func (c *cacheImpl) String() string {
stats := c.Stat()
size := c.Len()
return fmt.Sprintf("Size: %d, Stats: %+v (%0.1f%%)", size, stats, 100*float64(stats.Hits)/float64(stats.Hits+stats.Misses))
}
// Keys returns a slice of the keys in the cache, from oldest to newest. Has to be called with lock!
func (c *cacheImpl) keys() []string {
keys := make([]string, 0, len(c.items))
for ent := c.evictList.Back(); ent != nil; ent = ent.Prev() {
keys = append(keys, ent.Value.(*cacheItem).key)
}
return keys
}
// removeOldest removes the oldest item from the cache. Has to be called with lock!
func (c *cacheImpl) removeOldest() {
ent := c.evictList.Back()
if ent != nil {
c.removeElement(ent)
}
}
// removeOldest removes the oldest item from the cache in case it's already expired. Has to be called with lock!
func (c *cacheImpl) removeOldestIfExpired() {
ent := c.evictList.Back()
if ent != nil && time.Now().After(ent.Value.(*cacheItem).expiresAt) {
c.removeElement(ent)
}
}
// removeElement is used to remove a given list element from the cache. Has to be called with lock!
func (c *cacheImpl) removeElement(e *list.Element) {
c.evictList.Remove(e)
kv := e.Value.(*cacheItem)
delete(c.items, kv.key)
c.stat.Evicted++
if c.onEvicted != nil {
c.onEvicted(kv.key, kv.value)
}
}
// cacheItem is used to hold a value in the evictList
type cacheItem struct {
expiresAt time.Time
key string
value interface{}
}

8
vendor/github.com/go-pkgz/expirable-cache/go.mod generated vendored Normal file
View File

@ -0,0 +1,8 @@
module github.com/go-pkgz/expirable-cache
go 1.14
require (
github.com/pkg/errors v0.9.1
github.com/stretchr/testify v1.5.1
)

13
vendor/github.com/go-pkgz/expirable-cache/go.sum generated vendored Normal file
View File

@ -0,0 +1,13 @@
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

40
vendor/github.com/go-pkgz/expirable-cache/options.go generated vendored Normal file
View File

@ -0,0 +1,40 @@
package cache
import "time"
// Option func type
type Option func(lc *cacheImpl) error
// OnEvicted called automatically for automatically and manually deleted entries
func OnEvicted(fn func(key string, value interface{})) Option {
return func(lc *cacheImpl) error {
lc.onEvicted = fn
return nil
}
}
// MaxKeys functional option defines how many keys to keep.
// By default it is 0, which means unlimited.
func MaxKeys(max int) Option {
return func(lc *cacheImpl) error {
lc.maxKeys = max
return nil
}
}
// TTL functional option defines TTL for all cache entries.
// By default it is set to 10 years, sane option for expirable cache might be 5 minutes.
func TTL(ttl time.Duration) Option {
return func(lc *cacheImpl) error {
lc.ttl = ttl
return nil
}
}
// LRU sets cache to LRU (Least Recently Used) eviction mode.
func LRU() Option {
return func(lc *cacheImpl) error {
lc.isLRU = true
return nil
}
}

3
vendor/golang.org/x/time/AUTHORS generated vendored Normal file
View File

@ -0,0 +1,3 @@
# This source code refers to The Go Authors for copyright purposes.
# The master list of authors is in the main Go distribution,
# visible at http://tip.golang.org/AUTHORS.

3
vendor/golang.org/x/time/CONTRIBUTORS generated vendored Normal file
View File

@ -0,0 +1,3 @@
# This source code was written by the Go contributors.
# The master list of contributors is in the main Go distribution,
# visible at http://tip.golang.org/CONTRIBUTORS.

27
vendor/golang.org/x/time/LICENSE generated vendored Normal file
View File

@ -0,0 +1,27 @@
Copyright (c) 2009 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

22
vendor/golang.org/x/time/PATENTS generated vendored Normal file
View File

@ -0,0 +1,22 @@
Additional IP Rights Grant (Patents)
"This implementation" means the copyrightable works distributed by
Google as part of the Go project.
Google hereby grants to You a perpetual, worldwide, non-exclusive,
no-charge, royalty-free, irrevocable (except as stated in this section)
patent license to make, have made, use, offer to sell, sell, import,
transfer and otherwise run, modify and propagate the contents of this
implementation of Go, where such license applies only to those patent
claims, both currently owned or controlled by Google and acquired in
the future, licensable by Google that are necessarily infringed by this
implementation of Go. This grant does not include claims that would be
infringed only as a consequence of further modification of this
implementation. If you or your agent or exclusive licensee institute or
order or agree to the institution of patent litigation against any
entity (including a cross-claim or counterclaim in a lawsuit) alleging
that this implementation of Go or any code incorporated within this
implementation of Go constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any patent
rights granted to you under this License for this implementation of Go
shall terminate as of the date such litigation is filed.

400
vendor/golang.org/x/time/rate/rate.go generated vendored Normal file
View File

@ -0,0 +1,400 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package rate provides a rate limiter.
package rate
import (
"context"
"fmt"
"math"
"sync"
"time"
)
// Limit defines the maximum frequency of some events.
// Limit is represented as number of events per second.
// A zero Limit allows no events.
type Limit float64
// Inf is the infinite rate limit; it allows all events (even if burst is zero).
const Inf = Limit(math.MaxFloat64)
// Every converts a minimum time interval between events to a Limit.
func Every(interval time.Duration) Limit {
if interval <= 0 {
return Inf
}
return 1 / Limit(interval.Seconds())
}
// A Limiter controls how frequently events are allowed to happen.
// It implements a "token bucket" of size b, initially full and refilled
// at rate r tokens per second.
// Informally, in any large enough time interval, the Limiter limits the
// rate to r tokens per second, with a maximum burst size of b events.
// As a special case, if r == Inf (the infinite rate), b is ignored.
// See https://en.wikipedia.org/wiki/Token_bucket for more about token buckets.
//
// The zero value is a valid Limiter, but it will reject all events.
// Use NewLimiter to create non-zero Limiters.
//
// Limiter has three main methods, Allow, Reserve, and Wait.
// Most callers should use Wait.
//
// Each of the three methods consumes a single token.
// They differ in their behavior when no token is available.
// If no token is available, Allow returns false.
// If no token is available, Reserve returns a reservation for a future token
// and the amount of time the caller must wait before using it.
// If no token is available, Wait blocks until one can be obtained
// or its associated context.Context is canceled.
//
// The methods AllowN, ReserveN, and WaitN consume n tokens.
type Limiter struct {
limit Limit
burst int
mu sync.Mutex
tokens float64
// last is the last time the limiter's tokens field was updated
last time.Time
// lastEvent is the latest time of a rate-limited event (past or future)
lastEvent time.Time
}
// Limit returns the maximum overall event rate.
func (lim *Limiter) Limit() Limit {
lim.mu.Lock()
defer lim.mu.Unlock()
return lim.limit
}
// Burst returns the maximum burst size. Burst is the maximum number of tokens
// that can be consumed in a single call to Allow, Reserve, or Wait, so higher
// Burst values allow more events to happen at once.
// A zero Burst allows no events, unless limit == Inf.
func (lim *Limiter) Burst() int {
return lim.burst
}
// NewLimiter returns a new Limiter that allows events up to rate r and permits
// bursts of at most b tokens.
func NewLimiter(r Limit, b int) *Limiter {
return &Limiter{
limit: r,
burst: b,
}
}
// Allow is shorthand for AllowN(time.Now(), 1).
func (lim *Limiter) Allow() bool {
return lim.AllowN(time.Now(), 1)
}
// AllowN reports whether n events may happen at time now.
// Use this method if you intend to drop / skip events that exceed the rate limit.
// Otherwise use Reserve or Wait.
func (lim *Limiter) AllowN(now time.Time, n int) bool {
return lim.reserveN(now, n, 0).ok
}
// A Reservation holds information about events that are permitted by a Limiter to happen after a delay.
// A Reservation may be canceled, which may enable the Limiter to permit additional events.
type Reservation struct {
ok bool
lim *Limiter
tokens int
timeToAct time.Time
// This is the Limit at reservation time, it can change later.
limit Limit
}
// OK returns whether the limiter can provide the requested number of tokens
// within the maximum wait time. If OK is false, Delay returns InfDuration, and
// Cancel does nothing.
func (r *Reservation) OK() bool {
return r.ok
}
// Delay is shorthand for DelayFrom(time.Now()).
func (r *Reservation) Delay() time.Duration {
return r.DelayFrom(time.Now())
}
// InfDuration is the duration returned by Delay when a Reservation is not OK.
const InfDuration = time.Duration(1<<63 - 1)
// DelayFrom returns the duration for which the reservation holder must wait
// before taking the reserved action. Zero duration means act immediately.
// InfDuration means the limiter cannot grant the tokens requested in this
// Reservation within the maximum wait time.
func (r *Reservation) DelayFrom(now time.Time) time.Duration {
if !r.ok {
return InfDuration
}
delay := r.timeToAct.Sub(now)
if delay < 0 {
return 0
}
return delay
}
// Cancel is shorthand for CancelAt(time.Now()).
func (r *Reservation) Cancel() {
r.CancelAt(time.Now())
return
}
// CancelAt indicates that the reservation holder will not perform the reserved action
// and reverses the effects of this Reservation on the rate limit as much as possible,
// considering that other reservations may have already been made.
func (r *Reservation) CancelAt(now time.Time) {
if !r.ok {
return
}
r.lim.mu.Lock()
defer r.lim.mu.Unlock()
if r.lim.limit == Inf || r.tokens == 0 || r.timeToAct.Before(now) {
return
}
// calculate tokens to restore
// The duration between lim.lastEvent and r.timeToAct tells us how many tokens were reserved
// after r was obtained. These tokens should not be restored.
restoreTokens := float64(r.tokens) - r.limit.tokensFromDuration(r.lim.lastEvent.Sub(r.timeToAct))
if restoreTokens <= 0 {
return
}
// advance time to now
now, _, tokens := r.lim.advance(now)
// calculate new number of tokens
tokens += restoreTokens
if burst := float64(r.lim.burst); tokens > burst {
tokens = burst
}
// update state
r.lim.last = now
r.lim.tokens = tokens
if r.timeToAct == r.lim.lastEvent {
prevEvent := r.timeToAct.Add(r.limit.durationFromTokens(float64(-r.tokens)))
if !prevEvent.Before(now) {
r.lim.lastEvent = prevEvent
}
}
return
}
// Reserve is shorthand for ReserveN(time.Now(), 1).
func (lim *Limiter) Reserve() *Reservation {
return lim.ReserveN(time.Now(), 1)
}
// ReserveN returns a Reservation that indicates how long the caller must wait before n events happen.
// The Limiter takes this Reservation into account when allowing future events.
// The returned Reservation’s OK() method returns false if n exceeds the Limiter's burst size.
// Usage example:
// r := lim.ReserveN(time.Now(), 1)
// if !r.OK() {
// // Not allowed to act! Did you remember to set lim.burst to be > 0 ?
// return
// }
// time.Sleep(r.Delay())
// Act()
// Use this method if you wish to wait and slow down in accordance with the rate limit without dropping events.
// If you need to respect a deadline or cancel the delay, use Wait instead.
// To drop or skip events exceeding rate limit, use Allow instead.
func (lim *Limiter) ReserveN(now time.Time, n int) *Reservation {
r := lim.reserveN(now, n, InfDuration)
return &r
}
// Wait is shorthand for WaitN(ctx, 1).
func (lim *Limiter) Wait(ctx context.Context) (err error) {
return lim.WaitN(ctx, 1)
}
// WaitN blocks until lim permits n events to happen.
// It returns an error if n exceeds the Limiter's burst size, the Context is
// canceled, or the expected wait time exceeds the Context's Deadline.
// The burst limit is ignored if the rate limit is Inf.
func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) {
lim.mu.Lock()
burst := lim.burst
limit := lim.limit
lim.mu.Unlock()
if n > burst && limit != Inf {
return fmt.Errorf("rate: Wait(n=%d) exceeds limiter's burst %d", n, lim.burst)
}
// Check if ctx is already cancelled
select {
case <-ctx.Done():
return ctx.Err()
default:
}
// Determine wait limit
now := time.Now()
waitLimit := InfDuration
if deadline, ok := ctx.Deadline(); ok {
waitLimit = deadline.Sub(now)
}
// Reserve
r := lim.reserveN(now, n, waitLimit)
if !r.ok {
return fmt.Errorf("rate: Wait(n=%d) would exceed context deadline", n)
}
// Wait if necessary
delay := r.DelayFrom(now)
if delay == 0 {
return nil
}
t := time.NewTimer(delay)
defer t.Stop()
select {
case <-t.C:
// We can proceed.
return nil
case <-ctx.Done():
// Context was canceled before we could proceed. Cancel the
// reservation, which may permit other events to proceed sooner.
r.Cancel()
return ctx.Err()
}
}
// SetLimit is shorthand for SetLimitAt(time.Now(), newLimit).
func (lim *Limiter) SetLimit(newLimit Limit) {
lim.SetLimitAt(time.Now(), newLimit)
}
// SetLimitAt sets a new Limit for the limiter. The new Limit, and Burst, may be violated
// or underutilized by those which reserved (using Reserve or Wait) but did not yet act
// before SetLimitAt was called.
func (lim *Limiter) SetLimitAt(now time.Time, newLimit Limit) {
lim.mu.Lock()
defer lim.mu.Unlock()
now, _, tokens := lim.advance(now)
lim.last = now
lim.tokens = tokens
lim.limit = newLimit
}
// SetBurst is shorthand for SetBurstAt(time.Now(), newBurst).
func (lim *Limiter) SetBurst(newBurst int) {
lim.SetBurstAt(time.Now(), newBurst)
}
// SetBurstAt sets a new burst size for the limiter.
func (lim *Limiter) SetBurstAt(now time.Time, newBurst int) {
lim.mu.Lock()
defer lim.mu.Unlock()
now, _, tokens := lim.advance(now)
lim.last = now
lim.tokens = tokens
lim.burst = newBurst
}
// reserveN is a helper method for AllowN, ReserveN, and WaitN.
// maxFutureReserve specifies the maximum reservation wait duration allowed.
// reserveN returns Reservation, not *Reservation, to avoid allocation in AllowN and WaitN.
func (lim *Limiter) reserveN(now time.Time, n int, maxFutureReserve time.Duration) Reservation {
lim.mu.Lock()
if lim.limit == Inf {
lim.mu.Unlock()
return Reservation{
ok: true,
lim: lim,
tokens: n,
timeToAct: now,
}
}
now, last, tokens := lim.advance(now)
// Calculate the remaining number of tokens resulting from the request.
tokens -= float64(n)
// Calculate the wait duration
var waitDuration time.Duration
if tokens < 0 {
waitDuration = lim.limit.durationFromTokens(-tokens)
}
// Decide result
ok := n <= lim.burst && waitDuration <= maxFutureReserve
// Prepare reservation
r := Reservation{
ok: ok,
lim: lim,
limit: lim.limit,
}
if ok {
r.tokens = n
r.timeToAct = now.Add(waitDuration)
}
// Update state
if ok {
lim.last = now
lim.tokens = tokens
lim.lastEvent = r.timeToAct
} else {
lim.last = last
}
lim.mu.Unlock()
return r
}
// advance calculates and returns an updated state for lim resulting from the passage of time.
// lim is not changed.
func (lim *Limiter) advance(now time.Time) (newNow time.Time, newLast time.Time, newTokens float64) {
last := lim.last
if now.Before(last) {
last = now
}
// Avoid making delta overflow below when last is very old.
maxElapsed := lim.limit.durationFromTokens(float64(lim.burst) - lim.tokens)
elapsed := now.Sub(last)
if elapsed > maxElapsed {
elapsed = maxElapsed
}
// Calculate the new number of tokens, due to time that passed.
delta := lim.limit.tokensFromDuration(elapsed)
tokens := lim.tokens + delta
if burst := float64(lim.burst); tokens > burst {
tokens = burst
}
return now, last, tokens
}
// durationFromTokens is a unit conversion function from the number of tokens to the duration
// of time it takes to accumulate them at a rate of limit tokens per second.
func (limit Limit) durationFromTokens(tokens float64) time.Duration {
seconds := tokens / float64(limit)
return time.Nanosecond * time.Duration(1e9*seconds)
}
// tokensFromDuration is a unit conversion function from a time duration to the number of tokens
// which could be accumulated during that duration at a rate of limit tokens per second.
func (limit Limit) tokensFromDuration(d time.Duration) float64 {
// Split the integer and fractional parts ourself to minimize rounding errors.
// See golang.org/issues/34861.
sec := float64(d/time.Second) * float64(limit)
nsec := float64(d%time.Second) * float64(limit)
return sec + nsec/1e9
}

10
vendor/modules.txt vendored
View File

@ -4,8 +4,16 @@ github.com/beorn7/perks/quantile
github.com/cespare/xxhash/v2
# github.com/davecgh/go-spew v1.1.1
github.com/davecgh/go-spew/spew
# github.com/didip/tollbooth/v6 v6.1.0
## explicit
github.com/didip/tollbooth/v6
github.com/didip/tollbooth/v6/errors
github.com/didip/tollbooth/v6/libstring
github.com/didip/tollbooth/v6/limiter
# github.com/felixge/httpsnoop v1.0.1
github.com/felixge/httpsnoop
# github.com/go-pkgz/expirable-cache v0.0.3
github.com/go-pkgz/expirable-cache
# github.com/go-pkgz/lgr v0.10.4
## explicit
github.com/go-pkgz/lgr
@ -69,6 +77,8 @@ golang.org/x/text/secure/bidirule
golang.org/x/text/transform
golang.org/x/text/unicode/bidi
golang.org/x/text/unicode/norm
# golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1
golang.org/x/time/rate
# google.golang.org/protobuf v1.23.0
google.golang.org/protobuf/encoding/prototext
google.golang.org/protobuf/encoding/protowire