diff --git a/.gitignore b/.gitignore index 9008131..42ce9bf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ *.test *.out -.runconf +access.log var/ -dist/ \ No newline at end of file +dist/ diff --git a/app/discovery/discovery.go b/app/discovery/discovery.go index 039f9fb..22423e0 100644 --- a/app/discovery/discovery.go +++ b/app/discovery/discovery.go @@ -6,9 +6,10 @@ package discovery import ( "context" - "log" "regexp" "sync" + + log "github.com/go-pkgz/lgr" ) //go:generate moq -out provider_mock.go -fmt goimports . Provider diff --git a/app/main.go b/app/main.go index 30286fd..6a83370 100644 --- a/app/main.go +++ b/app/main.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "io" "os" "os/signal" "runtime" @@ -11,6 +12,7 @@ import ( docker "github.com/fsouza/go-dockerclient" log "github.com/go-pkgz/lgr" + "github.com/natefinch/lumberjack" "github.com/pkg/errors" "github.com/umputun/go-flags" @@ -40,6 +42,13 @@ var opts struct { WebRoot string `long:"root" env:"ROOT" default:"/" description:"assets web root"` } `group:"assets" namespace:"assets" env-namespace:"ASSETS"` + Logger struct { + Enabled bool `long:"enabled" env:"ENABLED" description:"enable access and error rotated logs"` + FileName string `long:"file" env:"FILE" default:"access.log" description:"location of access log"` + MaxSize int `long:"max-size" env:"MAX_SIZE" default:"100" description:"maximum size in megabytes before it gets rotated"` + MaxBackups int `long:"max-backups" env:"MAX_BACKUPS" default:"10" description:"maximum number of old log files to retain"` + } `group:"logger" namespace:"logger" env-namespace:"LOGGER"` + Docker struct { Enabled bool `long:"enabled" env:"ENABLED" description:"enable docker provider"` Host string `long:"host" env:"HOST" default:"unix:///var/run/docker.sock" description:"docker host"` @@ -65,7 +74,7 @@ var opts struct { var revision = "unknown" func main() { - fmt.Printf("docker-proxy (dpx) %s\n", revision) + fmt.Printf("reproxy %s\n", revision) p := flags.NewParser(&opts, flags.PrintErrors|flags.PassDoubleDash|flags.HelpFlag) p.SubcommandsOptional = true @@ -102,6 +111,14 @@ func main() { log.Fatalf("[ERROR] failed to make config of ssl server params, %v", err) } + accessLog := makeLogWriter() + defer func() { + if err := accessLog.Close(); err != nil { + log.Printf("[WARN] can't close access log, %v", err) + } + + }() + px := &proxy.Http{ Version: revision, Matcher: svc, @@ -113,6 +130,7 @@ func main() { GzEnabled: opts.GzipEnabled, SSLConfig: sslConfig, ProxyHeaders: opts.ProxyHeaders, + AccessLog: accessLog, } if err := px.Run(context.Background()); err != nil { log.Fatalf("[ERROR] proxy server failed, %v", err) @@ -172,6 +190,24 @@ func makeSSLConfig() (config proxy.SSLConfig, err error) { return config, err } +func makeLogWriter() (accessLog io.WriteCloser) { + if !opts.Logger.Enabled { + return nopWriteCloser{io.Discard} + } + log.Printf("[INFO] logger enabled for %s", opts.Logger.FileName) + return &lumberjack.Logger{ + Filename: opts.Logger.FileName, + MaxSize: opts.Logger.MaxSize, + MaxBackups: opts.Logger.MaxSize, + Compress: true, + LocalTime: true, + } +} + +type nopWriteCloser struct{ io.Writer } + +func (n nopWriteCloser) Close() error { return nil } + func setupLog(dbg bool) { if dbg { log.Setup(log.Debug, log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces) diff --git a/app/proxy/health.go b/app/proxy/health.go new file mode 100644 index 0000000..e9f6450 --- /dev/null +++ b/app/proxy/health.go @@ -0,0 +1,83 @@ +package proxy + +import ( + "fmt" + "net/http" + "strings" + "sync" + "sync/atomic" + "time" + + log "github.com/go-pkgz/lgr" + "github.com/umputun/reproxy/app/discovery" +) + +func (h *Http) healthMiddleware(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" && strings.HasSuffix(strings.ToLower(r.URL.Path), "/health") { + h.healthHandler(w, r) + return + } + next.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) +} + +func (h *Http) healthHandler(w http.ResponseWriter, r *http.Request) { + + // runs pings in parallel + check := func(mappers []discovery.UrlMapper) (ok bool, valid int, total int, errs []string) { + outCh := make(chan error, 8) + var pinged int32 + var wg sync.WaitGroup + for _, m := range mappers { + if m.PingURL == "" { + continue + } + wg.Add(1) + go func(m discovery.UrlMapper) { + defer wg.Done() + + atomic.AddInt32(&pinged, 1) + client := http.Client{Timeout: 100 * time.Millisecond} + resp, err := client.Get(m.PingURL) + if err != nil { + log.Printf("[WARN] failed to ping for health %s, %v", m.PingURL, err) + outCh <- fmt.Errorf("%s, %v", m.PingURL, err) + return + } + if resp.StatusCode != http.StatusOK { + log.Printf("[WARN] failed ping status for health %s (%s)", m.PingURL, resp.Status) + outCh <- fmt.Errorf("%s, %s", m.PingURL, resp.Status) + return + } + }(m) + } + + go func() { + wg.Wait() + close(outCh) + }() + + for e := range outCh { + errs = append(errs, e.Error()) + } + return len(errs) == 0, int(atomic.LoadInt32(&pinged)) - len(errs), len(mappers), errs + } + + w.Header().Set("Content-Type", "application/json; charset=UTF-8") + ok, valid, total, errs := check(h.Mappers()) + if !ok { + w.WriteHeader(http.StatusExpectationFailed) + _, err := fmt.Fprintf(w, `{"status": "failed", "passed": %d, "failed":%d, "errors": "%+v"}`, valid, total-valid, errs) + if err != nil { + log.Printf("[WARN] failed %v", err) + } + return + } + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprintf(w, `{"status": "ok", "services": %d}`, valid) + if err != nil { + log.Printf("[WARN] failed to send halth, %v", err) + } +} diff --git a/app/proxy/proxy.go b/app/proxy/proxy.go index 111db4e..3048aae 100644 --- a/app/proxy/proxy.go +++ b/app/proxy/proxy.go @@ -2,7 +2,7 @@ package proxy import ( "context" - "fmt" + "io" "net" "net/http" "net/http/httputil" @@ -10,8 +10,6 @@ import ( "regexp" "strconv" "strings" - "sync" - "sync/atomic" "time" "github.com/go-pkgz/lgr" @@ -19,7 +17,9 @@ import ( "github.com/go-pkgz/rest" R "github.com/go-pkgz/rest" "github.com/go-pkgz/rest/logger" + "github.com/gorilla/handlers" "github.com/pkg/errors" + "github.com/umputun/reproxy/app/discovery" ) @@ -35,6 +35,7 @@ type Http struct { ProxyHeaders []string SSLConfig SSLConfig Version string + AccessLog io.Writer } // Matcher source info (server and route) to the destination url @@ -74,6 +75,7 @@ func (h *Http) Run(ctx context.Context) error { R.Ping, h.healthMiddleware, logger.New(logger.Prefix("[DEBUG] PROXY")).Handler, + h.accessLogHandler(h.AccessLog), R.SizeLimit(h.MaxBodySize), R.Headers(h.ProxyHeaders...), h.gzipHandler(), @@ -122,23 +124,6 @@ func (h *Http) Run(ctx context.Context) error { return errors.Errorf("unknown SSL type %v", h.SSLConfig.SSLMode) } -func (h *Http) toHttp(address string, httpPort int) string { - rx := regexp.MustCompile(`(.*):(\d*)`) - return rx.ReplaceAllString(address, "$1:") + strconv.Itoa(httpPort) -} - -func (h *Http) gzipHandler() func(next http.Handler) http.Handler { - if h.GzEnabled { - return R.Gzip() - } - - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - next.ServeHTTP(w, r) - }) - } -} - func (h *Http) proxyHandler() http.HandlerFunc { type contextKey string @@ -197,6 +182,29 @@ func (h *Http) proxyHandler() http.HandlerFunc { } } +func (h *Http) toHttp(address string, httpPort int) string { + rx := regexp.MustCompile(`(.*):(\d*)`) + return rx.ReplaceAllString(address, "$1:") + strconv.Itoa(httpPort) +} + +func (h *Http) gzipHandler() func(next http.Handler) http.Handler { + if h.GzEnabled { + return R.Gzip() + } + + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r) + }) + } +} + +func (h *Http) accessLogHandler(wr io.Writer) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return handlers.CombinedLoggingHandler(wr, next) + } +} + func (h *Http) makeHTTPServer(addr string, router http.Handler) *http.Server { return &http.Server{ Addr: addr, @@ -220,73 +228,3 @@ func (h *Http) setXRealIP(r *http.Request) { } r.Header.Add("X-Real-IP", ip) } - -func (h *Http) healthMiddleware(next http.Handler) http.Handler { - fn := func(w http.ResponseWriter, r *http.Request) { - if r.Method == "GET" && strings.HasSuffix(strings.ToLower(r.URL.Path), "/health") { - h.healthHandler(w, r) - return - } - next.ServeHTTP(w, r) - } - return http.HandlerFunc(fn) -} - -func (h *Http) healthHandler(w http.ResponseWriter, r *http.Request) { - - // runs pings in parallel - check := func(mappers []discovery.UrlMapper) (ok bool, valid int, total int, errs []string) { - outCh := make(chan error, 8) - var pinged int32 - var wg sync.WaitGroup - for _, m := range mappers { - if m.PingURL == "" { - continue - } - wg.Add(1) - go func(m discovery.UrlMapper) { - defer wg.Done() - - atomic.AddInt32(&pinged, 1) - client := http.Client{Timeout: 100 * time.Millisecond} - resp, err := client.Get(m.PingURL) - if err != nil { - log.Printf("[WARN] failed to ping for health %s, %v", m.PingURL, err) - outCh <- fmt.Errorf("%s, %v", m.PingURL, err) - return - } - if resp.StatusCode != http.StatusOK { - log.Printf("[WARN] failed ping status for health %s (%s)", m.PingURL, resp.Status) - outCh <- fmt.Errorf("%s, %s", m.PingURL, resp.Status) - return - } - }(m) - } - - go func() { - wg.Wait() - close(outCh) - }() - - for e := range outCh { - errs = append(errs, e.Error()) - } - return len(errs) == 0, int(atomic.LoadInt32(&pinged)) - len(errs), len(mappers), errs - } - - w.Header().Set("Content-Type", "application/json; charset=UTF-8") - ok, valid, total, errs := check(h.Mappers()) - if !ok { - w.WriteHeader(http.StatusExpectationFailed) - _, err := fmt.Fprintf(w, `{"status": "failed", "passed": %d, "failed":%d, "errors": "%+v"}`, valid, total-valid, errs) - if err != nil { - log.Printf("[WARN] failed %v", err) - } - return - } - w.WriteHeader(http.StatusOK) - _, err := fmt.Fprintf(w, `{"status": "ok", "services": %d}`, valid) - if err != nil { - log.Printf("[WARN] failed to send halth, %v", err) - } -} diff --git a/go.mod b/go.mod index a374420..519d3ae 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,8 @@ require ( github.com/fsouza/go-dockerclient v1.7.2 github.com/go-pkgz/lgr v0.10.4 github.com/go-pkgz/rest v1.8.1 + github.com/gorilla/handlers v1.5.1 + github.com/natefinch/lumberjack v2.0.0+incompatible github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.7.0 github.com/umputun/go-flags v1.5.1 diff --git a/go.sum b/go.sum index a9c95d4..43a64d0 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5Xh github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo= github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8= github.com/go-pkgz/lgr v0.10.4 h1:l7qyFjqEZgwRgaQQSEp6tve4A3OU80VrfzpvtEX8ngw= @@ -61,6 +63,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= +github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -83,6 +87,8 @@ github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0 github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= +github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= diff --git a/vendor/github.com/natefinch/lumberjack/.gitignore b/vendor/github.com/natefinch/lumberjack/.gitignore new file mode 100644 index 0000000..8365624 --- /dev/null +++ b/vendor/github.com/natefinch/lumberjack/.gitignore @@ -0,0 +1,23 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test diff --git a/vendor/github.com/natefinch/lumberjack/.travis.yml b/vendor/github.com/natefinch/lumberjack/.travis.yml new file mode 100644 index 0000000..65dcbc5 --- /dev/null +++ b/vendor/github.com/natefinch/lumberjack/.travis.yml @@ -0,0 +1,6 @@ +language: go + +go: + - 1.8 + - 1.7 + - 1.6 \ No newline at end of file diff --git a/vendor/github.com/natefinch/lumberjack/LICENSE b/vendor/github.com/natefinch/lumberjack/LICENSE new file mode 100644 index 0000000..c3d4cc3 --- /dev/null +++ b/vendor/github.com/natefinch/lumberjack/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Nate Finch + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/natefinch/lumberjack/README.md b/vendor/github.com/natefinch/lumberjack/README.md new file mode 100644 index 0000000..060eae5 --- /dev/null +++ b/vendor/github.com/natefinch/lumberjack/README.md @@ -0,0 +1,179 @@ +# lumberjack [![GoDoc](https://godoc.org/gopkg.in/natefinch/lumberjack.v2?status.png)](https://godoc.org/gopkg.in/natefinch/lumberjack.v2) [![Build Status](https://travis-ci.org/natefinch/lumberjack.svg?branch=v2.0)](https://travis-ci.org/natefinch/lumberjack) [![Build status](https://ci.appveyor.com/api/projects/status/00gchpxtg4gkrt5d)](https://ci.appveyor.com/project/natefinch/lumberjack) [![Coverage Status](https://coveralls.io/repos/natefinch/lumberjack/badge.svg?branch=v2.0)](https://coveralls.io/r/natefinch/lumberjack?branch=v2.0) + +### Lumberjack is a Go package for writing logs to rolling files. + +Package lumberjack provides a rolling logger. + +Note that this is v2.0 of lumberjack, and should be imported using gopkg.in +thusly: + + import "gopkg.in/natefinch/lumberjack.v2" + +The package name remains simply lumberjack, and the code resides at +https://github.com/natefinch/lumberjack under the v2.0 branch. + +Lumberjack is intended to be one part of a logging infrastructure. +It is not an all-in-one solution, but instead is a pluggable +component at the bottom of the logging stack that simply controls the files +to which logs are written. + +Lumberjack plays well with any logging package that can write to an +io.Writer, including the standard library's log package. + +Lumberjack assumes that only one process is writing to the output files. +Using the same lumberjack configuration from multiple processes on the same +machine will result in improper behavior. + + +**Example** + +To use lumberjack with the standard library's log package, just pass it into the SetOutput function when your application starts. + +Code: + +```go +log.SetOutput(&lumberjack.Logger{ + Filename: "/var/log/myapp/foo.log", + MaxSize: 500, // megabytes + MaxBackups: 3, + MaxAge: 28, //days + Compress: true, // disabled by default +}) +``` + + + +## type Logger +``` go +type Logger struct { + // Filename is the file to write logs to. Backup log files will be retained + // in the same directory. It uses -lumberjack.log in + // os.TempDir() if empty. + Filename string `json:"filename" yaml:"filename"` + + // MaxSize is the maximum size in megabytes of the log file before it gets + // rotated. It defaults to 100 megabytes. + MaxSize int `json:"maxsize" yaml:"maxsize"` + + // MaxAge is the maximum number of days to retain old log files based on the + // timestamp encoded in their filename. Note that a day is defined as 24 + // hours and may not exactly correspond to calendar days due to daylight + // savings, leap seconds, etc. The default is not to remove old log files + // based on age. + MaxAge int `json:"maxage" yaml:"maxage"` + + // MaxBackups is the maximum number of old log files to retain. The default + // is to retain all old log files (though MaxAge may still cause them to get + // deleted.) + MaxBackups int `json:"maxbackups" yaml:"maxbackups"` + + // LocalTime determines if the time used for formatting the timestamps in + // backup files is the computer's local time. The default is to use UTC + // time. + LocalTime bool `json:"localtime" yaml:"localtime"` + + // Compress determines if the rotated log files should be compressed + // using gzip. The default is not to perform compression. + Compress bool `json:"compress" yaml:"compress"` + // contains filtered or unexported fields +} +``` +Logger is an io.WriteCloser that writes to the specified filename. + +Logger opens or creates the logfile on first Write. If the file exists and +is less than MaxSize megabytes, lumberjack will open and append to that file. +If the file exists and its size is >= MaxSize megabytes, the file is renamed +by putting the current time in a timestamp in the name immediately before the +file's extension (or the end of the filename if there's no extension). A new +log file is then created using original filename. + +Whenever a write would cause the current log file exceed MaxSize megabytes, +the current file is closed, renamed, and a new log file created with the +original name. Thus, the filename you give Logger is always the "current" log +file. + +Backups use the log file name given to Logger, in the form `name-timestamp.ext` +where name is the filename without the extension, timestamp is the time at which +the log was rotated formatted with the time.Time format of +`2006-01-02T15-04-05.000` and the extension is the original extension. For +example, if your Logger.Filename is `/var/log/foo/server.log`, a backup created +at 6:30pm on Nov 11 2016 would use the filename +`/var/log/foo/server-2016-11-04T18-30-00.000.log` + +### Cleaning Up Old Log Files +Whenever a new logfile gets created, old log files may be deleted. The most +recent files according to the encoded timestamp will be retained, up to a +number equal to MaxBackups (or all of them if MaxBackups is 0). Any files +with an encoded timestamp older than MaxAge days are deleted, regardless of +MaxBackups. Note that the time encoded in the timestamp is the rotation +time, which may differ from the last time that file was written to. + +If MaxBackups and MaxAge are both 0, no old log files will be deleted. + + + + + + + + + + + +### func (\*Logger) Close +``` go +func (l *Logger) Close() error +``` +Close implements io.Closer, and closes the current logfile. + + + +### func (\*Logger) Rotate +``` go +func (l *Logger) Rotate() error +``` +Rotate causes Logger to close the existing log file and immediately create a +new one. This is a helper function for applications that want to initiate +rotations outside of the normal rotation rules, such as in response to +SIGHUP. After rotating, this initiates a cleanup of old log files according +to the normal rules. + +**Example** + +Example of how to rotate in response to SIGHUP. + +Code: + +```go +l := &lumberjack.Logger{} +log.SetOutput(l) +c := make(chan os.Signal, 1) +signal.Notify(c, syscall.SIGHUP) + +go func() { + for { + <-c + l.Rotate() + } +}() +``` + +### func (\*Logger) Write +``` go +func (l *Logger) Write(p []byte) (n int, err error) +``` +Write implements io.Writer. If a write would cause the log file to be larger +than MaxSize, the file is closed, renamed to include a timestamp of the +current time, and a new log file is created using the original log file name. +If the length of the write is greater than MaxSize, an error is returned. + + + + + + + + + +- - - +Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) diff --git a/vendor/github.com/natefinch/lumberjack/chown.go b/vendor/github.com/natefinch/lumberjack/chown.go new file mode 100644 index 0000000..11d0669 --- /dev/null +++ b/vendor/github.com/natefinch/lumberjack/chown.go @@ -0,0 +1,11 @@ +// +build !linux + +package lumberjack + +import ( + "os" +) + +func chown(_ string, _ os.FileInfo) error { + return nil +} diff --git a/vendor/github.com/natefinch/lumberjack/chown_linux.go b/vendor/github.com/natefinch/lumberjack/chown_linux.go new file mode 100644 index 0000000..2758ec9 --- /dev/null +++ b/vendor/github.com/natefinch/lumberjack/chown_linux.go @@ -0,0 +1,19 @@ +package lumberjack + +import ( + "os" + "syscall" +) + +// os_Chown is a var so we can mock it out during tests. +var os_Chown = os.Chown + +func chown(name string, info os.FileInfo) error { + f, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode()) + if err != nil { + return err + } + f.Close() + stat := info.Sys().(*syscall.Stat_t) + return os_Chown(name, int(stat.Uid), int(stat.Gid)) +} diff --git a/vendor/github.com/natefinch/lumberjack/lumberjack.go b/vendor/github.com/natefinch/lumberjack/lumberjack.go new file mode 100644 index 0000000..46d97c5 --- /dev/null +++ b/vendor/github.com/natefinch/lumberjack/lumberjack.go @@ -0,0 +1,541 @@ +// Package lumberjack provides a rolling logger. +// +// Note that this is v2.0 of lumberjack, and should be imported using gopkg.in +// thusly: +// +// import "gopkg.in/natefinch/lumberjack.v2" +// +// The package name remains simply lumberjack, and the code resides at +// https://github.com/natefinch/lumberjack under the v2.0 branch. +// +// Lumberjack is intended to be one part of a logging infrastructure. +// It is not an all-in-one solution, but instead is a pluggable +// component at the bottom of the logging stack that simply controls the files +// to which logs are written. +// +// Lumberjack plays well with any logging package that can write to an +// io.Writer, including the standard library's log package. +// +// Lumberjack assumes that only one process is writing to the output files. +// Using the same lumberjack configuration from multiple processes on the same +// machine will result in improper behavior. +package lumberjack + +import ( + "compress/gzip" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" +) + +const ( + backupTimeFormat = "2006-01-02T15-04-05.000" + compressSuffix = ".gz" + defaultMaxSize = 100 +) + +// ensure we always implement io.WriteCloser +var _ io.WriteCloser = (*Logger)(nil) + +// Logger is an io.WriteCloser that writes to the specified filename. +// +// Logger opens or creates the logfile on first Write. If the file exists and +// is less than MaxSize megabytes, lumberjack will open and append to that file. +// If the file exists and its size is >= MaxSize megabytes, the file is renamed +// by putting the current time in a timestamp in the name immediately before the +// file's extension (or the end of the filename if there's no extension). A new +// log file is then created using original filename. +// +// Whenever a write would cause the current log file exceed MaxSize megabytes, +// the current file is closed, renamed, and a new log file created with the +// original name. Thus, the filename you give Logger is always the "current" log +// file. +// +// Backups use the log file name given to Logger, in the form +// `name-timestamp.ext` where name is the filename without the extension, +// timestamp is the time at which the log was rotated formatted with the +// time.Time format of `2006-01-02T15-04-05.000` and the extension is the +// original extension. For example, if your Logger.Filename is +// `/var/log/foo/server.log`, a backup created at 6:30pm on Nov 11 2016 would +// use the filename `/var/log/foo/server-2016-11-04T18-30-00.000.log` +// +// Cleaning Up Old Log Files +// +// Whenever a new logfile gets created, old log files may be deleted. The most +// recent files according to the encoded timestamp will be retained, up to a +// number equal to MaxBackups (or all of them if MaxBackups is 0). Any files +// with an encoded timestamp older than MaxAge days are deleted, regardless of +// MaxBackups. Note that the time encoded in the timestamp is the rotation +// time, which may differ from the last time that file was written to. +// +// If MaxBackups and MaxAge are both 0, no old log files will be deleted. +type Logger struct { + // Filename is the file to write logs to. Backup log files will be retained + // in the same directory. It uses -lumberjack.log in + // os.TempDir() if empty. + Filename string `json:"filename" yaml:"filename"` + + // MaxSize is the maximum size in megabytes of the log file before it gets + // rotated. It defaults to 100 megabytes. + MaxSize int `json:"maxsize" yaml:"maxsize"` + + // MaxAge is the maximum number of days to retain old log files based on the + // timestamp encoded in their filename. Note that a day is defined as 24 + // hours and may not exactly correspond to calendar days due to daylight + // savings, leap seconds, etc. The default is not to remove old log files + // based on age. + MaxAge int `json:"maxage" yaml:"maxage"` + + // MaxBackups is the maximum number of old log files to retain. The default + // is to retain all old log files (though MaxAge may still cause them to get + // deleted.) + MaxBackups int `json:"maxbackups" yaml:"maxbackups"` + + // LocalTime determines if the time used for formatting the timestamps in + // backup files is the computer's local time. The default is to use UTC + // time. + LocalTime bool `json:"localtime" yaml:"localtime"` + + // Compress determines if the rotated log files should be compressed + // using gzip. The default is not to perform compression. + Compress bool `json:"compress" yaml:"compress"` + + size int64 + file *os.File + mu sync.Mutex + + millCh chan bool + startMill sync.Once +} + +var ( + // currentTime exists so it can be mocked out by tests. + currentTime = time.Now + + // os_Stat exists so it can be mocked out by tests. + os_Stat = os.Stat + + // megabyte is the conversion factor between MaxSize and bytes. It is a + // variable so tests can mock it out and not need to write megabytes of data + // to disk. + megabyte = 1024 * 1024 +) + +// Write implements io.Writer. If a write would cause the log file to be larger +// than MaxSize, the file is closed, renamed to include a timestamp of the +// current time, and a new log file is created using the original log file name. +// If the length of the write is greater than MaxSize, an error is returned. +func (l *Logger) Write(p []byte) (n int, err error) { + l.mu.Lock() + defer l.mu.Unlock() + + writeLen := int64(len(p)) + if writeLen > l.max() { + return 0, fmt.Errorf( + "write length %d exceeds maximum file size %d", writeLen, l.max(), + ) + } + + if l.file == nil { + if err = l.openExistingOrNew(len(p)); err != nil { + return 0, err + } + } + + if l.size+writeLen > l.max() { + if err := l.rotate(); err != nil { + return 0, err + } + } + + n, err = l.file.Write(p) + l.size += int64(n) + + return n, err +} + +// Close implements io.Closer, and closes the current logfile. +func (l *Logger) Close() error { + l.mu.Lock() + defer l.mu.Unlock() + return l.close() +} + +// close closes the file if it is open. +func (l *Logger) close() error { + if l.file == nil { + return nil + } + err := l.file.Close() + l.file = nil + return err +} + +// Rotate causes Logger to close the existing log file and immediately create a +// new one. This is a helper function for applications that want to initiate +// rotations outside of the normal rotation rules, such as in response to +// SIGHUP. After rotating, this initiates compression and removal of old log +// files according to the configuration. +func (l *Logger) Rotate() error { + l.mu.Lock() + defer l.mu.Unlock() + return l.rotate() +} + +// rotate closes the current file, moves it aside with a timestamp in the name, +// (if it exists), opens a new file with the original filename, and then runs +// post-rotation processing and removal. +func (l *Logger) rotate() error { + if err := l.close(); err != nil { + return err + } + if err := l.openNew(); err != nil { + return err + } + l.mill() + return nil +} + +// openNew opens a new log file for writing, moving any old log file out of the +// way. This methods assumes the file has already been closed. +func (l *Logger) openNew() error { + err := os.MkdirAll(l.dir(), 0744) + if err != nil { + return fmt.Errorf("can't make directories for new logfile: %s", err) + } + + name := l.filename() + mode := os.FileMode(0644) + info, err := os_Stat(name) + if err == nil { + // Copy the mode off the old logfile. + mode = info.Mode() + // move the existing file + newname := backupName(name, l.LocalTime) + if err := os.Rename(name, newname); err != nil { + return fmt.Errorf("can't rename log file: %s", err) + } + + // this is a no-op anywhere but linux + if err := chown(name, info); err != nil { + return err + } + } + + // we use truncate here because this should only get called when we've moved + // the file ourselves. if someone else creates the file in the meantime, + // just wipe out the contents. + f, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) + if err != nil { + return fmt.Errorf("can't open new logfile: %s", err) + } + l.file = f + l.size = 0 + return nil +} + +// backupName creates a new filename from the given name, inserting a timestamp +// between the filename and the extension, using the local time if requested +// (otherwise UTC). +func backupName(name string, local bool) string { + dir := filepath.Dir(name) + filename := filepath.Base(name) + ext := filepath.Ext(filename) + prefix := filename[:len(filename)-len(ext)] + t := currentTime() + if !local { + t = t.UTC() + } + + timestamp := t.Format(backupTimeFormat) + return filepath.Join(dir, fmt.Sprintf("%s-%s%s", prefix, timestamp, ext)) +} + +// openExistingOrNew opens the logfile if it exists and if the current write +// would not put it over MaxSize. If there is no such file or the write would +// put it over the MaxSize, a new file is created. +func (l *Logger) openExistingOrNew(writeLen int) error { + l.mill() + + filename := l.filename() + info, err := os_Stat(filename) + if os.IsNotExist(err) { + return l.openNew() + } + if err != nil { + return fmt.Errorf("error getting log file info: %s", err) + } + + if info.Size()+int64(writeLen) >= l.max() { + return l.rotate() + } + + file, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + // if we fail to open the old log file for some reason, just ignore + // it and open a new log file. + return l.openNew() + } + l.file = file + l.size = info.Size() + return nil +} + +// genFilename generates the name of the logfile from the current time. +func (l *Logger) filename() string { + if l.Filename != "" { + return l.Filename + } + name := filepath.Base(os.Args[0]) + "-lumberjack.log" + return filepath.Join(os.TempDir(), name) +} + +// millRunOnce performs compression and removal of stale log files. +// Log files are compressed if enabled via configuration and old log +// files are removed, keeping at most l.MaxBackups files, as long as +// none of them are older than MaxAge. +func (l *Logger) millRunOnce() error { + if l.MaxBackups == 0 && l.MaxAge == 0 && !l.Compress { + return nil + } + + files, err := l.oldLogFiles() + if err != nil { + return err + } + + var compress, remove []logInfo + + if l.MaxBackups > 0 && l.MaxBackups < len(files) { + preserved := make(map[string]bool) + var remaining []logInfo + for _, f := range files { + // Only count the uncompressed log file or the + // compressed log file, not both. + fn := f.Name() + if strings.HasSuffix(fn, compressSuffix) { + fn = fn[:len(fn)-len(compressSuffix)] + } + preserved[fn] = true + + if len(preserved) > l.MaxBackups { + remove = append(remove, f) + } else { + remaining = append(remaining, f) + } + } + files = remaining + } + if l.MaxAge > 0 { + diff := time.Duration(int64(24*time.Hour) * int64(l.MaxAge)) + cutoff := currentTime().Add(-1 * diff) + + var remaining []logInfo + for _, f := range files { + if f.timestamp.Before(cutoff) { + remove = append(remove, f) + } else { + remaining = append(remaining, f) + } + } + files = remaining + } + + if l.Compress { + for _, f := range files { + if !strings.HasSuffix(f.Name(), compressSuffix) { + compress = append(compress, f) + } + } + } + + for _, f := range remove { + errRemove := os.Remove(filepath.Join(l.dir(), f.Name())) + if err == nil && errRemove != nil { + err = errRemove + } + } + for _, f := range compress { + fn := filepath.Join(l.dir(), f.Name()) + errCompress := compressLogFile(fn, fn+compressSuffix) + if err == nil && errCompress != nil { + err = errCompress + } + } + + return err +} + +// millRun runs in a goroutine to manage post-rotation compression and removal +// of old log files. +func (l *Logger) millRun() { + for _ = range l.millCh { + // what am I going to do, log this? + _ = l.millRunOnce() + } +} + +// mill performs post-rotation compression and removal of stale log files, +// starting the mill goroutine if necessary. +func (l *Logger) mill() { + l.startMill.Do(func() { + l.millCh = make(chan bool, 1) + go l.millRun() + }) + select { + case l.millCh <- true: + default: + } +} + +// oldLogFiles returns the list of backup log files stored in the same +// directory as the current log file, sorted by ModTime +func (l *Logger) oldLogFiles() ([]logInfo, error) { + files, err := ioutil.ReadDir(l.dir()) + if err != nil { + return nil, fmt.Errorf("can't read log file directory: %s", err) + } + logFiles := []logInfo{} + + prefix, ext := l.prefixAndExt() + + for _, f := range files { + if f.IsDir() { + continue + } + if t, err := l.timeFromName(f.Name(), prefix, ext); err == nil { + logFiles = append(logFiles, logInfo{t, f}) + continue + } + if t, err := l.timeFromName(f.Name(), prefix, ext+compressSuffix); err == nil { + logFiles = append(logFiles, logInfo{t, f}) + continue + } + // error parsing means that the suffix at the end was not generated + // by lumberjack, and therefore it's not a backup file. + } + + sort.Sort(byFormatTime(logFiles)) + + return logFiles, nil +} + +// timeFromName extracts the formatted time from the filename by stripping off +// the filename's prefix and extension. This prevents someone's filename from +// confusing time.parse. +func (l *Logger) timeFromName(filename, prefix, ext string) (time.Time, error) { + if !strings.HasPrefix(filename, prefix) { + return time.Time{}, errors.New("mismatched prefix") + } + if !strings.HasSuffix(filename, ext) { + return time.Time{}, errors.New("mismatched extension") + } + ts := filename[len(prefix) : len(filename)-len(ext)] + return time.Parse(backupTimeFormat, ts) +} + +// max returns the maximum size in bytes of log files before rolling. +func (l *Logger) max() int64 { + if l.MaxSize == 0 { + return int64(defaultMaxSize * megabyte) + } + return int64(l.MaxSize) * int64(megabyte) +} + +// dir returns the directory for the current filename. +func (l *Logger) dir() string { + return filepath.Dir(l.filename()) +} + +// prefixAndExt returns the filename part and extension part from the Logger's +// filename. +func (l *Logger) prefixAndExt() (prefix, ext string) { + filename := filepath.Base(l.filename()) + ext = filepath.Ext(filename) + prefix = filename[:len(filename)-len(ext)] + "-" + return prefix, ext +} + +// compressLogFile compresses the given log file, removing the +// uncompressed log file if successful. +func compressLogFile(src, dst string) (err error) { + f, err := os.Open(src) + if err != nil { + return fmt.Errorf("failed to open log file: %v", err) + } + defer f.Close() + + fi, err := os_Stat(src) + if err != nil { + return fmt.Errorf("failed to stat log file: %v", err) + } + + if err := chown(dst, fi); err != nil { + return fmt.Errorf("failed to chown compressed log file: %v", err) + } + + // If this file already exists, we presume it was created by + // a previous attempt to compress the log file. + gzf, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, fi.Mode()) + if err != nil { + return fmt.Errorf("failed to open compressed log file: %v", err) + } + defer gzf.Close() + + gz := gzip.NewWriter(gzf) + + defer func() { + if err != nil { + os.Remove(dst) + err = fmt.Errorf("failed to compress log file: %v", err) + } + }() + + if _, err := io.Copy(gz, f); err != nil { + return err + } + if err := gz.Close(); err != nil { + return err + } + if err := gzf.Close(); err != nil { + return err + } + + if err := f.Close(); err != nil { + return err + } + if err := os.Remove(src); err != nil { + return err + } + + return nil +} + +// logInfo is a convenience struct to return the filename and its embedded +// timestamp. +type logInfo struct { + timestamp time.Time + os.FileInfo +} + +// byFormatTime sorts by newest time formatted in the name. +type byFormatTime []logInfo + +func (b byFormatTime) Less(i, j int) bool { + return b[i].timestamp.After(b[j].timestamp) +} + +func (b byFormatTime) Swap(i, j int) { + b[i], b[j] = b[j], b[i] +} + +func (b byFormatTime) Len() int { + return len(b) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 0df04be..ede063b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -62,6 +62,8 @@ github.com/docker/docker/pkg/system github.com/docker/go-connections/nat # github.com/docker/go-units v0.4.0 github.com/docker/go-units +# github.com/felixge/httpsnoop v1.0.1 +github.com/felixge/httpsnoop # github.com/fsouza/go-dockerclient v1.7.2 ## explicit github.com/fsouza/go-dockerclient @@ -76,6 +78,9 @@ github.com/go-pkgz/rest/logger github.com/gogo/protobuf/gogoproto github.com/gogo/protobuf/proto github.com/gogo/protobuf/protoc-gen-gogo/descriptor +# github.com/gorilla/handlers v1.5.1 +## explicit +github.com/gorilla/handlers # github.com/hashicorp/golang-lru v0.5.1 github.com/hashicorp/golang-lru/simplelru # github.com/moby/sys/mount v0.2.0 @@ -87,6 +92,9 @@ github.com/moby/term github.com/moby/term/windows # github.com/morikuni/aec v1.0.0 github.com/morikuni/aec +# github.com/natefinch/lumberjack v2.0.0+incompatible +## explicit +github.com/natefinch/lumberjack # github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/go-digest # github.com/opencontainers/image-spec v1.0.1