From c590c3246d96c63c5db2c9af63dd7890fe5d5736 Mon Sep 17 00:00:00 2001 From: Umputun Date: Fri, 23 Apr 2021 02:02:36 -0500 Subject: [PATCH] Assets cache (#54) * add caching control for assets * regen site --- README.md | 7 ++++-- app/main.go | 31 ++++++++++++++------------ app/proxy/proxy.go | 43 ++++++++++++++++++++++-------------- app/proxy/proxy_test.go | 49 +++++++++++++++++++++++++++++++++++++++++ site/public/index.html | 5 +++-- 5 files changed, 100 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 6b5a9ce..eb00260 100644 --- a/README.md +++ b/README.md @@ -120,14 +120,16 @@ User can also turn stdout log on with `--logger.stdout`. It won't affect the fil ## Assets Server -User may turn assets server on (off by default) to serve static files. As long as `--assets.location` set it will treat every non-proxied request under `assets.root` as a request for static files. Assets server can be used without any proxy providers. In this mode reproxy acts as a simple web server for a static context. +Users may turn the assets server on (off by default) to serve static files. As long as `--assets.location` set it treats every non-proxied request under `assets.root` as a request for static files. The assets server can be used without any proxy providers; in this mode, reproxy acts as a simple web server for the static content. -In addition to the common assets server multiple custom static servers supported. Each provider has a different way to define such static rule and some providers may not support it at all. For example, multiple static server make sense in case of static (command line provide), file provider and can be even useful with docker provider. +In addition to the common assets server, multiple custom static servers are supported. Each provider has a different way to define such a static rule, and some providers may not support it at all. For example, multiple static servers make sense in static (command line provider), file provider, and even useful with docker providers. 1. static provider - if source element prefixed by `assets:` it will be treated as file-server. For example `*,assets:/web,/var/www,` will serve all `/web/*` request with a file server on top of `/var/www` directory. 2. file provider - setting optional field `assets: true` 3. docker provider - `reproxy.assets=web-root:location`, i.e. `reproxy.assets=/web:/var/www`. +Assets server supports caching control with the `--assets.cache=` parameter. `0s` duration (default) turns caching control off. + ## More options - `--gzip` enables gzip compression for responses. @@ -173,6 +175,7 @@ ssl: assets: -a, --assets.location= assets location [$ASSETS_LOCATION] --assets.root= assets web root (default: /) [$ASSETS_ROOT] + --assets.cache= cache duration for assets (default: 0s) [$ASSETS_CACHE] logger: --logger.stdout enable stdout logging [$LOGGER_STDOUT] diff --git a/app/main.go b/app/main.go index 4f61773..c5c7e56 100644 --- a/app/main.go +++ b/app/main.go @@ -40,8 +40,9 @@ var opts struct { } `group:"ssl" namespace:"ssl" env-namespace:"SSL"` Assets struct { - Location string `short:"a" long:"location" env:"LOCATION" default:"" description:"assets location"` - WebRoot string `long:"root" env:"ROOT" default:"/" description:"assets web root"` + Location string `short:"a" long:"location" env:"LOCATION" default:"" description:"assets location"` + WebRoot string `long:"root" env:"ROOT" default:"/" description:"assets web root"` + CacheDuration time.Duration `long:"cache" env:"CACHE" default:"0s" description:"cache duration for assets"` } `group:"assets" namespace:"assets" env-namespace:"ASSETS"` Logger struct { @@ -109,6 +110,7 @@ func main() { setupLog(opts.Dbg) + log.Printf("[DEBUG] options: %+v", opts) ctx, cancel := context.WithCancel(context.Background()) go func() { // catch signal and invoke graceful termination stop := make(chan os.Signal, 1) @@ -169,18 +171,19 @@ func main() { }() px := &proxy.Http{ - Version: revision, - Matcher: svc, - Address: opts.Listen, - MaxBodySize: opts.MaxSize, - AssetsLocation: opts.Assets.Location, - AssetsWebRoot: opts.Assets.WebRoot, - GzEnabled: opts.GzipEnabled, - SSLConfig: sslConfig, - ProxyHeaders: opts.ProxyHeaders, - AccessLog: accessLog, - StdOutEnabled: opts.Logger.StdOut, - Signature: opts.Signature, + Version: revision, + Matcher: svc, + Address: opts.Listen, + MaxBodySize: opts.MaxSize, + AssetsLocation: opts.Assets.Location, + AssetsWebRoot: opts.Assets.WebRoot, + AssetsCacheDuration: opts.Assets.CacheDuration, + GzEnabled: opts.GzipEnabled, + SSLConfig: sslConfig, + ProxyHeaders: opts.ProxyHeaders, + AccessLog: accessLog, + StdOutEnabled: opts.Logger.StdOut, + Signature: opts.Signature, Timeouts: proxy.Timeouts{ ReadHeader: opts.Timeouts.ReadHeader, Write: opts.Timeouts.Write, diff --git a/app/proxy/proxy.go b/app/proxy/proxy.go index 6811e10..a855e8b 100644 --- a/app/proxy/proxy.go +++ b/app/proxy/proxy.go @@ -24,19 +24,20 @@ import ( // Http is a proxy server for both http and https type Http struct { // nolint golint Matcher - Address string - AssetsLocation string - AssetsWebRoot string - MaxBodySize int64 - GzEnabled bool - ProxyHeaders []string - SSLConfig SSLConfig - Version string - AccessLog io.Writer - StdOutEnabled bool - Signature bool - Timeouts Timeouts - Metrics Metrics + Address string + AssetsLocation string + AssetsWebRoot string + AssetsCacheDuration time.Duration + MaxBodySize int64 + GzEnabled bool + ProxyHeaders []string + SSLConfig SSLConfig + Version string + AccessLog io.Writer + StdOutEnabled bool + Signature bool + Timeouts Timeouts + Metrics Metrics } // Matcher source info (server and route) to the destination url @@ -189,9 +190,7 @@ func (h *Http) proxyHandler() http.HandlerFunc { if h.AssetsLocation != "" && h.AssetsWebRoot != "" { fs, err := R.FileServer(h.AssetsWebRoot, h.AssetsLocation) if err == nil { - assetsHandler = func(w http.ResponseWriter, r *http.Request) { - fs.ServeHTTP(w, r) - } + assetsHandler = h.cachingHandler(fs).ServeHTTP } } @@ -229,7 +228,7 @@ func (h *Http) proxyHandler() http.HandlerFunc { http.Error(w, "Server error", http.StatusInternalServerError) return } - fs.ServeHTTP(w, r) + h.cachingHandler(fs).ServeHTTP(w, r) } } } @@ -311,6 +310,16 @@ func (h *Http) stdoutLogHandler(enable bool, lh func(next http.Handler) http.Han } } +func (h *Http) cachingHandler(next http.Handler) http.Handler { + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if h.AssetsCacheDuration > 0 { + w.Header().Set("Cache-Control", "public, max-age="+strconv.Itoa(int(h.AssetsCacheDuration.Seconds()))) + } + next.ServeHTTP(w, r) + }) +} + func (h *Http) makeHTTPServer(addr string, router http.Handler) *http.Server { return &http.Server{ Addr: addr, diff --git a/app/proxy/proxy_test.go b/app/proxy/proxy_test.go index e2b81cc..33026b3 100644 --- a/app/proxy/proxy_test.go +++ b/app/proxy/proxy_test.go @@ -4,13 +4,17 @@ import ( "context" "fmt" "io" + "io/ioutil" "math/rand" "net/http" "net/http/httptest" + "os" + "path" "strconv" "testing" "time" + R "github.com/go-pkgz/rest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -252,3 +256,48 @@ func TestHttp_toHttp(t *testing.T) { } } + +func TestHttp_cachingHandler(t *testing.T) { + + dir, e := ioutil.TempDir(os.TempDir(), "reproxy") + require.NoError(t, e) + e = ioutil.WriteFile(path.Join(dir, "1.html"), []byte("1.htm"), 0600) + assert.NoError(t, e) + e = ioutil.WriteFile(path.Join(dir, "2.html"), []byte("2.htm"), 0600) + assert.NoError(t, e) + + defer os.RemoveAll(dir) + + fh, e := R.FileServer("/static", dir) + require.NoError(t, e) + h := Http{AssetsCacheDuration: 10 * time.Second, AssetsLocation: dir, AssetsWebRoot: "/static"} + hh := R.Wrap(fh, h.cachingHandler) + ts := httptest.NewServer(hh) + defer ts.Close() + client := http.Client{Timeout: 599 * time.Second} + + { + resp, err := client.Get(ts.URL + "/static/1.html") + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + t.Logf("headers: %+v", resp.Header) + assert.Equal(t, "public, max-age=10", resp.Header.Get("Cache-Control")) + assert.NotEqual(t, "", resp.Header.Get("Last-Modified")) + } + { + resp, err := client.Get(ts.URL + "/static/bad.html") + require.NoError(t, err) + assert.Equal(t, http.StatusNotFound, resp.StatusCode) + t.Logf("headers: %+v", resp.Header) + assert.Equal(t, "public, max-age=10", resp.Header.Get("Cache-Control")) + assert.Equal(t, "", resp.Header.Get("Last-Modified")) + } + { + resp, err := client.Get(ts.URL + "/%2e%2e%2f%2e%2e%2f%2e%2e%2f/etc/passwd") + require.NoError(t, err) + assert.Equal(t, http.StatusNotFound, resp.StatusCode) + t.Logf("headers: %+v", resp.Header) + assert.Equal(t, "public, max-age=10", resp.Header.Get("Cache-Control")) + assert.Equal(t, "", resp.Header.Get("Last-Modified")) + } +} diff --git a/site/public/index.html b/site/public/index.html index c8cf2b1..050c311 100644 --- a/site/public/index.html +++ b/site/public/index.html @@ -9,7 +9,7 @@ srv.example.com: - { route: "^/api/svc2/(.*)", dest: "http://127.0.0.2:8080/blah2/$1/abc" }

This is a dynamic provider and file change will be applied automatically.

Docker

Docker provider supports a fully automatic discovery (with --docker.auto) with no extra configuration and by default redirects all requests like https://server/<container_name>/(.*) to the internal IP of the given container and the exposed port. Only active (running) containers will be detected.

This default can be changed with labels:

Pls note: without --docker.auto the destination container has to have at least one of reproxy.* labels to be considered as a potential destination.

With --docker.auto, all containers with exposed port will be considered as routing destinations. There are 3 ways to restrict it:

This is a dynamic provider and any change in container's status will be applied automatically.

SSL support

SSL mode (by default none) can be set to auto (ACME/LE certificates), static (existing certificate) or none. If auto turned on SSL certificate will be issued automatically for all discovered server names. User can override it by setting --ssl.fqdn value(s)

Logging

By default no request log generated. This can be turned on by setting --logger.enabled. The log (auto-rotated) has Apache Combined Log Format

User can also turn stdout log on with --logger.stdout. It won't affect the file logging but will output some minimal info about processed requests, something like this:

2021/04/16 01:17:25.601 [INFO]  GET - /echo/image.png - xxx.xxx.xxx.xxx - 200 (155400) - 371.661251ms
 2021/04/16 01:18:18.959 [INFO]  GET - /api/v1/params - xxx.xxx.xxx.xxx - 200 (74) - 1.217669m
-

Assets Server

User may turn assets server on (off by default) to serve static files. As long as --assets.location set it will treat every non-proxied request under assets.root as a request for static files. Assets server can be used without any proxy providers. In this mode reproxy acts as a simple web server for a static context.

In addition to the common assets server multiple custom static servers supported. Each provider has a different way to define such static rule and some providers may not support it at all. For example, multiple static server make sense in case of static (command line provide), file provider and can be even useful with docker provider.

  1. static provider - if source element prefixed by assets: it will be treated as file-server. For example *,assets:/web,/var/www, will serve all /web/* request with a file server on top of /var/www directory.
  2. file provider - setting optional field assets: true
  3. docker provider - reproxy.assets=web-root:location, i.e. reproxy.assets=/web:/var/www.

More options

Ping and health checks

reproxy provides 2 endpoints for this purpose:

Management API

Optional, can be turned on with --mgmt.enabled. Exposes 2 endpoints on mgmt.listen address:port:

see also examples/metrics

All Application Options 

  -l, --listen=                     listen on host:port (default: 127.0.0.1:8080) [$LISTEN]
+

Assets Server

Users may turn the assets server on (off by default) to serve static files. As long as --assets.location set it treats every non-proxied request under assets.root as a request for static files. The assets server can be used without any proxy providers; in this mode, reproxy acts as a simple web server for the static content.

In addition to the common assets server, multiple custom static servers are supported. Each provider has a different way to define such a static rule, and some providers may not support it at all. For example, multiple static servers make sense in static (command line provider), file provider, and even useful with docker providers.

  1. static provider - if source element prefixed by assets: it will be treated as file-server. For example *,assets:/web,/var/www, will serve all /web/* request with a file server on top of /var/www directory.
  2. file provider - setting optional field assets: true
  3. docker provider - reproxy.assets=web-root:location, i.e. reproxy.assets=/web:/var/www.

Assets server supports caching control with the --assets.cache=<duration> parameter. 0s duration (default) turns caching control off.

More options

Ping and health checks

reproxy provides 2 endpoints for this purpose:

Management API

Optional, can be turned on with --mgmt.enabled. Exposes 2 endpoints on mgmt.listen address:port:

see also examples/metrics

All Application Options 

  -l, --listen=                     listen on host:port (default: 127.0.0.1:8080) [$LISTEN]
   -m, --max=                        max request size (default: 64000) [$MAX_SIZE]
   -g, --gzip                        enable gz compression [$GZIP]
   -x, --header=                     proxy headers [$HEADER]
@@ -28,6 +28,7 @@ ssl:
 assets:
   -a, --assets.location=            assets location [$ASSETS_LOCATION]
       --assets.root=                assets web root (default: /) [$ASSETS_ROOT]
+      --assets.cache=               cache duration for assets (default: 0s) [$ASSETS_CACHE]
 
 logger:
       --logger.stdout               enable stdout logging [$LOGGER_STDOUT]
@@ -71,7 +72,7 @@ mgmt:
 Help Options:
   -h, --help                        Show this help message
 
-

Status

The project is under active development and may have breaking changes till v1 released.

Updated  Edit