mirror of
https://github.com/consbio/mbtileserver.git
synced 2024-10-05 19:47:41 +03:00
Add --enable-fs-watch to watch filesystem for changes to tilesets (#122)
This commit is contained in:
parent
fee83f0cc5
commit
26cd74b442
@ -1,4 +1,4 @@
|
||||
**/.git
|
||||
**/node_modules
|
||||
mbtiles/testdata/*
|
||||
mbtiles/testdata-bad/*
|
||||
testdata/*
|
||||
testdata-bad/*
|
@ -12,6 +12,10 @@
|
||||
- dropped internal mbtiles package in favor of github.com/brendan-ward/mbtiles-go,
|
||||
which wraps the SQlite-specific go package `crawshaw.io/sqlite`
|
||||
|
||||
### Command-line interface
|
||||
|
||||
- added support for watching filesystem for changes to tilesets using `--enable-fs-watch` option
|
||||
|
||||
### Breaking changes
|
||||
|
||||
- removes ArcGIS API layer info at the service root and layers endpoint (#116);
|
||||
|
33
README.md
33
README.md
@ -67,6 +67,7 @@ Flags:
|
||||
--domain string Domain name of this server. NOTE: only used for AutoTLS.
|
||||
--dsn string Sentry DSN
|
||||
--enable-arcgis Enable ArcGIS Mapserver endpoints
|
||||
--enable-fs-watch Enable reloading of tilesets by watching filesystem
|
||||
--enable-reload-signal Enable graceful reload using HUP signal to the server process
|
||||
--generate-ids Automatically generate tileset IDs instead of using relative path
|
||||
-h, --help help for mbtileserver
|
||||
@ -90,7 +91,7 @@ You can have multiple directories in your `tilesets` directory; these will be co
|
||||
If `--generate-ids` is provided, tileset IDs are automatically generated using a SHA1 hash of the path to each tileset.
|
||||
By default, tileset IDs are based on the relative path of each tileset to the base directory provided using `--dir`.
|
||||
|
||||
When you want to remove, modify, or add new tilesets, simply restart the server process or use the reloading process below.
|
||||
When you want to remove, modify, or add new tilesets, simply restart the server process or use one of the reloading processes below.
|
||||
|
||||
If a valid Sentry DSN is provided, warnings, errors, fatal errors, and panics will be reported to Sentry.
|
||||
|
||||
@ -136,7 +137,9 @@ mbtileserver:
|
||||
...
|
||||
```
|
||||
|
||||
### Reload
|
||||
### Reloading
|
||||
|
||||
#### Reload using a signal
|
||||
|
||||
mbtileserver optionally supports graceful reload (without interrupting any in-progress requests). This functionality
|
||||
must be enabled with the `--enable-reload-signal` flag. When enabled, the server can be reloaded by sending it a `HUP` signal:
|
||||
@ -148,6 +151,32 @@ $ kill -HUP <pid>
|
||||
Reloading the server will cause it to pick up changes to the tiles directory, adding new tilesets and removing any that
|
||||
are no longer present.
|
||||
|
||||
#### Reload using a filesystem watcher
|
||||
|
||||
mbtileserver optionally supports reload of individual tilesets by watching for filesystem changes. This functionality
|
||||
must be enabled with the `--enable-fs-watch` flag.
|
||||
|
||||
All directories specified by `-d` / `--dir` and any subdirectories that exist at the time the server is started
|
||||
will be watched for changes to the tilesets.
|
||||
|
||||
An existing tileset that is being updated will be locked while the file on disk
|
||||
is being updated. This will cause incoming requests to that tileset to stall
|
||||
for up to 30 seconds and will return as soon as the tileset is completely updated
|
||||
and unlocked. If it takes longer than 30 seconds for the tileset to be updated,
|
||||
HTTP 503 errors will be returned for that tileset until the tileset is completely
|
||||
updated and unlocked.
|
||||
|
||||
Under very high request volumes, requests that come in between when the file is
|
||||
first modified and when that modification is first detected (and tileset locked)
|
||||
may encounter errors.
|
||||
|
||||
WARNING: Do not remove the top-level watched directories while the server is running.
|
||||
|
||||
WARNING: Do not create or delete subdirectories within the watched directories while the server is running.
|
||||
|
||||
WARNING: do not generate tiles directly in the watched directories. Instead, create them in separate directories and
|
||||
copy them into the watched directories when complete.
|
||||
|
||||
### Using with a reverse proxy
|
||||
|
||||
You can use a reverse proxy in front of `mbtileserver` to intercept incoming requests, provide TLS, etc.
|
||||
|
2
go.mod
2
go.mod
@ -4,6 +4,7 @@ require (
|
||||
github.com/brendan-ward/mbtiles-go v0.0.0-20211210015813-553bc514bbdf
|
||||
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect
|
||||
github.com/evalphobia/logrus_sentry v0.8.2
|
||||
github.com/fsnotify/fsnotify v1.5.1
|
||||
github.com/getsentry/raven-go v0.2.0 // indirect
|
||||
github.com/labstack/echo/v4 v4.3.0
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
@ -12,7 +13,6 @@ require (
|
||||
github.com/stretchr/testify v1.7.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
|
||||
golang.org/x/net v0.0.0-20210510120150-4163338589ed // indirect
|
||||
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 // indirect
|
||||
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
|
||||
)
|
||||
|
6
go.sum
6
go.sum
@ -49,6 +49,8 @@ github.com/evalphobia/logrus_sentry v0.8.2 h1:dotxHq+YLZsT1Bb45bB5UQbfCh3gM/nFFe
|
||||
github.com/evalphobia/logrus_sentry v0.8.2/go.mod h1:pKcp+vriitUqu9KiWj/VRFbRfFNUwz95/UkgG8a6MNc=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
|
||||
github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
|
||||
github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs=
|
||||
github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
@ -279,8 +281,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 h1:hZR0X1kPW+nwyJ9xRxqZk1vx5RUObAPBdKVvXPDUH/E=
|
||||
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
@ -5,6 +5,7 @@ import (
|
||||
"math"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type arcGISLOD struct {
|
||||
@ -212,6 +213,12 @@ func (ts *Tileset) arcgisServiceJSON() ([]byte, error) {
|
||||
// arcgisServiceHandler is an http.HandlerFunc that returns standard ArcGIS
|
||||
// JSON for a given ArcGIS tile service
|
||||
func (ts *Tileset) arcgisServiceHandler(w http.ResponseWriter, r *http.Request) {
|
||||
// wait up to 30 seconds to see if tileset is ready and return it if possible
|
||||
if ts.isLockedWithTimeout(30 * time.Second) {
|
||||
tilesetLockedHandler(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
svcJSON, err := ts.arcgisServiceJSON()
|
||||
|
||||
if err != nil {
|
||||
@ -291,6 +298,12 @@ func (ts *Tileset) arcgisLayersJSON() ([]byte, error) {
|
||||
// arcgisLayersHandler is an http.HandlerFunc that returns standard ArcGIS
|
||||
// Layers JSON for a given ArcGIS tile service
|
||||
func (ts *Tileset) arcgisLayersHandler(w http.ResponseWriter, r *http.Request) {
|
||||
// wait up to 30 seconds to see if tileset is ready and return it if possible
|
||||
if ts.isLockedWithTimeout(30 * time.Second) {
|
||||
tilesetLockedHandler(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
layersJSON, err := ts.arcgisLayersJSON()
|
||||
if err != nil {
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
@ -343,6 +356,12 @@ func (ts *Tileset) arcgisLegendJSON() ([]byte, error) {
|
||||
// arcgisLegendHandler is an http.HandlerFunc that returns minimal ArcGIS
|
||||
// legend JSON for a given ArcGIS tile service
|
||||
func (ts *Tileset) arcgisLegendHandler(w http.ResponseWriter, r *http.Request) {
|
||||
// wait up to 30 seconds to see if tileset is ready and return it if possible
|
||||
if ts.isLockedWithTimeout(30 * time.Second) {
|
||||
tilesetLockedHandler(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
legendJSON, err := ts.arcgisLegendJSON()
|
||||
|
||||
if err != nil {
|
||||
@ -362,6 +381,13 @@ func (ts *Tileset) arcgisLegendHandler(w http.ResponseWriter, r *http.Request) {
|
||||
// arcgisTileHandler returns an image tile or blank image for a given
|
||||
// tile request within a given ArcGIS tile service
|
||||
func (ts *Tileset) arcgisTileHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// wait up to 30 seconds to see if tileset is ready and return it if possible
|
||||
if ts.isLockedWithTimeout(30 * time.Second) {
|
||||
tilesetLockedHandler(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
db := ts.db
|
||||
|
||||
// split path components to extract tile coordinates x, y and z
|
||||
|
@ -6,6 +6,8 @@ import (
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
type IDGenerator func(filename, baseDir string) (string, error)
|
||||
|
||||
// SHA1ID generates a URL safe base64 encoded SHA1 hash of the filename.
|
||||
func SHA1ID(filename string) string {
|
||||
// generate IDs from hash of full file path
|
||||
|
@ -60,7 +60,6 @@ func New(cfg *ServiceSetConfig) (*ServiceSet, error) {
|
||||
// AddTileset adds a single tileset identified by idGenerator using the filename.
|
||||
// If a service already exists with that ID, an error is returned.
|
||||
func (s *ServiceSet) AddTileset(filename, id string) error {
|
||||
|
||||
if _, ok := s.tilesets[id]; ok {
|
||||
return fmt.Errorf("Tileset already exists for ID: %q", id)
|
||||
}
|
||||
@ -114,6 +113,29 @@ func (s *ServiceSet) RemoveTileset(id string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// LockTileset sets a write mutex on the tileset to block reads while this
|
||||
// tileset is being updated.
|
||||
// This is ignored if the tileset does not exist.
|
||||
func (s *ServiceSet) LockTileset(id string) {
|
||||
ts, ok := s.tilesets[id]
|
||||
if !ok || ts == nil {
|
||||
return
|
||||
}
|
||||
|
||||
ts.locked = true
|
||||
}
|
||||
|
||||
// UnlockTileset removes the write mutex on the tileset.
|
||||
// This is ignored if the tileset does not exist.
|
||||
func (s *ServiceSet) UnlockTileset(id string) {
|
||||
ts, ok := s.tilesets[id]
|
||||
if !ok || ts == nil {
|
||||
return
|
||||
}
|
||||
|
||||
ts.locked = false
|
||||
}
|
||||
|
||||
// HasTileset returns true if the tileset identified by id exists within this
|
||||
// ServiceSet.
|
||||
func (s *ServiceSet) HasTileset(id string) bool {
|
||||
|
@ -2,11 +2,14 @@ package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
mbtiles "github.com/brendan-ward/mbtiles-go"
|
||||
)
|
||||
@ -19,6 +22,7 @@ type Tileset struct {
|
||||
name string
|
||||
tileformat mbtiles.TileFormat
|
||||
published bool
|
||||
locked bool
|
||||
router *http.ServeMux
|
||||
}
|
||||
|
||||
@ -97,7 +101,7 @@ func (ts *Tileset) reload() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete closes and deletes the mbtiles file for this tileset
|
||||
// Delete closes and deletes the mbtiles file connection for this tileset
|
||||
func (ts *Tileset) delete() error {
|
||||
if ts.db != nil {
|
||||
ts.db.Close()
|
||||
@ -164,6 +168,12 @@ func (ts *Tileset) tileJSONHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// wait up to 30 seconds to see if tileset is ready and return it if possible
|
||||
if ts.isLockedWithTimeout(30 * time.Second) {
|
||||
tilesetLockedHandler(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
query := ""
|
||||
if r.URL.RawQuery != "" {
|
||||
query = "?" + r.URL.RawQuery
|
||||
@ -201,6 +211,12 @@ func (ts *Tileset) tileHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// wait up to 30 seconds to see if tileset is ready and return it if possible
|
||||
if ts.isLockedWithTimeout(30 * time.Second) {
|
||||
tilesetLockedHandler(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
db := ts.db
|
||||
// split path components to extract tile coordinates x, y and z
|
||||
pcs := strings.Split(r.URL.Path[1:], "/")
|
||||
@ -238,8 +254,9 @@ func (ts *Tileset) tileHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
_, err = w.Write(data)
|
||||
|
||||
if err != nil {
|
||||
if err != nil && !errors.Is(err, syscall.EPIPE) {
|
||||
ts.svc.logError("Could not write tile data for %v: %v", r.URL.Path, err)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@ -252,6 +269,12 @@ func (ts *Tileset) previewHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// wait up to 30 seconds to see if tileset is ready and return it if possible
|
||||
if ts.isLockedWithTimeout(30 * time.Second) {
|
||||
tilesetLockedHandler(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
query := ""
|
||||
if r.URL.RawQuery != "" {
|
||||
query = "?" + r.URL.RawQuery
|
||||
@ -303,3 +326,32 @@ func tileNotFoundHandler(w http.ResponseWriter, r *http.Request, f mbtiles.TileF
|
||||
fmt.Fprint(w, `{"message": "Tile does not exist"}`)
|
||||
}
|
||||
}
|
||||
|
||||
// tilesetLockedHandler returns a 503 Service Unavailable response when
|
||||
// requests are made to a tileset that is beging updated
|
||||
func tilesetLockedHandler(w http.ResponseWriter, r *http.Request) {
|
||||
// send back service unavailable response with header to retry in 10 seconds
|
||||
w.Header().Set("Retry-After", "10")
|
||||
http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
|
||||
}
|
||||
|
||||
func (ts *Tileset) isLockedWithTimeout(timeout time.Duration) bool {
|
||||
if ts == nil || !ts.locked {
|
||||
return false
|
||||
}
|
||||
|
||||
timeoutReached := time.After(timeout)
|
||||
// poll locked status every 500 ms
|
||||
ticker := time.Tick(500 * time.Millisecond)
|
||||
for {
|
||||
select {
|
||||
case <-timeoutReached:
|
||||
return ts.locked
|
||||
case <-ticker:
|
||||
if !ts.locked {
|
||||
return false
|
||||
}
|
||||
// otherwise, still locked
|
||||
}
|
||||
}
|
||||
}
|
||||
|
95
main.go
95
main.go
@ -61,24 +61,25 @@ var rootCmd = &cobra.Command{
|
||||
}
|
||||
|
||||
var (
|
||||
port int
|
||||
tilePath string
|
||||
certificate string
|
||||
privateKey string
|
||||
rootURLStr string
|
||||
domain string
|
||||
secretKey string
|
||||
sentryDSN string
|
||||
verbose bool
|
||||
autotls bool
|
||||
redirect bool
|
||||
enableReloadSignal bool
|
||||
generateIDs bool
|
||||
enableArcGIS bool
|
||||
disablePreview bool
|
||||
disableTileJSON bool
|
||||
disableServiceList bool
|
||||
tilesOnly bool
|
||||
port int
|
||||
tilePath string
|
||||
certificate string
|
||||
privateKey string
|
||||
rootURLStr string
|
||||
domain string
|
||||
secretKey string
|
||||
sentryDSN string
|
||||
verbose bool
|
||||
autotls bool
|
||||
redirect bool
|
||||
enableReloadSignal bool
|
||||
enableReloadFSWatch bool
|
||||
generateIDs bool
|
||||
enableArcGIS bool
|
||||
disablePreview bool
|
||||
disableTileJSON bool
|
||||
disableServiceList bool
|
||||
tilesOnly bool
|
||||
)
|
||||
|
||||
func init() {
|
||||
@ -95,6 +96,7 @@ func init() {
|
||||
flags.BoolVarP(&redirect, "redirect", "r", false, "Redirect HTTP to HTTPS")
|
||||
|
||||
flags.BoolVarP(&enableArcGIS, "enable-arcgis", "", false, "Enable ArcGIS Mapserver endpoints")
|
||||
flags.BoolVarP(&enableReloadFSWatch, "enable-fs-watch", "", false, "Enable reloading of tilesets by watching filesystem")
|
||||
flags.BoolVarP(&enableReloadSignal, "enable-reload-signal", "", false, "Enable graceful reload using HUP signal to the server process")
|
||||
|
||||
flags.BoolVarP(&disablePreview, "disable-preview", "", false, "Disable map preview for each tileset (enabled by default)")
|
||||
@ -172,6 +174,22 @@ func init() {
|
||||
enableArcGIS = p
|
||||
}
|
||||
|
||||
if env := os.Getenv("ENABLE_FS_WATCH"); env != "" {
|
||||
p, err := strconv.ParseBool(env)
|
||||
if err != nil {
|
||||
log.Fatalln("ENABLE_FS_WATCH must be a bool(true/false)")
|
||||
}
|
||||
enableReloadFSWatch = p
|
||||
}
|
||||
|
||||
if env := os.Getenv("ENABLE_RELOAD_SIGNAL"); env != "" {
|
||||
p, err := strconv.ParseBool(env)
|
||||
if err != nil {
|
||||
log.Fatalln("ENABLE_RELOAD_SIGNAL must be a bool(true/false)")
|
||||
}
|
||||
enableReloadSignal = p
|
||||
}
|
||||
|
||||
if env := os.Getenv("VERBOSE"); env != "" {
|
||||
p, err := strconv.ParseBool(env)
|
||||
if err != nil {
|
||||
@ -249,6 +267,14 @@ func serve() {
|
||||
log.Infoln("An HMAC request authorization key was set. All incoming must be signed.")
|
||||
}
|
||||
|
||||
generateID := func(filename string, baseDir string) (string, error) {
|
||||
if generateIDs {
|
||||
return handlers.SHA1ID(filename), nil
|
||||
} else {
|
||||
return handlers.RelativePathID(filename, baseDir)
|
||||
}
|
||||
}
|
||||
|
||||
svcSet, err := handlers.New(&handlers.ServiceSetConfig{
|
||||
RootURL: rootURL,
|
||||
ErrorWriter: &errorLogger{log: log.New()},
|
||||
@ -274,16 +300,10 @@ func serve() {
|
||||
|
||||
// Register all tilesets
|
||||
for _, filename := range filenames {
|
||||
var id string
|
||||
var err error
|
||||
if generateIDs {
|
||||
id = handlers.SHA1ID(filename)
|
||||
} else {
|
||||
id, err = handlers.RelativePathID(filename, path)
|
||||
if err != nil {
|
||||
log.Errorf("Could not generate ID for tileset: %q", filename)
|
||||
continue
|
||||
}
|
||||
id, err := generateID(filename, path)
|
||||
if err != nil {
|
||||
log.Errorf("Could not generate ID for tileset: %q", filename)
|
||||
continue
|
||||
}
|
||||
|
||||
err = svcSet.AddTileset(filename, id)
|
||||
@ -296,6 +316,25 @@ func serve() {
|
||||
// print number of services
|
||||
log.Infof("Published %v services", svcSet.Size())
|
||||
|
||||
// watch filesystem for changes to tilesets
|
||||
if enableReloadFSWatch {
|
||||
watcher, err := NewFSWatcher(svcSet, generateID)
|
||||
if err != nil {
|
||||
log.Fatalln("Could not construct filesystem watcher")
|
||||
}
|
||||
defer watcher.Close()
|
||||
|
||||
for _, path := range strings.Split(tilePath, ",") {
|
||||
log.Infof("Watching %v\n", path)
|
||||
err = watcher.WatchDir((path))
|
||||
if err != nil {
|
||||
// If we cannot enable file watching, then this should be a fatal
|
||||
// error during server startup
|
||||
log.Fatalln("Could not enable filesystem watcher in", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
e := echo.New()
|
||||
e.HideBanner = true
|
||||
e.Pre(middleware.RemoveTrailingSlash())
|
||||
|
204
watch.go
Normal file
204
watch.go
Normal file
@ -0,0 +1,204 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
mbtiles "github.com/brendan-ward/mbtiles-go"
|
||||
"github.com/consbio/mbtileserver/handlers"
|
||||
"github.com/fsnotify/fsnotify"
|
||||
)
|
||||
|
||||
// debounce debounces requests to a callback function to occur no more
|
||||
// frequently than interval; once this is reached, the callback is called.
|
||||
//
|
||||
// Unique values sent to the channel are stored in an internal map and all
|
||||
// are processed once the the interval is up.
|
||||
func debounce(interval time.Duration, input chan string, firstCallback func(arg string), callback func(arg string)) {
|
||||
// keep a log of unique paths
|
||||
var items = make(map[string]bool)
|
||||
var item string
|
||||
timer := time.NewTimer(interval)
|
||||
for {
|
||||
select {
|
||||
case item = <-input:
|
||||
if _, ok := items[item]; !ok {
|
||||
// first time we see a given path, we need to call lockHandler
|
||||
// to lock it (unlocked by callback)
|
||||
firstCallback(item)
|
||||
}
|
||||
items[item] = true
|
||||
timer.Reset(interval)
|
||||
case <-timer.C:
|
||||
for path := range items {
|
||||
callback(path)
|
||||
delete(items, path)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// FSWatcher provides a filesystem watcher to detect when mbtiles files are
|
||||
// created, updated, or removed on the filesystem.
|
||||
type FSWatcher struct {
|
||||
watcher *fsnotify.Watcher
|
||||
svcSet *handlers.ServiceSet
|
||||
generateID handlers.IDGenerator
|
||||
}
|
||||
|
||||
// NewFSWatcher creates a new FSWatcher to watch the filesystem for changes to
|
||||
// mbtiles files and updates the ServiceSet accordingly.
|
||||
//
|
||||
// The generateID function needs to be of the same type used when the tilesets
|
||||
// were originally added to the ServiceSet.
|
||||
func NewFSWatcher(svcSet *handlers.ServiceSet, generateID handlers.IDGenerator) (*FSWatcher, error) {
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &FSWatcher{
|
||||
watcher: watcher,
|
||||
svcSet: svcSet,
|
||||
generateID: generateID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Close closes the FSWatcher and stops watching the filesystem.
|
||||
func (w *FSWatcher) Close() {
|
||||
if w.watcher != nil {
|
||||
w.watcher.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// WatchDir sets up the filesystem watcher for baseDir and all existing subdirectories
|
||||
func (w *FSWatcher) WatchDir(baseDir string) error {
|
||||
c := make(chan string)
|
||||
|
||||
// debounced call to create / update tileset
|
||||
go debounce(500*time.Millisecond, c, func(path string) {
|
||||
// callback for first time path is debounced
|
||||
id, err := w.generateID(path, baseDir)
|
||||
if err != nil {
|
||||
log.Errorf("Could not create ID for tileset %q\n%v", path, err)
|
||||
return
|
||||
}
|
||||
// lock tileset for writing, if it exists
|
||||
w.svcSet.LockTileset(id)
|
||||
}, func(path string) {
|
||||
// callback after debouncing incoming requests
|
||||
|
||||
// Verify that file can be opened with mbtiles-go, which runs
|
||||
// validation on open.
|
||||
// If file cannot be opened, assume it is still being written / copied.
|
||||
db, err := mbtiles.Open(path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
db.Close()
|
||||
|
||||
// determine file ID for tileset
|
||||
id, err := w.generateID(path, baseDir)
|
||||
if err != nil {
|
||||
log.Errorf("Could not create ID for tileset %q\n%v", path, err)
|
||||
return
|
||||
}
|
||||
|
||||
// update existing tileset
|
||||
if w.svcSet.HasTileset(id) {
|
||||
err = w.svcSet.UpdateTileset(id)
|
||||
if err != nil {
|
||||
log.Errorf("Could not update tileset %q with ID %q\n%v", path, id, err)
|
||||
} else {
|
||||
// only unlock if successfully updated
|
||||
w.svcSet.UnlockTileset(id)
|
||||
log.Infof("Updated tileset %q with ID %q\n", path, id)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// create new tileset
|
||||
err = w.svcSet.AddTileset(path, id)
|
||||
if err != nil {
|
||||
log.Errorf("Could not add tileset for %q with ID %q\n%v", path, id, err)
|
||||
} else {
|
||||
log.Infof("Updated tileset %q with ID %q\n", path, id)
|
||||
}
|
||||
return
|
||||
})
|
||||
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case event, ok := <-w.watcher.Events:
|
||||
if !ok {
|
||||
log.Errorf("error in filewatcher for %q, exiting filewatcher", event.Name)
|
||||
return
|
||||
}
|
||||
|
||||
if !((event.Op&fsnotify.Write == fsnotify.Write) ||
|
||||
(event.Op&fsnotify.Remove == fsnotify.Remove) ||
|
||||
(event.Op&fsnotify.Rename == fsnotify.Rename)) {
|
||||
continue
|
||||
}
|
||||
|
||||
path := event.Name
|
||||
|
||||
if ext := filepath.Ext(path); ext != ".mbtiles" {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err := os.Stat(path + "-journal"); err == nil {
|
||||
// Don't try to load .mbtiles files that are being written
|
||||
log.Debugf("Tileset %q is currently being created or is incomplete\n", path)
|
||||
continue
|
||||
}
|
||||
|
||||
if event.Op&fsnotify.Write == fsnotify.Write {
|
||||
// This event may get called multiple times while a file is being copied into a watched directory,
|
||||
// so we debounce this instead.
|
||||
c <- path
|
||||
}
|
||||
|
||||
if (event.Op&fsnotify.Remove == fsnotify.Remove) || (event.Op&fsnotify.Rename == fsnotify.Rename) {
|
||||
// remove tileset immediately so that there are not other errors in request handlers
|
||||
id, err := w.generateID(path, baseDir)
|
||||
if err != nil {
|
||||
log.Errorf("Could not create ID for tileset %q\n%v", path, err)
|
||||
}
|
||||
err = w.svcSet.RemoveTileset(id)
|
||||
if err != nil {
|
||||
log.Errorf("Could not remove tileset %q with ID %q\n%v", path, id, err)
|
||||
} else {
|
||||
log.Infof("Removed tileset %q with ID %q\n", path, id)
|
||||
}
|
||||
}
|
||||
|
||||
case err, ok := <-w.watcher.Errors:
|
||||
if !ok {
|
||||
log.Errorf("error in filewatcher, exiting filewatcher")
|
||||
return
|
||||
}
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
err := filepath.Walk(baseDir,
|
||||
func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode().IsDir() {
|
||||
return w.watcher.Add(path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
Loading…
Reference in New Issue
Block a user