From 26cd74b442bd484f18605cb725122a9b2bfad5ab Mon Sep 17 00:00:00 2001 From: Brendan Ward Date: Tue, 14 Dec 2021 11:23:55 -0800 Subject: [PATCH] Add --enable-fs-watch to watch filesystem for changes to tilesets (#122) --- .dockerignore | 4 +- CHANGELOG.md | 4 + README.md | 33 ++++++- go.mod | 2 +- go.sum | 6 +- handlers/arcgis.go | 26 ++++++ handlers/id.go | 2 + handlers/serviceset.go | 24 ++++- handlers/tileset.go | 56 ++++++++++- main.go | 95 +++++++++++++------ watch.go | 204 +++++++++++++++++++++++++++++++++++++++++ 11 files changed, 418 insertions(+), 38 deletions(-) create mode 100644 watch.go diff --git a/.dockerignore b/.dockerignore index 241b1cb..59f1aaf 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,4 @@ **/.git **/node_modules -mbtiles/testdata/* -mbtiles/testdata-bad/* \ No newline at end of file +testdata/* +testdata-bad/* \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ef0911..9651ad9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ - dropped internal mbtiles package in favor of github.com/brendan-ward/mbtiles-go, which wraps the SQlite-specific go package `crawshaw.io/sqlite` +### Command-line interface + +- added support for watching filesystem for changes to tilesets using `--enable-fs-watch` option + ### Breaking changes - removes ArcGIS API layer info at the service root and layers endpoint (#116); diff --git a/README.md b/README.md index 4d4a2cd..38be80c 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ Flags: --domain string Domain name of this server. NOTE: only used for AutoTLS. --dsn string Sentry DSN --enable-arcgis Enable ArcGIS Mapserver endpoints + --enable-fs-watch Enable reloading of tilesets by watching filesystem --enable-reload-signal Enable graceful reload using HUP signal to the server process --generate-ids Automatically generate tileset IDs instead of using relative path -h, --help help for mbtileserver @@ -90,7 +91,7 @@ You can have multiple directories in your `tilesets` directory; these will be co If `--generate-ids` is provided, tileset IDs are automatically generated using a SHA1 hash of the path to each tileset. By default, tileset IDs are based on the relative path of each tileset to the base directory provided using `--dir`. -When you want to remove, modify, or add new tilesets, simply restart the server process or use the reloading process below. +When you want to remove, modify, or add new tilesets, simply restart the server process or use one of the reloading processes below. If a valid Sentry DSN is provided, warnings, errors, fatal errors, and panics will be reported to Sentry. @@ -136,7 +137,9 @@ mbtileserver: ... ``` -### Reload +### Reloading + +#### Reload using a signal mbtileserver optionally supports graceful reload (without interrupting any in-progress requests). This functionality must be enabled with the `--enable-reload-signal` flag. When enabled, the server can be reloaded by sending it a `HUP` signal: @@ -148,6 +151,32 @@ $ kill -HUP Reloading the server will cause it to pick up changes to the tiles directory, adding new tilesets and removing any that are no longer present. +#### Reload using a filesystem watcher + +mbtileserver optionally supports reload of individual tilesets by watching for filesystem changes. This functionality +must be enabled with the `--enable-fs-watch` flag. + +All directories specified by `-d` / `--dir` and any subdirectories that exist at the time the server is started +will be watched for changes to the tilesets. + +An existing tileset that is being updated will be locked while the file on disk +is being updated. This will cause incoming requests to that tileset to stall +for up to 30 seconds and will return as soon as the tileset is completely updated +and unlocked. If it takes longer than 30 seconds for the tileset to be updated, +HTTP 503 errors will be returned for that tileset until the tileset is completely +updated and unlocked. + +Under very high request volumes, requests that come in between when the file is +first modified and when that modification is first detected (and tileset locked) +may encounter errors. + +WARNING: Do not remove the top-level watched directories while the server is running. + +WARNING: Do not create or delete subdirectories within the watched directories while the server is running. + +WARNING: do not generate tiles directly in the watched directories. Instead, create them in separate directories and +copy them into the watched directories when complete. + ### Using with a reverse proxy You can use a reverse proxy in front of `mbtileserver` to intercept incoming requests, provide TLS, etc. diff --git a/go.mod b/go.mod index 7e9de28..53e3d1c 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ require ( github.com/brendan-ward/mbtiles-go v0.0.0-20211210015813-553bc514bbdf github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect github.com/evalphobia/logrus_sentry v0.8.2 + github.com/fsnotify/fsnotify v1.5.1 github.com/getsentry/raven-go v0.2.0 // indirect github.com/labstack/echo/v4 v4.3.0 github.com/pkg/errors v0.9.1 // indirect @@ -12,7 +13,6 @@ require ( github.com/stretchr/testify v1.7.0 // indirect golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a golang.org/x/net v0.0.0-20210510120150-4163338589ed // indirect - golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 // indirect golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index 09e6421..5f652d0 100644 --- a/go.sum +++ b/go.sum @@ -49,6 +49,8 @@ github.com/evalphobia/logrus_sentry v0.8.2 h1:dotxHq+YLZsT1Bb45bB5UQbfCh3gM/nFFe github.com/evalphobia/logrus_sentry v0.8.2/go.mod h1:pKcp+vriitUqu9KiWj/VRFbRfFNUwz95/UkgG8a6MNc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -279,8 +281,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 h1:hZR0X1kPW+nwyJ9xRxqZk1vx5RUObAPBdKVvXPDUH/E= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/handlers/arcgis.go b/handlers/arcgis.go index b93a31b..48fcfcb 100644 --- a/handlers/arcgis.go +++ b/handlers/arcgis.go @@ -5,6 +5,7 @@ import ( "math" "net/http" "strings" + "time" ) type arcGISLOD struct { @@ -212,6 +213,12 @@ func (ts *Tileset) arcgisServiceJSON() ([]byte, error) { // arcgisServiceHandler is an http.HandlerFunc that returns standard ArcGIS // JSON for a given ArcGIS tile service func (ts *Tileset) arcgisServiceHandler(w http.ResponseWriter, r *http.Request) { + // wait up to 30 seconds to see if tileset is ready and return it if possible + if ts.isLockedWithTimeout(30 * time.Second) { + tilesetLockedHandler(w, r) + return + } + svcJSON, err := ts.arcgisServiceJSON() if err != nil { @@ -291,6 +298,12 @@ func (ts *Tileset) arcgisLayersJSON() ([]byte, error) { // arcgisLayersHandler is an http.HandlerFunc that returns standard ArcGIS // Layers JSON for a given ArcGIS tile service func (ts *Tileset) arcgisLayersHandler(w http.ResponseWriter, r *http.Request) { + // wait up to 30 seconds to see if tileset is ready and return it if possible + if ts.isLockedWithTimeout(30 * time.Second) { + tilesetLockedHandler(w, r) + return + } + layersJSON, err := ts.arcgisLayersJSON() if err != nil { http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) @@ -343,6 +356,12 @@ func (ts *Tileset) arcgisLegendJSON() ([]byte, error) { // arcgisLegendHandler is an http.HandlerFunc that returns minimal ArcGIS // legend JSON for a given ArcGIS tile service func (ts *Tileset) arcgisLegendHandler(w http.ResponseWriter, r *http.Request) { + // wait up to 30 seconds to see if tileset is ready and return it if possible + if ts.isLockedWithTimeout(30 * time.Second) { + tilesetLockedHandler(w, r) + return + } + legendJSON, err := ts.arcgisLegendJSON() if err != nil { @@ -362,6 +381,13 @@ func (ts *Tileset) arcgisLegendHandler(w http.ResponseWriter, r *http.Request) { // arcgisTileHandler returns an image tile or blank image for a given // tile request within a given ArcGIS tile service func (ts *Tileset) arcgisTileHandler(w http.ResponseWriter, r *http.Request) { + + // wait up to 30 seconds to see if tileset is ready and return it if possible + if ts.isLockedWithTimeout(30 * time.Second) { + tilesetLockedHandler(w, r) + return + } + db := ts.db // split path components to extract tile coordinates x, y and z diff --git a/handlers/id.go b/handlers/id.go index 4dd6f09..65f905f 100644 --- a/handlers/id.go +++ b/handlers/id.go @@ -6,6 +6,8 @@ import ( "path/filepath" ) +type IDGenerator func(filename, baseDir string) (string, error) + // SHA1ID generates a URL safe base64 encoded SHA1 hash of the filename. func SHA1ID(filename string) string { // generate IDs from hash of full file path diff --git a/handlers/serviceset.go b/handlers/serviceset.go index e802e91..cf0b300 100644 --- a/handlers/serviceset.go +++ b/handlers/serviceset.go @@ -60,7 +60,6 @@ func New(cfg *ServiceSetConfig) (*ServiceSet, error) { // AddTileset adds a single tileset identified by idGenerator using the filename. // If a service already exists with that ID, an error is returned. func (s *ServiceSet) AddTileset(filename, id string) error { - if _, ok := s.tilesets[id]; ok { return fmt.Errorf("Tileset already exists for ID: %q", id) } @@ -114,6 +113,29 @@ func (s *ServiceSet) RemoveTileset(id string) error { return nil } +// LockTileset sets a write mutex on the tileset to block reads while this +// tileset is being updated. +// This is ignored if the tileset does not exist. +func (s *ServiceSet) LockTileset(id string) { + ts, ok := s.tilesets[id] + if !ok || ts == nil { + return + } + + ts.locked = true +} + +// UnlockTileset removes the write mutex on the tileset. +// This is ignored if the tileset does not exist. +func (s *ServiceSet) UnlockTileset(id string) { + ts, ok := s.tilesets[id] + if !ok || ts == nil { + return + } + + ts.locked = false +} + // HasTileset returns true if the tileset identified by id exists within this // ServiceSet. func (s *ServiceSet) HasTileset(id string) bool { diff --git a/handlers/tileset.go b/handlers/tileset.go index ce51d5b..41b8fff 100644 --- a/handlers/tileset.go +++ b/handlers/tileset.go @@ -2,11 +2,14 @@ package handlers import ( "encoding/json" + "errors" "fmt" "html/template" "net/http" "path/filepath" "strings" + "syscall" + "time" mbtiles "github.com/brendan-ward/mbtiles-go" ) @@ -19,6 +22,7 @@ type Tileset struct { name string tileformat mbtiles.TileFormat published bool + locked bool router *http.ServeMux } @@ -97,7 +101,7 @@ func (ts *Tileset) reload() error { return nil } -// Delete closes and deletes the mbtiles file for this tileset +// Delete closes and deletes the mbtiles file connection for this tileset func (ts *Tileset) delete() error { if ts.db != nil { ts.db.Close() @@ -164,6 +168,12 @@ func (ts *Tileset) tileJSONHandler(w http.ResponseWriter, r *http.Request) { return } + // wait up to 30 seconds to see if tileset is ready and return it if possible + if ts.isLockedWithTimeout(30 * time.Second) { + tilesetLockedHandler(w, r) + return + } + query := "" if r.URL.RawQuery != "" { query = "?" + r.URL.RawQuery @@ -201,6 +211,12 @@ func (ts *Tileset) tileHandler(w http.ResponseWriter, r *http.Request) { return } + // wait up to 30 seconds to see if tileset is ready and return it if possible + if ts.isLockedWithTimeout(30 * time.Second) { + tilesetLockedHandler(w, r) + return + } + db := ts.db // split path components to extract tile coordinates x, y and z pcs := strings.Split(r.URL.Path[1:], "/") @@ -238,8 +254,9 @@ func (ts *Tileset) tileHandler(w http.ResponseWriter, r *http.Request) { _, err = w.Write(data) - if err != nil { + if err != nil && !errors.Is(err, syscall.EPIPE) { ts.svc.logError("Could not write tile data for %v: %v", r.URL.Path, err) + } } @@ -252,6 +269,12 @@ func (ts *Tileset) previewHandler(w http.ResponseWriter, r *http.Request) { return } + // wait up to 30 seconds to see if tileset is ready and return it if possible + if ts.isLockedWithTimeout(30 * time.Second) { + tilesetLockedHandler(w, r) + return + } + query := "" if r.URL.RawQuery != "" { query = "?" + r.URL.RawQuery @@ -303,3 +326,32 @@ func tileNotFoundHandler(w http.ResponseWriter, r *http.Request, f mbtiles.TileF fmt.Fprint(w, `{"message": "Tile does not exist"}`) } } + +// tilesetLockedHandler returns a 503 Service Unavailable response when +// requests are made to a tileset that is beging updated +func tilesetLockedHandler(w http.ResponseWriter, r *http.Request) { + // send back service unavailable response with header to retry in 10 seconds + w.Header().Set("Retry-After", "10") + http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable) +} + +func (ts *Tileset) isLockedWithTimeout(timeout time.Duration) bool { + if ts == nil || !ts.locked { + return false + } + + timeoutReached := time.After(timeout) + // poll locked status every 500 ms + ticker := time.Tick(500 * time.Millisecond) + for { + select { + case <-timeoutReached: + return ts.locked + case <-ticker: + if !ts.locked { + return false + } + // otherwise, still locked + } + } +} diff --git a/main.go b/main.go index 8eaa0f8..6520317 100644 --- a/main.go +++ b/main.go @@ -61,24 +61,25 @@ var rootCmd = &cobra.Command{ } var ( - port int - tilePath string - certificate string - privateKey string - rootURLStr string - domain string - secretKey string - sentryDSN string - verbose bool - autotls bool - redirect bool - enableReloadSignal bool - generateIDs bool - enableArcGIS bool - disablePreview bool - disableTileJSON bool - disableServiceList bool - tilesOnly bool + port int + tilePath string + certificate string + privateKey string + rootURLStr string + domain string + secretKey string + sentryDSN string + verbose bool + autotls bool + redirect bool + enableReloadSignal bool + enableReloadFSWatch bool + generateIDs bool + enableArcGIS bool + disablePreview bool + disableTileJSON bool + disableServiceList bool + tilesOnly bool ) func init() { @@ -95,6 +96,7 @@ func init() { flags.BoolVarP(&redirect, "redirect", "r", false, "Redirect HTTP to HTTPS") flags.BoolVarP(&enableArcGIS, "enable-arcgis", "", false, "Enable ArcGIS Mapserver endpoints") + flags.BoolVarP(&enableReloadFSWatch, "enable-fs-watch", "", false, "Enable reloading of tilesets by watching filesystem") flags.BoolVarP(&enableReloadSignal, "enable-reload-signal", "", false, "Enable graceful reload using HUP signal to the server process") flags.BoolVarP(&disablePreview, "disable-preview", "", false, "Disable map preview for each tileset (enabled by default)") @@ -172,6 +174,22 @@ func init() { enableArcGIS = p } + if env := os.Getenv("ENABLE_FS_WATCH"); env != "" { + p, err := strconv.ParseBool(env) + if err != nil { + log.Fatalln("ENABLE_FS_WATCH must be a bool(true/false)") + } + enableReloadFSWatch = p + } + + if env := os.Getenv("ENABLE_RELOAD_SIGNAL"); env != "" { + p, err := strconv.ParseBool(env) + if err != nil { + log.Fatalln("ENABLE_RELOAD_SIGNAL must be a bool(true/false)") + } + enableReloadSignal = p + } + if env := os.Getenv("VERBOSE"); env != "" { p, err := strconv.ParseBool(env) if err != nil { @@ -249,6 +267,14 @@ func serve() { log.Infoln("An HMAC request authorization key was set. All incoming must be signed.") } + generateID := func(filename string, baseDir string) (string, error) { + if generateIDs { + return handlers.SHA1ID(filename), nil + } else { + return handlers.RelativePathID(filename, baseDir) + } + } + svcSet, err := handlers.New(&handlers.ServiceSetConfig{ RootURL: rootURL, ErrorWriter: &errorLogger{log: log.New()}, @@ -274,16 +300,10 @@ func serve() { // Register all tilesets for _, filename := range filenames { - var id string - var err error - if generateIDs { - id = handlers.SHA1ID(filename) - } else { - id, err = handlers.RelativePathID(filename, path) - if err != nil { - log.Errorf("Could not generate ID for tileset: %q", filename) - continue - } + id, err := generateID(filename, path) + if err != nil { + log.Errorf("Could not generate ID for tileset: %q", filename) + continue } err = svcSet.AddTileset(filename, id) @@ -296,6 +316,25 @@ func serve() { // print number of services log.Infof("Published %v services", svcSet.Size()) + // watch filesystem for changes to tilesets + if enableReloadFSWatch { + watcher, err := NewFSWatcher(svcSet, generateID) + if err != nil { + log.Fatalln("Could not construct filesystem watcher") + } + defer watcher.Close() + + for _, path := range strings.Split(tilePath, ",") { + log.Infof("Watching %v\n", path) + err = watcher.WatchDir((path)) + if err != nil { + // If we cannot enable file watching, then this should be a fatal + // error during server startup + log.Fatalln("Could not enable filesystem watcher in", path, err) + } + } + } + e := echo.New() e.HideBanner = true e.Pre(middleware.RemoveTrailingSlash()) diff --git a/watch.go b/watch.go new file mode 100644 index 0000000..f3d28bd --- /dev/null +++ b/watch.go @@ -0,0 +1,204 @@ +package main + +import ( + "os" + "path/filepath" + "time" + + log "github.com/sirupsen/logrus" + + mbtiles "github.com/brendan-ward/mbtiles-go" + "github.com/consbio/mbtileserver/handlers" + "github.com/fsnotify/fsnotify" +) + +// debounce debounces requests to a callback function to occur no more +// frequently than interval; once this is reached, the callback is called. +// +// Unique values sent to the channel are stored in an internal map and all +// are processed once the the interval is up. +func debounce(interval time.Duration, input chan string, firstCallback func(arg string), callback func(arg string)) { + // keep a log of unique paths + var items = make(map[string]bool) + var item string + timer := time.NewTimer(interval) + for { + select { + case item = <-input: + if _, ok := items[item]; !ok { + // first time we see a given path, we need to call lockHandler + // to lock it (unlocked by callback) + firstCallback(item) + } + items[item] = true + timer.Reset(interval) + case <-timer.C: + for path := range items { + callback(path) + delete(items, path) + } + } + } +} + +// FSWatcher provides a filesystem watcher to detect when mbtiles files are +// created, updated, or removed on the filesystem. +type FSWatcher struct { + watcher *fsnotify.Watcher + svcSet *handlers.ServiceSet + generateID handlers.IDGenerator +} + +// NewFSWatcher creates a new FSWatcher to watch the filesystem for changes to +// mbtiles files and updates the ServiceSet accordingly. +// +// The generateID function needs to be of the same type used when the tilesets +// were originally added to the ServiceSet. +func NewFSWatcher(svcSet *handlers.ServiceSet, generateID handlers.IDGenerator) (*FSWatcher, error) { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + return &FSWatcher{ + watcher: watcher, + svcSet: svcSet, + generateID: generateID, + }, nil +} + +// Close closes the FSWatcher and stops watching the filesystem. +func (w *FSWatcher) Close() { + if w.watcher != nil { + w.watcher.Close() + } +} + +// WatchDir sets up the filesystem watcher for baseDir and all existing subdirectories +func (w *FSWatcher) WatchDir(baseDir string) error { + c := make(chan string) + + // debounced call to create / update tileset + go debounce(500*time.Millisecond, c, func(path string) { + // callback for first time path is debounced + id, err := w.generateID(path, baseDir) + if err != nil { + log.Errorf("Could not create ID for tileset %q\n%v", path, err) + return + } + // lock tileset for writing, if it exists + w.svcSet.LockTileset(id) + }, func(path string) { + // callback after debouncing incoming requests + + // Verify that file can be opened with mbtiles-go, which runs + // validation on open. + // If file cannot be opened, assume it is still being written / copied. + db, err := mbtiles.Open(path) + if err != nil { + return + } + db.Close() + + // determine file ID for tileset + id, err := w.generateID(path, baseDir) + if err != nil { + log.Errorf("Could not create ID for tileset %q\n%v", path, err) + return + } + + // update existing tileset + if w.svcSet.HasTileset(id) { + err = w.svcSet.UpdateTileset(id) + if err != nil { + log.Errorf("Could not update tileset %q with ID %q\n%v", path, id, err) + } else { + // only unlock if successfully updated + w.svcSet.UnlockTileset(id) + log.Infof("Updated tileset %q with ID %q\n", path, id) + } + return + } + + // create new tileset + err = w.svcSet.AddTileset(path, id) + if err != nil { + log.Errorf("Could not add tileset for %q with ID %q\n%v", path, id, err) + } else { + log.Infof("Updated tileset %q with ID %q\n", path, id) + } + return + }) + + go func() { + for { + select { + case event, ok := <-w.watcher.Events: + if !ok { + log.Errorf("error in filewatcher for %q, exiting filewatcher", event.Name) + return + } + + if !((event.Op&fsnotify.Write == fsnotify.Write) || + (event.Op&fsnotify.Remove == fsnotify.Remove) || + (event.Op&fsnotify.Rename == fsnotify.Rename)) { + continue + } + + path := event.Name + + if ext := filepath.Ext(path); ext != ".mbtiles" { + continue + } + + if _, err := os.Stat(path + "-journal"); err == nil { + // Don't try to load .mbtiles files that are being written + log.Debugf("Tileset %q is currently being created or is incomplete\n", path) + continue + } + + if event.Op&fsnotify.Write == fsnotify.Write { + // This event may get called multiple times while a file is being copied into a watched directory, + // so we debounce this instead. + c <- path + } + + if (event.Op&fsnotify.Remove == fsnotify.Remove) || (event.Op&fsnotify.Rename == fsnotify.Rename) { + // remove tileset immediately so that there are not other errors in request handlers + id, err := w.generateID(path, baseDir) + if err != nil { + log.Errorf("Could not create ID for tileset %q\n%v", path, err) + } + err = w.svcSet.RemoveTileset(id) + if err != nil { + log.Errorf("Could not remove tileset %q with ID %q\n%v", path, id, err) + } else { + log.Infof("Removed tileset %q with ID %q\n", path, id) + } + } + + case err, ok := <-w.watcher.Errors: + if !ok { + log.Errorf("error in filewatcher, exiting filewatcher") + return + } + log.Error(err) + } + } + }() + + err := filepath.Walk(baseDir, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.Mode().IsDir() { + return w.watcher.Add(path) + } + return nil + }) + if err != nil { + return err + } + + return nil +}