implement lru cache for tiles

This commit is contained in:
Stepan Kuzmin 2018-01-26 12:45:09 +03:00
parent 935b4f6a32
commit c6043d6b6d
5 changed files with 44 additions and 6 deletions

7
Cargo.lock generated
View File

@ -305,6 +305,11 @@ dependencies = [
"time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "lru"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "mapbox_expressions_to_sql" name = "mapbox_expressions_to_sql"
version = "0.1.0" version = "0.1.0"
@ -324,6 +329,7 @@ dependencies = [
"lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"logger 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "logger 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"lru 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
"mapbox_expressions_to_sql 0.1.0 (git+https://github.com/stepankuzmin/rust-mapbox-expressions-to-sql)", "mapbox_expressions_to_sql 0.1.0 (git+https://github.com/stepankuzmin/rust-mapbox-expressions-to-sql)",
"persistent 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "persistent 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"postgres 0.15.1 (registry+https://github.com/rust-lang/crates.io-index)", "postgres 0.15.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -933,6 +939,7 @@ dependencies = [
"checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" "checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b"
"checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2" "checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2"
"checksum logger 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6c9172cb4c2f6c52117e25570983edcbb322f130b1031ae5d5d6b1abe7eeb493" "checksum logger 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6c9172cb4c2f6c52117e25570983edcbb322f130b1031ae5d5d6b1abe7eeb493"
"checksum lru 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f1422de50f19f1e08676a563ba667974c0fe487a5c02eea23890983997c0184a"
"checksum mapbox_expressions_to_sql 0.1.0 (git+https://github.com/stepankuzmin/rust-mapbox-expressions-to-sql)" = "<none>" "checksum mapbox_expressions_to_sql 0.1.0 (git+https://github.com/stepankuzmin/rust-mapbox-expressions-to-sql)" = "<none>"
"checksum matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "100aabe6b8ff4e4a7e32c1c13523379802df0772b82466207ac25b013f193376" "checksum matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "100aabe6b8ff4e4a7e32c1c13523379802df0772b82466207ac25b013f193376"
"checksum md5 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "b6d9aab58e540f50b59d5cfa7f0da4c3d437476890e1e0b6206e230dce55a23c" "checksum md5 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "b6d9aab58e540f50b59d5cfa7f0da4c3d437476890e1e0b6206e230dce55a23c"

View File

@ -18,6 +18,7 @@ iron-test = "0.6.0"
lazy_static = "0.2.11" lazy_static = "0.2.11"
log = "0.4.1" log = "0.4.1"
logger = "0.4.0" logger = "0.4.0"
lru = "0.1.7"
mapbox_expressions_to_sql = { git = "https://github.com/stepankuzmin/rust-mapbox-expressions-to-sql" } mapbox_expressions_to_sql = { git = "https://github.com/stepankuzmin/rust-mapbox-expressions-to-sql" }
persistent = "0.4.0" persistent = "0.4.0"
postgres = { version = "0.15", features = ["with-time", "with-uuid", "with-serde_json"] } postgres = { version = "0.15", features = ["with-time", "with-uuid", "with-serde_json"] }

View File

@ -12,7 +12,11 @@ fn main() {
let conn_string: String = env::var("DATABASE_URL") let conn_string: String = env::var("DATABASE_URL")
.expect("DATABASE_URL must be set"); .expect("DATABASE_URL must be set");
let chain = martin_lib::chain(conn_string); let cache_size = env::var("CACHE_SIZE").ok()
.and_then(|cache_size| cache_size.parse::<usize>().ok())
.unwrap_or(16384);
let chain = martin_lib::chain(conn_string, cache_size);
let port = 3000; let port = 3000;
let bind_addr = format!("0.0.0.0:{}", port); let bind_addr = format!("0.0.0.0:{}", port);

View File

@ -1,6 +1,7 @@
extern crate iron_test; extern crate iron_test;
extern crate iron; extern crate iron;
extern crate logger; extern crate logger;
extern crate lru;
extern crate mapbox_expressions_to_sql; extern crate mapbox_expressions_to_sql;
extern crate persistent; extern crate persistent;
extern crate r2d2_postgres; extern crate r2d2_postgres;
@ -16,15 +17,17 @@ extern crate urlencoded;
use iron::prelude::Chain; use iron::prelude::Chain;
use logger::Logger; use logger::Logger;
use persistent::Read; use lru::LruCache;
use persistent::{Read, State};
use rererouter::RouterBuilder; use rererouter::RouterBuilder;
mod cache;
mod cors; mod cors;
mod db; mod db;
mod routes; mod routes;
mod tileset; mod tileset;
pub fn chain(conn_string: String) -> iron::Chain { pub fn chain(conn_string: String, cache_size: usize) -> iron::Chain {
let mut router_builder = RouterBuilder::new(); let mut router_builder = RouterBuilder::new();
router_builder.get(r"/index.json", routes::index); router_builder.get(r"/index.json", routes::index);
router_builder.get(r"/(?P<tileset>[\w|\.]*)\.json", routes::tileset); router_builder.get(r"/(?P<tileset>[\w|\.]*)\.json", routes::tileset);
@ -32,7 +35,7 @@ pub fn chain(conn_string: String) -> iron::Chain {
let router = router_builder.finalize(); let router = router_builder.finalize();
let mut chain = Chain::new(router); let mut chain = Chain::new(router);
let (logger_before, logger_after) = Logger::new(None); let (logger_before, logger_after) = Logger::new(None);
chain.link_before(logger_before); chain.link_before(logger_before);
@ -51,6 +54,9 @@ pub fn chain(conn_string: String) -> iron::Chain {
} }
}; };
let tile_cache = LruCache::new(cache_size);
chain.link(State::<cache::TileCache>::both(tile_cache));
chain.link_after(cors::Middleware); chain.link_after(cors::Middleware);
chain.link_after(logger_after); chain.link_after(logger_after);

View File

@ -1,13 +1,15 @@
use iron::{status, mime, Request, Response, IronResult}; use iron::{status, mime, Request, Response, IronResult};
use iron::headers::{Headers, parsing}; use iron::headers::{Headers, parsing};
use iron::prelude::{Plugin}; use iron::prelude::{Plugin};
use iron::url::Url;
use mapbox_expressions_to_sql; use mapbox_expressions_to_sql;
use persistent::Read; use persistent::{Read, State};
use regex::{Regex, Captures}; use regex::{Regex, Captures};
use serde_json; use serde_json;
use urlencoded::UrlEncodedQuery; use urlencoded::UrlEncodedQuery;
use super::db; use super::db;
use super::cache;
use super::tileset; use super::tileset;
use tilejson::TileJSONBuilder; use tilejson::TileJSONBuilder;
@ -72,6 +74,22 @@ fn get_filter<'a>(req: &'a mut Request) -> Option<&'a String> {
} }
pub fn tile(req: &mut Request, caps: Captures) -> IronResult<Response> { pub fn tile(req: &mut Request, caps: Captures) -> IronResult<Response> {
let url: Url = req.url.clone().into();
let lock = req.get::<State<cache::TileCache>>().unwrap();
let cached_tile = lock.write().ok().and_then(|mut guard|
guard.get(&url).cloned()
);
let content_type = "application/x-protobuf".parse::<mime::Mime>().unwrap();
if let Some(tile) = cached_tile {
debug!("{} hit", url);
return match tile.len() {
0 => Ok(Response::with((content_type, status::NoContent))),
_ => Ok(Response::with((content_type, status::Ok, tile)))
}
};
debug!("{} miss", url);
let tilesets = req.get::<Read<tileset::Tilesets>>().unwrap(); let tilesets = req.get::<Read<tileset::Tilesets>>().unwrap();
let tileset = match tilesets.get(&caps["tileset"]) { let tileset = match tilesets.get(&caps["tileset"]) {
Some(tileset) => tileset, Some(tileset) => tileset,
@ -112,7 +130,9 @@ pub fn tile(req: &mut Request, caps: Captures) -> IronResult<Response> {
} }
}; };
let content_type = "application/x-protobuf".parse::<mime::Mime>().unwrap(); let mut guard = lock.write().unwrap();
guard.put(req.url.clone().into(), tile.clone());
match tile.len() { match tile.len() {
0 => Ok(Response::with((content_type, status::NoContent))), 0 => Ok(Response::with((content_type, status::NoContent))),
_ => Ok(Response::with((content_type, status::Ok, tile))) _ => Ok(Response::with((content_type, status::Ok, tile)))