Add mechanism to configure cachelib process size shrinker

Summary:
We'll be running in Tupperware, and want to shrink when we get too
large to avoid OOM due to caches. Configure cachelib appropriately

Reviewed By: StanislavGlebik

Differential Revision: D8900371

fbshipit-source-id: 4f1f64c2508c64e4ce2d201e0a0e86446f84ffef

apiserver/src/main.rs

@@ -17,6 +17,7 @@ extern crate chrono;
 extern crate clap;
 #[macro_use]
 extern crate cloned;
+extern crate cmdlib;
 extern crate failure_ext as failure;
 extern crate futures;
 extern crate futures_ext;
@@ -223,88 +224,76 @@ struct HttpServerState {
 }
 
 fn main() -> Result<()> {
-    let matches = clap::App::new("Mononoke API Server")
-        .version("0.0.1")
-        .about("An API server serves requests for Mononoke")
-        .arg(
-            Arg::with_name("http-host")
-                .short("H")
-                .long("http-host")
-                .value_name("HOST")
-                .default_value("127.0.0.1")
-                .help("HTTP host to listen to"),
-        )
-        .arg(
-            Arg::with_name("http-port")
-                .short("p")
-                .long("http-port")
-                .value_name("PORT")
-                .default_value("8000")
-                .help("HTTP port to listen to"),
-        )
-        .arg(Arg::with_name("with-scuba").long("with-scuba"))
-        .arg(Arg::with_name("debug").short("p").long("debug"))
-        .arg(
-            Arg::with_name("stdlog")
-                .long("stdlog")
-                .help("print logs from third-party crates"),
-        )
-        .arg(
-            Arg::with_name("config-path")
-                .long("config-path")
-                .value_name("PATH")
-                .required(true)
-                .help("directory of the config repository"),
-        )
-        .arg(
-            Arg::with_name("config-bookmark")
-                .long("config-bookmark")
-                .value_name("BOOKMARK")
-                .required_unless("config-commit")
-                .help("bookmark of the config repository"),
-        )
-        .arg(
-            Arg::with_name("config-commit")
-                .long("config-commit")
-                .value_name("HASH")
-                .required_unless("config-bookmark")
-                .help("commit hash of the config repository"),
-        )
-        .arg(
-            Arg::with_name("ssl-certificate")
-                .long("ssl-certificate")
-                .value_name("PATH")
-                .help("path to the ssl certificate file"),
-        )
-        .arg(
-            Arg::with_name("ssl-private-key")
-                .long("ssl-private-key")
-                .value_name("PATH")
-                .help("path to the ssl private key file")
-                .requires("ssl-ca"),
-        )
-        .arg(
-            Arg::with_name("ssl-ca")
-                .long("ssl-ca")
-                .value_name("PATH")
-                .help("path to the ssl ca file"),
-        )
-        .arg(Arg::from_usage(
-            "--cache-size-gb [SIZE] 'size of the cachelib cache, in GiB'",
-        ))
-        .get_matches();
+    let matches = cmdlib::args::add_cachelib_args(
+        clap::App::new("Mononoke API Server")
+            .version("0.0.1")
+            .about("An API server serves requests for Mononoke")
+            .arg(
+                Arg::with_name("http-host")
+                    .short("H")
+                    .long("http-host")
+                    .value_name("HOST")
+                    .default_value("127.0.0.1")
+                    .help("HTTP host to listen to"),
+            )
+            .arg(
+                Arg::with_name("http-port")
+                    .short("p")
+                    .long("http-port")
+                    .value_name("PORT")
+                    .default_value("8000")
+                    .help("HTTP port to listen to"),
+            )
+            .arg(Arg::with_name("with-scuba").long("with-scuba"))
+            .arg(Arg::with_name("debug").short("p").long("debug"))
+            .arg(
+                Arg::with_name("stdlog")
+                    .long("stdlog")
+                    .help("print logs from third-party crates"),
+            )
+            .arg(
+                Arg::with_name("config-path")
+                    .long("config-path")
+                    .value_name("PATH")
+                    .required(true)
+                    .help("directory of the config repository"),
+            )
+            .arg(
+                Arg::with_name("config-bookmark")
+                    .long("config-bookmark")
+                    .value_name("BOOKMARK")
+                    .required_unless("config-commit")
+                    .help("bookmark of the config repository"),
+            )
+            .arg(
+                Arg::with_name("config-commit")
+                    .long("config-commit")
+                    .value_name("HASH")
+                    .required_unless("config-bookmark")
+                    .help("commit hash of the config repository"),
+            )
+            .arg(
+                Arg::with_name("ssl-certificate")
+                    .long("ssl-certificate")
+                    .value_name("PATH")
+                    .help("path to the ssl certificate file"),
+            )
+            .arg(
+                Arg::with_name("ssl-private-key")
+                    .long("ssl-private-key")
+                    .value_name("PATH")
+                    .help("path to the ssl private key file")
+                    .requires("ssl-ca"),
+            )
+            .arg(
+                Arg::with_name("ssl-ca")
+                    .long("ssl-ca")
+                    .value_name("PATH")
+                    .help("path to the ssl ca file"),
+            ),
+    ).get_matches();
 
-    let cache_size = matches
-        .value_of("cache-size-gb")
-        .unwrap_or("20")
-        .parse::<usize>()
-        .unwrap() * 1024 * 1024 * 1024;
-
-    cachelib::init_cache_once(cachelib::LruCacheConfig::new(cache_size)).unwrap();
-
-    cachelib::get_or_create_pool("blobstore-blobs", cachelib::get_available_space() * 3 / 4)
-        .unwrap();
-    cachelib::get_or_create_pool("blobstore-presence", cachelib::get_available_space()).unwrap();
+    cmdlib::args::init_cachelib(&matches);
 
     let host = matches.value_of("http-host").unwrap_or("127.0.0.1");
     let port = matches.value_of("http-port").unwrap_or("8000");

cmdlib/src/args.rs

@@ -6,6 +6,7 @@
 
 use std::fs;
 use std::path::Path;
+use std::time::Duration;
 
 use clap::{App, Arg, ArgMatches};
 use failure::{Result, ResultExt};
@@ -127,10 +128,9 @@ impl MononokeApp {
                     .default_value("5")
                     .hidden(hide_advanced_args)
                     .help("maximum open requests per Manifold IO thread")
-            )
-            .arg(Arg::from_usage(
-                    "--cache-size-gb [SIZE] 'size of the cachelib cache, in GiB'",
-            ));
+            );
+
+        app = add_cachelib_args(app);
 
         if self.local_instances {
             app = app.arg(
@@ -233,15 +233,60 @@ pub fn setup_blobrepo_dir<P: AsRef<Path>>(data_dir: P, create: bool) -> Result<(
     Ok(())
 }
 
+pub fn add_cachelib_args<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> {
+    app.arg(Arg::from_usage(
+            "--cache-size-gb [SIZE] 'size of the cachelib cache, in GiB'",
+    ))
+    .arg(Arg::from_usage(
+            "--max-process-size [SIZE] 'process size at which cachelib will shrink, in GiB'"
+    ))
+    .arg(Arg::from_usage(
+            "--min-process-size [SIZE] 'process size at which cachelib will grow back to cache-size-gb, in GiB'"
+    ))
+}
+
 pub fn init_cachelib<'a>(matches: &ArgMatches<'a>) {
     let cache_size = matches
         .value_of("cache-size-gb")
         .unwrap_or("20")
         .parse::<usize>()
         .unwrap() * 1024 * 1024 * 1024;
+    let max_process_size_gib = matches
+        .value_of("max-process-size")
+        .map(str::parse::<usize>)
+        // This is a fixed-point calculation. The process can grow to 1.2 times the size of the
+        // cache before the cache shrinks, but cachelib is inconsistent and wants cache size in
+        // bytes but process sizes in GiB
+        .unwrap_or(Ok(cache_size * 12 / (10 * 1024 * 1024 * 1024)))
+        .unwrap() as u32;
+    let min_process_size_gib = matches
+        .value_of("min-process-size")
+        .map(str::parse::<usize>)
+        // This is a fixed-point calculation. The process can fall to 0.8 times the size of the
+        // cache before the cache will regrow to target size, but cachelib is inconsistent
+        // and wants cache size in bytes but process sizes in GiB
+        .unwrap_or(Ok(cache_size * 8 / (10 * 1024 * 1024 * 1024)))
+        .unwrap() as u32;
 
-    cachelib::init_cache_once(cachelib::LruCacheConfig::new(cache_size)).unwrap();
-
+    cachelib::init_cache_once(cachelib::LruCacheConfig::new(cache_size).set_shrinker(
+        cachelib::ShrinkMonitor {
+            shrinker_type: cachelib::ShrinkMonitorType::ResidentSize {
+                max_process_size_gib: max_process_size_gib,
+                min_process_size_gib: min_process_size_gib,
+            },
+            interval: Duration::new(10, 0),
+            max_resize_per_iteration_percent: 25,
+            max_removed_percent: 50,
+            strategy: cachelib::RebalanceStrategy::HitsPerSlab {
+                // A small increase in hit ratio is desired
+                diff_ratio: 0.05,
+                min_retained_slabs: 1,
+                // Objects newer than 30 seconds old might be about to become interesting
+                min_tail_age: Duration::new(30, 0),
+                ignore_untouched_slabs: false,
+            },
+        },
+    )).unwrap();
     cachelib::get_or_create_pool("blobstore-blobs", cachelib::get_available_space() * 3 / 4)
         .unwrap();
     cachelib::get_or_create_pool("blobstore-presence", cachelib::get_available_space()).unwrap();

cmds/admin/main.rs

@@ -307,6 +307,8 @@ fn main() {
             let rev = sub_m.value_of("CHANGESET_ID").unwrap();
             let path = sub_m.value_of("PATH").unwrap();
 
+            args::init_cachelib(&matches);
+
             let repo = args::open_blobrepo(&logger, &matches);
             fetch_content(logger.clone(), Arc::new(repo), rev, path)
                 .and_then(|content| {

cmds/blobimport.rs

@@ -58,6 +58,7 @@ fn main() -> Result<()> {
 
     let logger = args::get_logger(&matches);
 
+    args::init_cachelib(&matches);
     let blobrepo = Arc::new(args::create_blobrepo(&logger, &matches));
 
     let revlogrepo_path = matches

cmds/runhook.rs

@@ -17,6 +17,7 @@ extern crate blobrepo;
 extern crate blobstore;
 extern crate cachelib;
 extern crate clap;
+extern crate cmdlib;
 extern crate failure_ext as failure;
 extern crate futures;
 #[macro_use]
@@ -62,16 +63,19 @@ fn run_hook(
     repo_creator: fn(&Logger, &ArgMatches) -> BlobRepo,
 ) -> BoxFuture<HookExecution, Error> {
     // Define command line args and parse command line
-    let matches = App::new("runhook")
-        .version("0.0.0")
-        .about("run a hook")
-        .args_from_usage(concat!(
-            "<REPO_NAME>           'name of repository\n",
-            "<HOOK_FILE>           'file containing hook code\n",
-            "<HOOK_TYPE>           'the type of the hook (perfile, percs)\n",
-            "<REV>                 'revision hash'"
-        ))
-        .get_matches_from(args);
+    let matches = cmdlib::args::add_cachelib_args(
+        App::new("runhook")
+            .version("0.0.0")
+            .about("run a hook")
+            .args_from_usage(concat!(
+                "<REPO_NAME>               'name of repository\n",
+                "<HOOK_FILE>               'file containing hook code\n",
+                "<HOOK_TYPE>               'the type of the hook (perfile, percs)\n",
+                "<REV>                     'revision hash'\n",
+            )),
+    ).get_matches_from(args);
+
+    cmdlib::args::init_cachelib(&matches);
 
     let logger = {
         let level = if matches.is_present("debug") {

server/src/main.rs

@@ -26,6 +26,7 @@ extern crate tracing_fb303;
 extern crate blobrepo;
 extern crate bookmarks;
 extern crate cachelib;
+extern crate cmdlib;
 extern crate mercurial_types;
 extern crate metaconfig;
 extern crate ready_state;
@@ -63,7 +64,7 @@ fn setup_panic_hook() {
 }
 
 fn setup_app<'a, 'b>() -> App<'a, 'b> {
-    App::new("mononoke server")
+    cmdlib::args::add_cachelib_args(App::new("mononoke server")
         .version("0.0.0")
         .about("serve repos")
         .args_from_usage(
@@ -83,9 +84,8 @@ fn setup_app<'a, 'b>() -> App<'a, 'b> {
             <ca_pem>      --ca-pem [PATH]                       'path to a file with CA certificate'
 
             -d, --debug                                          'print debug level output'
-            --cache-size-gb [SIZE]                                  'size of the cachelib cache, in GiB'
-        "#,
-        )
+            "#,
+        ))
 }
 
 fn setup_logger<'a>(matches: &ArgMatches<'a>) -> Logger {
@@ -153,16 +153,7 @@ fn main() {
     let matches = setup_app().get_matches();
     let root_log = setup_logger(&matches);
 
-    let cache_size = matches
-        .value_of("cache-size-gb")
-        .unwrap_or("20")
-        .parse::<usize>()
-        .unwrap() * 1024 * 1024 * 1024;
-    cachelib::init_cache_once(cachelib::LruCacheConfig::new(cache_size)).unwrap();
-
-    cachelib::get_or_create_pool("blobstore-blobs", cachelib::get_available_space() * 3 / 4)
-        .unwrap();
-    cachelib::get_or_create_pool("blobstore-presence", cachelib::get_available_space()).unwrap();
+    cmdlib::args::init_cachelib(&matches);
 
     fn run_server<'a>(root_log: &Logger, matches: ArgMatches<'a>) -> Result<!> {
         info!(root_log, "Starting up");