mononoke: move expected_item_size_byte into CachelibSettings

Summary:
Move expected_item_size_byte into CachelibSettings, seems like it should be there.

To enable its use also exposes a parse_and_init_cachelib method for callers that have different defaults to default cachelibe settings.

Reviewed By: krallin

Differential Revision: D24761024

fbshipit-source-id: 440082ab77b5b9f879c99b8f764e71bec68d270e
This commit is contained in:
Alex Hornby 2020-12-02 00:45:53 -08:00 committed by Facebook GitHub Bot
parent 687fddca2f
commit f077f69408
44 changed files with 64 additions and 60 deletions

View File

@ -159,7 +159,7 @@ fn main(fb: FacebookInit) -> Result<()> {
)
.get_matches();
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let logger = args::init_logging(fb, &matches);
let ctx = CoreContext::new_with_logger(fb, logger.clone());
let repo = new_benchmark_repo(fb, Default::default())?;

View File

@ -1247,7 +1247,7 @@ async fn save_reproducibility_under_load(fb: FacebookInit) -> Result<(), Error>
db_put_dist: Normal::new(0.002, 0.001).expect("Normal::new failed"),
db_get_dist: Normal::new(0.002, 0.001).expect("Normal::new failed"),
};
cmdlib::args::init_cachelib(fb, &Default::default(), None);
cmdlib::args::init_cachelib(fb, &Default::default());
let repo = new_benchmark_repo(fb, delay_settings)?;
let mut rng = XorShiftRng::seed_from_u64(1);

View File

@ -140,16 +140,21 @@ pub(crate) fn parse_caching<'a>(matches: &ArgMatches<'a>) -> Caching {
}
}
pub fn init_cachelib<'a>(
/// Usual entry point where binary is happy with CachelibSettings::default()
pub fn init_cachelib<'a>(fb: FacebookInit, matches: &ArgMatches<'a>) -> Caching {
parse_and_init_cachelib(fb, matches, CachelibSettings::default())
}
/// Provide a way for binaries to specify if they have different default cachelib settings
pub fn parse_and_init_cachelib<'a>(
fb: FacebookInit,
matches: &ArgMatches<'a>,
expected_item_size_bytes: Option<usize>,
mut settings: CachelibSettings,
) -> Caching {
let caching = parse_caching(matches);
match caching {
Caching::Enabled(..) | Caching::CachelibOnlyBlobstore(..) => {
let mut settings = CachelibSettings::default();
if let Some(cache_size) = matches.value_of(CACHE_SIZE_GB) {
settings.cache_size =
(cache_size.parse::<f64>().unwrap() * (1024 * 1024 * 1024) as f64) as usize;
@ -191,17 +196,12 @@ pub fn init_cachelib<'a>(
#[cfg(not(fbcode_build))]
{
let _ = (fb, expected_item_size_bytes);
let _ = fb;
unimplemented!("Initialization of cachelib works only for fbcode builds")
}
#[cfg(fbcode_build)]
{
super::facebook::init_cachelib_from_settings(
fb,
settings,
expected_item_size_bytes,
)
.unwrap();
super::facebook::init_cachelib_from_settings(fb, settings).unwrap();
}
}
Caching::Disabled => {
@ -212,7 +212,7 @@ pub fn init_cachelib<'a>(
caching
}
pub(crate) struct CachelibSettings {
pub struct CachelibSettings {
pub cache_size: usize,
pub max_process_size_gib: Option<u32>,
pub min_process_size_gib: Option<u32>,
@ -225,6 +225,7 @@ pub(crate) struct CachelibSettings {
pub idmapping_cache_size: Option<usize>,
pub blob_cache_size: Option<usize>,
pub phases_cache_size: Option<usize>,
pub expected_item_size_bytes: Option<usize>,
}
impl Default for CachelibSettings {
@ -242,6 +243,7 @@ impl Default for CachelibSettings {
idmapping_cache_size: None,
blob_cache_size: None,
phases_cache_size: None,
expected_item_size_bytes: None,
}
}
}

View File

@ -9,7 +9,7 @@ mod cache;
#[cfg(fbcode_build)]
mod facebook;
pub use self::cache::init_cachelib;
pub use self::cache::{init_cachelib, parse_and_init_cachelib, CachelibSettings};
use std::collections::{HashMap, HashSet};
use std::future::Future;
@ -1369,7 +1369,9 @@ pub fn init_mononoke<'a>(
let logger = init_logging(fb, matches);
debug!(logger, "Initialising cachelib...");
let caching = init_cachelib(fb, matches, expected_item_size_bytes);
let mut settings = CachelibSettings::default();
settings.expected_item_size_bytes = expected_item_size_bytes;
let caching = parse_and_init_cachelib(fb, matches, settings);
debug!(logger, "Initialising runtime...");
let runtime = init_runtime(matches)?;
init_tunables(fb, matches, logger.clone())?;

View File

@ -37,7 +37,7 @@ pub async fn subcommand_bonsai_fetch<'a>(
) -> Result<(), SubcommandError> {
let rev = sub_m.value_of("CHANGESET_ID").unwrap().to_string();
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let ctx = CoreContext::new_with_logger(fb, logger.clone());
let json_flag = sub_m.is_present("json");

View File

@ -41,7 +41,7 @@ pub async fn subcommand_content_fetch<'a>(
let rev = sub_m.value_of("CHANGESET_ID").unwrap().to_string();
let path = sub_m.value_of("PATH").unwrap().to_string();
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let ctx = CoreContext::new_with_logger(fb, logger.clone());

View File

@ -55,7 +55,7 @@ pub async fn subcommand_create_bonsai<'a>(
.read_to_string(&mut content)
.map_err(|e| SubcommandError::Error(anyhow!(e)))?;
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let ctx = CoreContext::new_with_logger(fb, logger.clone());

View File

@ -67,7 +67,7 @@ pub async fn subcommand_crossrepo<'a>(
let config_store = args::init_config_store(fb, &logger, &matches)?;
let live_commit_sync_config = CfgrLiveCommitSyncConfig::new(&logger, &config_store)?;
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let ctx = CoreContext::new_with_logger(fb, logger.clone());
match sub_m.subcommand() {
(MAP_SUBCOMMAND, Some(sub_sub_m)) => {

View File

@ -101,7 +101,7 @@ pub async fn subcommand_derived_data<'a>(
matches: &'a ArgMatches<'_>,
sub_m: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let ctx = CoreContext::new_with_logger(fb, logger.clone());
let repo = args::open_repo(fb, &logger, &matches).await?;

View File

@ -222,7 +222,7 @@ pub async fn subcommand_filenodes<'a>(
sub_m: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
let ctx = CoreContext::new_with_logger(fb, logger.clone());
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let repo = args::open_repo(fb, &ctx.logger(), &matches).await?;
let log_envelope = sub_m.is_present(ARG_ENVELOPE);

View File

@ -98,7 +98,7 @@ pub async fn execute_command<'a>(
matches: &'a ArgMatches<'_>,
sub_matches: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let blobrepo = args::open_repo(fb, &logger, &matches).await?;
let ctx = CoreContext::new_with_logger(fb, logger.clone());

View File

@ -63,7 +63,7 @@ pub async fn subcommand_hash_convert<'a>(
(source == "hg") ^ (target == "bonsai"),
"source and target should be different"
);
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let ctx = CoreContext::new_with_logger(fb, logger.clone());
let repo = args::open_repo(fb, &logger, &matches).await?;
if source == "hg" {

View File

@ -72,7 +72,7 @@ pub async fn subcommand_hg_changeset<'a>(
.ok_or(format_err!("RIGHT_CS argument expected"))
.and_then(HgChangesetId::from_str);
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let repo = args::open_repo(fb, &logger, &matches).await?;
(left_cs, right_cs)
.into_future()
@ -98,7 +98,7 @@ pub async fn subcommand_hg_changeset<'a>(
.ok_or(format_err!("STOP_CS argument expected"))
.and_then(HgChangesetId::from_str);
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let repo = args::open_repo(fb, &logger, &matches).await?;
(start_cs, stop_cs)
.into_future()

View File

@ -557,17 +557,17 @@ pub async fn subcommand_process_hg_sync<'a>(
remains(sub_m, &ctx, repo_id, &mutable_counters, &bookmarks).await?
}
(HG_SYNC_SHOW, Some(sub_m)) => {
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let repo = args::open_repo(fb, ctx.logger(), &matches).await?;
show(sub_m, &ctx, &repo, &mutable_counters, &bookmarks).await?
}
(HG_SYNC_FETCH_BUNDLE, Some(sub_m)) => {
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let repo = args::open_repo(fb, ctx.logger(), &matches).await?;
fetch_bundle(sub_m, &ctx, &repo, &bookmarks).await?
}
(HG_SYNC_INSPECT, Some(sub_m)) => {
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let repo = args::open_repo(fb, ctx.logger(), &matches).await?;
inspect(sub_m, &ctx, &repo, &bookmarks).await?
}

View File

@ -117,7 +117,7 @@ fn main(fb: FacebookInit) -> ExitCode {
subcommand_content_fetch(fb, logger, &matches, sub_m).await
}
(bookmarks_manager::BOOKMARKS, Some(sub_m)) => {
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let ctx = CoreContext::new_with_logger(fb, logger.clone());
let repo = args::open_repo(fb, &logger, &matches).await?;
bookmarks_manager::handle_command(ctx, repo, sub_m, logger.clone()).await

View File

@ -104,7 +104,7 @@ pub async fn subcommand_phases<'a>(
sub_m: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
let repo = args::open_repo(fb, &logger, &matches).await?;
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let ctx = CoreContext::new_with_logger(fb, logger.clone());
match sub_m.subcommand() {

View File

@ -50,7 +50,7 @@ pub async fn subcommand_pushrebase<'a>(
matches: &'a ArgMatches<'_>,
sub_matches: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let ctx = CoreContext::new_with_logger(fb, logger.clone());
let repo = args::open_repo(fb, &logger, &matches).await?;

View File

@ -231,7 +231,7 @@ async fn get_ctx_blobrepo_redacted_blobs_cs_id<'a>(
None => return Err(SubcommandError::InvalidArgs),
};
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let config_store = args::init_config_store(fb, &logger, matches)?;
let blobrepo = args::open_repo(fb, &logger, &matches);

View File

@ -145,7 +145,7 @@ pub async fn subcommand_rsync<'a>(
matches: &'a ArgMatches<'_>,
sub_matches: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let ctx = CoreContext::new_with_logger(fb, logger.clone());
let repo = args::open_repo(fb, &logger, &matches).await?;

View File

@ -105,7 +105,7 @@ pub async fn subcommand_skiplist<'a>(
let rebuild = sub_m.is_present("rebuild");
let skiplist_ty = SkiplistType::new(sub_m.is_present(ARG_SPARSE));
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let config_store = args::init_config_store(fb, &logger, matches)?;
let ctx = CoreContext::new_with_logger(fb, logger.clone());
let sql_changesets = args::open_sql::<SqlChangesets>(fb, config_store, &matches);
@ -129,7 +129,7 @@ pub async fn subcommand_skiplist<'a>(
.expect("blobstore key is not specified")
.to_string();
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let ctx = CoreContext::test_mock(fb);
let repo = args::open_repo(fb, &logger, &matches).await?;
let maybe_index = read_skiplist_index(ctx.clone(), repo, key, logger.clone())

View File

@ -104,7 +104,7 @@ pub async fn subcommand_blame<'a>(
matches: &'a ArgMatches<'_>,
sub_matches: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let ctx = CoreContext::new_with_logger(fb, logger.clone());

View File

@ -73,7 +73,7 @@ pub async fn subcommand_deleted_manifest<'a>(
matches: &'a ArgMatches<'_>,
sub_matches: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let repo = args::open_repo(fb, &logger, &matches).await?;
let ctx = CoreContext::new_with_logger(fb, logger.clone());

View File

@ -47,7 +47,7 @@ pub async fn subcommand_fsnodes<'a>(
matches: &'a ArgMatches<'_>,
sub_matches: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let repo = args::open_repo(fb, &logger, &matches).await?;
let ctx = CoreContext::new_with_logger(fb, logger.clone());

View File

@ -71,7 +71,7 @@ pub async fn subcommand_skeleton_manifests<'a>(
matches: &'a ArgMatches<'_>,
sub_matches: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let repo = args::open_repo(fb, &logger, &matches).await?;
let ctx = CoreContext::new_with_logger(fb, logger.clone());

View File

@ -91,7 +91,7 @@ pub async fn subcommand_unodes<'a>(
tracing::enable();
}
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let repo = args::open_repo(fb, &logger, &matches).await?;
let ctx = CoreContext::new_with_logger(fb, logger);

View File

@ -336,7 +336,7 @@ fn main(fb: FacebookInit) -> Result<()> {
let logger = args::init_logging(fb, &matches);
let ctx = CoreContext::new_with_logger(fb, logger.clone());
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let config_store = args::init_config_store(fb, &logger, &matches)?;
let mode = match matches.value_of("mode").expect("no default on mode") {

View File

@ -239,7 +239,7 @@ fn main(fb: FacebookInit) -> Result<()> {
)
);
let matches = app.get_matches();
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let logger = args::init_logging(fb, &matches);
args::init_config_store(fb, &logger, &matches)?;

View File

@ -479,7 +479,7 @@ async fn maybe_update_highest_imported_generation_number(
fn main(fb: FacebookInit) -> Result<()> {
let matches = setup_app().get_matches();
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let logger = args::init_logging(fb, &matches);
let ctx = &CoreContext::new_with_logger(fb, logger.clone());
args::init_config_store(fb, &logger, &matches)?;

View File

@ -116,7 +116,7 @@ fn subcommand_round_trip(
matches: &ArgMatches<'_>,
sub_m: &ArgMatches<'_>,
) -> Result<()> {
args::init_cachelib(ctx.fb, &matches, None);
args::init_cachelib(ctx.fb, &matches);
let mut runtime = args::init_runtime(&matches)?;
let repo = runtime.block_on_std(args::open_repo(ctx.fb, &logger, &matches))?;
@ -297,7 +297,7 @@ fn subcommmand_hg_manifest_verify(
matches: &ArgMatches<'_>,
sub_m: &ArgMatches<'_>,
) -> Result<()> {
args::init_cachelib(ctx.fb, &matches, None);
args::init_cachelib(ctx.fb, &matches);
let total = &AtomicUsize::new(0);
let total_millis = &AtomicU64::new(0);

View File

@ -59,7 +59,7 @@ fn main(fb: FacebookInit) -> Result<()> {
);
let matches = app.get_matches();
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let logger = args::init_logging(fb, &matches);
args::init_config_store(fb, &logger, &matches)?;

View File

@ -48,7 +48,7 @@ fn main(fb: FacebookInit) -> Result<(), Error> {
)
.get_matches();
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let logger = args::init_logging(fb, &matches);
args::init_config_store(fb, &logger, &matches)?;

View File

@ -523,7 +523,7 @@ async fn run_statistics<'a>(
fn main(fb: FacebookInit) -> Result<(), Error> {
let matches = setup_app().get_matches();
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let logger = args::init_logging(fb, &matches);
let ctx = CoreContext::new_with_logger(fb, logger.clone());

View File

@ -79,7 +79,7 @@ pub fn upload<P: AsRef<Path>>(
fn main(fb: FacebookInit) -> Result<(), Error> {
let matches = setup_app().get_matches();
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let logger = args::init_logging(fb, &matches);
let config_store = args::init_config_store(fb, &logger, &matches)?;

View File

@ -217,7 +217,7 @@ async fn run(
fn context_and_matches<'a>(fb: FacebookInit, app: App<'a, '_>) -> (CoreContext, ArgMatches<'a>) {
let matches = app.get_matches();
let logger = args::init_logging(fb, &matches);
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let ctx = CoreContext::new_with_logger(fb, logger);
(ctx, matches)
}

View File

@ -877,7 +877,7 @@ fn get_and_verify_repo_config<'a>(
fn main(fb: FacebookInit) -> Result<()> {
let app = setup_app();
let matches = app.get_matches();
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let logger = args::init_logging(fb, &matches);
let config_store = args::init_config_store(fb, &logger, &matches)?;
let ctx = CoreContext::new_with_logger(fb, logger.clone());

View File

@ -503,7 +503,7 @@ async fn run(
fn context_and_matches<'a>(fb: FacebookInit, app: App<'a, '_>) -> (CoreContext, ArgMatches<'a>) {
let matches = app.get_matches();
let logger = args::init_logging(fb, &matches);
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let ctx = CoreContext::new_with_logger(fb, logger);
(ctx, matches)
}

View File

@ -229,7 +229,7 @@ async fn bootstrap_repositories<'a>(
let config = args::load_repo_configs(config_store, &matches)?;
let mysql_options = cmdlib::args::parse_mysql_options(&matches);
let caching = cmdlib::args::init_cachelib(fb, &matches, None);
let caching = cmdlib::args::init_cachelib(fb, &matches);
let readonly_storage = cmdlib::args::parse_readonly_storage(&matches);
let blobstore_options = cmdlib::args::parse_blobstore_options(&matches);

View File

@ -117,7 +117,7 @@ fn main(fb: FacebookInit) -> Result<(), Error> {
let path = Path::new(matches.value_of(ARG_GIT_REPOSITORY_PATH).unwrap());
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let logger = args::init_logging(fb, &matches);
args::init_config_store(fb, &logger, &matches)?;
let ctx = CoreContext::new_with_logger(fb, logger.clone());

View File

@ -127,7 +127,7 @@ async fn run_hook_tailer<'a>(
let disabled_hooks = cmdlib::args::parse_disabled_hooks_no_repo_prefix(&matches, &logger);
let caching = cmdlib::args::init_cachelib(fb, &matches, None);
let caching = cmdlib::args::init_cachelib(fb, &matches);
let readonly_storage = cmdlib::args::parse_readonly_storage(&matches);
let builder = BlobrepoBuilder::new(
fb,

View File

@ -82,7 +82,7 @@ async fn do_main<'a>(
let mysql_options = cmdlib::args::parse_mysql_options(&matches);
let readonly_storage = cmdlib::args::parse_readonly_storage(&matches);
let blobstore_options = cmdlib::args::parse_blobstore_options(&matches);
let caching = cmdlib::args::init_cachelib(fb, &matches, None);
let caching = cmdlib::args::init_cachelib(fb, &matches);
let config_store = cmdlib::args::init_config_store(fb, logger, matches)?;
let RepoConfigs { repos, common } = args::load_repo_configs(config_store, &matches)?;

View File

@ -1179,7 +1179,7 @@ fn main(fb: FacebookInit) -> Result<()> {
let matches = app.get_matches();
let logger = args::init_logging(fb, &matches);
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
args::init_config_store(fb, &logger, &matches)?;
let ctx = CoreContext::new_with_logger(fb, logger.clone());

View File

@ -1283,7 +1283,7 @@ fn main(fb: FacebookInit) -> Result<(), Error> {
let app = setup_app();
let matches = app.get_matches();
args::init_cachelib(fb, &matches, None);
args::init_cachelib(fb, &matches);
let logger = args::init_logging(fb, &matches);
args::init_config_store(fb, &logger, &matches)?;
let ctx = CoreContext::new_with_logger(fb, logger.clone());

View File

@ -392,7 +392,7 @@ async fn do_main(
let mysql_options = args::parse_mysql_options(&matches);
let blobstore_options = args::parse_blobstore_options(&matches);
let readonly_storage = args::parse_readonly_storage(&matches);
let caching = args::init_cachelib(fb, &matches, None);
let caching = args::init_cachelib(fb, &matches);
let repo_id = args::get_repo_id(config_store, matches)?;
let (repo_name, repo_config) = args::get_config_by_repoid(config_store, &matches, repo_id)?;

View File

@ -810,7 +810,7 @@ pub fn setup_common<'a>(
Redaction::Disabled
};
let caching = cmdlib::args::init_cachelib(fb, &matches, None);
let caching = cmdlib::args::init_cachelib(fb, &matches);
let include_edge_types = parse_edge_types(
sub_m,