From f65ed936177b65fd65979a4537c49f99c96fa8b7 Mon Sep 17 00:00:00 2001 From: Rajiv Sharma Date: Mon, 12 Jun 2023 03:41:37 -0700 Subject: [PATCH] Migrate statistics_collector to new Mononoke App Summary: Now that the `blob_repo` dependency has been removed from `statistics_collector`, this diff migrates the tool from old Mononoke App to new Mononoke App getting rid of the old clap dependency. Differential Revision: D46478775 fbshipit-source-id: dd713eef2175fa01934508445866d64ef01657f0 --- eden/mononoke/cmds/statistics_collector.rs | 125 +++++++----------- eden/mononoke/statistics_collector/Cargo.toml | 4 +- 2 files changed, 50 insertions(+), 79 deletions(-) diff --git a/eden/mononoke/cmds/statistics_collector.rs b/eden/mononoke/cmds/statistics_collector.rs index 445b453222..f24116814c 100644 --- a/eden/mononoke/cmds/statistics_collector.rs +++ b/eden/mononoke/cmds/statistics_collector.rs @@ -26,12 +26,7 @@ use bytes::Bytes; use changesets::deserialize_cs_entries; use changesets::ChangesetEntry; use changesets::Changesets; -use clap_old::Arg; -use clap_old::SubCommand; -use cmdlib::args; -use cmdlib::args::MononokeClapApp; -use cmdlib::args::MononokeMatches; -use cmdlib::helpers::block_execute; +use clap::Parser; use context::CoreContext; use fbinit::FacebookInit; use filestore::FilestoreConfig; @@ -52,6 +47,12 @@ use mercurial_types::FileBytes; use mercurial_types::HgChangesetId; use mercurial_types::HgFileNodeId; use mercurial_types::HgManifestId; +use mononoke_app::args::AsRepoArg; +use mononoke_app::args::RepoArgs; +use mononoke_app::fb303::AliveService; +use mononoke_app::fb303::Fb303AppExtension; +use mononoke_app::MononokeApp; +use mononoke_app::MononokeAppBuilder; use mononoke_types::FileType; use mononoke_types::RepositoryId; use redactedblobstore::ErrorKind as RedactedBlobstoreError; @@ -61,7 +62,6 @@ use repo_derived_data::RepoDerivedData; use repo_derived_data::RepoDerivedDataRef; use scuba_ext::MononokeScubaSampleBuilder; use slog::info; -use slog::Logger; use stats::prelude::*; use tokio::time::sleep; @@ -83,52 +83,34 @@ pub struct Repo { filestore_config: FilestoreConfig, } +/// Tool to calculate repo statistic +#[derive(Parser)] +#[clap(about = "Tool to calculate repo statistic.")] +struct RepoStatisticsArgs { + /// The repo against which the repo statistic command needs to be executed + #[clap(flatten)] + repo: RepoArgs, + /// Bookmark from which we get statistics. Default is master. + #[clap(long, default_value = "master")] + bookmark: String, + /// If set, then statistics are logged to scuba + #[clap(long)] + log_to_scuba: bool, + /// A file with a list of bonsai changesets to calculate stats for. If this + /// argument is provided, then the statistics will be calculated for file based commit only. + #[clap(long)] + in_filename: Option, +} + define_stats! { prefix = "mononoke.statistics_collector"; calculated_changesets: timeseries(Rate, Sum), } -const ARG_IN_FILENAME: &str = "in-filename"; - -const SUBCOMMAND_STATISTICS_FROM_FILE: &str = "statistics-from-commits-in-file"; - const SCUBA_DATASET_NAME: &str = "mononoke_repository_statistics"; // Tool doesn't count number of lines from files with size greater than 10MB const BIG_FILE_THRESHOLD: u64 = 10000000; -fn setup_app<'a, 'b>() -> MononokeClapApp<'a, 'b> { - args::MononokeAppBuilder::new("Tool to calculate repo statistic") - .with_fb303_args() - .build() - .subcommand( - SubCommand::with_name(SUBCOMMAND_STATISTICS_FROM_FILE) - .about( - "calculate statistics for commits in provided file and save them to json file", - ) - .arg( - Arg::with_name(ARG_IN_FILENAME) - .long(ARG_IN_FILENAME) - .takes_value(true) - .required(true) - .help("a file with a list of bonsai changesets to calculate stats for"), - ), - ) - .arg( - Arg::with_name("bookmark") - .long("bookmark") - .takes_value(true) - .required(false) - .help("bookmark from which we get statistics"), - ) - .arg( - Arg::with_name("log-to-scuba") - .long("log-to-scuba") - .takes_value(false) - .required(false) - .help("if set then statistics are logged to scuba"), - ) -} - #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct RepoStatistics { num_files: i64, @@ -475,23 +457,18 @@ enum Operation { } #[allow(unreachable_code)] -async fn run_statistics<'a>( - fb: FacebookInit, +async fn run_statistics( + app: MononokeApp, + args: RepoStatisticsArgs, ctx: CoreContext, - logger: &Logger, scuba_logger: MononokeScubaSampleBuilder, - matches: &'a MononokeMatches<'a>, repo_name: String, bookmark: BookmarkKey, ) -> Result<(), Error> { - let repo: Repo = args::not_shardmanager_compatible::open_repo(fb, logger, matches).await?; + let repo: Repo = app.open_repo(&args.repo).await?; - if let (SUBCOMMAND_STATISTICS_FROM_FILE, Some(sub_m)) = matches.subcommand() { - // Both arguments are set to be required - let in_filename = sub_m - .value_of(ARG_IN_FILENAME) - .expect("missing required argument"); - return generate_statistics_from_file(&ctx, &repo, &in_filename).await; + if let Some(in_filename) = args.in_filename.as_ref() { + return generate_statistics_from_file(&ctx, &repo, in_filename).await; } let blobstore = Arc::new(repo.repo_blobstore().clone()); @@ -577,33 +554,27 @@ async fn run_statistics<'a>( Ok(()) } -#[fbinit::main] -fn main(fb: FacebookInit) -> Result<(), Error> { - let (matches, _runtime) = setup_app().get_matches(fb)?; +async fn async_main(app: MononokeApp) -> Result<(), Error> { + let logger = app.logger(); + let ctx = CoreContext::new_with_logger(app.fb, logger.clone()); - let logger = matches.logger(); - let ctx = CoreContext::new_with_logger(fb, logger.clone()); - let config_store = matches.config_store(); - let bookmark = match matches.value_of("bookmark") { - Some(name) => name.to_string(), - None => String::from("master"), - }; - let bookmark = BookmarkKey::new(bookmark)?; - let repo_name = args::not_shardmanager_compatible::get_repo_name(config_store, &matches)?; - let scuba_logger = if matches.is_present("log-to-scuba") { - MononokeScubaSampleBuilder::new(fb, SCUBA_DATASET_NAME)? + let args: RepoStatisticsArgs = app.args()?; + let bookmark = BookmarkKey::new(&args.bookmark)?; + let (repo_name, _) = app.repo_config(args.repo.as_repo_arg())?; + let scuba_logger = if args.log_to_scuba { + MononokeScubaSampleBuilder::new(app.fb, SCUBA_DATASET_NAME)? } else { MononokeScubaSampleBuilder::with_discard() }; + run_statistics(app, args, ctx, scuba_logger, repo_name, bookmark).await +} - block_execute( - run_statistics(fb, ctx, logger, scuba_logger, &matches, repo_name, bookmark), - fb, - "statistics_collector", - logger, - &matches, - cmdlib::monitoring::AliveService, - ) +#[fbinit::main] +fn main(fb: FacebookInit) -> Result<(), Error> { + let app = MononokeAppBuilder::new(fb) + .with_app_extension(Fb303AppExtension {}) + .build::()?; + app.run_with_monitoring_and_logging(async_main, "repo_statistics_collector", AliveService) } #[cfg(test)] diff --git a/eden/mononoke/statistics_collector/Cargo.toml b/eden/mononoke/statistics_collector/Cargo.toml index 733b8c2f93..99e1951dc0 100644 --- a/eden/mononoke/statistics_collector/Cargo.toml +++ b/eden/mononoke/statistics_collector/Cargo.toml @@ -20,8 +20,7 @@ bonsai_hg_mapping = { version = "0.1.0", path = "../bonsai_hg_mapping" } bookmarks = { version = "0.1.0", path = "../bookmarks" } bytes = { version = "1.1", features = ["serde"] } changesets = { version = "0.1.0", path = "../changesets" } -clap-old = { package = "clap", version = "2.33" } -cmdlib = { version = "0.1.0", path = "../cmdlib" } +clap = { version = "4.2.4", features = ["derive", "env", "string", "unicode", "wrap_help"] } context = { version = "0.1.0", path = "../server/context" } facet = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" } fbinit = { version = "0.1.2", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" } @@ -31,6 +30,7 @@ futures_ext = { package = "futures_01_ext", version = "0.1.0", git = "https://gi manifest = { version = "0.1.0", path = "../manifest" } mercurial_derivation = { version = "0.1.0", path = "../derived_data/mercurial_derivation" } mercurial_types = { version = "0.1.0", path = "../mercurial/types" } +mononoke_app = { version = "0.1.0", path = "../cmdlib/mononoke_app" } mononoke_types = { version = "0.1.0", path = "../mononoke_types" } redactedblobstore = { version = "0.1.0", path = "../blobstore/redactedblobstore" } repo_blobstore = { version = "0.1.0", path = "../blobrepo/repo_blobstore" }