From cec871cc9f6005a536b2ca23470c4aadc648e6f6 Mon Sep 17 00:00:00 2001 From: Arun Kulshreshtha Date: Thu, 10 Sep 2020 20:55:21 -0700 Subject: [PATCH] edenapi_server: log stats to ODS Summary: Add `OdsMiddleware` to the EdenAPI server to log various aggregate request stats to ODS. This middleware is directly based on the `OdsMiddleware` from the LFS server (which unfortunately can't be easily generalized due to the way in which the `stats` crate works). Reviewed By: krallin Differential Revision: D23619075 fbshipit-source-id: d361c73d18e0d1cb57347fd24c43bdb68fb7819d --- eden/mononoke/edenapi_server/Cargo.toml | 1 + eden/mononoke/edenapi_server/src/main.rs | 3 +- .../edenapi_server/src/middleware/mod.rs | 2 + .../edenapi_server/src/middleware/ods.rs | 95 +++++++++++++++++++ 4 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 eden/mononoke/edenapi_server/src/middleware/ods.rs diff --git a/eden/mononoke/edenapi_server/Cargo.toml b/eden/mononoke/edenapi_server/Cargo.toml index 4f8d1dcd6b..921631c9f9 100644 --- a/eden/mononoke/edenapi_server/Cargo.toml +++ b/eden/mononoke/edenapi_server/Cargo.toml @@ -22,6 +22,7 @@ cloned = { git = "https://github.com/facebookexperimental/rust-shed.git", branch fbinit = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" } scuba = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" } secure_utils = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" } +stats = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" } anyhow = "1.0" async-trait = "0.1.29" bytes = { version = "0.5", features = ["serde"] } diff --git a/eden/mononoke/edenapi_server/src/main.rs b/eden/mononoke/edenapi_server/src/main.rs index 8e822be70c..254a975843 100644 --- a/eden/mononoke/edenapi_server/src/main.rs +++ b/eden/mononoke/edenapi_server/src/main.rs @@ -54,7 +54,7 @@ mod utils; use crate::context::ServerContext; use crate::handlers::build_router; -use crate::middleware::RequestContextMiddleware; +use crate::middleware::{OdsMiddleware, RequestContextMiddleware}; const ARG_LISTEN_HOST: &str = "listen-host"; const ARG_LISTEN_PORT: &str = "listen-port"; @@ -171,6 +171,7 @@ async fn start( .add(RequestContextMiddleware::new(fb, logger.clone())) .add(LoadMiddleware::new()) .add(log_middleware) + .add(OdsMiddleware::new()) .add(>::new(scuba_logger)) .add(TimerMiddleware::new()) .build(router); diff --git a/eden/mononoke/edenapi_server/src/middleware/mod.rs b/eden/mononoke/edenapi_server/src/middleware/mod.rs index 56ea37480b..a9d5ab772b 100644 --- a/eden/mononoke/edenapi_server/src/middleware/mod.rs +++ b/eden/mononoke/edenapi_server/src/middleware/mod.rs @@ -5,6 +5,8 @@ * GNU General Public License version 2. */ +pub mod ods; pub mod request_context; +pub use self::ods::OdsMiddleware; pub use self::request_context::{RequestContext, RequestContextMiddleware}; diff --git a/eden/mononoke/edenapi_server/src/middleware/ods.rs b/eden/mononoke/edenapi_server/src/middleware/ods.rs new file mode 100644 index 0000000000..fe3c556efc --- /dev/null +++ b/eden/mononoke/edenapi_server/src/middleware/ods.rs @@ -0,0 +1,95 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This software may be used and distributed according to the terms of the + * GNU General Public License version 2. + */ + +use gotham::state::State; +use gotham_ext::middleware::{ClientIdentity, Middleware, PostRequestCallbacks}; +use hyper::StatusCode; +use hyper::{Body, Response}; +use stats::prelude::*; + +use crate::handlers::{EdenApiMethod, HandlerInfo}; + +define_stats! { + prefix = "mononoke.edenapi.request"; + requests: dynamic_timeseries("{}.requests", (repo_and_method: String); Rate, Sum), + success: dynamic_timeseries("{}.success", (repo_and_method: String); Rate, Sum), + failure_4xx: dynamic_timeseries("{}.failure_4xx", (repo_and_method: String); Rate, Sum), + failure_5xx: dynamic_timeseries("{}.failure_5xx", (repo_and_method: String); Rate, Sum), + response_bytes_sent: dynamic_histogram("{}.response_bytes_sent", (repo_and_method: String); 1_500_000, 0, 150_000_000, Average, Sum, Count; P 5; P 25; P 50; P 75; P 95; P 97; P 99), + files_duration: dynamic_histogram("{}.files_ms", (repo: String); 100, 0, 5000, Average, Sum, Count; P 5; P 25; P 50; P 75; P 95; P 97; P 99), + trees_duration: dynamic_histogram("{}.trees_ms", (repo: String); 100, 0, 5000, Average, Sum, Count; P 5; P 25; P 50; P 75; P 95; P 97; P 99), + complete_trees_duration: dynamic_histogram("{}.complete_trees_ms", (repo: String); 100, 0, 5000, Average, Sum, Count; P 5; P 25; P 50; P 75; P 95; P 97; P 99), + history_duration: dynamic_histogram("{}.history_ms", (repo: String); 100, 0, 5000, Average, Sum, Count; P 5; P 25; P 50; P 75; P 95; P 97; P 99), + commit_location_to_hash_duration: dynamic_histogram("{}.commit_location_to_hash_ms", (repo: String); 10, 0, 500, Average, Sum, Count; P 5; P 25; P 50; P 75; P 95; P 97; P 99), + commit_revlog_data_duration: dynamic_histogram("{}.commit_revlog_data_ms", (repo: String); 10, 0, 500, Average, Sum, Count; P 5; P 25; P 50; P 75; P 95; P 97; P 99), +} + +fn log_stats(state: &mut State, status: StatusCode) -> Option<()> { + // Proxygen can be configured to periodically send a preconfigured set of + // requests to check server health. These requests will look like ordinary + // user requests, but should be filtered out of the server's metrics. + match state.try_borrow::() { + Some(id) if id.is_proxygen_test_identity() => return None, + _ => {} + } + + let hander_info = state.try_borrow::()?; + let method = hander_info.method?; + let repo = hander_info.repo.clone()?; + let repo_and_method = format!("{}.{}", &repo, method.to_string()); + + let callbacks = state.try_borrow_mut::()?; + + callbacks.add(move |info| { + if let Some(duration) = info.duration { + let dur_ms = duration.as_millis() as i64; + + use EdenApiMethod::*; + match method { + Files => STATS::files_duration.add_value(dur_ms, (repo,)), + Trees => STATS::trees_duration.add_value(dur_ms, (repo,)), + CompleteTrees => STATS::complete_trees_duration.add_value(dur_ms, (repo,)), + History => STATS::history_duration.add_value(dur_ms, (repo,)), + CommitLocationToHash => { + STATS::commit_location_to_hash_duration.add_value(dur_ms, (repo,)) + } + CommitRevlogData => STATS::commit_revlog_data_duration.add_value(dur_ms, (repo,)), + } + } + + STATS::requests.add_value(1, (repo_and_method.clone(),)); + + if status.is_success() { + STATS::success.add_value(1, (repo_and_method.clone(),)); + } else if status.is_client_error() { + STATS::failure_4xx.add_value(1, (repo_and_method.clone(),)); + } else if status.is_server_error() { + STATS::failure_5xx.add_value(1, (repo_and_method.clone(),)); + } + + if let Some(response_bytes_sent) = info.bytes_sent { + STATS::response_bytes_sent.add_value(response_bytes_sent as i64, (repo_and_method,)) + } + }); + + Some(()) +} + +pub struct OdsMiddleware; + +impl OdsMiddleware { + pub fn new() -> Self { + OdsMiddleware + } +} + +#[async_trait::async_trait] +impl Middleware for OdsMiddleware { + async fn outbound(&self, state: &mut State, response: &mut Response) { + log_stats(state, response.status()); + } +}