From 61f9d6cf93514facaff4d575037b217f372b1fdb Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 15 Sep 2020 12:21:04 +1000 Subject: [PATCH] Add metrics --- Cargo.lock | 2 + beacon_node/http_api/Cargo.toml | 2 + beacon_node/http_api/src/lib.rs | 58 +++++++++++++++++++++++++++- beacon_node/http_api/src/metrics.rs | 19 +++++++++ common/lighthouse_metrics/src/lib.rs | 14 +++++++ 5 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 beacon_node/http_api/src/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index dc3848081b1..4e3ef2b6e24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2316,6 +2316,8 @@ dependencies = [ "eth2_libp2p", "fork_choice", "hex 0.4.2", + "lazy_static", + "lighthouse_metrics", "lighthouse_version", "network", "parking_lot 0.11.0", diff --git a/beacon_node/http_api/Cargo.toml b/beacon_node/http_api/Cargo.toml index 2eba8e73199..40528128c16 100644 --- a/beacon_node/http_api/Cargo.toml +++ b/beacon_node/http_api/Cargo.toml @@ -22,6 +22,8 @@ eth1 = { path = "../eth1" } fork_choice = { path = "../../consensus/fork_choice" } state_processing = { path = "../../consensus/state_processing" } lighthouse_version = { path = "../../common/lighthouse_version" } +lighthouse_metrics = { path = "../../common/lighthouse_metrics" } +lazy_static = "1.4.0" [dev-dependencies] store = { path = "../store" } diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 84e8a065bea..158f0199991 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -1,5 +1,6 @@ mod beacon_proposer_cache; mod block_id; +mod metrics; mod reject; mod state_id; @@ -109,6 +110,60 @@ pub fn slog_logging( }) } +pub fn prometheus_metrics() -> warp::filters::log::Log { + warp::log::custom(move |info| { + // Here we restrict the `info.path()` value to some predefined values. Without this, we end + // up with a new metric type each time someone includes something unique in the path (e.g., + // a block hash). + let path = { + let equals = |s: &'static str| -> Option<&'static str> { + if info.path() == &format!("{}/{}/{}", API_PREFIX, API_VERSION, s) { + Some(s) + } else { + None + } + }; + + let starts_with = |s: &'static str| -> Option<&'static str> { + if info + .path() + .starts_with(&format!("{}/{}/{}", API_PREFIX, API_VERSION, s)) + { + Some(s) + } else { + None + } + }; + + equals("beacon/blocks") + .or(starts_with("validator/duties/attester")) + .or(starts_with("validator/duties/proposer")) + .or(starts_with("validator/attestation_data")) + .or(starts_with("validator/aggregate_attestation")) + .or(starts_with("validator/aggregate_and_proofs")) + .or(starts_with("validator/beacon_committee_subscriptions")) + .or(starts_with("beacon/")) + .or(starts_with("config/")) + .or(starts_with("debug/")) + .or(starts_with("events/")) + .or(starts_with("node/")) + .or(starts_with("validator/")) + .unwrap_or("other") + }; + + metrics::inc_counter_vec(&metrics::HTTP_API_PATHS_TOTAL, &[path]); + metrics::inc_counter_vec( + &metrics::HTTP_API_STATUS_CODES_TOTAL, + &[&info.status().to_string()], + ); + metrics::observe_timer_vec( + &metrics::HTTP_API_PATHS_TIMES_TOTAL, + &[info.path()], + info.elapsed(), + ); + }) +} + pub fn serve( ctx: Arc>, shutdown: impl Future + Send + Sync + 'static, @@ -1291,7 +1346,8 @@ pub fn serve( )) .boxed() .recover(crate::reject::handle_rejection) - .with(slog_logging(log.clone())); + .with(slog_logging(log.clone())) + .with(prometheus_metrics()); let (listening_socket, server) = warp::serve(routes).try_bind_with_graceful_shutdown( SocketAddrV4::new(config.listen_addr, config.listen_port), diff --git a/beacon_node/http_api/src/metrics.rs b/beacon_node/http_api/src/metrics.rs new file mode 100644 index 00000000000..e2c753dd16a --- /dev/null +++ b/beacon_node/http_api/src/metrics.rs @@ -0,0 +1,19 @@ +pub use lighthouse_metrics::*; + +lazy_static::lazy_static! { + pub static ref HTTP_API_PATHS_TOTAL: Result = try_create_int_counter_vec( + "http_api_paths_total", + "Count of HTTP requests received", + &["path"] + ); + pub static ref HTTP_API_STATUS_CODES_TOTAL: Result = try_create_int_counter_vec( + "http_api_status_codes_total", + "Count of HTTP status codes returned", + &["status"] + ); + pub static ref HTTP_API_PATHS_TIMES_TOTAL: Result = try_create_histogram_vec( + "http_api_paths_times_total", + "Duration to process HTTP requests per path", + &["path"] + ); +} diff --git a/common/lighthouse_metrics/src/lib.rs b/common/lighthouse_metrics/src/lib.rs index 0a4251e06df..0637b973c74 100644 --- a/common/lighthouse_metrics/src/lib.rs +++ b/common/lighthouse_metrics/src/lib.rs @@ -55,6 +55,7 @@ //! ``` use prometheus::{HistogramOpts, HistogramTimer, Opts}; +use std::time::Duration; pub use prometheus::{ Encoder, Gauge, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, @@ -221,6 +222,19 @@ pub fn start_timer(histogram: &Result) -> Option { } } +/// Starts a timer on `vec` with the given `name`. +pub fn observe_timer_vec(vec: &Result, name: &[&str], duration: Duration) { + // This conversion was taken from here: + // + // https://docs.rs/prometheus/0.5.0/src/prometheus/histogram.rs.html#550-555 + let nanos = f64::from(duration.subsec_nanos()) / 1e9; + let secs = duration.as_secs() as f64 + nanos; + + if let Some(h) = get_histogram(vec, name) { + h.observe(secs) + } +} + /// Stops a timer created with `start_timer(..)`. pub fn stop_timer(timer: Option) { if let Some(t) = timer {