Add build_info metric & use it in generated queries (#69)

* Add build_info metric & use it in generated queries

* Include vergen as dev dependency to fix doctest

Author: emschwartz

autometrics-macros/src/lib.rs

@@ -10,6 +10,7 @@ mod parse;
 const COUNTER_NAME_PROMETHEUS: &str = "function_calls_count";
 const HISTOGRAM_BUCKET_NAME_PROMETHEUS: &str = "function_calls_duration_bucket";
 const GAUGE_NAME_PROMETHEUS: &str = "function_calls_concurrent";
+const ADD_BUILD_INFO_LABELS: &str = "* on (instance, job) group_left(version, commit) build_info";
 
 const DEFAULT_PROMETHEUS_URL: &str = "http://localhost:9090";
 
@@ -126,7 +127,11 @@ fn instrument_function(args: &AutometricsArgs, item: ItemFn) -> Result<TokenStre
 
         #vis #sig {
             let __autometrics_tracker = {
-                use autometrics::__private::{AutometricsTracker, TrackMetrics};
+                use autometrics::__private::{AutometricsTracker, BuildInfoLabels, TrackMetrics};
+                AutometricsTracker::set_build_info(&BuildInfoLabels::new(
+                    option_env!("AUTOMETRICS_VERSION").or(option_env!("CARGO_PKG_VERSION")).unwrap_or_default(),
+                    option_env!("AUTOMETRICS_COMMIT").or(option_env!("VERGEN_GIT_SHA")).unwrap_or_default()
+                ));
                 AutometricsTracker::start(#gauge_labels)
             };
 
@@ -260,18 +265,19 @@ fn make_prometheus_url(url: &str, query: &str, comment: &str) -> String {
 }
 
 fn request_rate_query(counter_name: &str, label_key: &str, label_value: &str) -> String {
-    format!("sum by (function, module) (rate({counter_name}{{{label_key}=\"{label_value}\"}}[5m]))")
+    format!("sum by (function, module, commit, version) (rate({counter_name}{{{label_key}=\"{label_value}\"}}[5m]) {ADD_BUILD_INFO_LABELS})")
 }
 
 fn error_ratio_query(counter_name: &str, label_key: &str, label_value: &str) -> String {
     let request_rate = request_rate_query(counter_name, label_key, label_value);
-    format!("sum by (function, module) (rate({counter_name}{{{label_key}=\"{label_value}\",result=\"error\"}}[5m])) /
+    format!("sum by (function, module, commit, version) (rate({counter_name}{{{label_key}=\"{label_value}\",result=\"error\"}}[5m]) {ADD_BUILD_INFO_LABELS})
+/
 {request_rate}", )
 }
 
 fn latency_query(bucket_name: &str, label_key: &str, label_value: &str) -> String {
     let latency = format!(
-        "sum by (le, function, module) (rate({bucket_name}{{{label_key}=\"{label_value}\"}}[5m]))"
+        "sum by (le, function, module, commit, version) (rate({bucket_name}{{{label_key}=\"{label_value}\"}}[5m]) {ADD_BUILD_INFO_LABELS})"
     );
     format!(
         "histogram_quantile(0.99, {latency}) or
@@ -280,5 +286,5 @@ histogram_quantile(0.95, {latency})"
 }
 
 fn concurrent_calls_query(gauge_name: &str, label_key: &str, label_value: &str) -> String {
-    format!("sum by (function, module) {gauge_name}{{{label_key}=\"{label_value}\"}}")
+    format!("sum by (function, module, commit, version) ({gauge_name}{{{label_key}=\"{label_value}\"}} {ADD_BUILD_INFO_LABELS})")
 }

autometrics/Cargo.toml

@@ -48,6 +48,7 @@ const_format = { version = "0.2", features = ["rust_1_51"], optional = true }
 [dev-dependencies]
 regex = "1.7"
 http = "0.2"
+vergen = { version = "8.1", features = ["git", "gitcl"] }
 
 [package.metadata.docs.rs]
 all-features = true

autometrics/README.md

@@ -29,6 +29,7 @@ Here is a demo of jumping from function docs to live Prometheus charts:
 - ✨ [`#[autometrics]`](https://docs.rs/autometrics/latest/autometrics/attr.autometrics.html) macro instruments any function or `impl` block to track the most useful metrics
 - 💡 Writes Prometheus queries so you can understand the data generated without knowing PromQL
 - 🔗 Injects links to live Prometheus charts directly into each function's doc comments
+- [🔍 Identify commits](#identifying-commits-that-introduced-problems) that introduced errors or increased latency
 - [🚨 Define alerts](#alerts--slos) using SLO best practices directly in your source code
 - [📊 Grafana dashboards](#dashboards) work out of the box to visualize the performance of instrumented functions & SLOs
 - [⚙️ Configurable](#metrics-libraries) metric collection library (`opentelemetry`, `prometheus`, or `metrics`)
@@ -96,6 +97,33 @@ Autometrics uses existing metrics libraries (see [below](#metrics-libraries)) to
 
 If you are already using one of these to collect metrics, simply configure autometrics to use the same library and the metrics it produces will be exported alongside yours. You do not need to use the Prometheus exporter functions this library provides and you do not need a separate endpoint for autometrics' metrics.
 
+## Identifying commits that introduced problems
+
+Autometrics makes it easy to identify if a specific version or commit introduced errors or increased latencies.
+
+It uses a separate metric (`build_info`) to track the version and, optionally, git commit of your service. It then writes queries that group metrics by the `version` and `commit` labels so you can spot correlations between those and potential issues.
+
+The `version` is collected from the `CARGO_PKG_VERSION` environment variable, which `cargo` sets by default. You can override this by setting the compile-time environment variable `AUTOMETRICS_VERSION`. This follows the method outlined in [Exposing the software version to Prometheus](https://www.robustperception.io/exposing-the-software-version-to-prometheus/).
+
+To set the `commit`, you can either set the compile-time environment variable `AUTOMETRICS_COMMIT`, or have it set automatically using the [vergen](https://crates.io/crates/vergen) crate:
+
+```toml
+# Cargo.toml
+
+[build-dependencies]
+vergen = { version = "8.1", features = ["git", "gitoxide"] }
+```
+
+```rust
+// build.rs
+fn main() {
+  vergen::EmitBuilder::builder()
+      .git_sha(true)
+      .emit()
+      .expect("Unable to generate build info");
+}
+```
+
 ## Dashboards
 
 Autometrics provides [Grafana dashboards](https://github.com/autometrics-dev/autometrics-shared#dashboards) that will work for any project instrumented with the library.

autometrics/src/constants.rs

@@ -2,11 +2,14 @@
 pub const COUNTER_NAME: &str = "function.calls.count";
 pub const HISTOGRAM_NAME: &str = "function.calls.duration";
 pub const GAUGE_NAME: &str = "function.calls.concurrent";
+pub const BUILD_INFO_NAME: &str = "build_info";
 
 // Descriptions
 pub const COUNTER_DESCRIPTION: &str = "Autometrics counter for tracking function calls";
 pub const HISTOGRAM_DESCRIPTION: &str = "Autometrics histogram for tracking function call duration";
 pub const GAUGE_DESCRIPTION: &str = "Autometrics gauge for tracking concurrent function calls";
+pub const BUILD_INFO_DESCRIPTION: &str =
+    "Autometrics info metric for tracking software version and build details";
 
 // Labels
 pub const FUNCTION_KEY: &'static str = "function";
@@ -18,3 +21,5 @@ pub const ERROR_KEY: &'static str = "error";
 pub const OBJECTIVE_NAME: &'static str = "objective.name";
 pub const OBJECTIVE_PERCENTILE: &'static str = "objective.percentile";
 pub const OBJECTIVE_LATENCY_THRESHOLD: &'static str = "objective.latency_threshold";
+pub const VERSION_KEY: &'static str = "version";
+pub const COMMIT_KEY: &'static str = "commit";

autometrics/src/labels.rs

@@ -4,6 +4,22 @@ use std::ops::Deref;
 pub(crate) type Label = (&'static str, &'static str);
 type ResultAndReturnTypeLabels = (&'static str, Option<&'static str>);
 
+/// These are the labels used for the `build_info` metric.
+pub struct BuildInfoLabels {
+    pub(crate) version: &'static str,
+    pub(crate) commit: &'static str,
+}
+
+impl BuildInfoLabels {
+    pub fn new(version: &'static str, commit: &'static str) -> Self {
+        Self { version, commit }
+    }
+
+    pub fn to_vec(&self) -> Vec<Label> {
+        vec![(COMMIT_KEY, self.commit), (VERSION_KEY, self.version)]
+    }
+}
+
 /// These are the labels used for the `function.calls.count` metric.
 pub struct CounterLabels {
     pub(crate) function: &'static str,

autometrics/src/tracker/metrics.rs

@@ -1,19 +1,21 @@
 use crate::constants::*;
-use crate::labels::{CounterLabels, GaugeLabels, HistogramLabels};
+use crate::labels::{BuildInfoLabels, CounterLabels, GaugeLabels, HistogramLabels};
 use crate::tracker::TrackMetrics;
 use metrics::{
     describe_counter, describe_gauge, describe_histogram, register_counter, register_gauge,
     register_histogram, Gauge,
 };
 use std::{sync::Once, time::Instant};
 
-static ONCE: Once = Once::new();
+const DESCRIBE_METRICS: Once = Once::new();
+const SET_BUILD_INFO: Once = Once::new();
 
 fn describe_metrics() {
-    ONCE.call_once(|| {
+    DESCRIBE_METRICS.call_once(|| {
         describe_counter!(COUNTER_NAME, COUNTER_DESCRIPTION);
         describe_histogram!(HISTOGRAM_NAME, HISTOGRAM_DESCRIPTION);
         describe_gauge!(GAUGE_NAME, GAUGE_DESCRIPTION);
+        describe_gauge!(BUILD_INFO_NAME, BUILD_INFO_DESCRIPTION);
     });
 }
 
@@ -48,4 +50,10 @@ impl TrackMetrics for MetricsTracker {
             gauge.decrement(1.0);
         }
     }
+
+    fn set_build_info(build_info_labels: &BuildInfoLabels) {
+        SET_BUILD_INFO.call_once(|| {
+            register_gauge!(BUILD_INFO_NAME, &build_info_labels.to_vec()).set(1.0);
+        });
+    }
 }

autometrics/src/tracker/mod.rs

@@ -1,4 +1,4 @@
-use crate::labels::{CounterLabels, GaugeLabels, HistogramLabels};
+use crate::labels::{BuildInfoLabels, CounterLabels, GaugeLabels, HistogramLabels};
 
 #[cfg(feature = "metrics")]
 mod metrics;
@@ -21,6 +21,7 @@ pub use self::metrics::MetricsTracker as AutometricsTracker;
 pub use self::prometheus::PrometheusTracker as AutometricsTracker;
 
 pub trait TrackMetrics {
+    fn set_build_info(build_info_labels: &BuildInfoLabels);
     fn start(gauge_labels: Option<&GaugeLabels>) -> Self;
     fn finish(self, counter_labels: &CounterLabels, histogram_labels: &HistogramLabels);
 }

autometrics/src/tracker/opentelemetry.rs

@@ -1,7 +1,9 @@
-use crate::labels::{CounterLabels, GaugeLabels, HistogramLabels, Label};
+use crate::labels::{BuildInfoLabels, CounterLabels, GaugeLabels, HistogramLabels, Label};
 use crate::{constants::*, tracker::TrackMetrics};
 use opentelemetry_api::{global, metrics::UpDownCounter, Context, KeyValue};
-use std::time::Instant;
+use std::{sync::Once, time::Instant};
+
+const SET_BUILD_INFO: Once = Once::new();
 
 /// Tracks the number of function calls, concurrent calls, and latency
 pub struct OpenTelemetryTracker {
@@ -58,6 +60,17 @@ impl TrackMetrics for OpenTelemetryTracker {
             concurrency_tracker.add(&self.context, -1, &gauge_labels);
         }
     }
+
+    fn set_build_info(build_info_labels: &BuildInfoLabels) {
+        SET_BUILD_INFO.call_once(|| {
+            let build_info_labels = to_key_values(build_info_labels.to_vec());
+            let build_info = global::meter("")
+                .f64_up_down_counter(BUILD_INFO_NAME)
+                .with_description(BUILD_INFO_DESCRIPTION)
+                .init();
+            build_info.add(&Context::current(), 1.0, &build_info_labels);
+        });
+    }
 }
 
 fn to_key_values(labels: impl IntoIterator<Item = Label>) -> Vec<KeyValue> {

autometrics/src/tracker/prometheus.rs

@@ -1,14 +1,13 @@
-use crate::labels::{CounterLabels, GaugeLabels, HistogramLabels};
+use crate::labels::{BuildInfoLabels, CounterLabels, GaugeLabels, HistogramLabels};
 use crate::{constants::*, tracker::TrackMetrics, HISTOGRAM_BUCKETS};
 use const_format::{formatcp, str_replace};
 use once_cell::sync::Lazy;
-use prometheus::histogram_opts;
+use prometheus::core::{AtomicI64, GenericGauge};
 use prometheus::{
-    core::{AtomicI64, GenericGauge},
-    register_histogram_vec, register_int_counter_vec, register_int_gauge_vec, HistogramVec,
-    IntCounterVec, IntGaugeVec,
+    histogram_opts, register_histogram_vec, register_int_counter_vec, register_int_gauge_vec,
+    HistogramVec, IntCounterVec, IntGaugeVec,
 };
-use std::time::Instant;
+use std::{sync::Once, time::Instant};
 
 const COUNTER_NAME_PROMETHEUS: &str = str_replace!(COUNTER_NAME, ".", "_");
 const HISTOGRAM_NAME_PROMETHEUS: &str = str_replace!(HISTOGRAM_NAME, ".", "_");
@@ -17,6 +16,8 @@ const OBJECTIVE_NAME_PROMETHEUS: &str = str_replace!(OBJECTIVE_NAME, ".", "_");
 const OBJECTIVE_PERCENTILE_PROMETHEUS: &str = str_replace!(OBJECTIVE_PERCENTILE, ".", "_");
 const OBJECTIVE_LATENCY_PROMETHEUS: &str = str_replace!(OBJECTIVE_LATENCY_THRESHOLD, ".", "_");
 
+const SET_BUILD_INFO: Once = Once::new();
+
 static COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
     register_int_counter_vec!(
         COUNTER_NAME_PROMETHEUS,
@@ -66,6 +67,14 @@ static GAUGE: Lazy<IntGaugeVec> = Lazy::new(|| {
     )
     .expect("Failed to register function_calls_concurrent gauge")
 });
+static BUILD_INFO: Lazy<IntGaugeVec> = Lazy::new(|| {
+    register_int_gauge_vec!(
+        BUILD_INFO_NAME,
+        BUILD_INFO_DESCRIPTION,
+        &[COMMIT_KEY, VERSION_KEY]
+    )
+    .expect("Failed to register build_info counter")
+});
 
 pub struct PrometheusTracker {
     start: Instant,
@@ -149,4 +158,12 @@ impl TrackMetrics for PrometheusTracker {
             gauge.dec();
         }
     }
+
+    fn set_build_info(build_info_labels: &BuildInfoLabels) {
+        SET_BUILD_INFO.call_once(|| {
+            BUILD_INFO
+                .with_label_values(&[build_info_labels.commit, build_info_labels.version])
+                .set(1);
+        });
+    }
 }

autometrics/tests/integration_test.rs

@@ -153,3 +153,19 @@ fn caller_label() {
     .unwrap();
     assert!(call_count.is_match(&metrics));
 }
+
+#[cfg(feature = "prometheus-exporter")]
+#[test]
+fn build_info() {
+    let _ = autometrics::global_metrics_exporter();
+
+    #[autometrics]
+    fn function_just_to_initialize_build_info() {}
+
+    function_just_to_initialize_build_info();
+    function_just_to_initialize_build_info();
+
+    let metrics = autometrics::encode_global_metrics().unwrap();
+    let build_info: Regex = Regex::new(r#"build_info\{commit="",version="\S+"\} 1"#).unwrap();
+    assert!(build_info.is_match(&metrics));
+}

examples/full-api/Cargo.toml

@@ -14,3 +14,6 @@ serde = { version = "1", features = ["derive"] }
 strum = { version = "0.24", features = ["derive"] }
 thiserror = "1"
 tokio = { version = "1", features = ["full"] }
+
+[build-dependencies]
+vergen = { version = "8.1", features = ["git", "gitoxide"] }

examples/full-api/build.rs

@@ -0,0 +1,6 @@
+fn main() {
+    vergen::EmitBuilder::builder()
+        .git_sha(true) // short commit hash
+        .emit()
+        .expect("Unable to generate build info");
+}