refactor: [#1446] WIP. Calculate global metrics from labeled metrics

We need to add a new label to make it easier to fileter by the server IP
family: IPV4 or IPv6.

Author: josecelano

Cargo.lock

@@ -32,7 +32,12 @@ pub async fn handle_event(event: Event, stats_repository: &Arc<Repository>, now:
                 .increase_counter(&metric_name!(HTTP_TRACKER_CORE_REQUESTS_RECEIVED_TOTAL), &label_set, now)
                 .await
             {
-                Ok(()) => {}
+                Ok(()) => {
+                    tracing::debug!(
+                        "Successfully increased the counter for HTTP announce requests received: {}",
+                        label_set
+                    );
+                }
                 Err(err) => tracing::error!("Failed to increase the counter: {}", err),
             };
         }
@@ -57,7 +62,12 @@ pub async fn handle_event(event: Event, stats_repository: &Arc<Repository>, now:
                 .increase_counter(&metric_name!(HTTP_TRACKER_CORE_REQUESTS_RECEIVED_TOTAL), &label_set, now)
                 .await
             {
-                Ok(()) => {}
+                Ok(()) => {
+                    tracing::debug!(
+                        "Successfully increased the counter for HTTP scrape requests received: {}",
+                        label_set
+                    );
+                }
                 Err(err) => tracing::error!("Failed to increase the counter: {}", err),
             };
         }

packages/http-tracker-core/src/statistics/event/handler.rs

@@ -8,7 +8,7 @@ use torrust_tracker_metrics::metric::description::MetricDescription;
 use torrust_tracker_metrics::metric_name;
 use torrust_tracker_metrics::unit::Unit;
 
-const HTTP_TRACKER_CORE_REQUESTS_RECEIVED_TOTAL: &str = "http_tracker_core_requests_received_total";
+pub const HTTP_TRACKER_CORE_REQUESTS_RECEIVED_TOTAL: &str = "http_tracker_core_requests_received_total";
 
 #[must_use]
 pub fn describe_metrics() -> Metrics {

packages/http-tracker-core/src/statistics/mod.rs

@@ -1,6 +1,6 @@
 use derive_more::Display;
 
-#[derive(Debug, Display, Clone, Copy, PartialEq)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Default)]
 pub struct AggregateValue(f64);
 
 impl AggregateValue {

packages/metrics/src/aggregate.rs

@@ -23,6 +23,7 @@ torrust-tracker-metrics = { version = "3.0.0-develop", path = "../metrics" }
 torrust-tracker-primitives = { version = "3.0.0-develop", path = "../primitives" }
 torrust-tracker-swarm-coordination-registry = { version = "3.0.0-develop", path = "../swarm-coordination-registry" }
 torrust-udp-tracker-server = { version = "3.0.0-develop", path = "../udp-tracker-server" }
+tracing = "0"
 
 [dev-dependencies]
 torrust-tracker-events = { version = "3.0.0-develop", path = "../events" }

packages/rest-tracker-api-core/Cargo.toml

@@ -1,11 +1,14 @@
 use std::sync::Arc;
 
+use bittorrent_http_tracker_core::statistics::HTTP_TRACKER_CORE_REQUESTS_RECEIVED_TOTAL;
 use bittorrent_tracker_core::torrent::repository::in_memory::InMemoryTorrentRepository;
 use bittorrent_udp_tracker_core::services::banning::BanService;
 use bittorrent_udp_tracker_core::{self};
 use tokio::sync::RwLock;
+use torrust_tracker_metrics::metric_collection::aggregate::Sum;
 use torrust_tracker_metrics::metric_collection::MetricCollection;
-use torrust_udp_tracker_server::statistics as udp_server_statistics;
+use torrust_tracker_metrics::metric_name;
+use torrust_udp_tracker_server::statistics::{self as udp_server_statistics, UDP_TRACKER_SERVER_REQUESTS_ACCEPTED_TOTAL};
 
 use super::metrics::TorrentsMetrics;
 use crate::statistics::metrics::ProtocolMetrics;
@@ -32,9 +35,38 @@ pub async fn get_metrics(
     http_stats_repository: Arc<bittorrent_http_tracker_core::statistics::repository::Repository>,
     udp_server_stats_repository: Arc<udp_server_statistics::repository::Repository>,
 ) -> TrackerMetrics {
+    let protocol_metrics_from_global_metrics = get_protocol_metrics(
+        ban_service.clone(),
+        http_stats_repository.clone(),
+        udp_server_stats_repository.clone(),
+    )
+    .await;
+
+    let protocol_metrics_from_labeled_metrics = get_protocol_metrics_from_labeled_metrics(
+        ban_service.clone(),
+        http_stats_repository.clone(),
+        udp_server_stats_repository.clone(),
+    )
+    .await;
+
+    // todo:
+    // We keep both metrics until we deploy to production and we can
+    // ensure that the protocol metrics from labeled metrics are correct.
+    // After that we can remove the `get_protocol_metrics` function and
+    // use only the `get_protocol_metrics_from_labeled_metrics` function.
+    // And also remove the code in repositories to generate the global metrics.
+    let protocol_metrics = if protocol_metrics_from_global_metrics == protocol_metrics_from_labeled_metrics {
+        protocol_metrics_from_labeled_metrics
+    } else {
+        // tracing::warn!("The protocol metrics from global metrics and labeled metrics are different");
+        // tracing::warn!("Global metrics: {:?}", protocol_metrics_from_global_metrics);
+        // tracing::warn!("Labeled metrics: {:?}", protocol_metrics_from_labeled_metrics);
+        protocol_metrics_from_global_metrics
+    };
+
     TrackerMetrics {
         torrents_metrics: get_torrents_metrics(in_memory_torrent_repository, tracker_core_stats_repository).await,
-        protocol_metrics: get_protocol_metrics(ban_service, http_stats_repository, udp_server_stats_repository).await,
+        protocol_metrics,
     }
 }
 
@@ -99,6 +131,87 @@ async fn get_protocol_metrics(
     }
 }
 
+#[allow(deprecated)]
+async fn get_protocol_metrics_from_labeled_metrics(
+    ban_service: Arc<RwLock<BanService>>,
+    http_stats_repository: Arc<bittorrent_http_tracker_core::statistics::repository::Repository>,
+    udp_server_stats_repository: Arc<udp_server_statistics::repository::Repository>,
+) -> ProtocolMetrics {
+    let udp_banned_ips_total = ban_service.read().await.get_banned_ips_total();
+    let http_stats = http_stats_repository.get_stats().await;
+    let udp_server_stats = udp_server_stats_repository.get_stats().await;
+
+    #[allow(clippy::cast_sign_loss)]
+    #[allow(clippy::cast_possible_truncation)]
+    let tcp4_announces_handled = http_stats
+        .metric_collection
+        .sum(
+            &metric_name!(HTTP_TRACKER_CORE_REQUESTS_RECEIVED_TOTAL),
+            &[("request_kind", "announce")].into(), // todo: add label for `server_binding_ip_family` with value `inet` (inet/inet6)
+        )
+        .unwrap_or_default()
+        .value() as u64;
+
+    #[allow(clippy::cast_sign_loss)]
+    #[allow(clippy::cast_possible_truncation)]
+    let udp4_announces_handled = udp_server_stats
+        .metric_collection
+        .sum(
+            &metric_name!(UDP_TRACKER_SERVER_REQUESTS_ACCEPTED_TOTAL),
+            &[("request_kind", "announce")].into(), // todo: add label for `server_binding_ip_family` with value `inet` (inet/inet6)
+        )
+        .unwrap_or_default()
+        .value() as u64;
+
+    /*
+    
+    todo:
+
+    - Add a label for `server_binding_ip_family` with value `inet` (inet/inet6)
+      to all metrics containing an IP address. This will allow us to distinguish 
+      between IPv4 and IPv6 metrics.
+    - Continue replacing the other metrics with the labeled metrics.
+    
+    */
+
+    // For backward compatibility we keep the `tcp4_connections_handled` and
+    // `tcp6_connections_handled` metrics. They don't make sense for the HTTP
+    // tracker, but we keep them for now. In new major versions we should remove
+    // them.
+
+    ProtocolMetrics {
+        // TCPv4
+        tcp4_connections_handled: tcp4_announces_handled + http_stats.tcp4_scrapes_handled,
+        tcp4_announces_handled,
+        tcp4_scrapes_handled: http_stats.tcp4_scrapes_handled,
+        // TCPv6
+        tcp6_connections_handled: http_stats.tcp6_announces_handled + http_stats.tcp6_scrapes_handled,
+        tcp6_announces_handled: http_stats.tcp6_announces_handled,
+        tcp6_scrapes_handled: http_stats.tcp6_scrapes_handled,
+        // UDP
+        udp_requests_aborted: udp_server_stats.udp_requests_aborted,
+        udp_requests_banned: udp_server_stats.udp_requests_banned,
+        udp_banned_ips_total: udp_banned_ips_total as u64,
+        udp_avg_connect_processing_time_ns: udp_server_stats.udp_avg_connect_processing_time_ns,
+        udp_avg_announce_processing_time_ns: udp_server_stats.udp_avg_announce_processing_time_ns,
+        udp_avg_scrape_processing_time_ns: udp_server_stats.udp_avg_scrape_processing_time_ns,
+        // UDPv4
+        udp4_requests: udp_server_stats.udp4_requests,
+        udp4_connections_handled: udp_server_stats.udp4_connections_handled,
+        udp4_announces_handled,
+        udp4_scrapes_handled: udp_server_stats.udp4_scrapes_handled,
+        udp4_responses: udp_server_stats.udp4_responses,
+        udp4_errors_handled: udp_server_stats.udp4_errors_handled,
+        // UDPv6
+        udp6_requests: udp_server_stats.udp6_requests,
+        udp6_connections_handled: udp_server_stats.udp6_connections_handled,
+        udp6_announces_handled: udp_server_stats.udp6_announces_handled,
+        udp6_scrapes_handled: udp_server_stats.udp6_scrapes_handled,
+        udp6_responses: udp_server_stats.udp6_responses,
+        udp6_errors_handled: udp_server_stats.udp6_errors_handled,
+    }
+}
+
 #[derive(Debug, PartialEq)]
 pub struct TrackerLabeledMetrics {
     pub metrics: MetricCollection,

packages/rest-tracker-api-core/src/statistics/services.rs

@@ -47,7 +47,9 @@ pub async fn handle_event(
         .increase_counter(&metric_name!(UDP_TRACKER_SERVER_REQUESTS_ACCEPTED_TOTAL), &label_set, now)
         .await
     {
-        Ok(()) => {}
+        Ok(()) => {
+            tracing::debug!("Successfully increased the counter for UDP requests accepted: {}", label_set);
+        }
         Err(err) => tracing::error!("Failed to increase the counter: {}", err),
     };
 }

packages/udp-tracker-server/src/statistics/event/handler/request_accepted.rs

@@ -13,7 +13,7 @@ const UDP_TRACKER_SERVER_REQUESTS_BANNED_TOTAL: &str = "udp_tracker_server_reque
 pub(crate) const UDP_TRACKER_SERVER_IPS_BANNED_TOTAL: &str = "udp_tracker_server_ips_banned_total";
 const UDP_TRACKER_SERVER_CONNECTION_ID_ERRORS_TOTAL: &str = "udp_tracker_server_connection_id_errors_total";
 const UDP_TRACKER_SERVER_REQUESTS_RECEIVED_TOTAL: &str = "udp_tracker_server_requests_received_total";
-const UDP_TRACKER_SERVER_REQUESTS_ACCEPTED_TOTAL: &str = "udp_tracker_server_requests_accepted_total";
+pub const UDP_TRACKER_SERVER_REQUESTS_ACCEPTED_TOTAL: &str = "udp_tracker_server_requests_accepted_total";
 const UDP_TRACKER_SERVER_RESPONSES_SENT_TOTAL: &str = "udp_tracker_server_responses_sent_total";
 const UDP_TRACKER_SERVER_ERRORS_TOTAL: &str = "udp_tracker_server_errors_total";
 const UDP_TRACKER_SERVER_PERFORMANCE_AVG_PROCESSING_TIME_NS: &str = "udp_tracker_server_performance_avg_processing_time_ns";