WIP Introduce xcp-metrics-plugin-squeezed plugin for xcp-rrdd

xcp-metrics-plugin-squeezed is meant as a drop-in replacement for
rrdp-squeezed, as a first Rust-written brick for XAPI.

Metrics are read in Xenstore using the xenstore-rs bindings of
libxenstore, structured in a suitable way for upcoming OpenMetrics
support, and communicated to xcp-rrdd using its v2 protocol.

NOTE: this currently references external git repository for
xenstore-rs, as we had to add support for xs_watch there and the PR is
still pending.

Signed-off-by: Teddy Astie <teddy.astie@outlook.fr>
Reviewed-by: Yann Dirson <yann.dirson@vates.fr>

.github/workflows/rust.yml

@@ -0,0 +1,79 @@
+name: Rust
+
+on:
+  push:
+  pull_request:
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    #container:
+    #  image: centos8 
+
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: 1.66
+        override: true
+    - name: Installing dependency packages
+      run: sudo apt update && sudo apt install -y libxen-dev protobuf-compiler
+
+    - name: Build
+      run: cargo build --verbose
+
+  doc:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: 1.66
+        override: true
+    - name: Installing dependency packages
+      run: sudo apt update && sudo apt install -y libxen-dev protobuf-compiler
+
+    - name: Build docs
+      run: cargo doc --workspace --no-deps
+
+    - name: Upload docs
+      uses: actions/upload-artifact@v3
+      with:
+        name: rustdoc-html
+        path: ./target/doc
+
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: nightly
+        override: true
+    - name: Installing dependency packages
+      run: sudo apt update && sudo apt install -y libxen-dev protobuf-compiler
+
+    - name: Run tests
+      run: |
+        cargo clean
+        cargo test --verbose
+      env:
+        CARGO_INCREMENTAL: '0'
+        RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests'
+        RUSTDOCFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests'
+
+    - name: Install grcov
+      run: cargo install grcov
+    - name: Test coverage report
+      run: grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existing -o ./target/debug/coverage/
+
+    - name: Upload test coverage report
+      uses: actions/upload-artifact@v3
+      with:
+        name: coverage-html
+        path: ./target/debug/coverage/

.gitignore

@@ -0,0 +1 @@
+/target

Cargo.toml

@@ -0,0 +1,12 @@
+[workspace]
+package.version = "0.1.0"
+package.repository = "https://github.com/xcp-ng/xcp-metrics"
+package.categories = ["virtualization"]
+members = [
+  "xcp-metrics-common",
+  "xapi-rs",
+  "plugins/xcp-metrics-plugin-common",
+  "plugins/xcp-metrics-plugin-squeezed",
+]
+[profile.release]
+lto = true

devscripts/gen-tarball.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -e
+
+# FIXME "git archive" wants $REF and "cargo vendor" wants $PWD
+
+REF="$1"
+PACKAGE="xcp-metrics"
+#VERSION=$(git describe --always "$REF")
+VERSION=0.0.0
+
+# FIXME use "cargo package"?  Requires all of our crates to include a version :/
+git archive "$REF" --prefix="$PACKAGE-$VERSION/" -o "../$PACKAGE-$VERSION.tar.gz"

devscripts/gen-vendor-tarball.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -e
+
+PACKAGE="xcp-metrics"
+#VERSION=$(git describe --always "HEAD")
+VERSION=0.0.0
+
+# # replace references to patched crates, for "cargo vendor"
+# find -name Cargo.toml | xargs sed -Ei '/^(git|branch) = / s/^/#GEN#/'
+
+cargo vendor --versioned-dirs third-party
+tar -zcf "../$PACKAGE-$VERSION-vendor.tar.gz" --xform="s,^,$PACKAGE-$VERSION/," third-party/
+
+# # unpatch references to patched crates
+# find -name Cargo.toml | xargs sed -Ei 's/^#GEN#//'

plugins/xcp-metrics-plugin-common/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+name = "xcp-metrics-plugin-common"
+description = "Library that helps making plugins for xcp-metrics and xcp-rrdd"
+version.workspace = true
+license = "AGPL-3.0-only"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+xcp-metrics-common = { path = "../../xcp-metrics-common" }
+xapi = { path = "../../xapi-rs" }
+
+anyhow = "1.0"
+tokio = "1"
+uuid = "1.4"
+tracing = "0.1"
+tracing-subscriber = "0.3"
+
+dashmap = "5.5"
+futures = "0.3"
+
+[dependencies.xenstore-rs]
+version = "0.3"
+optional = true
+
+[dependencies.xenstore-rs-wip]
+package = "xenstore-rs"
+optional = true
+
+[features]
+default = []
+xenstore = ["dep:xenstore-rs"]
+xenstore-wip = ["xenstore", "dep:xenstore-rs-wip"]

plugins/xcp-metrics-plugin-common/src/bridge/mod.rs

@@ -0,0 +1,3 @@
+//! Bridges for conversion beteen protocol v2 and protocol v3.
+
+pub mod v3_to_v2;

plugins/xcp-metrics-plugin-common/src/bridge/v3_to_v2.rs

@@ -0,0 +1,142 @@
+//! Adapter to convert from protocol v3 to protocol v2.
+use std::{collections::HashMap, iter};
+
+use xcp_metrics_common::{
+    metrics::{Label, Metric, MetricFamily, MetricPoint, MetricSet, MetricValue},
+    rrdd::{
+        protocol_common::{DataSourceMetadata, DataSourceValue},
+        protocol_v2::{indexmap::IndexMap, RrddMetadata},
+    },
+    utils::{
+        delta::MetricSetModel,
+        mapping::{CustomMapping, DefaultMapping, MetadataMapping},
+    },
+};
+
+/// Adapter to convert protocol v3 metrics set into protocol v2 metadata and data.
+pub struct BridgeToV2 {
+    model: MetricSetModel,
+    latest_set: MetricSet,
+    custom_mappings: HashMap<Box<str>, CustomMapping>,
+
+    metadata: RrddMetadata,
+    metadata_map: Vec<(Box<str>, Box<[Label]>)>,
+}
+
+/// Convert a MetricPoint into a protocol-v2 value.
+fn metric_point_to_v2(metric_point: &MetricPoint) -> DataSourceValue {
+    match metric_point.value {
+        MetricValue::Gauge(value) => DataSourceValue::from(value),
+        MetricValue::Counter { total, .. } => DataSourceValue::from(total),
+        _ => DataSourceValue::Undefined,
+    }
+}
+
+impl BridgeToV2 {
+    pub fn new() -> Self {
+        Self {
+            model: MetricSetModel::default(),
+            latest_set: MetricSet::default(),
+            custom_mappings: HashMap::default(),
+            metadata: RrddMetadata {
+                datasources: IndexMap::default(),
+            },
+            metadata_map: vec![],
+        }
+    }
+
+    pub fn with_mappings(custom_mappings: HashMap<Box<str>, CustomMapping>) -> Self {
+        Self {
+            custom_mappings,
+            ..Default::default()
+        }
+    }
+
+    fn metric_to_v2_metadata(
+        &self,
+        family_name: &str,
+        family: &MetricFamily,
+        metric: &Metric,
+    ) -> Option<(Box<str>, DataSourceMetadata)> {
+        if let Some(custom_mapping) = self.custom_mappings.get(family_name) {
+            custom_mapping.convert(family_name, family, metric)
+        } else {
+            DefaultMapping.convert(family_name, family, metric)
+        }
+    }
+
+    /// Update bridge information, returns true on metadata change.
+    pub fn update(&mut self, metrics_set: MetricSet) -> bool {
+        let delta = self.model.compute_delta(&metrics_set);
+        self.model.apply_delta(&delta);
+
+        if !delta.added_families.is_empty()
+            || !delta.added_metrics.is_empty()
+            || !delta.removed_metrics.is_empty()
+        {
+            self.latest_set = metrics_set;
+            self.reset_metadata();
+            true
+        } else {
+            self.latest_set = metrics_set;
+            false
+        }
+    }
+
+    pub fn get_data(&self) -> Box<[DataSourceValue]> {
+        self.metadata_map
+            .iter()
+            .filter_map(|(family_name, labels)| self.get_first_metric_point(family_name, labels))
+            .map(metric_point_to_v2)
+            .collect::<Box<[DataSourceValue]>>()
+    }
+
+    pub fn get_metadata(&self) -> &RrddMetadata {
+        &self.metadata
+    }
+
+    fn get_first_metric_point<'a>(
+        &'a self,
+        family_name: &str,
+        labels: &[Label],
+    ) -> Option<&'a MetricPoint> {
+        self.latest_set
+            .families
+            .get(family_name)
+            .and_then(|family| {
+                family
+                    .metrics
+                    .iter()
+                    // Filter by labels
+                    .filter(|(_, metric)| metric.labels.as_ref() == labels)
+                    // Only take first metrics_point
+                    .find_map(|(_, metric)| metric.metrics_point.first())
+            })
+    }
+
+    fn reset_metadata(&mut self) {
+        let (datasources, metadata_map) = self
+            .latest_set
+            .families
+            .iter()
+            // Combine family with family name and metrics.
+            .flat_map(|(name, family)| {
+                iter::zip(iter::repeat((name, family)), family.metrics.iter())
+            })
+            // Convert this data to protocol v2 metadata and mapping information.
+            .filter_map(|((family_name, family), (_, metric))| {
+                self.metric_to_v2_metadata(family_name, family, metric)
+                    .map(|mapping| (mapping, (family_name.clone(), metric.labels.clone())))
+            })
+            .unzip();
+
+        self.metadata = RrddMetadata { datasources };
+        self.metadata_map = metadata_map;
+    }
+}
+
+impl Default for BridgeToV2 {
+    fn default() -> Self {
+        Self::new()
+    }
+}

plugins/xcp-metrics-plugin-common/src/lib.rs

@@ -0,0 +1,7 @@
+pub mod bridge;
+pub mod plugin;
+pub mod protocol_v2;
+pub mod protocol_v3;
+
+#[cfg(feature = "xenstore")]
+pub mod xenstore;