more tls metrics

Cargo.toml

@@ -122,6 +122,7 @@ url = "2.5"
 uuid = { version = "1.8", features = ["v7"] }
 webpki-roots = "0.26"
 x509-parser = "0.16"
+zeroize = { version = "1.8", features = ["derive"] }
 
 [dev-dependencies]
 hex-literal = "0.4"

src/tls/mod.rs

@@ -52,19 +52,24 @@ pub fn prepare_server_config(
     session_storage: Arc<dyn StoresServerSessions + Send + Sync>,
     additional_alpn: Vec<Vec<u8>>,
     ticket_lifetime: Duration,
+    registry: &Registry,
 ) -> ServerConfig {
     let mut cfg = ServerConfig::builder_with_protocol_versions(&[&TLS13, &TLS12])
         .with_no_client_auth()
         .with_cert_resolver(resolver);
 
     // Set custom session storage with to allow effective TLS session resumption
-    cfg.session_storage = session_storage;
-
-    // Enable ticketer
-    let ticketer = TicketSwitcher::new(ticket_lifetime.as_secs() as u32, || {
-        Ok(Box::new(tickets::Ticketer::new()))
-    })
-    .unwrap();
+    let session_storage = sessions::WithMetrics(session_storage, sessions::Metrics::new(registry));
+    cfg.session_storage = Arc::new(session_storage);
+
+    // Enable ticketer to encrypt/decrypt TLS tickets
+    let ticketer = tickets::WithMetrics(
+        TicketSwitcher::new(ticket_lifetime.as_secs() as u32, move || {
+            Ok(Box::new(tickets::Ticketer::new()))
+        })
+        .unwrap(),
+        tickets::Metrics::new(registry),
+    );
     cfg.ticketer = Arc::new(ticketer);
 
     // Enable tickets
@@ -215,6 +220,7 @@ pub async fn setup(
             vec![]
         },
         cli.http_server.tls_ticket_lifetime,
+        registry,
     );
 
     Ok(config)

src/tls/sessions.rs

@@ -1,14 +1,21 @@
-use std::time::Duration;
+use std::{sync::Arc, time::Duration};
 
 use ahash::RandomState;
 use moka::sync::Cache;
+use prometheus::{register_int_counter_vec_with_registry, IntCounterVec, Registry};
 use rustls::server::StoresServerSessions;
+use zeroize::ZeroizeOnDrop;
 
 type Key = Vec<u8>;
-type Val = Vec<u8>;
+
+/// Sessions are considered highly sensitive data, so wipe the memory when
+/// they're removed from storage. We can't do anything with the returned Vec<u8>,
+/// but it's better than nothing.
+#[derive(Debug, PartialEq, Eq, Hash, Clone, ZeroizeOnDrop)]
+struct Val(Vec<u8>);
 
 fn weigher(k: &Key, v: &Val) -> u32 {
-    (k.len() + v.len()) as u32
+    (k.len() + v.0.len()) as u32
 }
 
 pub struct Stats {
@@ -46,23 +53,78 @@ impl Storage {
 
 impl StoresServerSessions for Storage {
     fn get(&self, key: &[u8]) -> Option<Vec<u8>> {
-        self.cache.get(key)
+        self.cache.get(key).map(|x| x.0.clone())
     }
 
     fn put(&self, key: Vec<u8>, value: Vec<u8>) -> bool {
-        self.cache.insert(key, value);
+        self.cache.insert(key, Val(value));
         true
     }
 
     fn take(&self, key: &[u8]) -> Option<Vec<u8>> {
-        self.cache.remove(key)
+        self.cache.remove(key).map(|x| x.0.clone())
     }
 
     fn can_cache(&self) -> bool {
         true
     }
 }
 
+#[derive(Debug)]
+pub struct Metrics {
+    processed: IntCounterVec,
+}
+
+impl Metrics {
+    pub fn new(registry: &Registry) -> Self {
+        Self {
+            processed: register_int_counter_vec_with_registry!(
+                format!("tls_sessions"),
+                format!("Number of TLS sessions that were processed"),
+                &["action", "found"],
+                registry
+            )
+            .unwrap(),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct WithMetrics(pub Arc<dyn StoresServerSessions + Send + Sync>, pub Metrics);
+
+impl WithMetrics {
+    fn record(&self, action: &str, ok: bool) {
+        self.1
+            .processed
+            .with_label_values(&[action, if ok { "yes" } else { "no" }])
+            .inc();
+    }
+}
+
+impl StoresServerSessions for WithMetrics {
+    fn get(&self, key: &[u8]) -> Option<Vec<u8>> {
+        let v = self.0.get(key);
+        self.record("get", v.is_some());
+        v
+    }
+
+    fn put(&self, key: Vec<u8>, value: Vec<u8>) -> bool {
+        let v = self.0.put(key, value);
+        self.record("put", v);
+        v
+    }
+
+    fn take(&self, key: &[u8]) -> Option<Vec<u8>> {
+        let v = self.0.take(key);
+        self.record("take", v.is_some());
+        v
+    }
+
+    fn can_cache(&self) -> bool {
+        self.0.can_cache()
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

src/tls/tickets.rs

@@ -7,15 +7,41 @@ use chacha20poly1305::{
     aead::{Aead, AeadCore, KeyInit, OsRng},
     XChaCha20Poly1305, XNonce,
 };
+use prometheus::{register_int_counter_vec_with_registry, IntCounterVec, Registry};
 use rustls::server::ProducesTickets;
+use zeroize::ZeroizeOnDrop;
 
 // We're using 192-bit nonce
 const NONCE_LEN: usize = 192 / 8;
 
+#[derive(Debug)]
+pub struct Metrics {
+    processed: IntCounterVec,
+}
+
+impl Metrics {
+    pub fn new(registry: &Registry) -> Self {
+        Self {
+            processed: register_int_counter_vec_with_registry!(
+                format!("tls_tickets"),
+                format!("Number of TLS tickets that were processed"),
+                &["action", "result"],
+                registry
+            )
+            .unwrap(),
+        }
+    }
+}
+
 /// Encrypts & decrypts tickets for TLS 1.3 session resumption.
-/// Must be used with rustls::ticketer::TicketSwitcher to facilitate key rotation.
-/// We're using XChaCha20Poly1305 authenicated encryption (AEAD)
+/// Must be used with `rustls::ticketer::TicketSwitcher` to facilitate key rotation.
+/// We're using XChaCha20Poly1305 authenicated encryption (AEAD).
+/// ZeroizeOnDrop is derived below to make sure the encryption keys are wiped from
+/// memory when the Ticketer is dropped.
+/// See https://docs.rs/zeroize/latest/zeroize/#what-guarantees-does-this-crate-provide
+#[derive(ZeroizeOnDrop)]
 pub struct Ticketer {
+    #[zeroize(skip)]
     counter: AtomicU32,
     cipher: XChaCha20Poly1305,
 }
@@ -28,11 +54,11 @@ impl fmt::Debug for Ticketer {
 
 impl Ticketer {
     pub fn new() -> Self {
-        // Generate random key that is valid for the lifetime of this ticketer
+        // Generate a random key that is valid for the lifetime of this ticketer
         let key = XChaCha20Poly1305::generate_key(&mut OsRng);
-        let cipher = XChaCha20Poly1305::new(&key);
+
         Self {
-            cipher,
+            cipher: XChaCha20Poly1305::new(&key),
             counter: AtomicU32::new(0),
         }
     }
@@ -63,8 +89,7 @@ impl ProducesTickets for Ticketer {
         let nonce = XNonce::from_slice(&cipher[0..NONCE_LEN]);
 
         // Try to decrypt
-        let plaintext = self.cipher.decrypt(nonce, &cipher[NONCE_LEN..]).ok()?;
-        Some(plaintext)
+        self.cipher.decrypt(nonce, &cipher[NONCE_LEN..]).ok()
     }
 
     fn encrypt(&self, plain: &[u8]) -> Option<Vec<u8>> {
@@ -87,6 +112,40 @@ impl ProducesTickets for Ticketer {
     }
 }
 
+#[derive(Debug)]
+pub struct WithMetrics<T: ProducesTickets>(pub T, pub Metrics);
+
+impl<T: ProducesTickets> WithMetrics<T> {
+    fn record(&self, action: &str, res: &Option<Vec<u8>>) {
+        self.1
+            .processed
+            .with_label_values(&[action, if res.is_some() { "ok" } else { "fail" }])
+            .inc();
+    }
+}
+
+impl<T: ProducesTickets> ProducesTickets for WithMetrics<T> {
+    fn enabled(&self) -> bool {
+        self.0.enabled()
+    }
+
+    fn lifetime(&self) -> u32 {
+        self.0.lifetime()
+    }
+
+    fn encrypt(&self, plain: &[u8]) -> Option<Vec<u8>> {
+        let res = self.0.encrypt(plain);
+        self.record("encrypt", &res);
+        res
+    }
+
+    fn decrypt(&self, cipher: &[u8]) -> Option<Vec<u8>> {
+        let res = self.0.decrypt(cipher);
+        self.record("decrypt", &res);
+        res
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
@@ -106,5 +165,8 @@ mod test {
         let ciphertext = t.encrypt(msg).unwrap();
         let plaintext = t.decrypt(&ciphertext).unwrap();
         assert_eq!(&msg[..], plaintext);
+
+        // Check that bad ciphertext fails to decrypt
+        assert!(t.decrypt(msg).is_none());
     }
 }