testing quorum

Author: benedettadavico

Cargo.lock

@@ -27,6 +27,8 @@ pub struct QuorumStateChecker {
     cancellation_token: CancellationToken,
     check_interval: Duration,
     quorum_state: QuorumState,
+    max_retries: u32,
+    retry_initial_delay: Duration,
 }
 
 impl QuorumStateChecker {
@@ -42,6 +44,8 @@ impl QuorumStateChecker {
             quorum_state: QuorumState {
                 available: Arc::new(Default::default()),
             },
+            max_retries: 3,
+            retry_initial_delay: Duration::from_secs(2),
         };
 
         // first check MUST succeed, otherwise we shouldn't start
@@ -56,7 +60,102 @@ impl QuorumStateChecker {
         self.quorum_state.clone()
     }
 
+    fn is_retryable_error(&self, err: &CredentialProxyError) -> bool {
+        let err_str = err.to_string().to_lowercase();
+
+        // Check for DNS-related errors
+        if err_str.contains("dns")
+            || err_str.contains("lookup")
+            || err_str.contains("name resolution")
+            || err_str.contains("temporary failure")
+            || err_str.contains("failed to lookup address")
+        {
+            return true;
+        }
+
+        // Check if it's a Tendermint RPC error (which could be DNS/timeout related)
+        if let CredentialProxyError::NyxdFailure { source: nyxd_err } = err {
+            let nyxd_err_str = nyxd_err.to_string().to_lowercase();
+            if nyxd_err_str.contains("tendermint rpc request failed") {
+                return true;
+            }
+
+            if nyxd_err.is_tendermint_response_timeout() {
+                return true;
+            }
+        }
+
+        false
+    }
+
     async fn check_quorum_state(&self) -> Result<bool, CredentialProxyError> {
+        self.check_quorum_state_with_retry().await
+    }
+
+    async fn check_quorum_state_with_retry(&self) -> Result<bool, CredentialProxyError> {
+        let mut last_error_msg = None;
+        let delay = self.retry_initial_delay;
+
+        for attempt in 0..=self.max_retries {
+            match self.check_quorum_state_once().await {
+                Ok(result) => {
+                    if attempt > 0 {
+                        info!("quorum check succeeded after {} retry attempt(s)", attempt);
+                    }
+                    return Ok(result);
+                }
+                Err(err) => {
+                    let err_msg = err.to_string();
+
+                    // Check if this error is retryable
+                    if !self.is_retryable_error(&err) {
+                        return Err(err);
+                    }
+
+                    last_error_msg = Some(err_msg.clone());
+
+                    if attempt >= self.max_retries {
+                        break;
+                    }
+
+                    // Log the retry attempt
+                    warn!(
+                        "quorum check failed (attempt {}/{}): {}. Retrying in {:?}...",
+                        attempt + 1,
+                        self.max_retries + 1,
+                        err_msg,
+                        delay
+                    );
+
+                    // Wait before retrying with exponential backoff
+                    tokio::time::sleep(delay).await;
+                }
+            }
+        }
+
+        // try one final time to get the actual error
+        match self.check_quorum_state_once().await {
+            Ok(result) => {
+                warn!(
+                    "quorum check succeeded on final attempt after {} retries",
+                    self.max_retries
+                );
+                Ok(result)
+            }
+            Err(err) => {
+                if let Some(error_msg) = last_error_msg {
+                    error!(
+                        "quorum check failed after {} retry attempts. Last error: {}",
+                        self.max_retries + 1,
+                        error_msg
+                    );
+                }
+                Err(err)
+            }
+        }
+    }
+
+    async fn check_quorum_state_once(&self) -> Result<bool, CredentialProxyError> {
         let client_guard = self.client.query_chain().await;
 
         // split the operation as we only need to hold the reference to chain client for the first part
@@ -93,7 +192,7 @@ impl QuorumStateChecker {
                     break
                 }
                 _ = tokio::time::sleep(self.check_interval) => {
-                    match self.check_quorum_state().await {
+                    match self.check_quorum_state_with_retry().await {
                         Ok(available) => self.quorum_state.available.store(available, Ordering::SeqCst),
                         Err(err) => error!("failed to check current quorum state: {err}"),
                     }

common/credential-proxy/src/quorum_checker.rs

@@ -1,6 +1,6 @@
 [package]
 name = "nym-credential-proxy"
-version = "0.3.0"
+version = "0.3.0-test"
 authors.workspace = true
 repository.workspace = true
 homepage.workspace = true