[typed store] add backoff to safe_drop_db (#21787)

## Description 

Mitigates the race between the destructor and metric threads. Reason:
`rocksdb::DB::Destroy` returns an error immediately if any other
reference is still alive

---

## Release notes

Check each box that your changes affect. If none of the boxes relate to
your changes, release notes aren't required.

For each box you select, include information after the relevant heading
that describes the impact of your changes that a user might notice and
any actions they must take to implement updates.

- [ ] Protocol: 
- [ ] Nodes (Validators and Full nodes): 
- [ ] gRPC:
- [ ] JSON-RPC: 
- [ ] GraphQL: 
- [ ] CLI: 
- [ ] Rust SDK:

Author: phoenix-o

Cargo.lock

@@ -9,6 +9,7 @@ edition = "2021"
 publish = false
 
 [dependencies]
+backoff.workspace = true
 bcs.workspace = true
 bincode.workspace = true
 collectable.workspace = true

crates/typed-store/Cargo.toml

@@ -22,6 +22,7 @@ use crate::{
     traits::{Map, TableSummary},
 };
 use crate::{DbIterator, TypedStoreError};
+use backoff::backoff::Backoff;
 use prometheus::{Histogram, HistogramTimer};
 use rocksdb::properties::num_files_at_level;
 use rocksdb::{checkpoint::Checkpoint, DBPinnableSlice, LiveFile};
@@ -1768,7 +1769,19 @@ pub fn open_cf_opts_secondary<P: AsRef<Path>>(
 }
 
 pub fn safe_drop_db(path: PathBuf) -> Result<(), rocksdb::Error> {
-    rocksdb::DB::destroy(&rocksdb::Options::default(), path)
+    let mut backoff = backoff::ExponentialBackoff {
+        max_elapsed_time: Some(Duration::from_secs(30)),
+        ..Default::default()
+    };
+    loop {
+        match rocksdb::DB::destroy(&rocksdb::Options::default(), path.clone()) {
+            Ok(()) => return Ok(()),
+            Err(err) => match backoff.next_backoff() {
+                Some(duration) => std::thread::sleep(duration),
+                None => return Err(err),
+            },
+        }
+    }
 }
 
 fn populate_missing_cfs(