feat(storage): add metric to monitor available split block tasks

Author: pashinov

scripts/gen-dashboard.py

@@ -840,6 +840,12 @@ def storage() -> RowPanel:
         create_heatmap_panel(
             "tycho_storage_cell_in_mem_store_time", "Time to store cell without write"
         ),
+        create_heatmap_quantile_panel(
+            "tycho_storage_split_block_available_permits",
+            "Split block available permits",
+            UNITS.NUMBER_FORMAT,
+            "0.999",
+        ),
         create_heatmap_panel(
             "tycho_storage_batch_write_time", "Time to write merge in write batch"
         ),

storage/src/store/block/mod.rs

@@ -1032,6 +1032,9 @@ impl BlockStorage {
         self.db.rocksdb().write(batch)?;
 
         // Start splitting block data
+        metrics::histogram!("tycho_storage_split_block_available_permits")
+            .record(self.split_block_semaphore.available_permits() as f64);
+
         let permit = self.split_block_semaphore.clone().acquire_owned().await?;
         let _handle = self.spawn_split_block_data(&id.block_id, data, permit);