make state pruner faster

mehdi-aouadi · Oct 16, 2024 · 6a42ffc · 6a42ffc
1 parent 2985a96
commit 6a42ffc
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 12 deletions.
diff --git a/...es/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java b/...es/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java
@@ -18,6 +18,7 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.Optional;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -134,12 +135,20 @@ protected SafeFuture<?> doStart() {
                 configureStatePruner(
                     config.getRetainedSlots(),
                     storagePrunerAsyncRunner,
+                    config.getStatePruningInterval(),
                     pruningTimingsLabelledGauge,
                     pruningActiveLabelledGauge);
               } else if (!config.getDataStorageMode().storesFinalizedStates()) {
+                final Duration statePruningInterval =
+                    config
+                            .getStatePruningInterval()
+                            .equals(StorageConfiguration.DEFAULT_STATE_PRUNING_INTERVAL)
+                        ? Duration.ofMinutes(1)
+                        : config.getStatePruningInterval();
                 configureStatePruner(
                     StorageConfiguration.DEFAULT_STORAGE_RETAINED_SLOTS,
                     storagePrunerAsyncRunner,
+                    statePruningInterval,
                     pruningTimingsLabelledGauge,
                     pruningActiveLabelledGauge);
               }
@@ -221,6 +230,7 @@ protected SafeFuture<?> doStart() {
   void configureStatePruner(
       final long slotsToRetain,
       final AsyncRunner storagePrunerAsyncRunner,
+      final Duration pruningInterval,
       final SettableLabelledGauge pruningTimingsLabelledGauge,
       final SettableLabelledGauge pruningActiveLabelledGauge) {
     if (config.getDataStorageCreateDbVersion() == DatabaseVersion.LEVELDB_TREE) {
@@ -240,7 +250,7 @@ void configureStatePruner(
                 config.getSpec(),
                 database,
                 storagePrunerAsyncRunner,
-                config.getStatePruningInterval(),
+                pruningInterval,
                 slotsToRetain,
                 config.getStatePruningLimit(),
                 "state",

diff --git a/...hainstorage/src/test/java/tech/pegasys/teku/services/chainstorage/StorageServiceTest.java b/...hainstorage/src/test/java/tech/pegasys/teku/services/chainstorage/StorageServiceTest.java
@@ -20,6 +20,7 @@
 import static org.mockito.Mockito.when;
 
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.Optional;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 import org.junit.jupiter.api.BeforeEach;
@@ -59,6 +60,10 @@ void setUp(@TempDir final Path tempDir) {
         .thenReturn(StorageConfiguration.DEFAULT_MAX_KNOWN_NODE_CACHE_SIZE);
     when(storageConfiguration.getDataStorageFrequency())
         .thenReturn(StorageConfiguration.DEFAULT_STORAGE_FREQUENCY);
+    when(storageConfiguration.getStatePruningLimit())
+        .thenReturn(StorageConfiguration.DEFAULT_STATE_PRUNING_LIMIT);
+    when(storageConfiguration.getStatePruningInterval())
+        .thenReturn(StorageConfiguration.DEFAULT_STATE_PRUNING_INTERVAL);
     when(storageConfiguration.getEth1DepositContract()).thenReturn(eth1DepositContract);
     when(storageConfiguration.isStoreNonCanonicalBlocksEnabled()).thenReturn(false);
     when(storageConfiguration.getSpec()).thenReturn(spec);
@@ -80,39 +85,60 @@ void setUp(@TempDir final Path tempDir) {
   void shouldNotSetupStatePrunerWhenArchiveMode() {
     when(storageConfiguration.getDataStorageMode()).thenReturn(StateStorageMode.ARCHIVE);
     final SafeFuture<?> future = storageService.doStart();
-    final Optional<StatePruner> statePruner = storageService.getStatePruner();
+    final Optional<StatePruner> maybeStatePruner = storageService.getStatePruner();
     assertThat(future).isCompleted();
-    assertThat(statePruner).isEmpty();
+    assertThat(maybeStatePruner).isEmpty();
   }
 
   @Test
   void shouldSetupStatePrunerWhenArchiveModeAndRetentionSlotsEnabled() {
     when(storageConfiguration.getDataStorageMode()).thenReturn(StateStorageMode.ARCHIVE);
     when(storageConfiguration.getRetainedSlots()).thenReturn(5L);
     final SafeFuture<?> future = storageService.doStart();
-    final Optional<StatePruner> statePruner = storageService.getStatePruner();
+    final Optional<StatePruner> maybeStatePruner = storageService.getStatePruner();
     assertThat(future).isCompleted();
-    assertThat(statePruner).isPresent();
-    assertThat(storageService.getStatePruner().get().isRunning()).isTrue();
+    assertThat(maybeStatePruner).isPresent();
+    final StatePruner statePruner = maybeStatePruner.get();
+    assertThat(statePruner.isRunning()).isTrue();
+    assertThat(statePruner.getPruneInterval())
+        .isEqualTo(StorageConfiguration.DEFAULT_STATE_PRUNING_INTERVAL);
   }
 
   @Test
   void shouldSetupStatePrunerWhenPruneMode() {
     when(storageConfiguration.getDataStorageMode()).thenReturn(StateStorageMode.PRUNE);
     final SafeFuture<?> future = storageService.doStart();
-    final Optional<StatePruner> statePruner = storageService.getStatePruner();
+    final Optional<StatePruner> maybeStatePruner = storageService.getStatePruner();
     assertThat(future).isCompleted();
-    assertThat(statePruner).isPresent();
-    assertThat(storageService.getStatePruner().get().isRunning()).isTrue();
+    assertThat(maybeStatePruner).isPresent();
+    final StatePruner statePruner = maybeStatePruner.get();
+    assertThat(statePruner.isRunning()).isTrue();
+    assertThat(statePruner.getPruneInterval()).isEqualTo(Duration.ofMinutes(1));
   }
 
   @Test
   void shouldSetupStatePrunerWhenMinimalMode() {
     when(storageConfiguration.getDataStorageMode()).thenReturn(StateStorageMode.MINIMAL);
     final SafeFuture<?> future = storageService.doStart();
-    final Optional<StatePruner> statePruner = storageService.getStatePruner();
+    final Optional<StatePruner> maybeStatePruner = storageService.getStatePruner();
     assertThat(future).isCompleted();
-    assertThat(statePruner).isPresent();
-    assertThat(storageService.getStatePruner().get().isRunning()).isTrue();
+    assertThat(maybeStatePruner).isPresent();
+    final StatePruner statePruner = maybeStatePruner.get();
+    assertThat(statePruner.isRunning()).isTrue();
+    assertThat(statePruner.getPruneInterval()).isEqualTo(Duration.ofMinutes(1));
+  }
+
+  @Test
+  void shouldSetupStatePrunerWithCustomInterval() {
+    when(storageConfiguration.getDataStorageMode()).thenReturn(StateStorageMode.MINIMAL);
+    final Duration customPruningInterval = Duration.ofSeconds(8);
+    when(storageConfiguration.getStatePruningInterval()).thenReturn(customPruningInterval);
+    final SafeFuture<?> future = storageService.doStart();
+    final Optional<StatePruner> maybeStatePruner = storageService.getStatePruner();
+    assertThat(future).isCompleted();
+    assertThat(maybeStatePruner).isPresent();
+    final StatePruner statePruner = maybeStatePruner.get();
+    assertThat(statePruner.isRunning()).isTrue();
+    assertThat(statePruner.getPruneInterval()).isEqualTo(customPruningInterval);
   }
 }
diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/StatePruner.java b/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/StatePruner.java
@@ -13,6 +13,7 @@
 
 package tech.pegasys.teku.storage.server.pruner;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.time.Duration;
 import java.util.Optional;
 import java.util.concurrent.RejectedExecutionException;
@@ -114,4 +115,9 @@ private void pruneStates() {
       LOG.debug("Shutting down", ex);
     }
   }
+
+  @VisibleForTesting
+  public Duration getPruneInterval() {
+    return pruneInterval;
+  }
 }