Skip to content

Commit

Permalink
Add metrics for trie log pruner (hyperledger#7352)
Browse files Browse the repository at this point in the history
---------

Signed-off-by: Simon Dudley <simon.dudley@consensys.net>
  • Loading branch information
siladu authored Jul 18, 2024
1 parent 592acb7 commit 7a905f8
Show file tree
Hide file tree
Showing 4 changed files with 60 additions and 9 deletions.
9 changes: 8 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,12 @@

## Next release

### Upcoming Breaking Changes
- Receipt compaction will be enabled by default in a future version of Besu. After this change it will not be possible to downgrade to the previous Besu version.
- --Xbonsai-limit-trie-logs-enabled is deprecated, use --bonsai-limit-trie-logs-enabled instead
- --Xbonsai-trie-logs-pruning-window-size is deprecated, use --bonsai-trie-logs-pruning-window-size instead
- `besu storage x-trie-log` subcommand is deprecated, use `besu storage trie-log` instead

### Breaking Changes
- Remove deprecated sync modes (X_SNAP and X_CHECKPOINT). Use SNAP and CHECKPOINT instead [#7309](https://github.com/hyperledger/besu/pull/7309)
- Remove PKI-backed QBFT (deprecated in 24.5.1) Other forms of QBFT remain unchanged. [#7293](https://github.com/hyperledger/besu/pull/7293)
Expand All @@ -15,7 +21,8 @@
- Implement gnark-crypto for eip-2537 [#7316](https://github.com/hyperledger/besu/pull/7316)
- Improve blob size transaction selector [#7312](https://github.com/hyperledger/besu/pull/7312)
- Added EIP-7702 [#7237](https://github.com/hyperledger/besu/pull/7237)
- implement gnark-crypto for eip-196 [#7262](https://github.com/hyperledger/besu/pull/7262)
- Implement gnark-crypto for eip-196 [#7262](https://github.com/hyperledger/besu/pull/7262)
- Add trie log pruner metrics [#7352](https://github.com/hyperledger/besu/pull/7352)

### Bug fixes
- Fix `eth_call` deserialization to correctly ignore unknown fields in the transaction object. [#7323](https://github.com/hyperledger/besu/pull/7323)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -793,7 +793,8 @@ private TrieLogPruner createTrieLogPruner(
scheduler::executeServiceTask,
dataStorageConfiguration.getBonsaiMaxLayersToLoad(),
dataStorageConfiguration.getBonsaiTrieLogPruningWindowSize(),
isProofOfStake);
isProofOfStake,
metricsSystem);
trieLogPruner.initialize();

return trieLogPruner;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader;
import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage;
import org.hyperledger.besu.metrics.BesuMetricCategory;
import org.hyperledger.besu.plugin.services.MetricsSystem;
import org.hyperledger.besu.plugin.services.metrics.Counter;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent;

import java.util.Comparator;
Expand All @@ -45,6 +48,9 @@ public class TrieLogPruner implements TrieLogEvent.TrieLogObserver {
private final Consumer<Runnable> executeAsync;
private final long numBlocksToRetain;
private final boolean requireFinalizedBlock;
private final Counter addedToPruneQueueCounter;
private final Counter prunedFromQueueCounter;
private final Counter prunedOrphanCounter;

private final Multimap<Long, Hash> trieLogBlocksAndForksByDescendingBlockNumber =
TreeMultimap.create(Comparator.reverseOrder(), Comparator.naturalOrder());
Expand All @@ -55,14 +61,26 @@ public TrieLogPruner(
final Consumer<Runnable> executeAsync,
final long numBlocksToRetain,
final int pruningLimit,
final boolean requireFinalizedBlock) {
final boolean requireFinalizedBlock,
final MetricsSystem metricsSystem) {
this.rootWorldStateStorage = rootWorldStateStorage;
this.blockchain = blockchain;
this.executeAsync = executeAsync;
this.numBlocksToRetain = numBlocksToRetain;
this.pruningLimit = pruningLimit;
this.loadingLimit = pruningLimit; // same as pruningLimit for now
this.requireFinalizedBlock = requireFinalizedBlock;
this.addedToPruneQueueCounter =
metricsSystem.createCounter(
BesuMetricCategory.PRUNER,
"trie_log_added_to_prune_queue",
"trie log added to prune queue");
this.prunedFromQueueCounter =
metricsSystem.createCounter(
BesuMetricCategory.PRUNER, "trie_log_pruned_from_queue", "trie log pruned from queue");
this.prunedOrphanCounter =
metricsSystem.createCounter(
BesuMetricCategory.PRUNER, "trie_log_pruned_orphan", "trie log pruned orphan");
}

public int initialize() {
Expand All @@ -88,6 +106,7 @@ private int preloadQueue() {
// prune orphaned blocks (sometimes created during block production)
rootWorldStateStorage.pruneTrieLog(blockHash);
orphansPruned.getAndIncrement();
prunedOrphanCounter.inc();
}
});
LOG.atDebug().log("Pruned {} orphaned trie logs from database...", orphansPruned.intValue());
Expand All @@ -106,6 +125,7 @@ public synchronized void addToPruneQueue(final long blockNumber, final Hash bloc
.addArgument(blockHash)
.log();
trieLogBlocksAndForksByDescendingBlockNumber.put(blockNumber, blockHash);
addedToPruneQueueCounter.inc();
}

public synchronized int pruneFromQueue() {
Expand Down Expand Up @@ -155,6 +175,7 @@ public synchronized int pruneFromQueue() {
});

wasPruned.keySet().forEach(trieLogBlocksAndForksByDescendingBlockNumber::removeAll);
prunedFromQueueCounter.inc(wasPruned.size());

LOG.atTrace()
.setMessage("pruned {} trie logs for blocks {}")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogAddedEvent;
import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer;
import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner;
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;

import java.util.Optional;
import java.util.function.Consumer;
Expand Down Expand Up @@ -72,7 +73,8 @@ public void initialize_preloads_queue_and_prunes_orphaned_blocks() {

// When
TrieLogPruner trieLogPruner =
new TrieLogPruner(worldState, blockchain, executeAsync, 3, loadingLimit, false);
new TrieLogPruner(
worldState, blockchain, executeAsync, 3, loadingLimit, false, new NoOpMetricsSystem());
trieLogPruner.initialize();

// Then
Expand All @@ -92,7 +94,13 @@ public void trieLogs_pruned_in_reverse_order_within_pruning_window() {
// requireFinalizedBlock = false means this is not a PoS chain
TrieLogPruner trieLogPruner =
new TrieLogPruner(
worldState, blockchain, executeAsync, blocksToRetain, pruningWindowSize, false);
worldState,
blockchain,
executeAsync,
blocksToRetain,
pruningWindowSize,
false,
new NoOpMetricsSystem());

trieLogPruner.addToPruneQueue(0, key(0)); // older block outside prune window
trieLogPruner.addToPruneQueue(1, key(1)); // block inside the prune window
Expand Down Expand Up @@ -201,7 +209,13 @@ public void skip_pruning_when_finalized_block_required_but_not_present() {
when(blockchain.getChainHeadBlockNumber()).thenReturn(chainHeight);
TrieLogPruner trieLogPruner =
new TrieLogPruner(
worldState, blockchain, executeAsync, blocksToRetain, pruningWindowSize, true);
worldState,
blockchain,
executeAsync,
blocksToRetain,
pruningWindowSize,
true,
new NoOpMetricsSystem());

trieLogPruner.addToPruneQueue(1, key(1));
trieLogPruner.addToPruneQueue(2, key(2));
Expand Down Expand Up @@ -240,7 +254,8 @@ public void onTrieLogAdded_should_prune() {
// Given
final TriggerableConsumer triggerableConsumer = new TriggerableConsumer();
TrieLogPruner trieLogPruner =
new TrieLogPruner(worldState, blockchain, triggerableConsumer, 0, 1, false);
new TrieLogPruner(
worldState, blockchain, triggerableConsumer, 0, 1, false, new NoOpMetricsSystem());
assertThat(trieLogPruner.pruneFromQueue()).isEqualTo(0);

final TrieLogLayer layer = new TrieLogLayer();
Expand All @@ -261,7 +276,8 @@ public void onTrieLogAdded_should_prune() {
public void onTrieLogAdded_should_not_prune_when_no_blockNumber() {
// Given
TrieLogPruner trieLogPruner =
new TrieLogPruner(worldState, blockchain, executeAsync, 0, 1, false);
new TrieLogPruner(
worldState, blockchain, executeAsync, 0, 1, false, new NoOpMetricsSystem());
assertThat(trieLogPruner.pruneFromQueue()).isEqualTo(0);

final TrieLogLayer layer = new TrieLogLayer();
Expand Down Expand Up @@ -289,7 +305,13 @@ private TrieLogPruner setupPrunerAndFinalizedBlock(
when(blockchain.getChainHeadBlockNumber()).thenReturn(chainHeight);
TrieLogPruner trieLogPruner =
new TrieLogPruner(
worldState, blockchain, executeAsync, blocksToRetain, pruningWindowSize, true);
worldState,
blockchain,
executeAsync,
blocksToRetain,
pruningWindowSize,
true,
new NoOpMetricsSystem());

trieLogPruner.addToPruneQueue(1, key(1));
trieLogPruner.addToPruneQueue(2, key(2));
Expand Down

0 comments on commit 7a905f8

Please sign in to comment.