Improve genesis state performance at startup (hyperledger#6977)

* Refactor genesis options file management Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Make loading allocation from genesis lazy Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Update tests Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Memory optimization with streaming Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Improve loading and storgin genesis state at startup Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Remove comments Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Avoid parsing genesis file allocations twice Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Update javadoc Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Fix Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Fix Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Ignore unknown objects in allocations Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * avoid keeping genesis allocation data in memory Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Update CHANGELOG Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> --------- Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> Signed-off-by: ahamlat <ameziane.hamlat@consensys.net> Co-authored-by: ahamlat <ameziane.hamlat@consensys.net>
gauravahuja · Jun 11, 2024 · c62f192 · c62f192
1 parent b1ac5ac
commit c62f192
Show file tree

Hide file tree

Showing 16 changed files with 713 additions and 357 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,6 +17,7 @@
 
 ### Additions and Improvements
 - Add two counters to DefaultBlockchain in order to be able to calculate TPS and Mgas/s [#7105](https://github.com/hyperledger/besu/pull/7105)
+- Improve genesis state performance at startup [#6977](https://github.com/hyperledger/besu/pull/6977)
 - Enable --Xbonsai-limit-trie-logs-enabled by default, unless sync-mode=FULL [#7181](https://github.com/hyperledger/besu/pull/7181)
 - Promote experimental --Xbonsai-limit-trie-logs-enabled to production-ready, --bonsai-limit-trie-logs-enabled [#7192](https://github.com/hyperledger/besu/pull/7192)
 - Promote experimental --Xbonsai-trie-logs-pruning-window-size to production-ready, --bonsai-trie-logs-pruning-window-size [#7192](https://github.com/hyperledger/besu/pull/7192)

diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java
@@ -41,6 +41,7 @@
 import org.hyperledger.besu.ethereum.chain.GenesisState;
 import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
 import org.hyperledger.besu.ethereum.chain.VariablesStorage;
+import org.hyperledger.besu.ethereum.core.BlockHeader;
 import org.hyperledger.besu.ethereum.core.Difficulty;
 import org.hyperledger.besu.ethereum.core.MiningParameters;
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
@@ -552,37 +553,23 @@ public BesuController build() {
     prepForBuild();
 
     final ProtocolSchedule protocolSchedule = createProtocolSchedule();
-    final GenesisState genesisState;
 
     final VariablesStorage variablesStorage = storageProvider.createVariablesStorage();
 
-    Optional<Hash> genesisStateHash = Optional.empty();
-    if (variablesStorage != null && this.genesisStateHashCacheEnabled) {
-      genesisStateHash = variablesStorage.getGenesisStateHash();
-    }
-
-    if (genesisStateHash.isPresent()) {
-      genesisState =
-          GenesisState.fromConfig(genesisStateHash.get(), genesisConfigFile, protocolSchedule);
-    } else {
-      genesisState =
-          GenesisState.fromConfig(dataStorageConfiguration, genesisConfigFile, protocolSchedule);
-      if (variablesStorage != null) {
-        VariablesStorage.Updater updater = variablesStorage.updater();
-        if (updater != null) {
-          updater.setGenesisStateHash(genesisState.getBlock().getHeader().getStateRoot());
-          updater.commit();
-        }
-      }
-    }
-
     final WorldStateStorageCoordinator worldStateStorageCoordinator =
         storageProvider.createWorldStateStorageCoordinator(dataStorageConfiguration);
 
     final BlockchainStorage blockchainStorage =
         storageProvider.createBlockchainStorage(
             protocolSchedule, variablesStorage, dataStorageConfiguration);
 
+    final var maybeStoredGenesisBlockHash = blockchainStorage.getBlockHash(0L);
+
+    final var genesisState =
+        getGenesisState(
+            maybeStoredGenesisBlockHash.flatMap(blockchainStorage::getBlockHeader),
+            protocolSchedule);
+
     final MutableBlockchain blockchain =
         DefaultBlockchain.createMutable(
             genesisState.getBlock(),
@@ -591,7 +578,6 @@ public BesuController build() {
             reorgLoggingThreshold,
             dataDirectory.toString(),
             numberOfBlocksToCache);
-
     final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader =
         besuComponent
             .map(BesuComponent::getCachedMerkleTrieLoader)
@@ -601,7 +587,7 @@ public BesuController build() {
         createWorldStateArchive(
             worldStateStorageCoordinator, blockchain, bonsaiCachedMerkleTrieLoader);
 
-    if (blockchain.getChainHeadBlockNumber() < 1) {
+    if (maybeStoredGenesisBlockHash.isEmpty()) {
       genesisState.writeStateTo(worldStateArchive.getMutable());
     }
 
@@ -772,6 +758,24 @@ public BesuController build() {
         dataStorageConfiguration);
   }
 
+  private GenesisState getGenesisState(
+      final Optional<BlockHeader> maybeGenesisBlockHeader,
+      final ProtocolSchedule protocolSchedule) {
+    final Optional<Hash> maybeGenesisStateRoot =
+        genesisStateHashCacheEnabled
+            ? maybeGenesisBlockHeader.map(BlockHeader::getStateRoot)
+            : Optional.empty();
+
+    return maybeGenesisStateRoot
+        .map(
+            genesisStateRoot ->
+                GenesisState.fromStorage(genesisStateRoot, genesisConfigFile, protocolSchedule))
+        .orElseGet(
+            () ->
+                GenesisState.fromConfig(
+                    dataStorageConfiguration, genesisConfigFile, protocolSchedule));
+  }
+
   private TrieLogPruner createTrieLogPruner(
       final WorldStateKeyValueStorage worldStateStorage,
       final Blockchain blockchain,

diff --git a/besu/src/test/java/org/hyperledger/besu/ForkIdsNetworkConfigTest.java b/besu/src/test/java/org/hyperledger/besu/ForkIdsNetworkConfigTest.java
@@ -18,7 +18,6 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-import org.hyperledger.besu.cli.config.EthNetworkConfig;
 import org.hyperledger.besu.cli.config.NetworkName;
 import org.hyperledger.besu.config.GenesisConfigFile;
 import org.hyperledger.besu.config.GenesisConfigOptions;
@@ -138,7 +137,7 @@ public static Collection<Object[]> parameters() {
   @MethodSource("parameters")
   public void testForkId(final NetworkName chainName, final List<ForkId> expectedForkIds) {
     final GenesisConfigFile genesisConfigFile =
-        GenesisConfigFile.fromConfig(EthNetworkConfig.jsonConfig(chainName));
+        GenesisConfigFile.fromResource(chainName.getGenesisFile());
     final MilestoneStreamingTransitionProtocolSchedule schedule = createSchedule(genesisConfigFile);
     final GenesisState genesisState = GenesisState.fromConfig(genesisConfigFile, schedule);
     final Blockchain mockBlockchain = mock(Blockchain.class);

diff --git a/config/src/main/java/org/hyperledger/besu/config/GenesisAccount.java b/config/src/main/java/org/hyperledger/besu/config/GenesisAccount.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright contributors to Hyperledger Besu.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.config;
+
+import org.hyperledger.besu.datatypes.Address;
+import org.hyperledger.besu.datatypes.Wei;
+
+import java.util.Map;
+
+import org.apache.tuweni.bytes.Bytes;
+import org.apache.tuweni.bytes.Bytes32;
+import org.apache.tuweni.units.bigints.UInt256;
+
+/**
+ * Genesis account
+ *
+ * @param address of the account
+ * @param nonce nonce of the account at genesis
+ * @param balance balance of the account at genesis
+ * @param code code of the account at genesis, can be null
+ * @param storage storage of the account at genesis
+ * @param privateKey of the account, only use for testing
+ */
+public record GenesisAccount(
+    Address address,
+    long nonce,
+    Wei balance,
+    Bytes code,
+    Map<UInt256, UInt256> storage,
+    Bytes32 privateKey) {}
diff --git a/config/src/main/java/org/hyperledger/besu/config/GenesisAllocation.java b/config/src/main/java/org/hyperledger/besu/config/GenesisAllocation.java
diff --git a/config/src/main/java/org/hyperledger/besu/config/GenesisConfigFile.java b/config/src/main/java/org/hyperledger/besu/config/GenesisConfigFile.java
@@ -14,8 +14,6 @@
  */
 package org.hyperledger.besu.config;
 
-import static org.hyperledger.besu.config.JsonUtil.normalizeKeys;
-
 import org.hyperledger.besu.datatypes.Wei;
 
 import java.net.URL;
@@ -30,22 +28,23 @@
 import java.util.stream.Stream;
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.collect.Streams;
 
 /** The Genesis config file. */
 public class GenesisConfigFile {
 
   /** The constant DEFAULT. */
   public static final GenesisConfigFile DEFAULT =
-      new GenesisConfigFile(JsonUtil.createEmptyObjectNode());
+      new GenesisConfigFile(new GenesisReader.FromObjectNode(JsonUtil.createEmptyObjectNode()));
 
   /** The constant BASEFEE_AT_GENESIS_DEFAULT_VALUE. */
   public static final Wei BASEFEE_AT_GENESIS_DEFAULT_VALUE = Wei.of(1_000_000_000L);
 
+  private final GenesisReader loader;
   private final ObjectNode genesisRoot;
 
-  private GenesisConfigFile(final ObjectNode config) {
-    this.genesisRoot = config;
+  private GenesisConfigFile(final GenesisReader loader) {
+    this.loader = loader;
+    this.genesisRoot = loader.getRoot();
   }
 
   /**
@@ -70,21 +69,31 @@ public static GenesisConfigFile fromSource(final URL jsonSource) {
   /**
    * Genesis file from resource.
    *
-   * @param jsonResource the resource name
+   * @param resourceName the resource name
+   * @return the genesis config file
+   */
+  public static GenesisConfigFile fromResource(final String resourceName) {
+    return fromConfig(GenesisConfigFile.class.getResource(resourceName));
+  }
+
+  /**
+   * From config genesis config file.
+   *
+   * @param jsonSource the json string
    * @return the genesis config file
    */
-  public static GenesisConfigFile fromResource(final String jsonResource) {
-    return fromSource(GenesisConfigFile.class.getResource(jsonResource));
+  public static GenesisConfigFile fromConfig(final URL jsonSource) {
+    return new GenesisConfigFile(new GenesisReader.FromURL(jsonSource));
   }
 
   /**
    * From config genesis config file.
    *
-   * @param jsonString the json string
+   * @param json the json string
    * @return the genesis config file
    */
-  public static GenesisConfigFile fromConfig(final String jsonString) {
-    return fromConfig(JsonUtil.objectNodeFromString(jsonString, false));
+  public static GenesisConfigFile fromConfig(final String json) {
+    return fromConfig(JsonUtil.objectNodeFromString(json, false));
   }
 
   /**
@@ -94,7 +103,7 @@ public static GenesisConfigFile fromConfig(final String jsonString) {
    * @return the genesis config file
    */
   public static GenesisConfigFile fromConfig(final ObjectNode config) {
-    return new GenesisConfigFile(normalizeKeys(config));
+    return new GenesisConfigFile(new GenesisReader.FromObjectNode(config));
   }
 
   /**
@@ -113,8 +122,7 @@ public GenesisConfigOptions getConfigOptions() {
    * @return the config options
    */
   public GenesisConfigOptions getConfigOptions(final Map<String, String> overrides) {
-    final ObjectNode config =
-        JsonUtil.getObjectNode(genesisRoot, "config").orElse(JsonUtil.createEmptyObjectNode());
+    final ObjectNode config = loader.getConfig();
 
     Map<String, String> overridesRef = overrides;
 
@@ -134,15 +142,8 @@ public GenesisConfigOptions getConfigOptions(final Map<String, String> overrides
    *
    * @return the stream
    */
-  public Stream<GenesisAllocation> streamAllocations() {
-    return JsonUtil.getObjectNode(genesisRoot, "alloc").stream()
-        .flatMap(
-            allocations ->
-                Streams.stream(allocations.fieldNames())
-                    .map(
-                        key ->
-                            new GenesisAllocation(
-                                key, JsonUtil.getObjectNode(allocations, key).get())));
+  public Stream<GenesisAccount> streamAllocations() {
+    return loader.streamAllocations();
   }
 
   /**
@@ -344,7 +345,7 @@ public String toString() {
         + "genesisRoot="
         + genesisRoot
         + ", allocations="
-        + streamAllocations().map(GenesisAllocation::toString).collect(Collectors.joining(","))
+        + loader.streamAllocations().map(GenesisAccount::toString).collect(Collectors.joining(","))
         + '}';
   }
 }