From 7ef6f0dbf37878d5589975999076763dc6a85949 Mon Sep 17 00:00:00 2001 From: Jason Frame Date: Tue, 11 Jun 2024 16:33:42 +1000 Subject: [PATCH 1/8] Update gradle to 8.8 (#7199) Signed-off-by: Jason Frame --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index b82aa23a4f0..a4413138c96 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME From 186b4c5d21a41edcd2ddaf166b37f14d41bab41e Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Tue, 11 Jun 2024 17:37:35 +1000 Subject: [PATCH 2/8] disable flaky test (#7202) Signed-off-by: Sally MacFarlane --- .../besu/ethereum/p2p/discovery/dns/DNSDaemonTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/dns/DNSDaemonTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/dns/DNSDaemonTest.java index 93074d37104..08539d47bf8 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/dns/DNSDaemonTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/dns/DNSDaemonTest.java @@ -29,6 +29,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -73,6 +74,7 @@ void testDNSDaemon(final Vertx vertx, final VertxTestContext testContext) } @Test + @Disabled("this test is flaky") @DisplayName("Test DNS Daemon with periodic lookup to a mock DNS server") void testDNSDaemonPeriodic(final Vertx vertx, final VertxTestContext testContext) throws InterruptedException { From 478b6d0184ef8154d68a45bcd3419295e35893e4 Mon Sep 17 00:00:00 2001 From: krsh24 Date: Tue, 11 Jun 2024 16:01:59 +0800 Subject: [PATCH 3/8] Support for eth_maxPriorityFeePerGasPrice (#7194) * Error out on permissions config accounts-allowlist validation errors. Signed-off-by: krishnannarayanan * Fixing compilation errors Signed-off-by: krishnannarayanan * Incorrect file check in Signed-off-by: krishnannarayanan * Support for eth_maxPriorityFeePerGas Signed-off-by: krishnannarayanan * Change log update for my PRs Signed-off-by: krishnannarayanan --------- Signed-off-by: krishnannarayanan --- CHANGELOG.md | 4 +- .../besu/ethereum/api/jsonrpc/RpcMethod.java | 1 + .../methods/EthMaxPriorityFeePerGas.java | 54 +++++++ .../jsonrpc/methods/EthJsonRpcMethods.java | 4 +- .../methods/EthMaxPriorityFeePerGasTest.java | 133 ++++++++++++++++++ 5 files changed, 193 insertions(+), 3 deletions(-) create mode 100644 ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthMaxPriorityFeePerGas.java create mode 100644 ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthMaxPriorityFeePerGasTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index c53b586a129..9b64946ae01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,10 +22,10 @@ - Promote experimental --Xbonsai-trie-logs-pruning-window-size to production-ready, --bonsai-trie-logs-pruning-window-size [#7192](https://github.com/hyperledger/besu/pull/7192) - `admin_nodeInfo` JSON/RPC call returns the currently active EVM version [#7127](https://github.com/hyperledger/besu/pull/7127) - Improve the selection of the most profitable built block [#7174](https://github.com/hyperledger/besu/pull/7174) - +- Support for eth_maxPriorityFeePerGas [#5658](https://github.com/hyperledger/besu/issues/5658) ### Bug fixes - Make `eth_gasPrice` aware of the base fee market [#7102](https://github.com/hyperledger/besu/pull/7102) - +- Validation errors ignored in accounts-allowlist and empty list [#7138](https://github.com/hyperledger/besu/issues/7138) ## 24.5.2 ### Upcoming Breaking Changes diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java index 36e4ff8b8a5..4174fdf8313 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java @@ -109,6 +109,7 @@ public enum RpcMethod { ETH_GET_FILTER_CHANGES("eth_getFilterChanges"), ETH_GET_FILTER_LOGS("eth_getFilterLogs"), ETH_GET_LOGS("eth_getLogs"), + ETH_GET_MAX_PRIORITY_FEE_PER_GAS("eth_maxPriorityFeePerGas"), ETH_GET_MINER_DATA_BY_BLOCK_HASH("eth_getMinerDataByBlockHash"), ETH_GET_MINER_DATA_BY_BLOCK_NUMBER("eth_getMinerDataByBlockNumber"), ETH_GET_PROOF("eth_getProof"), diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthMaxPriorityFeePerGas.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthMaxPriorityFeePerGas.java new file mode 100644 index 00000000000..2828fee7aca --- /dev/null +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthMaxPriorityFeePerGas.java @@ -0,0 +1,54 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods; + +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.Quantity; +import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; +import org.hyperledger.besu.ethereum.blockcreation.MiningCoordinator; + +import java.util.Optional; + +public class EthMaxPriorityFeePerGas implements JsonRpcMethod { + + private final BlockchainQueries blockchainQueries; + private final MiningCoordinator miningCoordinator; + + public EthMaxPriorityFeePerGas( + final BlockchainQueries blockchainQueries, final MiningCoordinator miningCoordinator) { + this.blockchainQueries = blockchainQueries; + this.miningCoordinator = miningCoordinator; + } + + @Override + public String getName() { + return RpcMethod.ETH_GET_MAX_PRIORITY_FEE_PER_GAS.getMethodName(); + } + + @Override + public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { + return new JsonRpcSuccessResponse( + requestContext.getRequest().getId(), Quantity.create(fetchAndLimitPriorityFeePerGas())); + } + + private Wei fetchAndLimitPriorityFeePerGas() { + final Optional gasPrice = blockchainQueries.gasPriorityFee(); + return gasPrice.orElseGet(miningCoordinator::getMinPriorityFeePerGas); + } +} diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthJsonRpcMethods.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthJsonRpcMethods.java index 678df31824e..c7575a5435a 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthJsonRpcMethods.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthJsonRpcMethods.java @@ -52,6 +52,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.EthGetUncleCountByBlockNumber; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.EthGetWork; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.EthHashrate; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.EthMaxPriorityFeePerGas; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.EthMining; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.EthNewBlockFilter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.EthNewFilter; @@ -183,6 +184,7 @@ protected Map create() { new EthChainId(protocolSchedule.getChainId()), new EthGetMinerDataByBlockHash(blockchainQueries, protocolSchedule), new EthGetMinerDataByBlockNumber(blockchainQueries, protocolSchedule), - new EthBlobBaseFee(blockchainQueries.getBlockchain(), protocolSchedule)); + new EthBlobBaseFee(blockchainQueries.getBlockchain(), protocolSchedule), + new EthMaxPriorityFeePerGas(blockchainQueries, miningCoordinator)); } } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthMaxPriorityFeePerGasTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthMaxPriorityFeePerGasTest.java new file mode 100644 index 00000000000..02d526107a1 --- /dev/null +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthMaxPriorityFeePerGasTest.java @@ -0,0 +1,133 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; +import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; +import org.hyperledger.besu.ethereum.blockcreation.MiningCoordinator; + +import java.util.Optional; +import java.util.stream.Stream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.internal.verification.VerificationModeFactory; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class EthMaxPriorityFeePerGasTest { + private static final String JSON_RPC_VERSION = "2.0"; + private static final String ETH_METHOD = + RpcMethod.ETH_GET_MAX_PRIORITY_FEE_PER_GAS.getMethodName(); + private EthMaxPriorityFeePerGas method; + + @Mock private BlockchainQueries blockchainQueries; + @Mock private MiningCoordinator miningCoordinator; + + @BeforeEach + public void setUp() { + method = createEthMaxPriorityFeePerGasMethod(); + } + + @Test + public void shouldReturnCorrectMethodName() { + assertThat(method.getName()).isEqualTo(ETH_METHOD); + } + + @Test + public void whenNoTransactionsExistReturnMinPriorityFeePerGasPrice() { + final JsonRpcRequestContext request = requestWithParams(); + final String expectedWei = Wei.ONE.toShortHexString(); + final JsonRpcResponse expectedResponse = + new JsonRpcSuccessResponse(request.getRequest().getId(), expectedWei); + when(miningCoordinator.getMinPriorityFeePerGas()).thenReturn(Wei.ONE); + + mockBlockchainQueriesMaxPriorityFeePerGasPrice(Optional.empty()); + final JsonRpcResponse actualResponse = method.response(request); + assertThat(actualResponse).usingRecursiveComparison().isEqualTo(expectedResponse); + verify(miningCoordinator, VerificationModeFactory.times(1)).getMinPriorityFeePerGas(); + } + + @ParameterizedTest + @MethodSource("minPriorityFeePerGasValues") + public void whenNoTransactionsExistReturnMinPriorityFeePerGasPriceExist( + final Wei minPriorityFeePerGasValue) { + final JsonRpcRequestContext request = requestWithParams(); + final String expectedWei = minPriorityFeePerGasValue.toShortHexString(); + final JsonRpcResponse expectedResponse = + new JsonRpcSuccessResponse(request.getRequest().getId(), expectedWei); + when(miningCoordinator.getMinPriorityFeePerGas()).thenReturn(minPriorityFeePerGasValue); + + mockBlockchainQueriesMaxPriorityFeePerGasPrice(Optional.empty()); + final JsonRpcResponse actualResponse = method.response(request); + assertThat(actualResponse).usingRecursiveComparison().isEqualTo(expectedResponse); + verify(miningCoordinator, VerificationModeFactory.times(1)).getMinPriorityFeePerGas(); + } + + @Test + public void whenNoTransactionsExistReturnNullMinPriorityFeePerGasPriceExist() { + final JsonRpcRequestContext request = requestWithParams(); + final JsonRpcResponse expectedResponse = + new JsonRpcSuccessResponse(request.getRequest().getId(), null); + when(miningCoordinator.getMinPriorityFeePerGas()).thenReturn(null); + + mockBlockchainQueriesMaxPriorityFeePerGasPrice(Optional.empty()); + final JsonRpcResponse actualResponse = method.response(request); + assertThat(actualResponse).usingRecursiveComparison().isEqualTo(expectedResponse); + verify(miningCoordinator, VerificationModeFactory.times(1)).getMinPriorityFeePerGas(); + } + + @Test + public void whenTransactionsExistReturnMaxPriorityFeePerGasPrice() { + final JsonRpcRequestContext request = requestWithParams(); + final String expectedWei = Wei.of(2000000000).toShortHexString(); + final JsonRpcResponse expectedResponse = + new JsonRpcSuccessResponse(request.getRequest().getId(), expectedWei); + mockBlockchainQueriesMaxPriorityFeePerGasPrice(Optional.of(Wei.of(2000000000))); + final JsonRpcResponse actualResponse = method.response(request); + assertThat(actualResponse).usingRecursiveComparison().isEqualTo(expectedResponse); + verify(miningCoordinator, VerificationModeFactory.times(0)).getMinPriorityFeePerGas(); + } + + private static Stream minPriorityFeePerGasValues() { + return Stream.of(Arguments.of(Wei.ONE), Arguments.of(Wei.ZERO)); + } + + private void mockBlockchainQueriesMaxPriorityFeePerGasPrice(final Optional result) { + when(blockchainQueries.gasPriorityFee()).thenReturn(result); + } + + private JsonRpcRequestContext requestWithParams(final Object... params) { + return new JsonRpcRequestContext(new JsonRpcRequest(JSON_RPC_VERSION, ETH_METHOD, params)); + } + + private EthMaxPriorityFeePerGas createEthMaxPriorityFeePerGasMethod() { + return new EthMaxPriorityFeePerGas(blockchainQueries, miningCoordinator); + } +} From 04f304fbb51fd18a8797164f0fc314c007d69e12 Mon Sep 17 00:00:00 2001 From: daniellehrner Date: Tue, 11 Jun 2024 10:25:17 +0200 Subject: [PATCH 4/8] move traceEndTransaction after coinbase refund, add self destruct set to the method (#7188) Signed-off-by: Daniel Lehrner Co-authored-by: Sally MacFarlane --- .../besu/services/TraceServiceImplTest.java | 15 ++++++- .../mainnet/MainnetTransactionProcessor.java | 39 +++++++++++++------ .../MainnetTransactionProcessorTest.java | 2 + .../hyperledger/besu/evmtool/T8nExecutor.java | 4 ++ .../besu/evm/tracing/OperationTracer.java | 4 ++ 5 files changed, 51 insertions(+), 13 deletions(-) diff --git a/besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java b/besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java index f7db3639ae1..1e734f62312 100644 --- a/besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java +++ b/besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java @@ -125,7 +125,7 @@ void shouldRetrieveStateUpdatePostTracingForOneBlock() { verify(opTracer).traceStartTransaction(any(), eq(tx)); verify(opTracer) .traceEndTransaction( - any(), eq(tx), anyBoolean(), any(), any(), anyLong(), anyLong()); + any(), eq(tx), anyBoolean(), any(), any(), anyLong(), any(), anyLong()); }); verify(opTracer).traceEndBlock(tracedBlock.getHeader(), tracedBlock.getBody()); @@ -173,7 +173,14 @@ void shouldRetrieveStateUpdatePostTracingForAllBlocks() { verify(opTracer).traceStartTransaction(any(), eq(tx)); verify(opTracer) .traceEndTransaction( - any(), eq(tx), anyBoolean(), any(), any(), anyLong(), anyLong()); + any(), + eq(tx), + anyBoolean(), + any(), + any(), + anyLong(), + any(), + anyLong()); }); verify(opTracer).traceEndBlock(tracedBlock.getHeader(), tracedBlock.getBody()); @@ -222,6 +229,7 @@ void shouldReturnTheCorrectWorldViewForTxStartEnd() { assertThat(txStartEndTracer.txEndStatus).isTrue(); assertThat(txStartEndTracer.txEndOutput).isEqualTo(Bytes.fromHexString("0x")); assertThat(txStartEndTracer.txEndGasUsed).isEqualTo(24303); + assertThat(txStartEndTracer.txEndSelfDestructs).isEmpty(); assertThat(txStartEndTracer.txEndTimeNs).isNotNull(); assertThat(txStartEndTracer.txEndLogs).isNotEmpty(); @@ -263,6 +271,7 @@ private static class TxStartEndTracer implements BlockAwareOperationTracer { public Bytes txEndOutput; public List txEndLogs; public long txEndGasUsed; + public Set
txEndSelfDestructs; public Long txEndTimeNs; private final Set traceStartTxCalled = new HashSet<>(); @@ -287,6 +296,7 @@ public void traceEndTransaction( final Bytes output, final List logs, final long gasUsed, + final Set
selfDestructs, final long timeNs) { if (!traceEndTxCalled.add(transaction)) { fail("traceEndTransaction already called for tx " + transaction); @@ -297,6 +307,7 @@ public void traceEndTransaction( txEndOutput = output; txEndLogs = logs; txEndGasUsed = gasUsed; + txEndSelfDestructs = selfDestructs; txEndTimeNs = timeNs; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java index 0027c51b79d..51d300cbc46 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java @@ -59,6 +59,8 @@ public class MainnetTransactionProcessor { private static final Logger LOG = LoggerFactory.getLogger(MainnetTransactionProcessor.class); + private static final Set
EMPTY_ADDRESS_SET = Set.of(); + protected final GasCalculator gasCalculator; protected final TransactionValidatorFactory transactionValidatorFactory; @@ -451,15 +453,6 @@ public TransactionProcessingResult processTransaction( .log(); final long gasUsedByTransaction = transaction.getGasLimit() - initialFrame.getRemainingGas(); - operationTracer.traceEndTransaction( - worldUpdater, - transaction, - initialFrame.getState() == MessageFrame.State.COMPLETED_SUCCESS, - initialFrame.getOutputData(), - initialFrame.getLogs(), - gasUsedByTransaction, - 0L); - // update the coinbase final var coinbase = worldState.getOrCreate(miningBeneficiary); final long usedGas = transaction.getGasLimit() - refundedGas; @@ -485,6 +478,16 @@ public TransactionProcessingResult processTransaction( coinbase.incrementBalance(coinbaseWeiDelta); + operationTracer.traceEndTransaction( + worldUpdater, + transaction, + initialFrame.getState() == MessageFrame.State.COMPLETED_SUCCESS, + initialFrame.getOutputData(), + initialFrame.getLogs(), + gasUsedByTransaction, + initialFrame.getSelfDestructs(), + 0L); + initialFrame.getSelfDestructs().forEach(worldState::deleteAccount); if (clearEmptyAccounts) { @@ -516,13 +519,27 @@ public TransactionProcessingResult processTransaction( } } catch (final MerkleTrieException re) { operationTracer.traceEndTransaction( - worldState.updater(), transaction, false, Bytes.EMPTY, List.of(), 0, 0L); + worldState.updater(), + transaction, + false, + Bytes.EMPTY, + List.of(), + 0, + EMPTY_ADDRESS_SET, + 0L); // need to throw to trigger the heal throw re; } catch (final RuntimeException re) { operationTracer.traceEndTransaction( - worldState.updater(), transaction, false, Bytes.EMPTY, List.of(), 0, 0L); + worldState.updater(), + transaction, + false, + Bytes.EMPTY, + List.of(), + 0, + EMPTY_ADDRESS_SET, + 0L); LOG.error("Critical Exception Processing Transaction", re); return TransactionProcessingResult.invalid( diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessorTest.java index 77466587314..d541cbd99aa 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessorTest.java @@ -41,6 +41,7 @@ import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Stream; @@ -216,6 +217,7 @@ public void traceEndTransaction( final Bytes output, final List logs, final long gasUsed, + final Set
selfDestructs, final long timeNs) { this.traceEndTxCalled = true; } diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java index 6e48aaf82ca..ccabce833a6 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java @@ -69,6 +69,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Optional; +import java.util.Set; import java.util.Spliterator; import java.util.Spliterators; import java.util.concurrent.TimeUnit; @@ -86,6 +87,8 @@ public class T8nExecutor { + private static final Set
EMPTY_ADDRESS_SET = Set.of(); + public record RejectedTransaction(int index, String error) {} protected static List extractTransactions( @@ -349,6 +352,7 @@ static T8nResult runTest( result.getOutput(), result.getLogs(), gasUsed - intrinsicGas, + EMPTY_ADDRESS_SET, timer.elapsed(TimeUnit.NANOSECONDS)); Bytes gasUsedInTransaction = Bytes.ofUnsignedLong(transactionGasUsed); receipts.add(receipt); diff --git a/evm/src/main/java/org/hyperledger/besu/evm/tracing/OperationTracer.java b/evm/src/main/java/org/hyperledger/besu/evm/tracing/OperationTracer.java index 5bd6de591e5..34e28bbb32f 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/tracing/OperationTracer.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/tracing/OperationTracer.java @@ -14,6 +14,7 @@ */ package org.hyperledger.besu.evm.tracing; +import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Transaction; import org.hyperledger.besu.evm.frame.ExceptionalHaltReason; import org.hyperledger.besu.evm.frame.MessageFrame; @@ -23,6 +24,7 @@ import java.util.List; import java.util.Optional; +import java.util.Set; import org.apache.tuweni.bytes.Bytes; @@ -92,6 +94,7 @@ default void traceStartTransaction(final WorldView worldView, final Transaction * @param output the bytes output from the transaction * @param logs the logs emitted by this transaction * @param gasUsed the gas used by the entire transaction + * @param selfDestructs the set of addresses that self-destructed during the transaction * @param timeNs the time in nanoseconds it took to execute the transaction */ default void traceEndTransaction( @@ -101,6 +104,7 @@ default void traceEndTransaction( final Bytes output, final List logs, final long gasUsed, + final Set
selfDestructs, final long timeNs) {} /** From b1ac5acd6067c32a85ed78c50c67da8042087c86 Mon Sep 17 00:00:00 2001 From: Matt Whitehead Date: Tue, 11 Jun 2024 11:46:36 +0100 Subject: [PATCH 5/8] Add new acceptance test to soak test BFT chains (#7023) * Add new acceptance test to soak test BFT chains Signed-off-by: Matthew Whitehead * Spotless Signed-off-by: Matthew Whitehead * Tidy up a little with re-usable start and stop functions with built in delays Signed-off-by: Matthew Whitehead * Add shanghai version of Simple Storage contract Signed-off-by: Matthew Whitehead * Put commented gradle code back in. Fix the web3j example commands in .sol files Signed-off-by: Matthew Whitehead * Spotless Signed-off-by: Matthew Whitehead * Set publication artifacts to avoid clash Signed-off-by: Matthew Whitehead * Exclude from regular acceptance tests Signed-off-by: Matthew Whitehead * Add shanghai fork to the test. Stall the chain for less time to reduce the time taken to mine new blocks Signed-off-by: Matthew Whitehead * Tidy up Signed-off-by: Matthew Whitehead * Update acceptance-tests/tests/shanghai/build.gradle Co-authored-by: Simon Dudley Signed-off-by: Matt Whitehead * Tidy up var names Signed-off-by: Matthew Whitehead * Fix ports for IBFT2 as well as QBFT Signed-off-by: Matthew Whitehead * Remove maven publish spec, disable jar building for shanghai contract project Signed-off-by: Matthew Whitehead * web3j version Signed-off-by: Matthew Whitehead * Make fixed port optional when creating a BFT node Signed-off-by: Matthew Whitehead * Only check artifact coordinates for those starting 'org.*' Signed-off-by: Matthew Whitehead --------- Signed-off-by: Matthew Whitehead Signed-off-by: Matt Whitehead Signed-off-by: Matt Whitehead Co-authored-by: Simon Dudley Co-authored-by: Sally MacFarlane --- .../node/configuration/BesuNodeFactory.java | 49 ++- acceptance-tests/tests/build.gradle | 32 ++ .../tests/contracts/CrossContractReader.sol | 2 +- .../tests/contracts/EventEmitter.sol | 2 +- .../tests/contracts/RemoteSimpleStorage.sol | 2 +- .../tests/contracts/RevertReason.sol | 2 +- .../tests/contracts/SimpleStorage.sol | 2 +- acceptance-tests/tests/shanghai/build.gradle | 21 ++ .../SimpleStorageShanghai.sol | 31 ++ .../BftAcceptanceTestParameterization.java | 15 +- .../acceptance/bftsoak/BftMiningSoakTest.java | 351 ++++++++++++++++++ build.gradle | 2 +- settings.gradle | 1 + 13 files changed, 494 insertions(+), 18 deletions(-) create mode 100644 acceptance-tests/tests/shanghai/build.gradle create mode 100644 acceptance-tests/tests/shanghai/shanghaicontracts/SimpleStorageShanghai.sol create mode 100644 acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/bftsoak/BftMiningSoakTest.java diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java index 6ed5ee55f33..96ab9cf6235 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java @@ -461,16 +461,30 @@ public BesuNode createIbft2Node(final String name, final String genesisFile) thr .build()); } - public BesuNode createIbft2Node(final String name) throws IOException { - return create( + public BesuNode createIbft2Node(final String name, final boolean fixedPort) throws IOException { + JsonRpcConfiguration rpcConfig = node.createJsonRpcWithIbft2EnabledConfig(false); + rpcConfig.addRpcApi("ADMIN,TXPOOL"); + if (fixedPort) { + rpcConfig.setPort( + Math.abs(name.hashCode() % 60000) + + 1024); // Generate a consistent port for p2p based on node name + } + BesuNodeConfigurationBuilder builder = new BesuNodeConfigurationBuilder() .name(name) .miningEnabled() - .jsonRpcConfiguration(node.createJsonRpcWithIbft2EnabledConfig(false)) + .jsonRpcConfiguration(rpcConfig) .webSocketConfiguration(node.createWebSocketEnabledConfig()) .devMode(false) - .genesisConfigProvider(GenesisConfigurationFactory::createIbft2GenesisConfig) - .build()); + .genesisConfigProvider(GenesisConfigurationFactory::createIbft2GenesisConfig); + if (fixedPort) { + builder.p2pPort( + Math.abs(name.hashCode() % 60000) + + 1024 + + 500); // Generate a consistent port for p2p based on node name (+ 500 to avoid + // clashing with RPC port or other nodes with a similar name) + } + return create(builder.build()); } public BesuNode createQbftNodeWithTLS(final String name, final String type) throws IOException { @@ -498,16 +512,31 @@ public BesuNode createQbftNodeWithTLSPKCS11(final String name) throws IOExceptio return createQbftNodeWithTLS(name, KeyStoreWrapper.KEYSTORE_TYPE_PKCS11); } - public BesuNode createQbftNode(final String name) throws IOException { - return create( + public BesuNode createQbftNode(final String name, final boolean fixedPort) throws IOException { + JsonRpcConfiguration rpcConfig = node.createJsonRpcWithQbftEnabledConfig(false); + rpcConfig.addRpcApi("ADMIN,TXPOOL"); + if (fixedPort) { + rpcConfig.setPort( + Math.abs(name.hashCode() % 60000) + + 1024); // Generate a consistent port for p2p based on node name + } + + BesuNodeConfigurationBuilder builder = new BesuNodeConfigurationBuilder() .name(name) .miningEnabled() - .jsonRpcConfiguration(node.createJsonRpcWithQbftEnabledConfig(false)) + .jsonRpcConfiguration(rpcConfig) .webSocketConfiguration(node.createWebSocketEnabledConfig()) .devMode(false) - .genesisConfigProvider(GenesisConfigurationFactory::createQbftGenesisConfig) - .build()); + .genesisConfigProvider(GenesisConfigurationFactory::createQbftGenesisConfig); + if (fixedPort) { + builder.p2pPort( + Math.abs(name.hashCode() % 60000) + + 1024 + + 500); // Generate a consistent port for p2p based on node name (+ 500 to avoid + // clashing with RPC port or other nodes with a similar name) + } + return create(builder.build()); } public BesuNode createCustomGenesisNode( diff --git a/acceptance-tests/tests/build.gradle b/acceptance-tests/tests/build.gradle index a9393b1ccf1..1bc0e55567f 100644 --- a/acceptance-tests/tests/build.gradle +++ b/acceptance-tests/tests/build.gradle @@ -24,6 +24,7 @@ solidity { resolvePackages = false // TODO: remove the forced version, when DEV network is upgraded to support latest forks version '0.8.19' + evmVersion 'london' } dependencies { @@ -79,6 +80,7 @@ dependencies { testImplementation 'org.web3j:besu' testImplementation 'org.web3j:core' testImplementation 'org.wiremock:wiremock' + testImplementation project(path: ':acceptance-tests:tests:shanghai') testRuntimeOnly 'org.junit.vintage:junit-vintage-engine' } @@ -153,6 +155,7 @@ task acceptanceTestMainnet(type: Test) { task acceptanceTestNotPrivacy(type: Test) { inputs.property "integration.date", LocalTime.now() // so it runs at every invocation exclude '**/privacy/**' + exclude '**/bftsoak/**' useJUnitPlatform {} @@ -205,6 +208,35 @@ task acceptanceTestCliqueBft(type: Test) { doFirst { mkdir "${buildDir}/jvmErrorLogs" } } +task acceptanceTestBftSoak(type: Test) { + inputs.property "integration.date", LocalTime.now() // so it runs at every invocation + include '**/bftsoak/**' + + useJUnitPlatform {} + + dependsOn(rootProject.installDist) + setSystemProperties(test.getSystemProperties()) + systemProperty 'acctests.runBesuAsProcess', 'true' + // Set to any time > 60 minutes to run the soak test for longer + // systemProperty 'acctests.soakTimeMins', '120' + systemProperty 'java.security.properties', "${buildDir}/resources/test/acceptanceTesting.security" + mustRunAfter rootProject.subprojects*.test + description = 'Runs BFT soak test.' + group = 'verification' + + jvmArgs "-XX:ErrorFile=${buildDir}/jvmErrorLogs/java_err_pid%p.log" + + testLogging { + exceptionFormat = 'full' + showStackTraces = true + showStandardStreams = true + showExceptions = true + showCauses = true + } + + doFirst { mkdir "${buildDir}/jvmErrorLogs" } +} + task acceptanceTestPrivacy(type: Test) { inputs.property "integration.date", LocalTime.now() // so it runs at every invocation include '**/privacy/**' diff --git a/acceptance-tests/tests/contracts/CrossContractReader.sol b/acceptance-tests/tests/contracts/CrossContractReader.sol index f43ce8b621e..9524d5bde86 100644 --- a/acceptance-tests/tests/contracts/CrossContractReader.sol +++ b/acceptance-tests/tests/contracts/CrossContractReader.sol @@ -19,7 +19,7 @@ import "./EventEmitter.sol"; // compile with: // solc CrossContractReader.sol --bin --abi --optimize --overwrite -o . // then create web3j wrappers with: -// web3j solidity generate -b ./generated/CrossContractReader.bin -a ./generated/CrossContractReader.abi -o ../../../../../ -p org.hyperledger.besu.tests.web3j.generated +// web3j generate solidity -b ./generated/CrossContractReader.bin -a ./generated/CrossContractReader.abi -o ../../../../../ -p org.hyperledger.besu.tests.web3j.generated contract CrossContractReader { uint counter; diff --git a/acceptance-tests/tests/contracts/EventEmitter.sol b/acceptance-tests/tests/contracts/EventEmitter.sol index 2e6e29e59d6..05b8868eee9 100644 --- a/acceptance-tests/tests/contracts/EventEmitter.sol +++ b/acceptance-tests/tests/contracts/EventEmitter.sol @@ -17,7 +17,7 @@ pragma solidity >=0.7.0 <0.9.0; // compile with: // solc EventEmitter.sol --bin --abi --optimize --overwrite -o . // then create web3j wrappers with: -// web3j solidity generate -b ./generated/EventEmitter.bin -a ./generated/EventEmitter.abi -o ../../../../../ -p org.hyperledger.besu.tests.web3j.generated +// web3j generate solidity -b ./generated/EventEmitter.bin -a ./generated/EventEmitter.abi -o ../../../../../ -p org.hyperledger.besu.tests.web3j.generated contract EventEmitter { address owner; event stored(address _to, uint _amount); diff --git a/acceptance-tests/tests/contracts/RemoteSimpleStorage.sol b/acceptance-tests/tests/contracts/RemoteSimpleStorage.sol index 03c95dc2bd1..f399658789f 100644 --- a/acceptance-tests/tests/contracts/RemoteSimpleStorage.sol +++ b/acceptance-tests/tests/contracts/RemoteSimpleStorage.sol @@ -19,7 +19,7 @@ import "./SimpleStorage.sol"; // compile with: // solc RemoteSimpleStorage.sol --bin --abi --optimize --overwrite -o . // then create web3j wrappers with: -// web3j solidity generate -b ./generated/RemoteSimpleStorage.bin -a ./generated/RemoteSimpleStorage.abi -o ../../../../../ -p org.hyperledger.besu.tests.web3j.generated +// web3j generate solidity -b ./generated/RemoteSimpleStorage.bin -a ./generated/RemoteSimpleStorage.abi -o ../../../../../ -p org.hyperledger.besu.tests.web3j.generated contract RemoteSimpleStorage { SimpleStorage public simpleStorage; diff --git a/acceptance-tests/tests/contracts/RevertReason.sol b/acceptance-tests/tests/contracts/RevertReason.sol index b1270fe4ccd..2d42cafe3c2 100644 --- a/acceptance-tests/tests/contracts/RevertReason.sol +++ b/acceptance-tests/tests/contracts/RevertReason.sol @@ -17,7 +17,7 @@ pragma solidity >=0.7.0 <0.9.0; // compile with: // solc RevertReason.sol --bin --abi --optimize --overwrite -o . // then create web3j wrappers with: -// web3j solidity generate -b ./generated/RevertReason.bin -a ./generated/RevertReason.abi -o ../../../../../ -p org.hyperledger.besu.tests.web3j.generated +// web3j generate solidity -b ./generated/RevertReason.bin -a ./generated/RevertReason.abi -o ../../../../../ -p org.hyperledger.besu.tests.web3j.generated contract RevertReason { function revertWithRevertReason() public pure returns (bool) { diff --git a/acceptance-tests/tests/contracts/SimpleStorage.sol b/acceptance-tests/tests/contracts/SimpleStorage.sol index 9303def9e05..712e2a87456 100644 --- a/acceptance-tests/tests/contracts/SimpleStorage.sol +++ b/acceptance-tests/tests/contracts/SimpleStorage.sol @@ -17,7 +17,7 @@ pragma solidity >=0.7.0 <0.8.20; // compile with: // solc SimpleStorage.sol --bin --abi --optimize --overwrite -o . // then create web3j wrappers with: -// web3j solidity generate -b ./generated/SimpleStorage.bin -a ./generated/SimpleStorage.abi -o ../../../../../ -p org.hyperledger.besu.tests.web3j.generated +// web3j generate solidity -b ./generated/SimpleStorage.bin -a ./generated/SimpleStorage.abi -o ../../../../../ -p org.hyperledger.besu.tests.web3j.generated contract SimpleStorage { uint data; diff --git a/acceptance-tests/tests/shanghai/build.gradle b/acceptance-tests/tests/shanghai/build.gradle new file mode 100644 index 00000000000..c309d9e9914 --- /dev/null +++ b/acceptance-tests/tests/shanghai/build.gradle @@ -0,0 +1,21 @@ + +plugins { + id 'org.web3j' version '4.11.3' + id 'org.web3j.solidity' version '0.4.1' +} + +jar { enabled = true } + +web3j { + generatedPackageName = 'org.hyperledger.besu.tests.web3j.generated' +} + +sourceSets.main.solidity.srcDirs = [ + "$projectDir/shanghaicontracts" +] + +solidity { + resolvePackages = false + version '0.8.25' + evmVersion 'shanghai' +} diff --git a/acceptance-tests/tests/shanghai/shanghaicontracts/SimpleStorageShanghai.sol b/acceptance-tests/tests/shanghai/shanghaicontracts/SimpleStorageShanghai.sol new file mode 100644 index 00000000000..32a7d9a2a09 --- /dev/null +++ b/acceptance-tests/tests/shanghai/shanghaicontracts/SimpleStorageShanghai.sol @@ -0,0 +1,31 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +pragma solidity >=0.8.20; + +// compile with: +// solc SimpleStorageShanghai.sol --bin --abi --optimize --overwrite -o . +// then create web3j wrappers with: +// web3j generate solidity -b ./SimpleStorageShanghai.bin -a ./SimpleStorageShanghai.abi -o ../../../../../ -p org.hyperledger.besu.tests.web3j.generated +contract SimpleStorageShanghai { + uint data; + + function set(uint value) public { + data = value; + } + + function get() public view returns (uint) { + return data; + } +} diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/bft/BftAcceptanceTestParameterization.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/bft/BftAcceptanceTestParameterization.java index c9fcf364840..15872070d00 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/bft/BftAcceptanceTestParameterization.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/bft/BftAcceptanceTestParameterization.java @@ -38,7 +38,14 @@ public static Stream getFactories() { @FunctionalInterface public interface NodeCreator { - BesuNode create(BesuNodeFactory factory, String name) throws Exception; + BesuNode create(BesuNodeFactory factory, String name, boolean fixedPort) throws Exception; + } + + @FunctionalInterface + public interface FixedPortNodeCreator { + + BesuNode createFixedPort(BesuNodeFactory factory, String name, boolean fixedPort) + throws Exception; } @FunctionalInterface @@ -57,7 +64,11 @@ public BftAcceptanceTestParameterization( } public BesuNode createNode(BesuNodeFactory factory, String name) throws Exception { - return creatorFn.create(factory, name); + return creatorFn.create(factory, name, false); + } + + public BesuNode createNodeFixedPort(BesuNodeFactory factory, String name) throws Exception { + return creatorFn.create(factory, name, true); } public BesuNode createNodeWithValidators( diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/bftsoak/BftMiningSoakTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/bftsoak/BftMiningSoakTest.java new file mode 100644 index 00000000000..9861a7dab65 --- /dev/null +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/bftsoak/BftMiningSoakTest.java @@ -0,0 +1,351 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.tests.acceptance.bftsoak; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.config.JsonUtil; +import org.hyperledger.besu.tests.acceptance.bft.BftAcceptanceTestParameterization; +import org.hyperledger.besu.tests.acceptance.bft.ParameterizedBftTestBase; +import org.hyperledger.besu.tests.acceptance.dsl.node.BesuNode; +import org.hyperledger.besu.tests.web3j.generated.SimpleStorage; +import org.hyperledger.besu.tests.web3j.generated.SimpleStorageShanghai; + +import java.math.BigInteger; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +public class BftMiningSoakTest extends ParameterizedBftTestBase { + + private final int NUM_STEPS = 5; + + private final int MIN_TEST_TIME_MINS = 60; + + private static final long ONE_MINUTE = Duration.of(1, ChronoUnit.MINUTES).toMillis(); + + private static final long THREE_MINUTES = Duration.of(1, ChronoUnit.MINUTES).toMillis(); + + private static final long TEN_SECONDS = Duration.of(10, ChronoUnit.SECONDS).toMillis(); + + static int getTestDurationMins() { + // Use a default soak time of 60 mins + return Integer.getInteger("acctests.soakTimeMins", 60); + } + + @ParameterizedTest(name = "{index}: {0}") + @MethodSource("factoryFunctions") + public void shouldBeStableDuringLongTest( + final String testName, final BftAcceptanceTestParameterization nodeFactory) throws Exception { + setUp(testName, nodeFactory); + + // There is a minimum amount of time the test can be run for, due to hard coded delays + // in between certain steps. There should be no upper-limit to how long the test is run for + assertThat(getTestDurationMins()).isGreaterThanOrEqualTo(MIN_TEST_TIME_MINS); + + final BesuNode minerNode1 = nodeFactory.createNodeFixedPort(besu, "miner1"); + final BesuNode minerNode2 = nodeFactory.createNodeFixedPort(besu, "miner2"); + final BesuNode minerNode3 = nodeFactory.createNodeFixedPort(besu, "miner3"); + final BesuNode minerNode4 = nodeFactory.createNodeFixedPort(besu, "miner4"); + + // Each step should be given a minimum of 3 minutes to complete successfully. If the time + // give to run the soak test results in a time-per-step lower than this then the time + // needs to be increased. + assertThat(getTestDurationMins() / NUM_STEPS).isGreaterThanOrEqualTo(3); + + cluster.start(minerNode1, minerNode2, minerNode3, minerNode4); + + cluster.verify(blockchain.reachesHeight(minerNode1, 1, 85)); + + // Setup + // Deploy a contract that we'll invoke periodically to ensure state + // is correct during the test, especially after stopping nodes and + // applying new forks. + SimpleStorage simpleStorageContract = + minerNode1.execute(contractTransactions.createSmartContract(SimpleStorage.class)); + + // Check the contract address is as expected for this sender & nonce + contractVerifier + .validTransactionReceipt("0x42699a7612a82f1d9c36148af9c77354759b210b") + .verify(simpleStorageContract); + + // Before upgrading to newer forks, try creating a shanghai-evm contract and check that + // the transaction fails + try { + minerNode1.execute(contractTransactions.createSmartContract(SimpleStorageShanghai.class)); + Assertions.fail("Shanghai transaction should not be executed on a pre-shanghai chain"); + } catch (RuntimeException e) { + assertThat(e.getMessage()) + .contains( + "Revert reason: 'Transaction processing could not be completed due to an exception'"); + } + + // Should initially be set to 0 + assertThat(simpleStorageContract.get().send()).isEqualTo(BigInteger.valueOf(0)); + + // Set to something new + simpleStorageContract.set(BigInteger.valueOf(101)).send(); + + // Check the state of the contract has updated correctly. We'll set & get this several times + // during the test + assertThat(simpleStorageContract.get().send()).isEqualTo(BigInteger.valueOf(101)); + + // Step 1 + // Run for the configured time period, periodically checking that + // the chain is progressing as expected + BigInteger chainHeight = minerNode1.execute(ethTransactions.blockNumber()); + assertThat(chainHeight.compareTo(BigInteger.ZERO)).isGreaterThanOrEqualTo(1); + BigInteger lastChainHeight = chainHeight; + + Instant startTime = Instant.now(); + Instant nextStepEndTime = startTime.plus(getTestDurationMins() / NUM_STEPS, ChronoUnit.MINUTES); + + while (System.currentTimeMillis() < nextStepEndTime.toEpochMilli()) { + Thread.sleep(ONE_MINUTE); + chainHeight = minerNode1.execute(ethTransactions.blockNumber()); + + // With 1-second block period chain height should have moved on by at least 50 blocks + assertThat(chainHeight.compareTo(lastChainHeight.add(BigInteger.valueOf(50)))) + .isGreaterThanOrEqualTo(1); + lastChainHeight = chainHeight; + } + Instant previousStepEndTime = Instant.now(); + + // Step 2 + // Stop one of the nodes, check that the chain continues mining + // blocks + stopNode(minerNode4); + + nextStepEndTime = + previousStepEndTime.plus(getTestDurationMins() / NUM_STEPS, ChronoUnit.MINUTES); + + while (System.currentTimeMillis() < nextStepEndTime.toEpochMilli()) { + Thread.sleep(ONE_MINUTE); + chainHeight = minerNode1.execute(ethTransactions.blockNumber()); + + // Chain height should have moved on by at least 5 blocks + assertThat(chainHeight.compareTo(lastChainHeight.add(BigInteger.valueOf(20)))) + .isGreaterThanOrEqualTo(1); + lastChainHeight = chainHeight; + } + previousStepEndTime = Instant.now(); + + // Step 3 + // Stop another one of the nodes, check that the chain now stops + // mining blocks + stopNode(minerNode3); + + chainHeight = minerNode1.execute(ethTransactions.blockNumber()); + lastChainHeight = chainHeight; + + // Leave the chain stalled for 3 minutes. Check no new blocks are mined. Then + // resume the other validators. + nextStepEndTime = previousStepEndTime.plus(3, ChronoUnit.MINUTES); + while (System.currentTimeMillis() < nextStepEndTime.toEpochMilli()) { + Thread.sleep(ONE_MINUTE); + chainHeight = minerNode1.execute(ethTransactions.blockNumber()); + + // Chain height should not have moved on + assertThat(chainHeight.equals(lastChainHeight)).isTrue(); + } + + // Step 4 + // Restart both of the stopped nodes. Check that the chain resumes + // mining blocks + startNode(minerNode3); + + startNode(minerNode4); + + previousStepEndTime = Instant.now(); + + // This step gives the stalled chain time to re-sync and agree on the next BFT round + chainHeight = minerNode1.execute(ethTransactions.blockNumber()); + nextStepEndTime = + previousStepEndTime.plus((getTestDurationMins() / NUM_STEPS), ChronoUnit.MINUTES); + lastChainHeight = chainHeight; + + while (System.currentTimeMillis() < nextStepEndTime.toEpochMilli()) { + Thread.sleep(ONE_MINUTE); + chainHeight = minerNode1.execute(ethTransactions.blockNumber()); + lastChainHeight = chainHeight; + } + previousStepEndTime = Instant.now(); + + // By this loop it should be producing blocks again + nextStepEndTime = + previousStepEndTime.plus(getTestDurationMins() / NUM_STEPS, ChronoUnit.MINUTES); + + while (System.currentTimeMillis() < nextStepEndTime.toEpochMilli()) { + Thread.sleep(ONE_MINUTE); + chainHeight = minerNode1.execute(ethTransactions.blockNumber()); + + // Chain height should have moved on by at least 1 block + assertThat(chainHeight.compareTo(lastChainHeight.add(BigInteger.valueOf(1)))) + .isGreaterThanOrEqualTo(1); + lastChainHeight = chainHeight; + } + + // Update our smart contract before upgrading from berlin to london + assertThat(simpleStorageContract.get().send()).isEqualTo(BigInteger.valueOf(101)); + simpleStorageContract.set(BigInteger.valueOf(201)).send(); + assertThat(simpleStorageContract.get().send()).isEqualTo(BigInteger.valueOf(201)); + + // Upgrade the chain from berlin to london in 120 blocks time + upgradeToLondon( + minerNode1, minerNode2, minerNode3, minerNode4, lastChainHeight.intValue() + 120); + + previousStepEndTime = Instant.now(); + + chainHeight = minerNode1.execute(ethTransactions.blockNumber()); + nextStepEndTime = + previousStepEndTime.plus(getTestDurationMins() / NUM_STEPS, ChronoUnit.MINUTES); + lastChainHeight = chainHeight; + + while (System.currentTimeMillis() < nextStepEndTime.toEpochMilli()) { + Thread.sleep(ONE_MINUTE); + chainHeight = minerNode1.execute(ethTransactions.blockNumber()); + + // Chain height should have moved on by at least 50 blocks + assertThat(chainHeight.compareTo(lastChainHeight.add(BigInteger.valueOf(50)))) + .isGreaterThanOrEqualTo(1); + lastChainHeight = chainHeight; + } + + // Check that the state of our smart contract is still correct + assertThat(simpleStorageContract.get().send()).isEqualTo(BigInteger.valueOf(201)); + + // Update it once more to check new transactions are mined OK + simpleStorageContract.set(BigInteger.valueOf(301)).send(); + assertThat(simpleStorageContract.get().send()).isEqualTo(BigInteger.valueOf(301)); + + // Upgrade the chain to shanghai in 120 seconds. Then try to deploy a shanghai contract + upgradeToShanghai( + minerNode1, minerNode2, minerNode3, minerNode4, Instant.now().getEpochSecond() + 120); + + Thread.sleep(THREE_MINUTES); + + SimpleStorageShanghai simpleStorageContractShanghai = + minerNode1.execute(contractTransactions.createSmartContract(SimpleStorageShanghai.class)); + + // Check the contract address is as expected for this sender & nonce + contractVerifier + .validTransactionReceipt("0x05d91b9031a655d08e654177336d08543ac4b711") + .verify(simpleStorageContractShanghai); + } + + private static void updateGenesisConfigToLondon( + final BesuNode minerNode, final boolean zeroBaseFeeEnabled, final int blockNumber) { + + if (minerNode.getGenesisConfig().isPresent()) { + final ObjectNode genesisConfigNode = + JsonUtil.objectNodeFromString(minerNode.getGenesisConfig().get()); + final ObjectNode config = (ObjectNode) genesisConfigNode.get("config"); + config.put("londonBlock", blockNumber); + config.put("zeroBaseFee", zeroBaseFeeEnabled); + minerNode.setGenesisConfig(genesisConfigNode.toString()); + } + } + + private static void updateGenesisConfigToShanghai( + final BesuNode minerNode, final long blockTimestamp) { + + if (minerNode.getGenesisConfig().isPresent()) { + final ObjectNode genesisConfigNode = + JsonUtil.objectNodeFromString(minerNode.getGenesisConfig().get()); + final ObjectNode config = (ObjectNode) genesisConfigNode.get("config"); + config.put("shanghaiTime", blockTimestamp); + minerNode.setGenesisConfig(genesisConfigNode.toString()); + } + } + + private void upgradeToLondon( + final BesuNode minerNode1, + final BesuNode minerNode2, + final BesuNode minerNode3, + final BesuNode minerNode4, + final int londonBlockNumber) + throws InterruptedException { + // Node 1 + stopNode(minerNode1); + updateGenesisConfigToLondon(minerNode1, true, londonBlockNumber); + startNode(minerNode1); + + // Node 2 + stopNode(minerNode2); + updateGenesisConfigToLondon(minerNode2, true, londonBlockNumber); + startNode(minerNode2); + + // Node 3 + stopNode(minerNode3); + updateGenesisConfigToLondon(minerNode3, true, londonBlockNumber); + startNode(minerNode3); + + // Node 4 + stopNode(minerNode4); + updateGenesisConfigToLondon(minerNode4, true, londonBlockNumber); + startNode(minerNode4); + } + + private void upgradeToShanghai( + final BesuNode minerNode1, + final BesuNode minerNode2, + final BesuNode minerNode3, + final BesuNode minerNode4, + final long shanghaiTime) + throws InterruptedException { + // Node 1 + stopNode(minerNode1); + updateGenesisConfigToShanghai(minerNode1, shanghaiTime); + startNode(minerNode1); + + // Node 2 + stopNode(minerNode2); + updateGenesisConfigToShanghai(minerNode2, shanghaiTime); + startNode(minerNode2); + + // Node 3 + stopNode(minerNode3); + updateGenesisConfigToShanghai(minerNode3, shanghaiTime); + startNode(minerNode3); + + // Node 4 + stopNode(minerNode4); + updateGenesisConfigToShanghai(minerNode4, shanghaiTime); + startNode(minerNode4); + } + + // Start a node with a delay before returning to give it time to start + private void startNode(final BesuNode node) throws InterruptedException { + cluster.startNode(node); + Thread.sleep(TEN_SECONDS); + } + + // Stop a node with a delay before returning to give it time to stop + private void stopNode(final BesuNode node) throws InterruptedException { + cluster.stopNode(node); + Thread.sleep(TEN_SECONDS); + } + + @Override + public void tearDownAcceptanceTestBase() { + cluster.stop(); + super.tearDownAcceptanceTestBase(); + } +} diff --git a/build.gradle b/build.gradle index 905e8b2c8af..4f34e67985d 100644 --- a/build.gradle +++ b/build.gradle @@ -386,7 +386,7 @@ task checkMavenCoordinateCollisions { getAllprojects().forEach { if (it.properties.containsKey('publishing') && it.jar?.enabled) { def coordinate = it.publishing?.publications[0].coordinates - if (coordinates.containsKey(coordinate)) { + if (coordinate.toString().startsWith("org") && coordinates.containsKey(coordinate)) { throw new GradleException("Duplicate maven coordinates detected, ${coordinate} is used by " + "both ${coordinates[coordinate]} and ${it.path}.\n" + "Please add a `publishing` script block to one or both subprojects.") diff --git a/settings.gradle b/settings.gradle index 81f4cb8736b..09a8d20d4c6 100644 --- a/settings.gradle +++ b/settings.gradle @@ -28,6 +28,7 @@ rootProject.name='besu' include 'acceptance-tests:test-plugins' include 'acceptance-tests:dsl' include 'acceptance-tests:tests' +include 'acceptance-tests:tests:shanghai' include 'besu' include 'config' include 'consensus:clique' From c62f192459626a602b90a4959eab047b326c1b4a Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 11 Jun 2024 15:28:30 +0200 Subject: [PATCH 6/8] Improve genesis state performance at startup (#6977) * Refactor genesis options file management Signed-off-by: Fabio Di Fabio * Make loading allocation from genesis lazy Signed-off-by: Fabio Di Fabio * Update tests Signed-off-by: Fabio Di Fabio * Memory optimization with streaming Signed-off-by: Fabio Di Fabio * Improve loading and storgin genesis state at startup Signed-off-by: Fabio Di Fabio * Remove comments Signed-off-by: Fabio Di Fabio * Avoid parsing genesis file allocations twice Signed-off-by: Fabio Di Fabio * Update javadoc Signed-off-by: Fabio Di Fabio * Fix Signed-off-by: Fabio Di Fabio * Fix Signed-off-by: Fabio Di Fabio * Ignore unknown objects in allocations Signed-off-by: Fabio Di Fabio * avoid keeping genesis allocation data in memory Signed-off-by: Fabio Di Fabio * Update CHANGELOG Signed-off-by: Fabio Di Fabio --------- Signed-off-by: Fabio Di Fabio Signed-off-by: ahamlat Co-authored-by: ahamlat --- CHANGELOG.md | 1 + .../controller/BesuControllerBuilder.java | 50 ++-- .../besu/ForkIdsNetworkConfigTest.java | 3 +- .../besu/config/GenesisAccount.java | 42 +++ .../besu/config/GenesisAllocation.java | 109 -------- .../besu/config/GenesisConfigFile.java | 51 ++-- .../besu/config/GenesisReader.java | 242 ++++++++++++++++++ .../org/hyperledger/besu/config/JsonUtil.java | 148 ++++++++++- .../besu/config/GenesisConfigFileTest.java | 59 +++-- .../besu/config/GenesisReaderTest.java | 98 +++++++ .../hyperledger/besu/config/JsonUtilTest.java | 23 +- .../MergeGenesisConfigHelper.java | 7 +- .../besu/ethereum/chain/GenesisState.java | 156 +++-------- .../besu/ethereum/chain/GenesisStateTest.java | 64 ++--- .../bonsai/AbstractIsolationTests.java | 15 +- .../bonsai/BonsaiSnapshotIsolationTests.java | 2 +- 16 files changed, 713 insertions(+), 357 deletions(-) create mode 100644 config/src/main/java/org/hyperledger/besu/config/GenesisAccount.java delete mode 100644 config/src/main/java/org/hyperledger/besu/config/GenesisAllocation.java create mode 100644 config/src/main/java/org/hyperledger/besu/config/GenesisReader.java create mode 100644 config/src/test/java/org/hyperledger/besu/config/GenesisReaderTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b64946ae01..435ecea9e35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ ### Additions and Improvements - Add two counters to DefaultBlockchain in order to be able to calculate TPS and Mgas/s [#7105](https://github.com/hyperledger/besu/pull/7105) +- Improve genesis state performance at startup [#6977](https://github.com/hyperledger/besu/pull/6977) - Enable --Xbonsai-limit-trie-logs-enabled by default, unless sync-mode=FULL [#7181](https://github.com/hyperledger/besu/pull/7181) - Promote experimental --Xbonsai-limit-trie-logs-enabled to production-ready, --bonsai-limit-trie-logs-enabled [#7192](https://github.com/hyperledger/besu/pull/7192) - Promote experimental --Xbonsai-trie-logs-pruning-window-size to production-ready, --bonsai-trie-logs-pruning-window-size [#7192](https://github.com/hyperledger/besu/pull/7192) diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 4e3929e8ae5..aab0000592f 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -41,6 +41,7 @@ import org.hyperledger.besu.ethereum.chain.GenesisState; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.chain.VariablesStorage; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.Difficulty; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; @@ -552,30 +553,9 @@ public BesuController build() { prepForBuild(); final ProtocolSchedule protocolSchedule = createProtocolSchedule(); - final GenesisState genesisState; final VariablesStorage variablesStorage = storageProvider.createVariablesStorage(); - Optional genesisStateHash = Optional.empty(); - if (variablesStorage != null && this.genesisStateHashCacheEnabled) { - genesisStateHash = variablesStorage.getGenesisStateHash(); - } - - if (genesisStateHash.isPresent()) { - genesisState = - GenesisState.fromConfig(genesisStateHash.get(), genesisConfigFile, protocolSchedule); - } else { - genesisState = - GenesisState.fromConfig(dataStorageConfiguration, genesisConfigFile, protocolSchedule); - if (variablesStorage != null) { - VariablesStorage.Updater updater = variablesStorage.updater(); - if (updater != null) { - updater.setGenesisStateHash(genesisState.getBlock().getHeader().getStateRoot()); - updater.commit(); - } - } - } - final WorldStateStorageCoordinator worldStateStorageCoordinator = storageProvider.createWorldStateStorageCoordinator(dataStorageConfiguration); @@ -583,6 +563,13 @@ public BesuController build() { storageProvider.createBlockchainStorage( protocolSchedule, variablesStorage, dataStorageConfiguration); + final var maybeStoredGenesisBlockHash = blockchainStorage.getBlockHash(0L); + + final var genesisState = + getGenesisState( + maybeStoredGenesisBlockHash.flatMap(blockchainStorage::getBlockHeader), + protocolSchedule); + final MutableBlockchain blockchain = DefaultBlockchain.createMutable( genesisState.getBlock(), @@ -591,7 +578,6 @@ public BesuController build() { reorgLoggingThreshold, dataDirectory.toString(), numberOfBlocksToCache); - final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader = besuComponent .map(BesuComponent::getCachedMerkleTrieLoader) @@ -601,7 +587,7 @@ public BesuController build() { createWorldStateArchive( worldStateStorageCoordinator, blockchain, bonsaiCachedMerkleTrieLoader); - if (blockchain.getChainHeadBlockNumber() < 1) { + if (maybeStoredGenesisBlockHash.isEmpty()) { genesisState.writeStateTo(worldStateArchive.getMutable()); } @@ -772,6 +758,24 @@ public BesuController build() { dataStorageConfiguration); } + private GenesisState getGenesisState( + final Optional maybeGenesisBlockHeader, + final ProtocolSchedule protocolSchedule) { + final Optional maybeGenesisStateRoot = + genesisStateHashCacheEnabled + ? maybeGenesisBlockHeader.map(BlockHeader::getStateRoot) + : Optional.empty(); + + return maybeGenesisStateRoot + .map( + genesisStateRoot -> + GenesisState.fromStorage(genesisStateRoot, genesisConfigFile, protocolSchedule)) + .orElseGet( + () -> + GenesisState.fromConfig( + dataStorageConfiguration, genesisConfigFile, protocolSchedule)); + } + private TrieLogPruner createTrieLogPruner( final WorldStateKeyValueStorage worldStateStorage, final Blockchain blockchain, diff --git a/besu/src/test/java/org/hyperledger/besu/ForkIdsNetworkConfigTest.java b/besu/src/test/java/org/hyperledger/besu/ForkIdsNetworkConfigTest.java index 7d7c6391ff8..ae338f27cac 100644 --- a/besu/src/test/java/org/hyperledger/besu/ForkIdsNetworkConfigTest.java +++ b/besu/src/test/java/org/hyperledger/besu/ForkIdsNetworkConfigTest.java @@ -18,7 +18,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import org.hyperledger.besu.cli.config.EthNetworkConfig; import org.hyperledger.besu.cli.config.NetworkName; import org.hyperledger.besu.config.GenesisConfigFile; import org.hyperledger.besu.config.GenesisConfigOptions; @@ -138,7 +137,7 @@ public static Collection parameters() { @MethodSource("parameters") public void testForkId(final NetworkName chainName, final List expectedForkIds) { final GenesisConfigFile genesisConfigFile = - GenesisConfigFile.fromConfig(EthNetworkConfig.jsonConfig(chainName)); + GenesisConfigFile.fromResource(chainName.getGenesisFile()); final MilestoneStreamingTransitionProtocolSchedule schedule = createSchedule(genesisConfigFile); final GenesisState genesisState = GenesisState.fromConfig(genesisConfigFile, schedule); final Blockchain mockBlockchain = mock(Blockchain.class); diff --git a/config/src/main/java/org/hyperledger/besu/config/GenesisAccount.java b/config/src/main/java/org/hyperledger/besu/config/GenesisAccount.java new file mode 100644 index 00000000000..70bb80e0422 --- /dev/null +++ b/config/src/main/java/org/hyperledger/besu/config/GenesisAccount.java @@ -0,0 +1,42 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.config; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Wei; + +import java.util.Map; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; + +/** + * Genesis account + * + * @param address of the account + * @param nonce nonce of the account at genesis + * @param balance balance of the account at genesis + * @param code code of the account at genesis, can be null + * @param storage storage of the account at genesis + * @param privateKey of the account, only use for testing + */ +public record GenesisAccount( + Address address, + long nonce, + Wei balance, + Bytes code, + Map storage, + Bytes32 privateKey) {} diff --git a/config/src/main/java/org/hyperledger/besu/config/GenesisAllocation.java b/config/src/main/java/org/hyperledger/besu/config/GenesisAllocation.java deleted file mode 100644 index c53ded9e865..00000000000 --- a/config/src/main/java/org/hyperledger/besu/config/GenesisAllocation.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.config; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import com.fasterxml.jackson.databind.node.ObjectNode; - -/** The Genesis allocation configuration. */ -public class GenesisAllocation { - private final String address; - private final ObjectNode data; - - /** - * Instantiates a new Genesis allocation. - * - * @param address the address - * @param data the data - */ - GenesisAllocation(final String address, final ObjectNode data) { - this.address = address; - this.data = data; - } - - /** - * Gets address. - * - * @return the address - */ - public String getAddress() { - return address; - } - - /** - * Gets private key. - * - * @return the private key - */ - public Optional getPrivateKey() { - return Optional.ofNullable(JsonUtil.getString(data, "privatekey", null)); - } - - /** - * Gets balance. - * - * @return the balance - */ - public String getBalance() { - return JsonUtil.getValueAsString(data, "balance", "0"); - } - - /** - * Gets code. - * - * @return the code - */ - public String getCode() { - return JsonUtil.getString(data, "code", null); - } - - /** - * Gets nonce. - * - * @return the nonce - */ - public String getNonce() { - return JsonUtil.getValueAsString(data, "nonce", "0"); - } - - /** - * Gets version. - * - * @return the version - */ - public String getVersion() { - return JsonUtil.getValueAsString(data, "version", null); - } - - /** - * Gets storage map. - * - * @return fields under storage as a map - */ - public Map getStorage() { - final Map map = new HashMap<>(); - JsonUtil.getObjectNode(data, "storage") - .orElse(JsonUtil.createEmptyObjectNode()) - .fields() - .forEachRemaining( - (entry) -> { - map.put(entry.getKey(), entry.getValue().asText()); - }); - return map; - } -} diff --git a/config/src/main/java/org/hyperledger/besu/config/GenesisConfigFile.java b/config/src/main/java/org/hyperledger/besu/config/GenesisConfigFile.java index cf5a13d8558..bd1e117773b 100644 --- a/config/src/main/java/org/hyperledger/besu/config/GenesisConfigFile.java +++ b/config/src/main/java/org/hyperledger/besu/config/GenesisConfigFile.java @@ -14,8 +14,6 @@ */ package org.hyperledger.besu.config; -import static org.hyperledger.besu.config.JsonUtil.normalizeKeys; - import org.hyperledger.besu.datatypes.Wei; import java.net.URL; @@ -30,22 +28,23 @@ import java.util.stream.Stream; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.Streams; /** The Genesis config file. */ public class GenesisConfigFile { /** The constant DEFAULT. */ public static final GenesisConfigFile DEFAULT = - new GenesisConfigFile(JsonUtil.createEmptyObjectNode()); + new GenesisConfigFile(new GenesisReader.FromObjectNode(JsonUtil.createEmptyObjectNode())); /** The constant BASEFEE_AT_GENESIS_DEFAULT_VALUE. */ public static final Wei BASEFEE_AT_GENESIS_DEFAULT_VALUE = Wei.of(1_000_000_000L); + private final GenesisReader loader; private final ObjectNode genesisRoot; - private GenesisConfigFile(final ObjectNode config) { - this.genesisRoot = config; + private GenesisConfigFile(final GenesisReader loader) { + this.loader = loader; + this.genesisRoot = loader.getRoot(); } /** @@ -70,21 +69,31 @@ public static GenesisConfigFile fromSource(final URL jsonSource) { /** * Genesis file from resource. * - * @param jsonResource the resource name + * @param resourceName the resource name + * @return the genesis config file + */ + public static GenesisConfigFile fromResource(final String resourceName) { + return fromConfig(GenesisConfigFile.class.getResource(resourceName)); + } + + /** + * From config genesis config file. + * + * @param jsonSource the json string * @return the genesis config file */ - public static GenesisConfigFile fromResource(final String jsonResource) { - return fromSource(GenesisConfigFile.class.getResource(jsonResource)); + public static GenesisConfigFile fromConfig(final URL jsonSource) { + return new GenesisConfigFile(new GenesisReader.FromURL(jsonSource)); } /** * From config genesis config file. * - * @param jsonString the json string + * @param json the json string * @return the genesis config file */ - public static GenesisConfigFile fromConfig(final String jsonString) { - return fromConfig(JsonUtil.objectNodeFromString(jsonString, false)); + public static GenesisConfigFile fromConfig(final String json) { + return fromConfig(JsonUtil.objectNodeFromString(json, false)); } /** @@ -94,7 +103,7 @@ public static GenesisConfigFile fromConfig(final String jsonString) { * @return the genesis config file */ public static GenesisConfigFile fromConfig(final ObjectNode config) { - return new GenesisConfigFile(normalizeKeys(config)); + return new GenesisConfigFile(new GenesisReader.FromObjectNode(config)); } /** @@ -113,8 +122,7 @@ public GenesisConfigOptions getConfigOptions() { * @return the config options */ public GenesisConfigOptions getConfigOptions(final Map overrides) { - final ObjectNode config = - JsonUtil.getObjectNode(genesisRoot, "config").orElse(JsonUtil.createEmptyObjectNode()); + final ObjectNode config = loader.getConfig(); Map overridesRef = overrides; @@ -134,15 +142,8 @@ public GenesisConfigOptions getConfigOptions(final Map overrides * * @return the stream */ - public Stream streamAllocations() { - return JsonUtil.getObjectNode(genesisRoot, "alloc").stream() - .flatMap( - allocations -> - Streams.stream(allocations.fieldNames()) - .map( - key -> - new GenesisAllocation( - key, JsonUtil.getObjectNode(allocations, key).get()))); + public Stream streamAllocations() { + return loader.streamAllocations(); } /** @@ -344,7 +345,7 @@ public String toString() { + "genesisRoot=" + genesisRoot + ", allocations=" - + streamAllocations().map(GenesisAllocation::toString).collect(Collectors.joining(",")) + + loader.streamAllocations().map(GenesisAccount::toString).collect(Collectors.joining(",")) + '}'; } } diff --git a/config/src/main/java/org/hyperledger/besu/config/GenesisReader.java b/config/src/main/java/org/hyperledger/besu/config/GenesisReader.java new file mode 100644 index 00000000000..316aa73d043 --- /dev/null +++ b/config/src/main/java/org/hyperledger/besu/config/GenesisReader.java @@ -0,0 +1,242 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.config; + +import static org.hyperledger.besu.config.JsonUtil.normalizeKey; +import static org.hyperledger.besu.config.JsonUtil.normalizeKeys; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Wei; + +import java.io.IOException; +import java.math.BigInteger; +import java.net.URL; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Streams; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; + +interface GenesisReader { + String CONFIG_FIELD = "config"; + String ALLOCATION_FIELD = "alloc"; + + ObjectNode getRoot(); + + ObjectNode getConfig(); + + Stream streamAllocations(); + + class FromObjectNode implements GenesisReader { + private final ObjectNode allocations; + private final ObjectNode rootWithoutAllocations; + + public FromObjectNode(final ObjectNode root) { + final var removedAllocations = root.remove(ALLOCATION_FIELD); + this.allocations = + removedAllocations != null + ? (ObjectNode) removedAllocations + : JsonUtil.createEmptyObjectNode(); + this.rootWithoutAllocations = normalizeKeys(root); + } + + @Override + public ObjectNode getRoot() { + return rootWithoutAllocations; + } + + @Override + public ObjectNode getConfig() { + return JsonUtil.getObjectNode(rootWithoutAllocations, CONFIG_FIELD) + .orElse(JsonUtil.createEmptyObjectNode()); + } + + @Override + public Stream streamAllocations() { + return Streams.stream(allocations.fields()) + .map( + entry -> { + final var on = normalizeKeys((ObjectNode) entry.getValue()); + return new GenesisAccount( + Address.fromHexString(entry.getKey()), + JsonUtil.getString(on, "nonce").map(ParserUtils::parseUnsignedLong).orElse(0L), + JsonUtil.getString(on, "balance") + .map(ParserUtils::parseBalance) + .orElse(Wei.ZERO), + JsonUtil.getBytes(on, "code", null), + ParserUtils.getStorageMap(on, "storage"), + JsonUtil.getBytes(on, "privatekey").map(Bytes32::wrap).orElse(null)); + }); + } + } + + class FromURL implements GenesisReader { + private final URL url; + private final ObjectNode rootWithoutAllocations; + + public FromURL(final URL url) { + this.url = url; + this.rootWithoutAllocations = + normalizeKeys(JsonUtil.objectNodeFromURL(url, false, ALLOCATION_FIELD)); + } + + @Override + public ObjectNode getRoot() { + return rootWithoutAllocations; + } + + @Override + public ObjectNode getConfig() { + return JsonUtil.getObjectNode(rootWithoutAllocations, CONFIG_FIELD) + .orElse(JsonUtil.createEmptyObjectNode()); + } + + @Override + public Stream streamAllocations() { + final var parser = JsonUtil.jsonParserFromURL(url, false); + + try { + parser.nextToken(); + while (parser.nextToken() != JsonToken.END_OBJECT) { + if (ALLOCATION_FIELD.equals(parser.getCurrentName())) { + parser.nextToken(); + parser.nextToken(); + break; + } else { + parser.skipChildren(); + } + } + } catch (final IOException e) { + throw new RuntimeException(e); + } + + return Streams.stream(new AllocationIterator(parser)); + } + + private static class AllocationIterator implements Iterator { + final JsonParser parser; + + public AllocationIterator(final JsonParser parser) { + this.parser = parser; + } + + @Override + public boolean hasNext() { + final var end = parser.currentToken() == JsonToken.END_OBJECT; + if (end) { + try { + parser.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return !end; + } + + @Override + public GenesisAccount next() { + try { + final Address address = Address.fromHexString(parser.currentName()); + long nonce = 0; + Wei balance = Wei.ZERO; + Bytes code = null; + Map storage = Map.of(); + Bytes32 privateKey = null; + parser.nextToken(); // consume start object + while (parser.nextToken() != JsonToken.END_OBJECT) { + switch (normalizeKey(parser.currentName())) { + case "nonce": + parser.nextToken(); + nonce = ParserUtils.parseUnsignedLong(parser.getText()); + break; + case "balance": + parser.nextToken(); + balance = ParserUtils.parseBalance(parser.getText()); + break; + case "code": + parser.nextToken(); + code = Bytes.fromHexStringLenient(parser.getText()); + break; + case "privatekey": + parser.nextToken(); + privateKey = Bytes32.fromHexStringLenient(parser.getText()); + break; + case "storage": + parser.nextToken(); + storage = new HashMap<>(); + while (parser.nextToken() != JsonToken.END_OBJECT) { + final var key = UInt256.fromHexString(parser.currentName()); + parser.nextToken(); + final var value = UInt256.fromHexString(parser.getText()); + storage.put(key, value); + } + break; + } + if (parser.currentToken() == JsonToken.START_OBJECT) { + // ignore any unknown nested object + parser.skipChildren(); + } + } + parser.nextToken(); + return new GenesisAccount(address, nonce, balance, code, storage, privateKey); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + } + + class ParserUtils { + static long parseUnsignedLong(final String value) { + String v = value.toLowerCase(Locale.US); + if (v.startsWith("0x")) { + v = v.substring(2); + } + return Long.parseUnsignedLong(v, 16); + } + + static Wei parseBalance(final String balance) { + final BigInteger val; + if (balance.startsWith("0x")) { + val = new BigInteger(1, Bytes.fromHexStringLenient(balance).toArrayUnsafe()); + } else { + val = new BigInteger(balance); + } + + return Wei.of(val); + } + + static Map getStorageMap(final ObjectNode json, final String key) { + return JsonUtil.getObjectNode(json, key) + .map( + storageMap -> + Streams.stream(storageMap.fields()) + .collect( + Collectors.toMap( + e -> UInt256.fromHexString(e.getKey()), + e -> UInt256.fromHexString(e.getValue().asText())))) + .orElse(Map.of()); + } + } +} diff --git a/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java b/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java index fba62dd9060..bcb89c64edb 100644 --- a/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java +++ b/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java @@ -23,17 +23,30 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; +import java.util.Set; +import java.util.function.Function; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonParser.Feature; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.filter.FilteringParserDelegate; +import com.fasterxml.jackson.core.filter.TokenFilter; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeType; import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.tuweni.bytes.Bytes; /** The Json util class. */ public class JsonUtil { + private static final JsonFactory JSON_FACTORY = + JsonFactory.builder() + .disable(JsonFactory.Feature.INTERN_FIELD_NAMES) + .disable(JsonFactory.Feature.CANONICALIZE_FIELD_NAMES) + .build(); + /** Default constructor. */ private JsonUtil() {} @@ -53,7 +66,7 @@ public static ObjectNode normalizeKeys(final ObjectNode objectNode) { entry -> { final String key = entry.getKey(); final JsonNode value = entry.getValue(); - final String normalizedKey = key.toLowerCase(Locale.US); + final String normalizedKey = normalizeKey(key); if (value instanceof ObjectNode) { normalized.set(normalizedKey, normalizeKeys((ObjectNode) value)); } else if (value instanceof ArrayNode) { @@ -65,6 +78,17 @@ public static ObjectNode normalizeKeys(final ObjectNode objectNode) { return normalized; } + /** + * Converts the key to lowercase for easier lookup. This is useful in cases such as the + * 'genesis.json' file where all keys are assumed to be case insensitive. + * + * @param key the key to be normalized + * @return key in lower case. + */ + public static String normalizeKey(final String key) { + return key.toLowerCase(Locale.US); + } + private static ArrayNode normalizeKeysInArray(final ArrayNode arrayNode) { final ArrayNode normalizedArray = JsonUtil.createEmptyArrayNode(); arrayNode.forEach( @@ -263,6 +287,35 @@ public static boolean getBoolean( return getBoolean(node, key).orElse(defaultValue); } + /** + * Gets Bytes. + * + * @param json the json + * @param key the key + * @return the Bytes + */ + public static Optional getBytes(final ObjectNode json, final String key) { + return getParsedValue(json, key, Bytes::fromHexString); + } + + /** + * Gets Wei. + * + * @param json the json + * @param key the key + * @param defaultValue the default value + * @return the Wei + */ + public static Bytes getBytes(final ObjectNode json, final String key, final Bytes defaultValue) { + return getBytes(json, key).orElse(defaultValue); + } + + private static Optional getParsedValue( + final ObjectNode json, final String name, final Function parser) { + + return getValue(json, name).map(JsonNode::asText).map(parser); + } + /** * Create empty object node object node. * @@ -308,18 +361,75 @@ public static ObjectNode objectNodeFromString(final String jsonData) { * * @param jsonData the json data * @param allowComments true to allow comments + * @param excludeFields names of the fields to not read * @return the object node */ public static ObjectNode objectNodeFromString( - final String jsonData, final boolean allowComments) { - final ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.configure(Feature.ALLOW_COMMENTS, allowComments); + final String jsonData, final boolean allowComments, final String... excludeFields) { try { - final JsonNode jsonNode = objectMapper.readTree(jsonData); + return objectNodeFromParser( + JSON_FACTORY.createParser(jsonData), allowComments, excludeFields); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Object node from string object node. + * + * @param jsonSource the json data + * @param allowComments true to allow comments + * @param excludeFields names of the fields to not read + * @return the object node + */ + public static ObjectNode objectNodeFromURL( + final URL jsonSource, final boolean allowComments, final String... excludeFields) { + try { + return objectNodeFromParser( + JSON_FACTORY.createParser(jsonSource).enable(Feature.AUTO_CLOSE_SOURCE), + allowComments, + excludeFields); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Get a JsonParser to parse JSON from URL. + * + * @param jsonSource the json source + * @param allowComments true to allow comments + * @return the json parser + */ + public static JsonParser jsonParserFromURL(final URL jsonSource, final boolean allowComments) { + try { + return JSON_FACTORY + .createParser(jsonSource) + .enable(Feature.AUTO_CLOSE_SOURCE) + .configure(Feature.ALLOW_COMMENTS, allowComments); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static ObjectNode objectNodeFromParser( + final JsonParser baseParser, final boolean allowComments, final String... excludeFields) { + try { + final var parser = + excludeFields.length > 0 + ? new FilteringParserDelegate( + baseParser, + new NameExcludeFilter(excludeFields), + TokenFilter.Inclusion.INCLUDE_ALL_AND_PATH, + true) + : baseParser; + parser.configure(Feature.ALLOW_COMMENTS, allowComments); + + final ObjectMapper objectMapper = new ObjectMapper(); + final JsonNode jsonNode = objectMapper.readTree(parser); validateType(jsonNode, JsonNodeType.OBJECT); return (ObjectNode) jsonNode; } catch (final IOException e) { - // Reading directly from a string should not raise an IOException, just catch and rethrow throw new RuntimeException(e); } } @@ -490,4 +600,30 @@ private static boolean validateInt(final JsonNode node) { } return true; } + + private static class NameExcludeFilter extends TokenFilter { + private final Set names; + + public NameExcludeFilter(final String... names) { + this.names = Set.of(names); + } + + @Override + public TokenFilter includeProperty(final String name) { + if (names.contains(name)) { + return null; + } + return this; + } + + @Override + public boolean includeEmptyObject(final boolean contentsFiltered) { + return !contentsFiltered; + } + + @Override + public boolean includeEmptyArray(final boolean contentsFiltered) { + return !contentsFiltered; + } + } } diff --git a/config/src/test/java/org/hyperledger/besu/config/GenesisConfigFileTest.java b/config/src/test/java/org/hyperledger/besu/config/GenesisConfigFileTest.java index 2605f0d5e89..9f014b7d4d2 100644 --- a/config/src/test/java/org/hyperledger/besu/config/GenesisConfigFileTest.java +++ b/config/src/test/java/org/hyperledger/besu/config/GenesisConfigFileTest.java @@ -19,6 +19,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.hyperledger.besu.config.GenesisConfigFile.fromConfig; +import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; import java.io.IOException; @@ -50,7 +51,11 @@ void shouldLoadMainnetConfigFile() { // Sanity check some basic properties to confirm this is the mainnet file. assertThat(config.getConfigOptions().isEthHash()).isTrue(); assertThat(config.getConfigOptions().getChainId()).hasValue(MAINNET_CHAIN_ID); - assertThat(config.streamAllocations().map(GenesisAllocation::getAddress)) + assertThat( + config + .streamAllocations() + .map(GenesisAccount::address) + .map(Address::toUnprefixedHexString)) .contains( "000d836201318ec6899a67540690382780743280", "001762430ea9c3a26e5749afdb70da5f78ddbb8c", @@ -63,7 +68,11 @@ void shouldLoadDevelopmentConfigFile() { // Sanity check some basic properties to confirm this is the dev file. assertThat(config.getConfigOptions().isEthHash()).isTrue(); assertThat(config.getConfigOptions().getChainId()).hasValue(DEVELOPMENT_CHAIN_ID); - assertThat(config.streamAllocations().map(GenesisAllocation::getAddress)) + assertThat( + config + .streamAllocations() + .map(GenesisAccount::address) + .map(Address::toUnprefixedHexString)) .contains( "fe3b557e8fb62b89f4916b721be55ceb828dbd73", "627306090abab3a6e1400e9345bc60c78a8bef57", @@ -271,31 +280,41 @@ void shouldGetAllocations() { + " }" + "}"); - final Map allocations = + final Map allocations = config .streamAllocations() - .collect(Collectors.toMap(GenesisAllocation::getAddress, Function.identity())); - assertThat(allocations) - .containsOnlyKeys( + .collect(Collectors.toMap(GenesisAccount::address, Function.identity())); + assertThat(allocations.keySet()) + .map(Address::toUnprefixedHexString) + .containsOnly( "fe3b557e8fb62b89f4916b721be55ceb828dbd73", "627306090abab3a6e1400e9345bc60c78a8bef57", "f17f52151ebef6c7334fad080c5704d77216b732"); - final GenesisAllocation alloc1 = allocations.get("fe3b557e8fb62b89f4916b721be55ceb828dbd73"); - final GenesisAllocation alloc2 = allocations.get("627306090abab3a6e1400e9345bc60c78a8bef57"); - final GenesisAllocation alloc3 = allocations.get("f17f52151ebef6c7334fad080c5704d77216b732"); - - assertThat(alloc1.getBalance()).isEqualTo("0xad78ebc5ac6200000"); - assertThat(alloc2.getBalance()).isEqualTo("1000"); - assertThat(alloc3.getBalance()).isEqualTo("90000000000000000000000"); - assertThat(alloc3.getStorage()).hasSize(2); - assertThat(alloc3.getStorage()) + final GenesisAccount alloc1 = + allocations.get(Address.fromHexString("fe3b557e8fb62b89f4916b721be55ceb828dbd73")); + final GenesisAccount alloc2 = + allocations.get(Address.fromHexString("627306090abab3a6e1400e9345bc60c78a8bef57")); + final GenesisAccount alloc3 = + allocations.get(Address.fromHexString("f17f52151ebef6c7334fad080c5704d77216b732")); + + assertThat(alloc1.balance()) + .isEqualTo(GenesisReader.ParserUtils.parseBalance("0xad78ebc5ac6200000")); + assertThat(alloc2.balance()).isEqualTo(GenesisReader.ParserUtils.parseBalance("1000")); + assertThat(alloc3.balance()) + .isEqualTo(GenesisReader.ParserUtils.parseBalance("90000000000000000000000")); + assertThat(alloc3.storage()).hasSize(2); + assertThat(alloc3.storage()) .containsEntry( - "0xc4c3a3f99b26e5e534b71d6f33ca6ea5c174decfb16dd7237c60eff9774ef4a4", - "0x937307647bd3b9a82abe2974e1407241d54947bbb39763a4cac9f77166ad92a0"); - assertThat(alloc3.getStorage()) + UInt256.fromHexString( + "0xc4c3a3f99b26e5e534b71d6f33ca6ea5c174decfb16dd7237c60eff9774ef4a4"), + UInt256.fromHexString( + "0x937307647bd3b9a82abe2974e1407241d54947bbb39763a4cac9f77166ad92a0")); + assertThat(alloc3.storage()) .containsEntry( - "0xc4c3a3f99b26e5e534b71d6f33ca6ea5c174decfb16dd7237c60eff9774ef4a3", - "0x6f8a80d14311c39f35f516fa664deaaaa13e85b2f7493f37f6144d86991ec012"); + UInt256.fromHexString( + "0xc4c3a3f99b26e5e534b71d6f33ca6ea5c174decfb16dd7237c60eff9774ef4a3"), + UInt256.fromHexString( + "0x6f8a80d14311c39f35f516fa664deaaaa13e85b2f7493f37f6144d86991ec012")); } @Test diff --git a/config/src/test/java/org/hyperledger/besu/config/GenesisReaderTest.java b/config/src/test/java/org/hyperledger/besu/config/GenesisReaderTest.java new file mode 100644 index 00000000000..5d73a2829f5 --- /dev/null +++ b/config/src/test/java/org/hyperledger/besu/config/GenesisReaderTest.java @@ -0,0 +1,98 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.config; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.config.GenesisReader.ALLOCATION_FIELD; +import static org.hyperledger.besu.config.GenesisReader.CONFIG_FIELD; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Wei; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class GenesisReaderTest { + private final ObjectMapper mapper = new ObjectMapper(); + + @Test + public void readGenesisFromObjectNode() { + final var configNode = mapper.createObjectNode(); + configNode.put("londonBlock", 1); + final var allocNode = mapper.createObjectNode(); + allocNode.put(Address.BLS12_G2MUL.toUnprefixedHexString(), generateAllocation(Wei.ONE)); + final var rootNode = mapper.createObjectNode(); + rootNode.put("chainId", 12); + rootNode.put(CONFIG_FIELD, configNode); + rootNode.put(ALLOCATION_FIELD, allocNode); + final var genesisReader = new GenesisReader.FromObjectNode(rootNode); + + assertThat(genesisReader.getRoot().get("chainid").asInt()).isEqualTo(12); + assertThat(genesisReader.getRoot().has(ALLOCATION_FIELD)).isFalse(); + assertThat(genesisReader.getConfig().get("londonblock").asInt()).isEqualTo(1); + assertThat(genesisReader.streamAllocations()) + .containsExactly(new GenesisAccount(Address.BLS12_G2MUL, 0, Wei.ONE, null, Map.of(), null)); + } + + @Test + public void readGenesisFromURL(@TempDir final Path folder) throws IOException { + final String jsonStr = + """ + { + "chainId":11, + "config": { + "londonBlock":1 + }, + "alloc": { + "000d836201318ec6899a67540690382780743280": { + "balance": "0xad78ebc5ac6200000" + } + }, + "gasLimit": "0x1" + } + """; + + final var genesisFile = Files.writeString(folder.resolve("genesis.json"), jsonStr); + + final var genesisReader = new GenesisReader.FromURL(genesisFile.toUri().toURL()); + + assertThat(genesisReader.getRoot().get("chainid").asInt()).isEqualTo(11); + assertThat(genesisReader.getRoot().get("gaslimit").asText()).isEqualTo("0x1"); + assertThat(genesisReader.getRoot().has(ALLOCATION_FIELD)).isFalse(); + assertThat(genesisReader.getConfig().get("londonblock").asInt()).isEqualTo(1); + assertThat(genesisReader.streamAllocations()) + .containsExactly( + new GenesisAccount( + Address.fromHexString("000d836201318ec6899a67540690382780743280"), + 0, + Wei.fromHexString("0xad78ebc5ac6200000"), + null, + Map.of(), + null)); + } + + private ObjectNode generateAllocation(final Wei balance) { + final ObjectNode entry = mapper.createObjectNode(); + entry.put("balance", balance.toShortHexString()); + return entry; + } +} diff --git a/config/src/test/java/org/hyperledger/besu/config/JsonUtilTest.java b/config/src/test/java/org/hyperledger/besu/config/JsonUtilTest.java index 1b251d5cd1d..1e1df382e71 100644 --- a/config/src/test/java/org/hyperledger/besu/config/JsonUtilTest.java +++ b/config/src/test/java/org/hyperledger/besu/config/JsonUtilTest.java @@ -659,7 +659,24 @@ public void objectNodeFromString_withComments_commentsEnabled() { } @Test - public void objectNodeFromURL(@TempDir final Path folder) throws IOException { + public void objectNodeFromString_excludingField() { + final String jsonStr = + """ + { + "a":1, + "b":2, + "c":3 + } + """; + + final ObjectNode result = JsonUtil.objectNodeFromString(jsonStr, false, "b"); + assertThat(result.get("a").asInt()).isEqualTo(1); + assertThat(result.has("b")).isFalse(); + assertThat(result.get("c").asInt()).isEqualTo(3); + } + + @Test + public void objectNodeFromURL_excludingField(@TempDir final Path folder) throws IOException { final String jsonStr = """ { @@ -670,9 +687,9 @@ public void objectNodeFromURL(@TempDir final Path folder) throws IOException { """; final var genesisFile = Files.writeString(folder.resolve("genesis.json"), jsonStr); - final ObjectNode result = JsonUtil.objectNodeFromURL(genesisFile.toUri().toURL(), false); + final ObjectNode result = JsonUtil.objectNodeFromURL(genesisFile.toUri().toURL(), false, "b"); assertThat(result.get("a").asInt()).isEqualTo(1); - assertThat(result.get("b").asInt()).isEqualTo(2); + assertThat(result.has("b")).isFalse(); assertThat(result.get("c").asInt()).isEqualTo(3); } diff --git a/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeGenesisConfigHelper.java b/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeGenesisConfigHelper.java index 4f510ec7e30..54099f868cb 100644 --- a/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeGenesisConfigHelper.java +++ b/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeGenesisConfigHelper.java @@ -14,7 +14,7 @@ */ package org.hyperledger.besu.consensus.merge.blockcreation; -import org.hyperledger.besu.config.GenesisAllocation; +import org.hyperledger.besu.config.GenesisAccount; import org.hyperledger.besu.config.GenesisConfigFile; import org.hyperledger.besu.consensus.merge.MergeProtocolSchedule; import org.hyperledger.besu.datatypes.Address; @@ -48,10 +48,7 @@ default GenesisConfigFile getPowGenesisConfigFile() { } default Stream
genesisAllocations(final GenesisConfigFile configFile) { - return configFile - .streamAllocations() - .map(GenesisAllocation::getAddress) - .map(Address::fromHexString); + return configFile.streamAllocations().map(GenesisAccount::address); } default ProtocolSchedule getMergeProtocolSchedule() { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java index ae9acc89a69..d42f2d39a3f 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java @@ -17,12 +17,11 @@ import static java.util.Collections.emptyList; import static org.hyperledger.besu.ethereum.trie.common.GenesisWorldStateProvider.createGenesisWorldState; -import org.hyperledger.besu.config.GenesisAllocation; +import org.hyperledger.besu.config.GenesisAccount; import org.hyperledger.besu.config.GenesisConfigFile; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.BlobGas; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockBody; import org.hyperledger.besu.ethereum.core.BlockHeader; @@ -38,29 +37,27 @@ import org.hyperledger.besu.evm.log.LogsBloomFilter; import org.hyperledger.besu.evm.worldstate.WorldUpdater; -import java.math.BigInteger; -import java.util.HashMap; +import java.net.URL; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Optional; import java.util.OptionalLong; import java.util.function.Function; import java.util.stream.Stream; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; public final class GenesisState { private final Block block; - private final List genesisAccounts; + private final GenesisConfigFile genesisConfigFile; - private GenesisState(final Block block, final List genesisAccounts) { + private GenesisState(final Block block, final GenesisConfigFile genesisConfigFile) { this.block = block; - this.genesisAccounts = genesisAccounts; + this.genesisConfigFile = genesisConfigFile; } /** @@ -75,24 +72,25 @@ public static GenesisState fromJson(final String json, final ProtocolSchedule pr } /** - * Construct a {@link GenesisState} from a JSON string. + * Construct a {@link GenesisState} from a URL * * @param dataStorageConfiguration A {@link DataStorageConfiguration} describing the storage * configuration - * @param json A JSON string describing the genesis block + * @param jsonSource A URL pointing to JSON genesis file * @param protocolSchedule A protocol Schedule associated with * @return A new {@link GenesisState}. */ - public static GenesisState fromJson( + @VisibleForTesting + static GenesisState fromJsonSource( final DataStorageConfiguration dataStorageConfiguration, - final String json, + final URL jsonSource, final ProtocolSchedule protocolSchedule) { return fromConfig( - dataStorageConfiguration, GenesisConfigFile.fromConfig(json), protocolSchedule); + dataStorageConfiguration, GenesisConfigFile.fromConfig(jsonSource), protocolSchedule); } /** - * Construct a {@link GenesisState} from a JSON object. + * Construct a {@link GenesisState} from a genesis file object. * * @param config A {@link GenesisConfigFile} describing the genesis block. * @param protocolSchedule A protocol Schedule associated with @@ -108,41 +106,40 @@ public static GenesisState fromConfig( * * @param dataStorageConfiguration A {@link DataStorageConfiguration} describing the storage * configuration - * @param config A {@link GenesisConfigFile} describing the genesis block. + * @param genesisConfigFile A {@link GenesisConfigFile} describing the genesis block. * @param protocolSchedule A protocol Schedule associated with * @return A new {@link GenesisState}. */ public static GenesisState fromConfig( final DataStorageConfiguration dataStorageConfiguration, - final GenesisConfigFile config, + final GenesisConfigFile genesisConfigFile, final ProtocolSchedule protocolSchedule) { - final List genesisAccounts = parseAllocations(config).toList(); + final var genesisStateRoot = + calculateGenesisStateRoot(dataStorageConfiguration, genesisConfigFile); final Block block = new Block( - buildHeader( - config, - calculateGenesisStateHash(dataStorageConfiguration, genesisAccounts), - protocolSchedule), - buildBody(config)); - return new GenesisState(block, genesisAccounts); + buildHeader(genesisConfigFile, genesisStateRoot, protocolSchedule), + buildBody(genesisConfigFile)); + return new GenesisState(block, genesisConfigFile); } /** * Construct a {@link GenesisState} from a JSON object. * - * @param genesisStateHash The hash of the genesis state. - * @param config A {@link GenesisConfigFile} describing the genesis block. + * @param genesisStateRoot The root of the genesis state. + * @param genesisConfigFile A {@link GenesisConfigFile} describing the genesis block. * @param protocolSchedule A protocol Schedule associated with * @return A new {@link GenesisState}. */ - public static GenesisState fromConfig( - final Hash genesisStateHash, - final GenesisConfigFile config, + public static GenesisState fromStorage( + final Hash genesisStateRoot, + final GenesisConfigFile genesisConfigFile, final ProtocolSchedule protocolSchedule) { - final List genesisAccounts = parseAllocations(config).toList(); final Block block = - new Block(buildHeader(config, genesisStateHash, protocolSchedule), buildBody(config)); - return new GenesisState(block, genesisAccounts); + new Block( + buildHeader(genesisConfigFile, genesisStateRoot, protocolSchedule), + buildBody(genesisConfigFile)); + return new GenesisState(block, genesisConfigFile); } private static BlockBody buildBody(final GenesisConfigFile config) { @@ -164,31 +161,31 @@ public Block getBlock() { * @param target WorldView to write genesis state to */ public void writeStateTo(final MutableWorldState target) { - writeAccountsTo(target, genesisAccounts, block.getHeader()); + writeAccountsTo(target, genesisConfigFile.streamAllocations(), block.getHeader()); } private static void writeAccountsTo( final MutableWorldState target, - final List genesisAccounts, + final Stream genesisAccounts, final BlockHeader rootHeader) { final WorldUpdater updater = target.updater(); genesisAccounts.forEach( genesisAccount -> { - final MutableAccount account = updater.getOrCreate(genesisAccount.address); - account.setNonce(genesisAccount.nonce); - account.setBalance(genesisAccount.balance); - account.setCode(genesisAccount.code); - genesisAccount.storage.forEach(account::setStorageValue); + final MutableAccount account = updater.createAccount(genesisAccount.address()); + account.setNonce(genesisAccount.nonce()); + account.setBalance(genesisAccount.balance()); + account.setCode(genesisAccount.code()); + genesisAccount.storage().forEach(account::setStorageValue); }); updater.commit(); target.persist(rootHeader); } - private static Hash calculateGenesisStateHash( + private static Hash calculateGenesisStateRoot( final DataStorageConfiguration dataStorageConfiguration, - final List genesisAccounts) { + final GenesisConfigFile genesisConfigFile) { try (var worldState = createGenesisWorldState(dataStorageConfiguration)) { - writeAccountsTo(worldState, genesisAccounts, null); + writeAccountsTo(worldState, genesisConfigFile.streamAllocations(), null); return worldState.rootHash(); } catch (Exception e) { throw new RuntimeException(e); @@ -265,10 +262,6 @@ private static Hash parseMixHash(final GenesisConfigFile genesis) { return withNiceErrorMessage("mixHash", genesis.getMixHash(), Hash::fromHexStringLenient); } - private static Stream parseAllocations(final GenesisConfigFile genesis) { - return genesis.streamAllocations().map(GenesisAccount::fromAllocation); - } - private static long parseNonce(final GenesisConfigFile genesis) { return withNiceErrorMessage("nonce", genesis.getNonce(), GenesisState::parseUnsignedLong); } @@ -340,75 +333,6 @@ private static boolean isExperimentalEipsTimeAtGenesis(final GenesisConfigFile g @Override public String toString() { - return MoreObjects.toStringHelper(this) - .add("block", block) - .add("genesisAccounts", genesisAccounts) - .toString(); - } - - private static final class GenesisAccount { - - final long nonce; - final Address address; - final Wei balance; - final Map storage; - final Bytes code; - - static GenesisAccount fromAllocation(final GenesisAllocation allocation) { - return new GenesisAccount( - allocation.getNonce(), - allocation.getAddress(), - allocation.getBalance(), - allocation.getStorage(), - allocation.getCode()); - } - - private GenesisAccount( - final String hexNonce, - final String hexAddress, - final String balance, - final Map storage, - final String hexCode) { - this.nonce = withNiceErrorMessage("nonce", hexNonce, GenesisState::parseUnsignedLong); - this.address = withNiceErrorMessage("address", hexAddress, Address::fromHexString); - this.balance = withNiceErrorMessage("balance", balance, this::parseBalance); - this.code = hexCode != null ? Bytes.fromHexString(hexCode) : null; - this.storage = parseStorage(storage); - } - - private Wei parseBalance(final String balance) { - final BigInteger val; - if (balance.startsWith("0x")) { - val = new BigInteger(1, Bytes.fromHexStringLenient(balance).toArrayUnsafe()); - } else { - val = new BigInteger(balance); - } - - return Wei.of(val); - } - - private Map parseStorage(final Map storage) { - final Map parsedStorage = new HashMap<>(); - storage.forEach( - (key1, value1) -> { - final UInt256 key = withNiceErrorMessage("storage key", key1, UInt256::fromHexString); - final UInt256 value = - withNiceErrorMessage("storage value", value1, UInt256::fromHexString); - parsedStorage.put(key, value); - }); - - return parsedStorage; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("address", address) - .add("nonce", nonce) - .add("balance", balance) - .add("storage", storage) - .add("code", code) - .toString(); - } + return MoreObjects.toStringHelper(this).add("block", block).toString(); } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/GenesisStateTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/GenesisStateTest.java index 6e6217e08bc..ea1eb044149 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/GenesisStateTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/GenesisStateTest.java @@ -29,8 +29,6 @@ import java.util.stream.Stream; -import com.google.common.base.Charsets; -import com.google.common.io.Resources; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.units.bigints.UInt256; import org.bouncycastle.util.encoders.Hex; @@ -64,12 +62,11 @@ public Stream provideArguments(final ExtensionContext conte @ParameterizedTest @ArgumentsSource(GenesisStateTestArguments.class) - public void createFromJsonWithAllocs(final DataStorageConfiguration dataStorageConfiguration) - throws Exception { + public void createFromJsonWithAllocs(final DataStorageConfiguration dataStorageConfiguration) { final GenesisState genesisState = - GenesisState.fromJson( + GenesisState.fromJsonSource( dataStorageConfiguration, - Resources.toString(GenesisStateTest.class.getResource("genesis1.json"), Charsets.UTF_8), + GenesisStateTest.class.getResource("genesis1.json"), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); assertThat(header.getStateRoot()) @@ -95,12 +92,11 @@ public void createFromJsonWithAllocs(final DataStorageConfiguration dataStorageC @ParameterizedTest @ArgumentsSource(GenesisStateTestArguments.class) - void createFromJsonNoAllocs(final DataStorageConfiguration dataStorageConfiguration) - throws Exception { + void createFromJsonNoAllocs(final DataStorageConfiguration dataStorageConfiguration) { final GenesisState genesisState = - GenesisState.fromJson( + GenesisState.fromJsonSource( dataStorageConfiguration, - Resources.toString(GenesisStateTest.class.getResource("genesis2.json"), Charsets.UTF_8), + GenesisStateTest.class.getResource("genesis2.json"), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); assertThat(header.getStateRoot()).isEqualTo(Hash.EMPTY_TRIE_HASH); @@ -114,12 +110,11 @@ void createFromJsonNoAllocs(final DataStorageConfiguration dataStorageConfigurat private void assertContractInvariants( final DataStorageConfiguration dataStorageConfiguration, final String sourceFile, - final String blockHash) - throws Exception { + final String blockHash) { final GenesisState genesisState = - GenesisState.fromJson( + GenesisState.fromJsonSource( dataStorageConfiguration, - Resources.toString(GenesisStateTest.class.getResource(sourceFile), Charsets.UTF_8), + GenesisStateTest.class.getResource(sourceFile), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); assertThat(header.getHash()).isEqualTo(Hash.fromHexString(blockHash)); @@ -141,8 +136,7 @@ private void assertContractInvariants( @ParameterizedTest @ArgumentsSource(GenesisStateTestArguments.class) - void createFromJsonWithContract(final DataStorageConfiguration dataStorageConfiguration) - throws Exception { + void createFromJsonWithContract(final DataStorageConfiguration dataStorageConfiguration) { assertContractInvariants( dataStorageConfiguration, "genesis3.json", @@ -151,13 +145,11 @@ void createFromJsonWithContract(final DataStorageConfiguration dataStorageConfig @ParameterizedTest @ArgumentsSource(GenesisStateTestArguments.class) - void createFromJsonWithNonce(final DataStorageConfiguration dataStorageConfiguration) - throws Exception { + void createFromJsonWithNonce(final DataStorageConfiguration dataStorageConfiguration) { final GenesisState genesisState = - GenesisState.fromJson( + GenesisState.fromJsonSource( dataStorageConfiguration, - Resources.toString( - GenesisStateTest.class.getResource("genesisNonce.json"), Charsets.UTF_8), + GenesisStateTest.class.getResource("genesisNonce.json"), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); assertThat(header.getHash()) @@ -168,13 +160,11 @@ void createFromJsonWithNonce(final DataStorageConfiguration dataStorageConfigura @ParameterizedTest @ArgumentsSource(GenesisStateTestArguments.class) - void encodeOlympicBlock(final DataStorageConfiguration dataStorageConfiguration) - throws Exception { + void encodeOlympicBlock(final DataStorageConfiguration dataStorageConfiguration) { final GenesisState genesisState = - GenesisState.fromJson( + GenesisState.fromJsonSource( dataStorageConfiguration, - Resources.toString( - GenesisStateTest.class.getResource("genesis-olympic.json"), Charsets.UTF_8), + GenesisStateTest.class.getResource("genesis-olympic.json"), ProtocolScheduleFixture.MAINNET); final BytesValueRLPOutput tmp = new BytesValueRLPOutput(); genesisState.getBlock().writeTo(tmp); @@ -190,13 +180,11 @@ private void assertStorageValue(final Account contract, final String key, final @ParameterizedTest @ArgumentsSource(GenesisStateTestArguments.class) - void genesisFromShanghai(final DataStorageConfiguration dataStorageConfiguration) - throws Exception { + void genesisFromShanghai(final DataStorageConfiguration dataStorageConfiguration) { final GenesisState genesisState = - GenesisState.fromJson( + GenesisState.fromJsonSource( dataStorageConfiguration, - Resources.toString( - GenesisStateTest.class.getResource("genesis_shanghai.json"), Charsets.UTF_8), + GenesisStateTest.class.getResource("genesis_shanghai.json"), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); assertThat(header.getHash()) @@ -241,12 +229,11 @@ void genesisFromShanghai(final DataStorageConfiguration dataStorageConfiguration @ParameterizedTest @ArgumentsSource(GenesisStateTestArguments.class) - void genesisFromCancun(final DataStorageConfiguration dataStorageConfiguration) throws Exception { + void genesisFromCancun(final DataStorageConfiguration dataStorageConfiguration) { final GenesisState genesisState = - GenesisState.fromJson( + GenesisState.fromJsonSource( dataStorageConfiguration, - Resources.toString( - GenesisStateTest.class.getResource("genesis_cancun.json"), Charsets.UTF_8), + GenesisStateTest.class.getResource("genesis_cancun.json"), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); assertThat(header.getHash()) @@ -292,12 +279,11 @@ void genesisFromCancun(final DataStorageConfiguration dataStorageConfiguration) @ParameterizedTest @ArgumentsSource(GenesisStateTestArguments.class) - void genesisFromPrague(final DataStorageConfiguration dataStorageConfiguration) throws Exception { + void genesisFromPrague(final DataStorageConfiguration dataStorageConfiguration) { final GenesisState genesisState = - GenesisState.fromJson( + GenesisState.fromJsonSource( dataStorageConfiguration, - Resources.toString( - GenesisStateTest.class.getResource("genesis_prague.json"), Charsets.UTF_8), + GenesisStateTest.class.getResource("genesis_prague.json"), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); assertThat(header.getHash()) diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java index d9b9d5d1955..62adf26eccb 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java @@ -20,7 +20,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import org.hyperledger.besu.config.GenesisAllocation; +import org.hyperledger.besu.config.GenesisAccount; import org.hyperledger.besu.config.GenesisConfigFile; import org.hyperledger.besu.crypto.KeyPair; import org.hyperledger.besu.crypto.SECPPrivateKey; @@ -85,7 +85,6 @@ import java.util.Optional; import java.util.function.BiFunction; import java.util.function.Function; -import java.util.stream.Collectors; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; @@ -98,10 +97,10 @@ public abstract class AbstractIsolationTests { protected ProtocolContext protocolContext; protected EthContext ethContext; protected EthScheduler ethScheduler = new DeterministicEthScheduler(); - final Function asKeyPair = + final Function asKeyPair = key -> SignatureAlgorithmFactory.getInstance() - .createKeyPair(SECPPrivateKey.create(Bytes32.fromHexString(key), "ECDSA")); + .createKeyPair(SECPPrivateKey.create(key, "ECDSA")); protected final ProtocolSchedule protocolSchedule = MainnetProtocolSchedule.fromConfig( GenesisConfigFile.fromResource("/dev.json").getConfigOptions(), @@ -139,13 +138,13 @@ public abstract class AbstractIsolationTests { new BlobCache(), MiningParameters.newDefault())); - protected final List accounts = + protected final List accounts = GenesisConfigFile.fromResource("/dev.json") .streamAllocations() - .filter(ga -> ga.getPrivateKey().isPresent()) - .collect(Collectors.toList()); + .filter(ga -> ga.privateKey() != null) + .toList(); - KeyPair sender1 = asKeyPair.apply(accounts.get(0).getPrivateKey().get()); + KeyPair sender1 = Optional.ofNullable(accounts.get(0).privateKey()).map(asKeyPair).orElseThrow(); TransactionPool transactionPool; @TempDir private Path tempData; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java index ffdc3fa897f..322aacb7eb7 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java @@ -39,7 +39,7 @@ public void ensureTruncateDoesNotCauseSegfault() { var postTruncatedWorldState = archive.getMutable(genesisState.getBlock().getHeader(), false); assertThat(postTruncatedWorldState).isEmpty(); // assert that trying to access pre-worldstate does not segfault after truncating - preTruncatedWorldState.get().get(Address.fromHexString(accounts.get(0).getAddress())); + preTruncatedWorldState.get().get(accounts.get(0).address()); assertThat(true).isTrue(); } From a8621f4962fb28f9067093a69b926f793b5dbecb Mon Sep 17 00:00:00 2001 From: ahamlat Date: Tue, 11 Jun 2024 16:57:26 +0200 Subject: [PATCH 7/8] Enable JFR continuous profiling with default setting (#7006) Enable JFR continuous profiling with default setting Signed-off-by: Ameziane H --- CHANGELOG.md | 2 ++ build.gradle | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 435ecea9e35..c3efddef70d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ - `admin_nodeInfo` JSON/RPC call returns the currently active EVM version [#7127](https://github.com/hyperledger/besu/pull/7127) - Improve the selection of the most profitable built block [#7174](https://github.com/hyperledger/besu/pull/7174) - Support for eth_maxPriorityFeePerGas [#5658](https://github.com/hyperledger/besu/issues/5658) +- Enable continuous profiling with default setting [#7006](https://github.com/hyperledger/besu/pull/7006) + ### Bug fixes - Make `eth_gasPrice` aware of the base fee market [#7102](https://github.com/hyperledger/besu/pull/7102) - Validation errors ignored in accounts-allowlist and empty list [#7138](https://github.com/hyperledger/besu/issues/7138) diff --git a/build.gradle b/build.gradle index 4f34e67985d..b26f36b5194 100644 --- a/build.gradle +++ b/build.gradle @@ -634,7 +634,9 @@ startScripts { defaultJvmOpts = applicationDefaultJvmArgs + [ "-XX:G1ConcRefinementThreads=2", "-XX:G1HeapWastePercent=15", - "-XX:MaxGCPauseMillis=100" + "-XX:MaxGCPauseMillis=100", + "-XX:StartFlightRecording,dumponexit=true,settings=default.jfc", + "-Xlog:jfr*=off" ] unixStartScriptGenerator.template = resources.text.fromFile("${projectDir}/besu/src/main/scripts/unixStartScript.txt") windowsStartScriptGenerator.template = resources.text.fromFile("${projectDir}/besu/src/main/scripts/windowsStartScript.txt") From c7e2e4dbbf1a3348992d2701465fa0b3521ad96a Mon Sep 17 00:00:00 2001 From: Usman Saleem Date: Wed, 12 Jun 2024 10:40:32 +1000 Subject: [PATCH 8/8] fix(test): Fix dns daemon periodic test (#7200) * fix(test): Fix dns daemon periodic test Increase second run to 3 seconds Signed-off-by: Usman Saleem * fix(test): Fix dns daemon periodic test change vertx testContext verify to failNow Signed-off-by: Usman Saleem * fix(test): Fix dns daemon periodic test Remove Disabled Signed-off-by: Usman Saleem * fix(test): Code formatting Signed-off-by: Usman Saleem --------- Signed-off-by: Usman Saleem --- .../p2p/discovery/dns/DNSDaemonTest.java | 52 +++++++++++++------ 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/dns/DNSDaemonTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/dns/DNSDaemonTest.java index 08539d47bf8..fd8ba1382d8 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/dns/DNSDaemonTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/dns/DNSDaemonTest.java @@ -29,13 +29,13 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @ExtendWith(VertxExtension.class) class DNSDaemonTest { + private static final int EXPECTED_SEQ = 932; private static final String holeskyEnr = "enrtree://AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@all.holesky.ethdisco.net"; private final MockDnsServerVerticle mockDnsServerVerticle = new MockDnsServerVerticle(); @@ -54,13 +54,24 @@ void prepare(final Vertx vertx, final VertxTestContext vertxTestContext) { @Test @DisplayName("Test DNS Daemon with a mock DNS server") - void testDNSDaemon(final Vertx vertx, final VertxTestContext testContext) - throws InterruptedException { + void testDNSDaemon(final Vertx vertx, final VertxTestContext testContext) { final Checkpoint checkpoint = testContext.checkpoint(); dnsDaemon = new DNSDaemon( holeskyEnr, - (seq, records) -> checkpoint.flag(), + (seq, records) -> { + if (seq != EXPECTED_SEQ) { + testContext.failNow( + String.format( + "Expecting sequence to be %d in first pass but got: %d", + EXPECTED_SEQ, seq)); + } + if (records.size() != 115) { + testContext.failNow( + "Expecting 115 records in first pass but got: " + records.size()); + } + checkpoint.flag(); + }, 0, 0, 0, @@ -74,7 +85,6 @@ void testDNSDaemon(final Vertx vertx, final VertxTestContext testContext) } @Test - @Disabled("this test is flaky") @DisplayName("Test DNS Daemon with periodic lookup to a mock DNS server") void testDNSDaemonPeriodic(final Vertx vertx, final VertxTestContext testContext) throws InterruptedException { @@ -87,18 +97,28 @@ void testDNSDaemonPeriodic(final Vertx vertx, final VertxTestContext testContext (seq, records) -> { switch (pass.incrementAndGet()) { case 1: - testContext.verify( - () -> { - assertThat(seq).isEqualTo(932); - assertThat(records).hasSize(115); - }); + if (seq != EXPECTED_SEQ) { + testContext.failNow( + String.format( + "Expecting sequence to be %d in first pass but got: %d", + EXPECTED_SEQ, seq)); + } + if (records.size() != 115) { + testContext.failNow( + "Expecting 115 records in first pass but got: " + records.size()); + } break; case 2: - testContext.verify( - () -> { - assertThat(seq).isEqualTo(932); - assertThat(records).isEmpty(); - }); + if (seq != EXPECTED_SEQ) { + testContext.failNow( + String.format( + "Expecting sequence to be %d in second pass but got: %d", + EXPECTED_SEQ, seq)); + } + if (!records.isEmpty()) { + testContext.failNow( + "Expecting 0 records in second pass but got: " + records.size()); + } break; default: testContext.failNow("Third pass is not expected"); @@ -107,7 +127,7 @@ void testDNSDaemonPeriodic(final Vertx vertx, final VertxTestContext testContext }, 0, 1, // initial delay - 300, // second lookup after 300 ms (due to Mock DNS server, we are very quick). + 3000, // second lookup after 3 seconds (the thread scheduling can be slower in CI) "localhost:" + mockDnsServerVerticle.port()); final DeploymentOptions options =