generalised requests flat encoding and engine api changes

Signed-off-by: Jason Frame <jason.frame@consensys.net>

consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/PayloadWrapper.java

@@ -18,12 +18,17 @@
 import org.hyperledger.besu.datatypes.Wei;
 import org.hyperledger.besu.ethereum.core.BlockValueCalculator;
 import org.hyperledger.besu.ethereum.core.BlockWithReceipts;
+import org.hyperledger.besu.ethereum.core.Request;
+
+import java.util.List;
+import java.util.Optional;
 
 /** Wrapper for payload plus extra info. */
 public class PayloadWrapper {
   private final PayloadIdentifier payloadIdentifier;
   private final BlockWithReceipts blockWithReceipts;
   private final Wei blockValue;
+  private final Optional<List<Request>> requests;
 
   /**
    * Construct a wrapper with the following fields.
@@ -32,10 +37,13 @@ public class PayloadWrapper {
    * @param blockWithReceipts Block with receipts
    */
   public PayloadWrapper(
-      final PayloadIdentifier payloadIdentifier, final BlockWithReceipts blockWithReceipts) {
+      final PayloadIdentifier payloadIdentifier,
+      final BlockWithReceipts blockWithReceipts,
+      final Optional<List<Request>> requests) {
     this.blockWithReceipts = blockWithReceipts;
     this.payloadIdentifier = payloadIdentifier;
     this.blockValue = BlockValueCalculator.calculateBlockValue(blockWithReceipts);
+    this.requests = requests;
   }
 
   /**
@@ -64,4 +72,13 @@ public PayloadIdentifier payloadIdentifier() {
   public BlockWithReceipts blockWithReceipts() {
     return blockWithReceipts;
   }
+
+  /**
+   * Get the requests
+   *
+   * @return requests
+   */
+  public Optional<List<Request>> requests() {
+    return requests;
+  }
 }

consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java

@@ -22,6 +22,7 @@
 import org.hyperledger.besu.datatypes.Address;
 import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.datatypes.Wei;
+import org.hyperledger.besu.ethereum.BlockProcessingOutputs;
 import org.hyperledger.besu.ethereum.BlockProcessingResult;
 import org.hyperledger.besu.ethereum.ProtocolContext;
 import org.hyperledger.besu.ethereum.blockcreation.BlockCreator.BlockCreationResult;
@@ -299,10 +300,13 @@ public PayloadIdentifier preparePayload(
             .getBlock();
 
     BlockProcessingResult result = validateProposedBlock(emptyBlock);
+    // TODO include the requests in the payload wrapper
     if (result.isSuccessful()) {
       mergeContext.putPayloadById(
           new PayloadWrapper(
-              payloadIdentifier, new BlockWithReceipts(emptyBlock, result.getReceipts())));
+              payloadIdentifier,
+              new BlockWithReceipts(emptyBlock, result.getReceipts()),
+              result.getYield().flatMap(BlockProcessingOutputs::getRequests)));
       LOG.info(
           "Start building proposals for block {} identified by {}",
           emptyBlock.getHeader().getNumber(),
@@ -469,7 +473,9 @@ private void evaluateNewBlock(
 
       mergeContext.putPayloadById(
           new PayloadWrapper(
-              payloadIdentifier, new BlockWithReceipts(bestBlock, resultBest.getReceipts())));
+              payloadIdentifier,
+              new BlockWithReceipts(bestBlock, resultBest.getReceipts()),
+              resultBest.getYield().flatMap(BlockProcessingOutputs::getRequests)));
       LOG.atDebug()
           .setMessage(
               "Successfully built block {} for proposal identified by {}, with {} transactions, in {}ms")

consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/messagewrappers/ProposalTest.java

@@ -54,8 +54,7 @@ public class ProposalTest {
           new BlockBody(
               Collections.emptyList(),
               Collections.emptyList(),
-              Optional.of(Collections.emptyList()),
-              Optional.empty()));
+              Optional.of(Collections.emptyList())));
 
   @Test
   public void canRoundTripProposalMessage() {

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/execution/TracedJsonRpcProcessor.java

@@ -120,7 +120,7 @@ public JsonRpcResponse process(
         case INVALID_TRANSACTION_INDEX_PARAMS:
         case INVALID_TRANSACTION_LIMIT_PARAMS:
         case INVALID_TRANSACTION_TRACE_PARAMS:
-        case INVALID_VERSIONED_HASH_PARAMS:
+        case INVALID_REQUESTS_PARAMS:
         case INVALID_VOTE_TYPE_PARAMS:
         case INVALID_WITHDRAWALS_PARAMS:
           metricSpan.setStatus(StatusCode.ERROR, "Invalid Params");

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractEngineNewPayload.java

@@ -20,28 +20,23 @@
 import static org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.ExecutionEngineJsonRpcMethod.EngineStatus.INVALID_BLOCK_HASH;
 import static org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.ExecutionEngineJsonRpcMethod.EngineStatus.SYNCING;
 import static org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.ExecutionEngineJsonRpcMethod.EngineStatus.VALID;
-import static org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.RequestValidatorProvider.getConsolidationRequestValidator;
-import static org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.RequestValidatorProvider.getDepositRequestValidator;
-import static org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.RequestValidatorProvider.getWithdrawalRequestValidator;
 import static org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.WithdrawalsValidatorProvider.getWithdrawalsValidator;
 
 import org.hyperledger.besu.consensus.merge.blockcreation.MergeMiningCoordinator;
 import org.hyperledger.besu.datatypes.Address;
 import org.hyperledger.besu.datatypes.BlobGas;
 import org.hyperledger.besu.datatypes.Hash;
+import org.hyperledger.besu.datatypes.RequestType;
 import org.hyperledger.besu.datatypes.VersionedHash;
 import org.hyperledger.besu.datatypes.Wei;
 import org.hyperledger.besu.ethereum.BlockProcessingResult;
 import org.hyperledger.besu.ethereum.ProtocolContext;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcRequestException;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.ExecutionEngineJsonRpcMethod;
-import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.ConsolidationRequestParameter;
-import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.DepositRequestParameter;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.EnginePayloadParameter;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonRpcParameter.JsonRpcParameterException;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.WithdrawalParameter;
-import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.WithdrawalRequestParameter;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse;
@@ -64,7 +59,6 @@
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec;
 import org.hyperledger.besu.ethereum.mainnet.ValidationResult;
 import org.hyperledger.besu.ethereum.mainnet.feemarket.ExcessBlobGasCalculator;
-import org.hyperledger.besu.ethereum.mainnet.requests.RequestUtil;
 import org.hyperledger.besu.ethereum.rlp.RLPException;
 import org.hyperledger.besu.ethereum.trie.MerkleTrieException;
 import org.hyperledger.besu.plugin.services.exception.StorageException;
@@ -75,6 +69,7 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import io.vertx.core.Vertx;
 import io.vertx.core.json.Json;
@@ -141,8 +136,22 @@ public JsonRpcResponse syncResponse(final JsonRpcRequestContext requestContext)
     final Optional<Bytes32> maybeParentBeaconBlockRoot =
         maybeParentBeaconBlockRootParam.map(Bytes32::fromHexString);
 
+    final Optional<List<String>> maybeRequestsParam;
+    try {
+      maybeRequestsParam = requestContext.getOptionalList(3, String.class);
+    } catch (JsonRpcParameterException e) {
+      throw new InvalidJsonRpcRequestException(
+          "Invalid execution request parameters (index 3)",
+          RpcErrorType.INVALID_REQUESTS_PARAMS,
+          e);
+    }
+
     final ValidationResult<RpcErrorType> parameterValidationResult =
-        validateParameters(blockParam, maybeVersionedHashParam, maybeParentBeaconBlockRootParam);
+        validateParameters(
+            blockParam,
+            maybeVersionedHashParam,
+            maybeParentBeaconBlockRootParam,
+            maybeRequestsParam);
     if (!parameterValidationResult.isValid()) {
       return new JsonRpcErrorResponse(reqId, parameterValidationResult);
     }
@@ -183,44 +192,56 @@ public JsonRpcResponse syncResponse(final JsonRpcRequestContext requestContext)
       return new JsonRpcErrorResponse(reqId, RpcErrorType.INVALID_WITHDRAWALS_PARAMS);
     }
 
-    final Optional<List<Request>> maybeDepositRequests =
-        Optional.ofNullable(blockParam.getDepositRequests())
-            .map(ds -> ds.stream().map(DepositRequestParameter::toDeposit).collect(toList()));
-    if (!getDepositRequestValidator(
-            protocolSchedule.get(), blockParam.getTimestamp(), blockParam.getBlockNumber())
-        .validateParameter(maybeDepositRequests)) {
-      return new JsonRpcErrorResponse(reqId, RpcErrorType.INVALID_DEPOSIT_REQUEST_PARAMS);
-    }
-
-    final Optional<List<Request>> maybeWithdrawalRequests =
-        Optional.ofNullable(blockParam.getWithdrawalRequests())
-            .map(
-                withdrawalRequest ->
-                    withdrawalRequest.stream()
-                        .map(WithdrawalRequestParameter::toWithdrawalRequest)
-                        .collect(toList()));
-    if (!getWithdrawalRequestValidator(
-            protocolSchedule.get(), blockParam.getTimestamp(), blockParam.getBlockNumber())
-        .validateParameter(maybeWithdrawalRequests)) {
-      return new JsonRpcErrorResponse(reqId, RpcErrorType.INVALID_WITHDRAWALS_PARAMS);
-    }
-
-    final Optional<List<Request>> maybeConsolidationRequests =
-        Optional.ofNullable(blockParam.getConsolidationRequests())
-            .map(
-                consolidationRequest ->
-                    consolidationRequest.stream()
-                        .map(ConsolidationRequestParameter::toConsolidationRequest)
-                        .collect(toList()));
-    if (!getConsolidationRequestValidator(
-            protocolSchedule.get(), blockParam.getTimestamp(), blockParam.getBlockNumber())
-        .validateParameter(maybeConsolidationRequests)) {
-      return new JsonRpcErrorResponse(reqId, RpcErrorType.INVALID_CONSOLIDATION_REQUEST_PARAMS);
-    }
-
-    Optional<List<Request>> maybeRequests =
-        RequestUtil.combine(
-            maybeDepositRequests, maybeWithdrawalRequests, maybeConsolidationRequests);
+    final Optional<List<Request>> maybeRequests;
+    try {
+      maybeRequests = extractRequests(maybeVersionedHashParam);
+    } catch (RuntimeException ex) {
+      return respondWithInvalid(
+          reqId,
+          blockParam,
+          mergeCoordinator.getLatestValidAncestor(blockParam.getParentHash()).orElse(null),
+          INVALID,
+          "Invalid requests");
+    }
+
+    // TODO re-enable validation with requests validation verifying
+    //  that request_hash = sha256(sha256(0x00 || request_data_00) || sha256(0x01 ||
+    // request_data_01) || ...)
+    //    final Optional<List<Request>> maybeDepositRequests =
+    //        Optional.ofNullable(blockParam.getDepositRequests())
+    //            .map(ds -> ds.stream().map(DepositRequestParameter::toDeposit).collect(toList()));
+    //    if (!getDepositRequestValidator(
+    //            protocolSchedule.get(), blockParam.getTimestamp(), blockParam.getBlockNumber())
+    //        .validateParameter(maybeDepositRequests)) {
+    //      return new JsonRpcErrorResponse(reqId, RpcErrorType.INVALID_DEPOSIT_REQUEST_PARAMS);
+    //    }
+    //
+    //    final Optional<List<Request>> maybeWithdrawalRequests =
+    //        Optional.ofNullable(blockParam.getWithdrawalRequests())
+    //            .map(
+    //                withdrawalRequest ->
+    //                    withdrawalRequest.stream()
+    //                        .map(WithdrawalRequestParameter::toWithdrawalRequest)
+    //                        .collect(toList()));
+    //    if (!getWithdrawalRequestValidator(
+    //            protocolSchedule.get(), blockParam.getTimestamp(), blockParam.getBlockNumber())
+    //        .validateParameter(maybeWithdrawalRequests)) {
+    //      return new JsonRpcErrorResponse(reqId, RpcErrorType.INVALID_WITHDRAWALS_PARAMS);
+    //    }
+    //
+    //    final Optional<List<Request>> maybeConsolidationRequests =
+    //        Optional.ofNullable(blockParam.getConsolidationRequests())
+    //            .map(
+    //                consolidationRequest ->
+    //                    consolidationRequest.stream()
+    //                        .map(ConsolidationRequestParameter::toConsolidationRequest)
+    //                        .collect(toList()));
+    //    if (!getConsolidationRequestValidator(
+    //            protocolSchedule.get(), blockParam.getTimestamp(), blockParam.getBlockNumber())
+    //        .validateParameter(maybeConsolidationRequests)) {
+    //      return new JsonRpcErrorResponse(reqId,
+    // RpcErrorType.INVALID_CONSOLIDATION_REQUEST_PARAMS);
+    //    }
 
     if (mergeContext.get().isSyncing()) {
       LOG.debug("We are syncing");
@@ -351,8 +372,7 @@ public JsonRpcResponse syncResponse(final JsonRpcRequestContext requestContext)
 
     final var block =
         new Block(
-            newBlockHeader,
-            new BlockBody(transactions, Collections.emptyList(), maybeWithdrawals, maybeRequests));
+            newBlockHeader, new BlockBody(transactions, Collections.emptyList(), maybeWithdrawals));
 
     if (maybeParentHeader.isEmpty()) {
       LOG.atDebug()
@@ -466,7 +486,8 @@ protected EngineStatus getInvalidBlockHashStatus() {
   protected ValidationResult<RpcErrorType> validateParameters(
       final EnginePayloadParameter parameter,
       final Optional<List<String>> maybeVersionedHashParam,
-      final Optional<String> maybeBeaconBlockRootParam) {
+      final Optional<String> maybeBeaconBlockRootParam,
+      final Optional<List<String>> maybeRequestsParam) {
     return ValidationResult.valid();
   }
 
@@ -490,15 +511,15 @@ protected ValidationResult<RpcErrorType> validateBlobs(
 
     if (maybeVersionedHashes.isEmpty() && !transactionVersionedHashes.isEmpty()) {
       return ValidationResult.invalid(
-          RpcErrorType.INVALID_VERSIONED_HASH_PARAMS,
+          RpcErrorType.INVALID_REQUESTS_PARAMS,
           "Payload must contain versioned hashes for transactions");
     }
 
     // Validate versionedHashesParam
     if (maybeVersionedHashes.isPresent()
         && !maybeVersionedHashes.get().equals(transactionVersionedHashes)) {
       return ValidationResult.invalid(
-          RpcErrorType.INVALID_VERSIONED_HASH_PARAMS,
+          RpcErrorType.INVALID_REQUESTS_PARAMS,
           "Versioned hashes from blob transactions do not match expected values");
     }
 
@@ -562,6 +583,18 @@ private Optional<List<VersionedHash>> extractVersionedHashes(
                 .collect(Collectors.toList()));
   }
 
+  private Optional<List<Request>> extractRequests(final Optional<List<String>> maybeRequestsParam) {
+    if (maybeRequestsParam.isEmpty()) {
+      return Optional.empty();
+    }
+
+    return maybeRequestsParam.map(
+        requests ->
+            IntStream.range(0, requests.size())
+                .mapToObj(i -> new Request(RequestType.of(i), Bytes.fromHexString(requests.get(i))))
+                .collect(Collectors.toList()));
+  }
+
   private void logImportedBlockInfo(final Block block, final int blobCount, final double timeInS) {
     final StringBuilder message = new StringBuilder();
     message.append("Imported #%,d / %d tx");
@@ -572,10 +605,6 @@ private void logImportedBlockInfo(final Block block, final int blobCount, final
       message.append(" / %d ws");
       messageArgs.add(block.getBody().getWithdrawals().get().size());
     }
-    if (block.getBody().getRequests().isPresent()) {
-      message.append(" / %d rs");
-      messageArgs.add(block.getBody().getRequests().get().size());
-    }
     message.append(" / %d blobs / base fee %s / %,d (%01.1f%%) gas / (%s) in %01.3fs. Peers: %d");
     messageArgs.addAll(
         List.of(

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV2.java

@@ -57,7 +57,8 @@ public String getName() {
   protected ValidationResult<RpcErrorType> validateParameters(
       final EnginePayloadParameter payloadParameter,
       final Optional<List<String>> maybeVersionedHashParam,
-      final Optional<String> maybeBeaconBlockRootParam) {
+      final Optional<String> maybeBeaconBlockRootParam,
+      final Optional<List<String>> maybeRequestsParam) {
     if (payloadParameter.getBlobGasUsed() != null) {
       return ValidationResult.invalid(
           RpcErrorType.INVALID_BLOB_GAS_USED_PARAMS, "Missing blob gas used field");

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV3.java

@@ -55,7 +55,8 @@ public String getName() {
   protected ValidationResult<RpcErrorType> validateParameters(
       final EnginePayloadParameter payloadParameter,
       final Optional<List<String>> maybeVersionedHashParam,
-      final Optional<String> maybeBeaconBlockRootParam) {
+      final Optional<String> maybeBeaconBlockRootParam,
+      final Optional<List<String>> maybeRequestsParam) {
     if (payloadParameter.getBlobGasUsed() == null) {
       return ValidationResult.invalid(
           RpcErrorType.INVALID_BLOB_GAS_USED_PARAMS, "Missing blob gas used field");

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV4.java

@@ -55,7 +55,8 @@ public String getName() {
   protected ValidationResult<RpcErrorType> validateParameters(
       final EnginePayloadParameter payloadParameter,
       final Optional<List<String>> maybeVersionedHashParam,
-      final Optional<String> maybeBeaconBlockRootParam) {
+      final Optional<String> maybeBeaconBlockRootParam,
+      final Optional<List<String>> maybeRequestsParam) {
     if (payloadParameter.getBlobGasUsed() == null) {
       return ValidationResult.invalid(
           RpcErrorType.INVALID_BLOB_GAS_USED_PARAMS, "Missing blob gas used field");
@@ -69,8 +70,9 @@ protected ValidationResult<RpcErrorType> validateParameters(
       return ValidationResult.invalid(
           RpcErrorType.INVALID_PARENT_BEACON_BLOCK_ROOT_PARAMS,
           "Missing parent beacon block root field");
-    } else if (payloadParameter.getDepositRequests() == null) {
-      return ValidationResult.invalid(RpcErrorType.INVALID_PARAMS, "Missing deposit field");
+    } else if (maybeRequestsParam.isEmpty()) {
+      return ValidationResult.invalid(
+          RpcErrorType.INVALID_REQUESTS_PARAMS, "Missing execution requests field");
     } else {
       return ValidationResult.valid();
     }