feat: de-duplicate payloads from persisted beacon blocks

packages/beacon-node/src/api/impl/beacon/blocks/index.ts

@@ -4,9 +4,10 @@ import {
   computeEpochAtSlot,
   computeTimeAtSlot,
   reconstructFullBlockOrContents,
-  signedBeaconBlockToBlinded,
+  blindedOrFullBlockHashTreeRoot,
+  fullOrBlindedSignedBlockToBlinded,
 } from "@lodestar/state-transition";
-import {ForkExecution, SLOTS_PER_HISTORICAL_ROOT, isForkExecution, isForkPostElectra} from "@lodestar/params";
+import {SLOTS_PER_HISTORICAL_ROOT, isForkExecution, isForkPostElectra} from "@lodestar/params";
 import {sleep, fromHex, toRootHex} from "@lodestar/utils";
 import {
   deneb,
@@ -331,15 +332,12 @@ export function getBeaconBlockApi({
         if (slot > headSlot) {
           return {data: [], meta: {executionOptimistic: false, finalized: false}};
         }
-
         const canonicalBlock = await chain.getCanonicalBlockAtSlot(slot);
         // skip slot
         if (!canonicalBlock) {
           return {data: [], meta: {executionOptimistic: false, finalized: false}};
         }
-        const canonicalRoot = config
-          .getForkTypes(canonicalBlock.block.message.slot)
-          .BeaconBlock.hashTreeRoot(canonicalBlock.block.message);
+        const canonicalRoot = blindedOrFullBlockHashTreeRoot(config, canonicalBlock.block.message);
         result.push(toBeaconHeaderResponse(config, canonicalBlock.block, true));
         if (!canonicalBlock.finalized) {
           finalized = false;
@@ -381,7 +379,7 @@ export function getBeaconBlockApi({
     async getBlockV2({blockId}) {
       const {block, executionOptimistic, finalized} = await getBlockResponse(chain, blockId);
       return {
-        data: block,
+        data: await chain.fullOrBlindedSignedBeaconBlockToFull(block),
         meta: {
           executionOptimistic,
           finalized,
@@ -394,9 +392,7 @@ export function getBeaconBlockApi({
       const {block, executionOptimistic, finalized} = await getBlockResponse(chain, blockId);
       const fork = config.getForkName(block.message.slot);
       return {
-        data: isForkExecution(fork)
-          ? signedBeaconBlockToBlinded(config, block as SignedBeaconBlock<ForkExecution>)
-          : block,
+        data: isForkExecution(fork) ? fullOrBlindedSignedBlockToBlinded(config, block) : block,
         meta: {
           executionOptimistic,
           finalized,
@@ -464,7 +460,7 @@ export function getBeaconBlockApi({
       // Slow path
       const {block, executionOptimistic, finalized} = await getBlockResponse(chain, blockId);
       return {
-        data: {root: config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message)},
+        data: {root: blindedOrFullBlockHashTreeRoot(config, block.message)},
         meta: {executionOptimistic, finalized},
       };
     },
@@ -482,7 +478,7 @@ export function getBeaconBlockApi({
 
     async getBlobSidecars({blockId, indices}) {
       const {block, executionOptimistic, finalized} = await getBlockResponse(chain, blockId);
-      const blockRoot = config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message);
+      const blockRoot = blindedOrFullBlockHashTreeRoot(config, block.message);
 
       let {blobSidecars} = (await db.blobSidecars.get(blockRoot)) ?? {};
       if (!blobSidecars) {

packages/beacon-node/src/api/impl/beacon/blocks/utils.ts

@@ -1,7 +1,7 @@
 import {routes} from "@lodestar/api";
-import {blockToHeader} from "@lodestar/state-transition";
+import {blockToHeader, blindedOrFullBlockHashTreeRoot} from "@lodestar/state-transition";
 import {ChainForkConfig} from "@lodestar/config";
-import {RootHex, SignedBeaconBlock, Slot} from "@lodestar/types";
+import {RootHex, SignedBeaconBlock, SignedBlindedBeaconBlock, Slot} from "@lodestar/types";
 import {IForkChoice} from "@lodestar/fork-choice";
 import {GENESIS_SLOT} from "../../../../constants/index.js";
 import {ApiError, ValidationError} from "../../errors.js";
@@ -10,11 +10,12 @@ import {rootHexRegex} from "../../../../eth1/provider/utils.js";
 
 export function toBeaconHeaderResponse(
   config: ChainForkConfig,
-  block: SignedBeaconBlock,
+  block: SignedBeaconBlock | SignedBlindedBeaconBlock,
   canonical = false
 ): routes.beacon.BlockHeaderResponse {
+  const root = blindedOrFullBlockHashTreeRoot(config, block.message);
   return {
-    root: config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message),
+    root,
     canonical,
     header: {
       message: blockToHeader(config, block.message),
@@ -59,7 +60,7 @@ export function resolveBlockId(forkChoice: IForkChoice, blockId: routes.beacon.B
 export async function getBlockResponse(
   chain: IBeaconChain,
   blockId: routes.beacon.BlockId
-): Promise<{block: SignedBeaconBlock; executionOptimistic: boolean; finalized: boolean}> {
+): Promise<{block: SignedBeaconBlock | SignedBlindedBeaconBlock; executionOptimistic: boolean; finalized: boolean}> {
   const rootOrSlot = resolveBlockId(chain.forkChoice, blockId);
 
   const res =

packages/beacon-node/src/api/impl/proof/index.ts

@@ -1,6 +1,7 @@
 import {CompactMultiProof, createProof, ProofType} from "@chainsafe/persistent-merkle-tree";
 import {routes} from "@lodestar/api";
 import {ApplicationMethods} from "@lodestar/api/server";
+import {isBlindedBlock} from "@lodestar/state-transition";
 import {ApiModules} from "../types.js";
 import {getStateResponse} from "../beacon/state/utils.js";
 import {getBlockResponse} from "../beacon/blocks/utils.js";
@@ -43,7 +44,9 @@ export function getProofApi(
       const {block} = await getBlockResponse(chain, blockId);
 
       // Commit any changes before computing the state root. In normal cases the state should have no changes here
-      const blockNode = config.getForkTypes(block.message.slot).BeaconBlock.toView(block.message).node;
+      const blockNode = isBlindedBlock(block.message)
+        ? config.getExecutionForkTypes(block.message.slot).BlindedBeaconBlock.toView(block.message).node
+        : config.getForkTypes(block.message.slot).BeaconBlock.toView(block.message).node;
 
       const proof = createProof(blockNode, {type: ProofType.compactMulti, descriptor});
 

packages/beacon-node/src/api/impl/validator/index.ts

@@ -8,7 +8,8 @@ import {
   getBlockRootAtSlot,
   computeEpochAtSlot,
   getCurrentSlot,
-  beaconBlockToBlinded,
+  blindedOrFullSignedBlockHashTreeRoot,
+  fullOrBlindedBlockToBlinded,
 } from "@lodestar/state-transition";
 import {
   GENESIS_SLOT,
@@ -33,7 +34,6 @@ import {
   ProducedBlockSource,
   bellatrix,
   BLSSignature,
-  isBlindedBeaconBlock,
   isBlockContents,
   phase0,
   Wei,
@@ -135,12 +135,9 @@ export function getValidatorApi(
       if (state.slot < SLOTS_PER_HISTORICAL_ROOT) {
         genesisBlockRoot = state.blockRoots.get(0);
       }
-
       const blockRes = await chain.getCanonicalBlockAtSlot(GENESIS_SLOT);
       if (blockRes) {
-        genesisBlockRoot = config
-          .getForkTypes(blockRes.block.message.slot)
-          .SignedBeaconBlock.hashTreeRoot(blockRes.block);
+        genesisBlockRoot = blindedOrFullSignedBlockHashTreeRoot(config, blockRes.block);
       }
     }
 
@@ -767,13 +764,13 @@ export function getValidatorApi(
         } else {
           if (isBlockContents(data)) {
             const {block} = data;
-            const blindedBlock = beaconBlockToBlinded(config, block as BeaconBlock<ForkExecution>);
+            const blindedBlock = fullOrBlindedBlockToBlinded(config, block);
             return {
               data: blindedBlock,
               meta: {...meta, executionPayloadBlinded: true},
             };
           } else {
-            const blindedBlock = beaconBlockToBlinded(config, data as BeaconBlock<ForkExecution>);
+            const blindedBlock = fullOrBlindedBlockToBlinded(config, data);
             return {
               data: blindedBlock,
               meta: {...meta, executionPayloadBlinded: true},
@@ -790,17 +787,8 @@ export function getValidatorApi(
       if (!isForkExecution(version)) {
         throw Error(`Invalid fork=${version} for produceBlindedBlock`);
       }
-
-      if (isBlockContents(data)) {
-        const {block} = data;
-        const blindedBlock = beaconBlockToBlinded(config, block as BeaconBlock<ForkExecution>);
-        return {data: blindedBlock, meta: {version}};
-      } else if (isBlindedBeaconBlock(data)) {
-        return {data, meta: {version}};
-      } else {
-        const blindedBlock = beaconBlockToBlinded(config, data as BeaconBlock<ForkExecution>);
-        return {data: blindedBlock, meta: {version}};
-      }
+      const blindedBlock = fullOrBlindedBlockToBlinded(config, isBlockContents(data) ? data.block : data);
+      return {data: blindedBlock, meta: {version}};
     },
 
     async produceAttestationData({committeeIndex, slot}) {

packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts

@@ -13,17 +13,11 @@ export async function writeBlockInputToDb(this: BeaconChain, blocksInput: BlockI
   const fnPromises: Promise<void>[] = [];
 
   for (const blockInput of blocksInput) {
-    const {block, blockBytes} = blockInput;
+    const {block} = blockInput;
     const blockRoot = this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message);
     const blockRootHex = toRootHex(blockRoot);
-    if (blockBytes) {
-      // skip serializing data if we already have it
-      this.metrics?.importBlock.persistBlockWithSerializedDataCount.inc();
-      fnPromises.push(this.db.block.putBinary(this.db.block.getId(block), blockBytes));
-    } else {
-      this.metrics?.importBlock.persistBlockNoSerializedDataCount.inc();
-      fnPromises.push(this.db.block.add(block));
-    }
+    this.metrics?.importBlock.persistBlockNoSerializedDataCount.inc();
+    fnPromises.push(this.db.block.add(block));
     this.logger.debug("Persist block to hot DB", {
       slot: block.message.slot,
       root: blockRootHex,

packages/beacon-node/src/chain/chain.ts

@@ -13,6 +13,8 @@ import {
   PubkeyIndexMap,
   EpochShuffling,
   computeEndSlotAtEpoch,
+  blindedOrFullBlockHashTreeRoot,
+  isBlindedBlock,
 } from "@lodestar/state-transition";
 import {BeaconConfig} from "@lodestar/config";
 import {
@@ -26,16 +28,16 @@ import {
   deneb,
   Wei,
   bellatrix,
-  isBlindedBeaconBlock,
   BeaconBlock,
   SignedBeaconBlock,
   ExecutionPayload,
   BlindedBeaconBlock,
   BlindedBeaconBlockBody,
+  SignedBlindedBeaconBlock,
 } from "@lodestar/types";
 import {CheckpointWithHex, ExecutionStatus, IForkChoice, ProtoBlock, UpdateHeadOpt} from "@lodestar/fork-choice";
 import {ProcessShutdownCallback} from "@lodestar/validator";
-import {Logger, fromHex, gweiToWei, isErrorAborted, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils";
+import {Logger, fromHex, toHex, gweiToWei, isErrorAborted, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils";
 import {ForkSeq, GENESIS_SLOT, SLOTS_PER_EPOCH} from "@lodestar/params";
 
 import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js";
@@ -47,6 +49,8 @@ import {Clock, ClockEvent, IClock} from "../util/clock.js";
 import {ensureDir, writeIfNotExist} from "../util/file.js";
 import {isOptimisticBlock} from "../util/forkChoice.js";
 import {BufferPool} from "../util/bufferPool.js";
+import {Eth1Error, Eth1ErrorCode} from "../eth1/errors.js";
+import {blindedOrFullBlockToFull} from "../util/fullOrBlindedBlock.js";
 import {BlockProcessor, ImportBlockOpts} from "./blocks/index.js";
 import {ChainEventEmitter, ChainEvent} from "./emitter.js";
 import {
@@ -561,16 +565,25 @@ export class BeaconChain implements IBeaconChain {
   }
 
   async getCanonicalBlockAtSlot(
-    slot: Slot
-  ): Promise<{block: SignedBeaconBlock; executionOptimistic: boolean; finalized: boolean} | null> {
+    slot: Slot,
+    getFull = true
+  ): Promise<{
+    block: SignedBeaconBlock | SignedBlindedBeaconBlock;
+    executionOptimistic: boolean;
+    finalized: boolean;
+  } | null> {
     const finalizedBlock = this.forkChoice.getFinalizedBlock();
     if (slot > finalizedBlock.slot) {
       // Unfinalized slot, attempt to find in fork-choice
       const block = this.forkChoice.getCanonicalBlockAtSlot(slot);
       if (block) {
         const data = await this.db.block.get(fromHex(block.blockRoot));
         if (data) {
-          return {block: data, executionOptimistic: isOptimisticBlock(block), finalized: false};
+          return {
+            block: getFull ? await this.fullOrBlindedSignedBeaconBlockToFull(data) : data,
+            executionOptimistic: isOptimisticBlock(block),
+            finalized: false,
+          };
         }
       }
       // A non-finalized slot expected to be found in the hot db, could be archived during
@@ -579,24 +592,45 @@ export class BeaconChain implements IBeaconChain {
     }
 
     const data = await this.db.blockArchive.get(slot);
-    return data && {block: data, executionOptimistic: false, finalized: true};
+    return (
+      data && {
+        block: getFull ? await this.fullOrBlindedSignedBeaconBlockToFull(data) : data,
+        executionOptimistic: false,
+        finalized: true,
+      }
+    );
   }
 
   async getBlockByRoot(
-    root: string
-  ): Promise<{block: SignedBeaconBlock; executionOptimistic: boolean; finalized: boolean} | null> {
+    root: string,
+    getFull = true
+  ): Promise<{
+    block: SignedBeaconBlock | SignedBlindedBeaconBlock;
+    executionOptimistic: boolean;
+    finalized: boolean;
+  } | null> {
     const block = this.forkChoice.getBlockHex(root);
     if (block) {
       const data = await this.db.block.get(fromHex(root));
       if (data) {
-        return {block: data, executionOptimistic: isOptimisticBlock(block), finalized: false};
+        return {
+          block: getFull ? await this.fullOrBlindedSignedBeaconBlockToFull(data) : data,
+          executionOptimistic: isOptimisticBlock(block),
+          finalized: false,
+        };
       }
       // If block is not found in hot db, try cold db since there could be an archive cycle happening
       // TODO: Add a lock to the archiver to have deterministic behavior on where are blocks
     }
 
     const data = await this.db.blockArchive.getByRoot(fromHex(root));
-    return data && {block: data, executionOptimistic: false, finalized: true};
+    return (
+      data && {
+        block: getFull ? await this.fullOrBlindedSignedBeaconBlockToFull(data) : data,
+        executionOptimistic: false,
+        finalized: true,
+      }
+    );
   }
 
   async produceCommonBlockBody(blockAttributes: BlockAttributes): Promise<CommonBlockBody> {
@@ -836,7 +870,7 @@ export class BeaconChain implements IBeaconChain {
 
   persistBlock(data: BeaconBlock | BlindedBeaconBlock, suffix?: string): void {
     const slot = data.slot;
-    if (isBlindedBeaconBlock(data)) {
+    if (isBlindedBlock(data)) {
       const sszType = this.config.getExecutionForkTypes(slot).BlindedBeaconBlock;
       void this.persistSszObject("BlindedBeaconBlock", sszType.serialize(data), sszType.hashTreeRoot(data), suffix);
     } else {
@@ -995,6 +1029,25 @@ export class BeaconChain implements IBeaconChain {
     return {state: blockState, stateId: "block_state_any_epoch", shouldWarn: true};
   }
 
+  async fullOrBlindedSignedBeaconBlockToFull(
+    block: SignedBeaconBlock | SignedBlindedBeaconBlock
+  ): Promise<SignedBeaconBlock> {
+    if (!isBlindedBlock(block)) {
+      return block;
+    }
+    const blockHash = toHex(blindedOrFullBlockHashTreeRoot(this.config, block.message));
+    const [payload] = await this.executionEngine.getPayloadBodiesByHash(this.config.getForkName(block.message.slot), [
+      blockHash,
+    ]);
+    if (!payload) {
+      throw new Eth1Error(
+        {code: Eth1ErrorCode.INVALID_PAYLOAD_BODY, blockHash},
+        `Execution PayloadBody not found by eth1 engine for ${blockHash}`
+      );
+    }
+    return blindedOrFullBlockToFull(this.config, block, payload);
+  }
+
   private async persistSszObject(
     typeName: string,
     bytes: Uint8Array,