feat: enable builder proposals post deneb with blobs (#5933)

implement missing blindedblock publishing

remove the throw

refactor the type reconstructions for builder

improv

Author: g11tech

packages/api/src/builder/routes.ts

@@ -1,6 +1,6 @@
 import {fromHexString, toHexString} from "@chainsafe/ssz";
 import {ssz, allForks, bellatrix, Slot, Root, BLSPubkey} from "@lodestar/types";
-import {ForkName, isForkExecution} from "@lodestar/params";
+import {ForkName, isForkExecution, isForkBlobs} from "@lodestar/params";
 import {ChainForkConfig} from "@lodestar/config";
 
 import {
@@ -34,11 +34,14 @@ export type Api = {
       HttpStatusCode.NOT_FOUND | HttpStatusCode.BAD_REQUEST
     >
   >;
-  submitBlindedBlock(
-    signedBlock: allForks.SignedBlindedBeaconBlockOrContents
-  ): Promise<
+  submitBlindedBlock(signedBlock: allForks.SignedBlindedBeaconBlockOrContents): Promise<
     ApiClientResponse<
-      {[HttpStatusCode.OK]: {data: allForks.ExecutionPayload; version: ForkName}},
+      {
+        [HttpStatusCode.OK]: {
+          data: allForks.ExecutionPayload | allForks.ExecutionPayloadAndBlobsBundle;
+          version: ForkName;
+        };
+      },
       HttpStatusCode.SERVICE_UNAVAILABLE
     >
   >;
@@ -84,8 +87,13 @@ export function getReturnTypes(): ReturnTypes<Api> {
     getHeader: WithVersion((fork: ForkName) =>
       isForkExecution(fork) ? ssz.allForksExecution[fork].SignedBuilderBid : ssz.bellatrix.SignedBuilderBid
     ),
-    submitBlindedBlock: WithVersion((fork: ForkName) =>
-      isForkExecution(fork) ? ssz.allForksExecution[fork].ExecutionPayload : ssz.bellatrix.ExecutionPayload
+    submitBlindedBlock: WithVersion<allForks.ExecutionPayload | allForks.ExecutionPayloadAndBlobsBundle>(
+      (fork: ForkName) =>
+        isForkBlobs(fork)
+          ? ssz.allForksBlobs[fork].ExecutionPayloadAndBlobsBundle
+          : isForkExecution(fork)
+          ? ssz.allForksExecution[fork].ExecutionPayload
+          : ssz.bellatrix.ExecutionPayload
     ),
   };
 }

packages/beacon-node/src/api/impl/beacon/blocks/index.ts

@@ -1,9 +1,13 @@
 import {fromHexString, toHexString} from "@chainsafe/ssz";
 import {routes, ServerApi, ResponseFormat} from "@lodestar/api";
-import {computeTimeAtSlot, signedBlindedBlockToFull, signedBlindedBlobSidecarsToFull} from "@lodestar/state-transition";
+import {
+  computeTimeAtSlot,
+  parseSignedBlindedBlockOrContents,
+  reconstructFullBlockOrContents,
+} from "@lodestar/state-transition";
 import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
-import {sleep, toHex, LogDataBasic} from "@lodestar/utils";
-import {allForks, deneb, isSignedBlockContents, isSignedBlindedBlockContents} from "@lodestar/types";
+import {sleep, toHex} from "@lodestar/utils";
+import {allForks, deneb, isSignedBlockContents, ProducedBlockSource} from "@lodestar/types";
 import {BlockSource, getBlockInput, ImportBlockOpts, BlockInput} from "../../../../chain/blocks/types.js";
 import {promiseAllMaybeAsync} from "../../../../util/promises.js";
 import {isOptimisticBlock} from "../../../../util/forkChoice.js";
@@ -15,11 +19,6 @@ import {resolveBlockId, toBeaconHeaderResponse} from "./utils.js";
 
 type PublishBlockOpts = ImportBlockOpts & {broadcastValidation?: routes.beacon.BroadcastValidation};
 
-type ParsedSignedBlindedBlockOrContents = {
-  signedBlindedBlock: allForks.SignedBlindedBeaconBlock;
-  signedBlindedBlobSidecars: deneb.SignedBlindedBlobSidecars | null;
-};
-
 /**
  * Validator clock may be advanced from beacon's clock. If the validator requests a resource in a
  * future slot, wait some time instead of rejecting the request because it's in the future
@@ -152,27 +151,28 @@ export function getBeaconBlockApi({
         .getBlindedForkTypes(signedBlindedBlock.message.slot)
         .BeaconBlock.hashTreeRoot(signedBlindedBlock.message)
     );
-    const logCtx = {blockRoot, slot};
 
     // Either the payload/blobs are cached from i) engine locally or ii) they are from the builder
     //
-    // executionPayload can be null or a real payload in locally produced, its only undefined when
-    // the block came from the builder
-    const executionPayload = chain.producedBlockRoot.get(blockRoot);
+    // executionPayload can be null or a real payload in locally produced so check for presence of root
+    const source = chain.producedBlockRoot.has(blockRoot) ? ProducedBlockSource.engine : ProducedBlockSource.builder;
+
+    const executionPayload = chain.producedBlockRoot.get(blockRoot) ?? null;
+    const blobSidecars = executionPayload
+      ? chain.producedBlobSidecarsCache.get(toHex(executionPayload.blockHash))
+      : undefined;
+    const blobs = blobSidecars ? blobSidecars.map((blobSidecar) => blobSidecar.blob) : null;
+
     const signedBlockOrContents =
-      executionPayload !== undefined
-        ? reconstructLocalBlockOrContents(
-            chain,
-            {signedBlindedBlock, signedBlindedBlobSidecars},
-            executionPayload,
-            logCtx
-          )
-        : await reconstructBuilderBlockOrContents(chain, signedBlindedBlockOrContents, logCtx);
+      source === ProducedBlockSource.engine
+        ? reconstructFullBlockOrContents({signedBlindedBlock, signedBlindedBlobSidecars}, {executionPayload, blobs})
+        : await reconstructBuilderBlockOrContents(chain, signedBlindedBlockOrContents);
 
     // the full block is published by relay and it's possible that the block is already known to us
     // by gossip
     //
     // see: https://github.com/ChainSafe/lodestar/issues/5404
+    chain.logger.info("Publishing assembled block", {blockRoot, slot, source});
     return publishBlock(signedBlockOrContents, {...opts, ignoreIfKnown: true});
   };
 
@@ -365,73 +365,15 @@ export function getBeaconBlockApi({
   };
 }
 
-function parseSignedBlindedBlockOrContents(
-  signedBlindedBlockOrContents: allForks.SignedBlindedBeaconBlockOrContents
-): ParsedSignedBlindedBlockOrContents {
-  if (isSignedBlindedBlockContents(signedBlindedBlockOrContents)) {
-    const signedBlindedBlock = signedBlindedBlockOrContents.signedBlindedBlock;
-    const signedBlindedBlobSidecars = signedBlindedBlockOrContents.signedBlindedBlobSidecars;
-    return {signedBlindedBlock, signedBlindedBlobSidecars};
-  } else {
-    return {signedBlindedBlock: signedBlindedBlockOrContents, signedBlindedBlobSidecars: null};
-  }
-}
-
-function reconstructLocalBlockOrContents(
-  chain: ApiModules["chain"],
-  {signedBlindedBlock, signedBlindedBlobSidecars}: ParsedSignedBlindedBlockOrContents,
-  executionPayload: allForks.ExecutionPayload | null,
-  logCtx: Record<string, LogDataBasic>
-): allForks.SignedBeaconBlockOrContents {
-  const signedBlock = signedBlindedBlockToFull(signedBlindedBlock, executionPayload);
-  if (executionPayload !== null) {
-    Object.assign(logCtx, {transactions: executionPayload.transactions.length});
-  }
-
-  if (signedBlindedBlobSidecars !== null) {
-    if (executionPayload === null) {
-      throw Error("Missing locally produced executionPayload for deneb+ publishBlindedBlock");
-    }
-
-    const blockHash = toHex(executionPayload.blockHash);
-    const blobSidecars = chain.producedBlobSidecarsCache.get(blockHash);
-    if (blobSidecars === undefined) {
-      throw Error("Missing blobSidecars from the local execution cache");
-    }
-    if (blobSidecars.length !== signedBlindedBlobSidecars.length) {
-      throw Error(
-        `Length mismatch signedBlindedBlobSidecars=${signedBlindedBlobSidecars.length} blobSidecars=${blobSidecars.length}`
-      );
-    }
-    const signedBlobSidecars = signedBlindedBlobSidecarsToFull(
-      signedBlindedBlobSidecars,
-      blobSidecars.map((blobSidecar) => blobSidecar.blob)
-    );
-
-    Object.assign(logCtx, {blobs: signedBlindedBlobSidecars.length});
-    chain.logger.verbose("Block & blobs assembled from locally cached payload", logCtx);
-    return {signedBlock, signedBlobSidecars} as allForks.SignedBeaconBlockOrContents;
-  } else {
-    chain.logger.verbose("Block assembled from locally cached payload", logCtx);
-    return signedBlock as allForks.SignedBeaconBlockOrContents;
-  }
-}
-
 async function reconstructBuilderBlockOrContents(
   chain: ApiModules["chain"],
-  signedBlindedBlockOrContents: allForks.SignedBlindedBeaconBlockOrContents,
-  logCtx: Record<string, LogDataBasic>
+  signedBlindedBlockOrContents: allForks.SignedBlindedBeaconBlockOrContents
 ): Promise<allForks.SignedBeaconBlockOrContents> {
-  // Mechanism for blobs & blocks on builder is implemenented separately in a followup deneb-builder PR
-  if (isSignedBlindedBlockContents(signedBlindedBlockOrContents)) {
-    throw Error("exeutionBuilder not yet implemented for deneb+ forks");
-  }
   const executionBuilder = chain.executionBuilder;
   if (!executionBuilder) {
     throw Error("exeutionBuilder required to publish SignedBlindedBeaconBlock");
   }
 
   const signedBlockOrContents = await executionBuilder.submitBlindedBlock(signedBlindedBlockOrContents);
-  chain.logger.verbose("Publishing block assembled from the builder", logCtx);
   return signedBlockOrContents;
 }

packages/beacon-node/src/api/impl/validator/index.ts

@@ -310,14 +310,17 @@ export function getValidatorApi({
 
       const version = config.getForkName(block.slot);
       if (isForkBlobs(version)) {
-        if (!isBlindedBlockContents(block)) {
-          throw Error(`Expected BlockContents response at fork=${version}`);
+        const blockHash = toHex((block as bellatrix.BlindedBeaconBlock).body.executionPayloadHeader.blockHash);
+        const blindedBlobSidecars = chain.producedBlindedBlobSidecarsCache.get(blockHash);
+        if (blindedBlobSidecars === undefined) {
+          throw Error("blobSidecars missing in cache");
         }
-        return {data: block, version, executionPayloadValue};
+        return {
+          data: {blindedBlock: block, blindedBlobSidecars} as allForks.BlindedBlockContents,
+          version,
+          executionPayloadValue,
+        };
       } else {
-        if (isBlindedBlockContents(block)) {
-          throw Error(`Invalid BlockContents response at fork=${version}`);
-        }
         return {data: block, version, executionPayloadValue};
       }
     } finally {

packages/beacon-node/src/chain/chain.ts

@@ -131,6 +131,7 @@ export class BeaconChain implements IBeaconChain {
   readonly checkpointBalancesCache: CheckpointBalancesCache;
   /** Map keyed by executionPayload.blockHash of the block for those blobs */
   readonly producedBlobSidecarsCache = new Map<BlockHash, deneb.BlobSidecars>();
+  readonly producedBlindedBlobSidecarsCache = new Map<BlockHash, deneb.BlindedBlobSidecars>();
 
   // Cache payload from the local execution so that produceBlindedBlock or produceBlockV3 and
   // send and get signed/published blinded versions which beacon can assemble into full before
@@ -522,7 +523,7 @@ export class BeaconChain implements IBeaconChain {
     // publishing the blinded block's full version
     if (blobs.type === BlobsResultType.produced) {
       // body is of full type here
-      const blockHash = toHex((block as bellatrix.BeaconBlock).body.executionPayload.blockHash);
+      const blockHash = blobs.blockHash;
       const blobSidecars = blobs.blobSidecars.map((blobSidecar) => ({
         ...blobSidecar,
         blockRoot,
@@ -533,6 +534,21 @@ export class BeaconChain implements IBeaconChain {
 
       this.producedBlobSidecarsCache.set(blockHash, blobSidecars);
       this.metrics?.blockProductionCaches.producedBlobSidecarsCache.set(this.producedBlobSidecarsCache.size);
+    } else if (blobs.type === BlobsResultType.blinded) {
+      // body is of blinded type here
+      const blockHash = blobs.blockHash;
+      const blindedBlobSidecars = blobs.blobSidecars.map((blindedBlobSidecar) => ({
+        ...blindedBlobSidecar,
+        blockRoot,
+        slot,
+        blockParentRoot: parentBlockRoot,
+        proposerIndex,
+      }));
+
+      this.producedBlindedBlobSidecarsCache.set(blockHash, blindedBlobSidecars);
+      this.metrics?.blockProductionCaches.producedBlindedBlobSidecarsCache.set(
+        this.producedBlindedBlobSidecarsCache.size
+      );
     }
 
     return {block, executionPayloadValue};
@@ -792,6 +808,14 @@ export class BeaconChain implements IBeaconChain {
         this.opts.maxCachedBlobSidecars ?? DEFAULT_MAX_CACHED_BLOB_SIDECARS
       );
       this.metrics?.blockProductionCaches.producedBlobSidecarsCache.set(this.producedBlobSidecarsCache.size);
+
+      pruneSetToMax(
+        this.producedBlindedBlobSidecarsCache,
+        this.opts.maxCachedBlobSidecars ?? DEFAULT_MAX_CACHED_BLOB_SIDECARS
+      );
+      this.metrics?.blockProductionCaches.producedBlindedBlobSidecarsCache.set(
+        this.producedBlindedBlobSidecarsCache.size
+      );
     }
 
     const metrics = this.metrics;

packages/beacon-node/src/chain/interface.ts

@@ -95,6 +95,7 @@ export interface IBeaconChain {
   readonly checkpointBalancesCache: CheckpointBalancesCache;
   readonly producedBlobSidecarsCache: Map<BlockHash, deneb.BlobSidecars>;
   readonly producedBlockRoot: Map<RootHex, allForks.ExecutionPayload | null>;
+  readonly producedBlindedBlobSidecarsCache: Map<BlockHash, deneb.BlindedBlobSidecars>;
   readonly producedBlindedBlockRoot: Set<RootHex>;
   readonly opts: IChainOptions;
 

packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts

@@ -35,7 +35,10 @@ import {PayloadId, IExecutionEngine, IExecutionBuilder, PayloadAttributes} from
 import {ZERO_HASH, ZERO_HASH_HEX} from "../../constants/index.js";
 import {IEth1ForBlockProduction} from "../../eth1/index.js";
 import {numToQuantity} from "../../eth1/provider/utils.js";
-import {validateBlobsAndKzgCommitments} from "./validateBlobsAndKzgCommitments.js";
+import {
+  validateBlobsAndKzgCommitments,
+  validateBlindedBlobsAndKzgCommitments,
+} from "./validateBlobsAndKzgCommitments.js";
 
 // Time to provide the EL to generate a payload from new payload id
 const PAYLOAD_GENERATION_TIME_MS = 500;
@@ -70,8 +73,9 @@ export enum BlobsResultType {
 }
 
 export type BlobsResult =
-  | {type: BlobsResultType.preDeneb | BlobsResultType.blinded}
-  | {type: BlobsResultType.produced; blobSidecars: deneb.BlobSidecars; blockHash: RootHex};
+  | {type: BlobsResultType.preDeneb}
+  | {type: BlobsResultType.produced; blobSidecars: deneb.BlobSidecars; blockHash: RootHex}
+  | {type: BlobsResultType.blinded; blobSidecars: deneb.BlindedBlobSidecars; blockHash: RootHex};
 
 export async function produceBlockBody<T extends BlockType>(
   this: BeaconChain,
@@ -195,16 +199,47 @@ export async function produceBlockBody<T extends BlockType>(
       );
       (blockBody as allForks.BlindedBeaconBlockBody).executionPayloadHeader = builderRes.header;
       executionPayloadValue = builderRes.executionPayloadValue;
-      this.logger.verbose("Fetched execution payload header from builder", {slot: blockSlot, executionPayloadValue});
+
+      const fetchedTime = Date.now() / 1000 - computeTimeAtSlot(this.config, blockSlot, this.genesisTime);
+      const prepType = "blinded";
+      this.metrics?.blockPayload.payloadFetchedTime.observe({prepType}, fetchedTime);
+      this.logger.verbose("Fetched execution payload header from builder", {
+        slot: blockSlot,
+        executionPayloadValue,
+        prepType,
+        fetchedTime,
+      });
+
       if (ForkSeq[fork] >= ForkSeq.deneb) {
-        const {blobKzgCommitments} = builderRes;
-        if (blobKzgCommitments === undefined) {
-          throw Error(`Invalid builder getHeader response for fork=${fork}, missing blobKzgCommitments`);
+        const {blindedBlobsBundle} = builderRes;
+        if (blindedBlobsBundle === undefined) {
+          throw Error(`Invalid builder getHeader response for fork=${fork}, missing blindedBlobsBundle`);
         }
-        (blockBody as deneb.BlindedBeaconBlockBody).blobKzgCommitments = blobKzgCommitments;
-        blobsResult = {type: BlobsResultType.blinded};
 
-        Object.assign(logMeta, {blobs: blobKzgCommitments.length});
+        // validate blindedBlobsBundle
+        if (this.opts.sanityCheckExecutionEngineBlobs) {
+          validateBlindedBlobsAndKzgCommitments(builderRes.header, blindedBlobsBundle);
+        }
+
+        (blockBody as deneb.BlindedBeaconBlockBody).blobKzgCommitments = blindedBlobsBundle.commitments;
+        const blockHash = toHex(builderRes.header.blockHash);
+
+        const blobSidecars = Array.from({length: blindedBlobsBundle.blobRoots.length}, (_v, index) => {
+          const blobRoot = blindedBlobsBundle.blobRoots[index];
+          const commitment = blindedBlobsBundle.commitments[index];
+          const proof = blindedBlobsBundle.proofs[index];
+          const blindedBlobSidecar = {
+            index,
+            blobRoot,
+            kzgProof: proof,
+            kzgCommitment: commitment,
+          };
+          // Other fields will be injected after postState is calculated
+          return blindedBlobSidecar;
+        }) as deneb.BlindedBlobSidecars;
+        blobsResult = {type: BlobsResultType.blinded, blobSidecars, blockHash};
+
+        Object.assign(logMeta, {blobs: blindedBlobsBundle.commitments.length});
       } else {
         blobsResult = {type: BlobsResultType.preDeneb};
       }
@@ -270,7 +305,7 @@ export async function produceBlockBody<T extends BlockType>(
               throw Error(`Missing blobsBundle response from getPayload at fork=${fork}`);
             }
 
-            // Optionally sanity-check that the KZG commitments match the versioned hashes in the transactions
+            // validate blindedBlobsBundle
             if (this.opts.sanityCheckExecutionEngineBlobs) {
               validateBlobsAndKzgCommitments(executionPayload, blobsBundle);
             }
@@ -288,6 +323,7 @@ export async function produceBlockBody<T extends BlockType>(
                 kzgProof: proof,
                 kzgCommitment: commitment,
               };
+              // Other fields will be injected after postState is calculated
               return blobSidecar;
             }) as deneb.BlobSidecars;
             blobsResult = {type: BlobsResultType.produced, blobSidecars, blockHash};
@@ -443,21 +479,19 @@ async function prepareExecutionPayloadHeader(
 ): Promise<{
   header: allForks.ExecutionPayloadHeader;
   executionPayloadValue: Wei;
-  blobKzgCommitments?: deneb.BlobKzgCommitments;
+  blindedBlobsBundle?: deneb.BlindedBlobsBundle;
 }> {
   if (!chain.executionBuilder) {
     throw Error("executionBuilder required");
   }
 
   const parentHashRes = await getExecutionPayloadParentHash(chain, state);
-
   if (parentHashRes.isPremerge) {
-    // TODO: Is this okay?
     throw Error("Execution builder disabled pre-merge");
   }
 
   const {parentHash} = parentHashRes;
-  return chain.executionBuilder.getHeader(state.slot, parentHash, proposerPubKey);
+  return chain.executionBuilder.getHeader(fork, state.slot, parentHash, proposerPubKey);
 }
 
 export async function getExecutionPayloadParentHash(