feat: generalize blobs to data for extension ready for ils and/or data columns (#6693)

rename types and fix references to them

update forkchoice and availability flow

propagate changes to the codebase

lint and tsc

some fixes

cleanup

test run fixes

fix the tests

fix

apply feedback

Author: g11tech

packages/beacon-node/src/api/impl/beacon/blocks/index.ts

@@ -4,7 +4,14 @@ import {computeEpochAtSlot, computeTimeAtSlot, reconstructFullBlockOrContents} f
 import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
 import {sleep, fromHex, toHex} from "@lodestar/utils";
 import {allForks, deneb, isSignedBlockContents, ProducedBlockSource} from "@lodestar/types";
-import {BlockSource, getBlockInput, ImportBlockOpts, BlockInput, BlobsSource} from "../../../../chain/blocks/types.js";
+import {
+  BlockSource,
+  getBlockInput,
+  ImportBlockOpts,
+  BlockInput,
+  BlobsSource,
+  BlockInputDataBlobs,
+} from "../../../../chain/blocks/types.js";
 import {promiseAllMaybeAsync} from "../../../../util/promises.js";
 import {isOptimisticBlock} from "../../../../util/forkChoice.js";
 import {computeBlobSidecars} from "../../../../util/blobs.js";
@@ -51,20 +58,24 @@ export function getBeaconBlockApi({
     if (isSignedBlockContents(signedBlockOrContents)) {
       ({signedBlock} = signedBlockOrContents);
       blobSidecars = computeBlobSidecars(config, signedBlock, signedBlockOrContents);
-      blockForImport = getBlockInput.postDeneb(
+      const blockData = {
+        fork: config.getForkName(signedBlock.message.slot),
+        blobs: blobSidecars,
+        blobsSource: BlobsSource.api,
+        blobsBytes: blobSidecars.map(() => null),
+      } as BlockInputDataBlobs;
+      blockForImport = getBlockInput.availableData(
         config,
         signedBlock,
         BlockSource.api,
-        blobSidecars,
-        BlobsSource.api,
         // don't bundle any bytes for block and blobs
         null,
-        blobSidecars.map(() => null)
+        blockData
       );
     } else {
       signedBlock = signedBlockOrContents;
       blobSidecars = [];
-      blockForImport = getBlockInput.preDeneb(config, signedBlock, BlockSource.api, context?.sszBytes ?? null);
+      blockForImport = getBlockInput.preData(config, signedBlock, BlockSource.api, context?.sszBytes ?? null);
     }
 
     // check what validations have been requested before broadcasting and publishing the block

packages/beacon-node/src/api/impl/validator/index.ts

@@ -38,7 +38,7 @@ import {
   phase0,
   Wei,
 } from "@lodestar/types";
-import {ExecutionStatus} from "@lodestar/fork-choice";
+import {ExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice";
 import {fromHex, toHex, resolveOrRacePromises, prettyWeiToEth} from "@lodestar/utils";
 import {
   AttestationError,
@@ -324,7 +324,7 @@ export function getValidatorApi({
   function notOnOptimisticBlockRoot(beaconBlockRoot: Root): void {
     const protoBeaconBlock = chain.forkChoice.getBlock(beaconBlockRoot);
     if (!protoBeaconBlock) {
-      throw new ApiError(400, "Block not in forkChoice");
+      throw new ApiError(400, `Block not in forkChoice, beaconBlockRoot=${toHex(beaconBlockRoot)}`);
     }
 
     if (protoBeaconBlock.executionStatus === ExecutionStatus.Syncing)
@@ -333,6 +333,16 @@ export function getValidatorApi({
       );
   }
 
+  function notOnOutOfRangeData(beaconBlockRoot: Root): void {
+    const protoBeaconBlock = chain.forkChoice.getBlock(beaconBlockRoot);
+    if (!protoBeaconBlock) {
+      throw new ApiError(400, `Block not in forkChoice, beaconBlockRoot=${toHex(beaconBlockRoot)}`);
+    }
+
+    if (protoBeaconBlock.dataAvailabilityStatus === DataAvailabilityStatus.OutOfRange)
+      throw new NodeIsSyncing("Block's data is out of range and not validated");
+  }
+
   async function produceBuilderBlindedBlock(
     slot: Slot,
     randaoReveal: BLSSignature,
@@ -385,6 +395,7 @@ export function getValidatorApi({
     } else {
       parentBlockRoot = inParentBlockRoot;
     }
+    notOnOutOfRangeData(parentBlockRoot);
 
     let timer;
     try {
@@ -452,6 +463,7 @@ export function getValidatorApi({
     } else {
       parentBlockRoot = inParentBlockRoot;
     }
+    notOnOutOfRangeData(parentBlockRoot);
 
     let timer;
     try {
@@ -522,6 +534,7 @@ export function getValidatorApi({
     // handler, in which case this should just return.
     chain.forkChoice.updateTime(slot);
     const parentBlockRoot = fromHex(chain.getProposerHead(slot).blockRoot);
+    notOnOutOfRangeData(parentBlockRoot);
 
     const fork = config.getForkName(slot);
     // set some sensible opts
@@ -825,11 +838,15 @@ export function getValidatorApi({
       // Check the execution status as validator shouldn't vote on an optimistic head
       // Check on target is sufficient as a valid target would imply a valid source
       notOnOptimisticBlockRoot(targetRoot);
+      notOnOutOfRangeData(targetRoot);
 
       // To get the correct source we must get a state in the same epoch as the attestation's epoch.
       // An epoch transition may change state.currentJustifiedCheckpoint
       const attEpochState = await chain.getHeadStateAtEpoch(attEpoch, RegenCaller.produceAttestationData);
 
+      // TODO confirm if the below is correct assertion
+      // notOnOutOfRangeData(attEpochState.currentJustifiedCheckpoint.root);
+
       return {
         data: {
           slot,
@@ -865,6 +882,7 @@ export function getValidatorApi({
 
       // Check the execution status as validator shouldn't contribute on an optimistic head
       notOnOptimisticBlockRoot(beaconBlockRoot);
+      notOnOutOfRangeData(beaconBlockRoot);
 
       const contribution = chain.syncCommitteeMessagePool.getContribution(subcommitteeIndex, slot, beaconBlockRoot);
       if (!contribution) throw new ApiError(500, "No contribution available");

packages/beacon-node/src/chain/blocks/importBlock.ts

@@ -58,7 +58,7 @@ export async function importBlock(
   fullyVerifiedBlock: FullyVerifiedBlock,
   opts: ImportBlockOpts
 ): Promise<void> {
-  const {blockInput, postState, parentBlockSlot, executionStatus} = fullyVerifiedBlock;
+  const {blockInput, postState, parentBlockSlot, executionStatus, dataAvailabilityStatus} = fullyVerifiedBlock;
   const {block, source} = blockInput;
   const {slot: blockSlot} = block.message;
   const blockRoot = this.config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(block.message);
@@ -70,8 +70,8 @@ export async function importBlock(
   const blockDelaySec = (fullyVerifiedBlock.seenTimestampSec - postState.genesisTime) % this.config.SECONDS_PER_SLOT;
   const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000);
 
-  // this is just a type assertion since blockinput with blobsPromise type will not end up here
-  if (blockInput.type === BlockInputType.blobsPromise) {
+  // this is just a type assertion since blockinput with dataPromise type will not end up here
+  if (blockInput.type === BlockInputType.dataPromise) {
     throw Error("Unavailable block can not be imported in forkchoice");
   }
 
@@ -90,7 +90,8 @@ export async function importBlock(
     postState,
     blockDelaySec,
     this.clock.currentSlot,
-    executionStatus
+    executionStatus,
+    dataAvailabilityStatus
   );
 
   // This adds the state necessary to process the next block
@@ -108,11 +109,11 @@ export async function importBlock(
       executionOptimistic: blockSummary != null && isOptimisticBlock(blockSummary),
     });
 
-    // blobsPromise will not end up here, but preDeneb could. In future we might also allow syncing
+    // dataPromise will not end up here, but preDeneb could. In future we might also allow syncing
     // out of data range blocks and import then in forkchoice although one would not be able to
     // attest and propose with such head similar to optimistic sync
-    if (blockInput.type === BlockInputType.postDeneb) {
-      const {blobsSource, blobs} = blockInput;
+    if (blockInput.type === BlockInputType.availableData) {
+      const {blobsSource, blobs} = blockInput.blockData;
 
       this.metrics?.importBlock.blobsBySource.inc({blobsSource});
       for (const blobSidecar of blobs) {

packages/beacon-node/src/chain/blocks/index.ts

@@ -87,9 +87,8 @@ export async function processBlocks(
         postState: postStates[i],
         parentBlockSlot: parentSlots[i],
         executionStatus: executionStatuses[i],
-        // Currently dataAvailableStatus is not used upstream but that can change if we
         // start supporting optimistic syncing/processing
-        dataAvailableStatus: dataAvailabilityStatuses[i],
+        dataAvailabilityStatus: dataAvailabilityStatuses[i],
         proposerBalanceDelta: proposerBalanceDeltas[i],
         // TODO: Make this param mandatory and capture in gossip
         seenTimestampSec: opts.seenTimestampSec ?? Math.floor(Date.now() / 1000),

packages/beacon-node/src/chain/blocks/types.ts

@@ -1,13 +1,16 @@
-import {CachedBeaconStateAllForks, computeEpochAtSlot, DataAvailableStatus} from "@lodestar/state-transition";
-import {MaybeValidExecutionStatus} from "@lodestar/fork-choice";
+import {CachedBeaconStateAllForks, computeEpochAtSlot} from "@lodestar/state-transition";
+import {MaybeValidExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice";
 import {allForks, deneb, Slot, RootHex} from "@lodestar/types";
-import {ForkSeq} from "@lodestar/params";
+import {ForkSeq, ForkName} from "@lodestar/params";
 import {ChainForkConfig} from "@lodestar/config";
 
 export enum BlockInputType {
-  preDeneb = "preDeneb",
-  postDeneb = "postDeneb",
-  blobsPromise = "blobsPromise",
+  // preData is preDeneb
+  preData = "preData",
+  // data is out of available window, can be used to sync forward and keep adding to forkchoice
+  outOfRangeData = "outOfRangeData",
+  availableData = "availableData",
+  dataPromise = "dataPromise",
 }
 
 /** Enum to represent where blocks come from */
@@ -31,21 +34,28 @@ export enum GossipedInputType {
   blob = "blob",
 }
 
-export type BlobsCache = Map<number, {blobSidecar: deneb.BlobSidecar; blobBytes: Uint8Array | null}>;
+type BlobsCacheMap = Map<number, {blobSidecar: deneb.BlobSidecar; blobBytes: Uint8Array | null}>;
+
+type ForkBlobsInfo = {fork: ForkName.deneb};
+type BlobsData = {blobs: deneb.BlobSidecars; blobsBytes: (Uint8Array | null)[]; blobsSource: BlobsSource};
+export type BlockInputDataBlobs = ForkBlobsInfo & BlobsData;
+export type BlockInputData = BlockInputDataBlobs;
+
 export type BlockInputBlobs = {blobs: deneb.BlobSidecars; blobsBytes: (Uint8Array | null)[]; blobsSource: BlobsSource};
-type CachedBlobs = {
-  blobsCache: BlobsCache;
-  availabilityPromise: Promise<BlockInputBlobs>;
-  resolveAvailability: (blobs: BlockInputBlobs) => void;
-};
+type Availability<T> = {availabilityPromise: Promise<T>; resolveAvailability: (data: T) => void};
+
+type CachedBlobs = {blobsCache: BlobsCacheMap} & Availability<BlockInputDataBlobs>;
+export type CachedData = ForkBlobsInfo & CachedBlobs;
 
 export type BlockInput = {block: allForks.SignedBeaconBlock; source: BlockSource; blockBytes: Uint8Array | null} & (
-  | {type: BlockInputType.preDeneb}
-  | ({type: BlockInputType.postDeneb} & BlockInputBlobs)
+  | {type: BlockInputType.preData | BlockInputType.outOfRangeData}
+  | ({type: BlockInputType.availableData} & {blockData: BlockInputData})
   // the blobsSource here is added to BlockInputBlobs when availability is resolved
-  | ({type: BlockInputType.blobsPromise} & CachedBlobs)
+  | ({type: BlockInputType.dataPromise} & {cachedData: CachedData})
 );
-export type NullBlockInput = {block: null; blockRootHex: RootHex; blockInputPromise: Promise<BlockInput>} & CachedBlobs;
+export type NullBlockInput = {block: null; blockRootHex: RootHex; blockInputPromise: Promise<BlockInput>} & {
+  cachedData: CachedData;
+};
 
 export function blockRequiresBlobs(config: ChainForkConfig, blockSlot: Slot, clockSlot: Slot): boolean {
   return (
@@ -56,7 +66,7 @@ export function blockRequiresBlobs(config: ChainForkConfig, blockSlot: Slot, clo
 }
 
 export const getBlockInput = {
-  preDeneb(
+  preData(
     config: ChainForkConfig,
     block: allForks.SignedBeaconBlock,
     source: BlockSource,
@@ -66,61 +76,76 @@ export const getBlockInput = {
       throw Error(`Post Deneb block slot ${block.message.slot}`);
     }
     return {
-      type: BlockInputType.preDeneb,
+      type: BlockInputType.preData,
+      block,
+      source,
+      blockBytes,
+    };
+  },
+
+  // This isn't used right now but we might enable importing blobs into forkchoice from a point
+  // where data is not guaranteed to be available to hopefully reach a point where we have
+  // available data. Hence the validator duties can't be performed on outOfRangeData
+  //
+  // This can help with some of the requests of syncing without data for some use cases for e.g.
+  // building states or where importing data isn't important if valid child exists like ILs
+  outOfRangeData(
+    config: ChainForkConfig,
+    block: allForks.SignedBeaconBlock,
+    source: BlockSource,
+    blockBytes: Uint8Array | null
+  ): BlockInput {
+    if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) {
+      throw Error(`Pre Deneb block slot ${block.message.slot}`);
+    }
+    return {
+      type: BlockInputType.outOfRangeData,
       block,
       source,
       blockBytes,
     };
   },
 
-  postDeneb(
+  availableData(
     config: ChainForkConfig,
     block: allForks.SignedBeaconBlock,
     source: BlockSource,
-    blobs: deneb.BlobSidecars,
-    blobsSource: BlobsSource,
     blockBytes: Uint8Array | null,
-    blobsBytes: (Uint8Array | null)[]
+    blockData: BlockInputData
   ): BlockInput {
     if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) {
       throw Error(`Pre Deneb block slot ${block.message.slot}`);
     }
     return {
-      type: BlockInputType.postDeneb,
+      type: BlockInputType.availableData,
       block,
       source,
-      blobs,
-      blobsSource,
       blockBytes,
-      blobsBytes,
+      blockData,
     };
   },
 
-  blobsPromise(
+  dataPromise(
     config: ChainForkConfig,
     block: allForks.SignedBeaconBlock,
     source: BlockSource,
-    blobsCache: BlobsCache,
     blockBytes: Uint8Array | null,
-    availabilityPromise: Promise<BlockInputBlobs>,
-    resolveAvailability: (blobs: BlockInputBlobs) => void
+    cachedData: CachedData
   ): BlockInput {
     if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) {
       throw Error(`Pre Deneb block slot ${block.message.slot}`);
     }
     return {
-      type: BlockInputType.blobsPromise,
+      type: BlockInputType.dataPromise,
       block,
       source,
-      blobsCache,
       blockBytes,
-      availabilityPromise,
-      resolveAvailability,
+      cachedData,
     };
   },
 };
 
-export function getBlockInputBlobs(blobsCache: BlobsCache): Omit<BlockInputBlobs, "blobsSource"> {
+export function getBlockInputBlobs(blobsCache: BlobsCacheMap): Omit<BlobsData, "blobsSource"> {
   const blobs = [];
   const blobsBytes = [];
 
@@ -206,7 +231,7 @@ export type FullyVerifiedBlock = {
    * used in optimistic sync or for merge block
    */
   executionStatus: MaybeValidExecutionStatus;
-  dataAvailableStatus: DataAvailableStatus;
+  dataAvailabilityStatus: DataAvailabilityStatus;
   /** Seen timestamp seconds */
   seenTimestampSec: number;
 };

packages/beacon-node/src/chain/blocks/verifyBlock.ts

@@ -7,7 +7,7 @@ import {
 } from "@lodestar/state-transition";
 import {bellatrix, deneb} from "@lodestar/types";
 import {ForkName} from "@lodestar/params";
-import {ProtoBlock, ExecutionStatus} from "@lodestar/fork-choice";
+import {ProtoBlock, ExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice";
 import {ChainForkConfig} from "@lodestar/config";
 import {Logger} from "@lodestar/utils";
 import {BlockError, BlockErrorCode} from "../errors/index.js";
@@ -44,7 +44,7 @@ export async function verifyBlocksInEpoch(
   postStates: CachedBeaconStateAllForks[];
   proposerBalanceDeltas: number[];
   segmentExecStatus: SegmentExecStatus;
-  dataAvailabilityStatuses: DataAvailableStatus[];
+  dataAvailabilityStatuses: DataAvailabilityStatus[];
   availableBlockInputs: BlockInput[];
 }> {
   const blocks = blocksInput.map(({block}) => block);
@@ -169,7 +169,7 @@ export async function verifyBlocksInEpoch(
         blocksInput.length === 1 &&
         // gossip blocks have seenTimestampSec
         opts.seenTimestampSec !== undefined &&
-        blocksInput[0].type !== BlockInputType.preDeneb &&
+        blocksInput[0].type !== BlockInputType.preData &&
         executionStatuses[0] === ExecutionStatus.Valid
       ) {
         // Find the max time when the block was actually verified