Merge 8fcb43d into 3a6702e

Author: matthewkeil

packages/beacon-node/src/api/impl/beacon/blocks/utils.ts

@@ -1,23 +1,38 @@
 import {allForks} from "@lodestar/types";
 import {routes} from "@lodestar/api";
-import {blockToHeader} from "@lodestar/state-transition";
 import {ChainForkConfig} from "@lodestar/config";
 import {GENESIS_SLOT} from "../../../../constants/index.js";
 import {ApiError, ValidationError} from "../../errors.js";
 import {IBeaconChain} from "../../../../chain/interface.js";
 import {rootHexRegex} from "../../../../eth1/provider/utils.js";
+import {isBlinded} from "../../../../util/fullOrBlindedBlock.js";
 
 export function toBeaconHeaderResponse(
   config: ChainForkConfig,
-  block: allForks.SignedBeaconBlock,
+  block: allForks.FullOrBlindedSignedBeaconBlock,
   canonical = false
 ): routes.beacon.BlockHeaderResponse {
+  // need to have ts-ignore below to pull type here so it only happens once and
+  // gets used twice
+  const types = isBlinded(block)
+    ? config.getBlindedForkTypes(block.message.slot)
+    : config.getForkTypes(block.message.slot);
   return {
-    root: config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message),
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore
+    root: types.BeaconBlock.hashTreeRoot(block.message),
     canonical,
     header: {
-      message: blockToHeader(config, block.message),
       signature: block.signature,
+      message: {
+        stateRoot: block.message.stateRoot,
+        proposerIndex: block.message.proposerIndex,
+        slot: block.message.slot,
+        parentRoot: block.message.parentRoot,
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        // @ts-ignore
+        bodyRoot: types.BeaconBlockBody.hashTreeRoot(block.message.body),
+      },
     },
   };
 }

packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts

@@ -1,5 +1,6 @@
 import {toHex} from "@lodestar/utils";
 import {BeaconChain} from "../chain.js";
+import {blindedOrFullBlockToBlinded} from "../../util/fullOrBlindedBlock.js";
 import {BlockInput, BlockInputType} from "./types.js";
 
 /**
@@ -13,17 +14,11 @@ export async function writeBlockInputToDb(this: BeaconChain, blocksInput: BlockI
   const fnPromises: Promise<void>[] = [];
 
   for (const blockInput of blocksInput) {
-    const {block, blockBytes, type} = blockInput;
+    const {block, type} = blockInput;
     const blockRoot = this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message);
     const blockRootHex = toHex(blockRoot);
-    if (blockBytes) {
-      // skip serializing data if we already have it
-      this.metrics?.importBlock.persistBlockWithSerializedDataCount.inc();
-      fnPromises.push(this.db.block.putBinary(this.db.block.getId(block), blockBytes));
-    } else {
-      this.metrics?.importBlock.persistBlockNoSerializedDataCount.inc();
-      fnPromises.push(this.db.block.add(block));
-    }
+    this.metrics?.importBlock.persistBlockNoSerializedDataCount.inc();
+    fnPromises.push(this.db.block.add(blindedOrFullBlockToBlinded(this.config, block)));
     this.logger.debug("Persist block to hot DB", {
       slot: block.message.slot,
       root: blockRootHex,

packages/beacon-node/src/chain/chain.ts

@@ -31,6 +31,7 @@ import {ProcessShutdownCallback} from "@lodestar/validator";
 import {Logger, isErrorAborted, pruneSetToMax, sleep, toHex} from "@lodestar/utils";
 import {ForkSeq, SLOTS_PER_EPOCH, MAX_BLOBS_PER_BLOCK} from "@lodestar/params";
 
+import {toHexString} from "@lodestar/utils";
 import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js";
 import {IBeaconDb} from "../db/index.js";
 import {Metrics} from "../metrics/index.js";
@@ -39,6 +40,14 @@ import {IExecutionEngine, IExecutionBuilder} from "../execution/index.js";
 import {Clock, ClockEvent, IClock} from "../util/clock.js";
 import {ensureDir, writeIfNotExist} from "../util/file.js";
 import {isOptimisticBlock} from "../util/forkChoice.js";
+import {
+  blindedOrFullBlockToFull,
+  deserializeFullOrBlindedSignedBeaconBlock,
+  getEth1BlockHashFromSerializedBlock,
+  serializeFullOrBlindedSignedBeaconBlock,
+} from "../util/fullOrBlindedBlock.js";
+import {ExecutionPayloadBody} from "../execution/engine/types.js";
+import {Eth1Error, Eth1ErrorCode} from "../eth1/errors.js";
 import {CheckpointStateCache, StateContextCache} from "./stateCache/index.js";
 import {BlockProcessor, ImportBlockOpts} from "./blocks/index.js";
 import {ChainEventEmitter, ChainEvent} from "./emitter.js";
@@ -433,7 +442,7 @@ export class BeaconChain implements IBeaconChain {
       if (block) {
         const data = await this.db.block.get(fromHexString(block.blockRoot));
         if (data) {
-          return {block: data, executionOptimistic: isOptimisticBlock(block)};
+          return {block: await this.blindedOrFullBlockToFull(data), executionOptimistic: isOptimisticBlock(block)};
         }
       }
       // A non-finalized slot expected to be found in the hot db, could be archived during
@@ -442,7 +451,7 @@ export class BeaconChain implements IBeaconChain {
     }
 
     const data = await this.db.blockArchive.get(slot);
-    return data && {block: data, executionOptimistic: false};
+    return data && {block: await this.blindedOrFullBlockToFull(data), executionOptimistic: false};
   }
 
   async getBlockByRoot(
@@ -452,7 +461,7 @@ export class BeaconChain implements IBeaconChain {
     if (block) {
       const data = await this.db.block.get(fromHexString(root));
       if (data) {
-        return {block: data, executionOptimistic: isOptimisticBlock(block)};
+        return {block: await this.blindedOrFullBlockToFull(data), executionOptimistic: isOptimisticBlock(block)};
       }
       // If block is not found in hot db, try cold db since there could be an archive cycle happening
       // TODO: Add a lock to the archiver to have determinstic behaviour on where are blocks
@@ -462,6 +471,28 @@ export class BeaconChain implements IBeaconChain {
     return data && {block: data, executionOptimistic: false};
   }
 
+  async blindedOrFullBlockToFull(block: allForks.FullOrBlindedSignedBeaconBlock): Promise<allForks.SignedBeaconBlock> {
+    const info = this.config.getForkInfo(block.message.slot);
+    return blindedOrFullBlockToFull(
+      this.config,
+      info.seq,
+      block,
+      await this.getTransactionsAndWithdrawals(info.seq, toHexString(block.message.body.eth1Data.blockHash))
+    );
+  }
+
+  async blindedOrFullBlockToFullBytes(forkSeq: ForkSeq, block: Uint8Array): Promise<Uint8Array> {
+    return serializeFullOrBlindedSignedBeaconBlock(
+      this.config,
+      blindedOrFullBlockToFull(
+        this.config,
+        forkSeq,
+        deserializeFullOrBlindedSignedBeaconBlock(this.config, block),
+        await this.getTransactionsAndWithdrawals(forkSeq, toHexString(getEth1BlockHashFromSerializedBlock(block)))
+      )
+    );
+  }
+
   produceBlock(blockAttributes: BlockAttributes): Promise<{block: allForks.BeaconBlock; blockValue: Wei}> {
     return this.produceBlockWrapper<BlockType.Full>(BlockType.Full, blockAttributes);
   }
@@ -626,6 +657,23 @@ export class BeaconChain implements IBeaconChain {
     }
   }
 
+  private async getTransactionsAndWithdrawals(
+    forkSeq: ForkSeq,
+    blockHash: string
+  ): Promise<Partial<ExecutionPayloadBody>> {
+    if (forkSeq < ForkSeq.bellatrix) {
+      return {};
+    }
+    const [payload] = await this.executionEngine.getPayloadBodiesByHash([blockHash]);
+    if (!payload) {
+      throw new Eth1Error(
+        {code: Eth1ErrorCode.INVALID_PAYLOAD_BODY, blockHash},
+        "payload body not found by eth1 engine"
+      );
+    }
+    return payload;
+  }
+
   /**
    * `ForkChoice.onBlock` must never throw for a block that is valid with respect to the network
    * `justifiedBalancesGetter()` must never throw and it should always return a state.

packages/beacon-node/src/chain/interface.ts

@@ -10,6 +10,7 @@ import {BeaconConfig} from "@lodestar/config";
 import {Logger} from "@lodestar/utils";
 
 import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
+import {ForkSeq} from "@lodestar/params";
 import {IEth1ForBlockProduction} from "../eth1/index.js";
 import {IExecutionEngine, IExecutionBuilder} from "../execution/index.js";
 import {Metrics} from "../metrics/metrics.js";
@@ -138,6 +139,9 @@ export interface IBeaconChain {
 
   getBlobSidecars(beaconBlock: deneb.BeaconBlock): deneb.BlobSidecars;
 
+  blindedOrFullBlockToFull(block: allForks.FullOrBlindedSignedBeaconBlock): Promise<allForks.SignedBeaconBlock>;
+  blindedOrFullBlockToFullBytes(forkSeq: ForkSeq, block: Uint8Array): Promise<Uint8Array>;
+
   produceBlock(blockAttributes: BlockAttributes): Promise<{block: allForks.BeaconBlock; blockValue: Wei}>;
   produceBlindedBlock(blockAttributes: BlockAttributes): Promise<{block: allForks.BlindedBeaconBlock; blockValue: Wei}>;
 

packages/beacon-node/src/db/repositories/block.ts

@@ -1,15 +1,19 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {Db, Repository} from "@lodestar/db";
 import {allForks, ssz} from "@lodestar/types";
-import {getSignedBlockTypeFromBytes} from "../../util/multifork.js";
+import {blindedOrFullBlockHashTreeRoot} from "@lodestar/state-transition";
+import {
+  deserializeFullOrBlindedSignedBeaconBlock,
+  serializeFullOrBlindedSignedBeaconBlock,
+} from "../../util/fullOrBlindedBlock.js";
 import {Bucket, getBucketNameByValue} from "../buckets.js";
 
 /**
  * Blocks by root
  *
  * Used to store unfinalized blocks
  */
-export class BlockRepository extends Repository<Uint8Array, allForks.SignedBeaconBlock> {
+export class BlockRepository extends Repository<Uint8Array, allForks.FullOrBlindedSignedBeaconBlock> {
   constructor(config: ChainForkConfig, db: Db) {
     const bucket = Bucket.allForks_block;
     const type = ssz.phase0.SignedBeaconBlock; // Pick some type but won't be used
@@ -19,15 +23,15 @@ export class BlockRepository extends Repository<Uint8Array, allForks.SignedBeaco
   /**
    * Id is hashTreeRoot of unsigned BeaconBlock
    */
-  getId(value: allForks.SignedBeaconBlock): Uint8Array {
-    return this.config.getForkTypes(value.message.slot).BeaconBlock.hashTreeRoot(value.message);
+  getId(value: allForks.FullOrBlindedSignedBeaconBlock): Uint8Array {
+    return blindedOrFullBlockHashTreeRoot(this.config, value.message);
   }
 
-  encodeValue(value: allForks.SignedBeaconBlock): Buffer {
-    return this.config.getForkTypes(value.message.slot).SignedBeaconBlock.serialize(value) as Buffer;
+  encodeValue(value: allForks.FullOrBlindedSignedBeaconBlock): Buffer {
+    return serializeFullOrBlindedSignedBeaconBlock(this.config, value) as Buffer;
   }
 
-  decodeValue(data: Buffer): allForks.SignedBeaconBlock {
-    return getSignedBlockTypeFromBytes(this.config, data).deserialize(data);
+  decodeValue(data: Buffer): allForks.FullOrBlindedSignedBeaconBlock {
+    return deserializeFullOrBlindedSignedBeaconBlock(this.config, data);
   }
 }

packages/beacon-node/src/db/repositories/blockArchive.ts

@@ -3,7 +3,11 @@ import {ChainForkConfig} from "@lodestar/config";
 import {Db, Repository, KeyValue, FilterOptions} from "@lodestar/db";
 import {Slot, Root, allForks, ssz} from "@lodestar/types";
 import {bytesToInt} from "@lodestar/utils";
-import {getSignedBlockTypeFromBytes} from "../../util/multifork.js";
+import {blindedOrFullBlockHashTreeRoot} from "@lodestar/state-transition";
+import {
+  deserializeFullOrBlindedSignedBeaconBlock,
+  serializeFullOrBlindedSignedBeaconBlock,
+} from "../../util/fullOrBlindedBlock.js";
 import {Bucket, getBucketNameByValue} from "../buckets.js";
 import {getRootIndexKey, getParentRootIndexKey} from "./blockArchiveIndex.js";
 import {deleteParentRootIndex, deleteRootIndex, storeParentRootIndex, storeRootIndex} from "./blockArchiveIndex.js";
@@ -21,7 +25,7 @@ export type BlockArchiveBatchPutBinaryItem = KeyValue<Slot, Uint8Array> & {
 /**
  * Stores finalized blocks. Block slot is identifier.
  */
-export class BlockArchiveRepository extends Repository<Slot, allForks.SignedBeaconBlock> {
+export class BlockArchiveRepository extends Repository<Slot, allForks.FullOrBlindedSignedBeaconBlock> {
   constructor(config: ChainForkConfig, db: Db) {
     const bucket = Bucket.allForks_blockArchive;
     const type = ssz.phase0.SignedBeaconBlock; // Pick some type but won't be used
@@ -30,17 +34,17 @@ export class BlockArchiveRepository extends Repository<Slot, allForks.SignedBeac
 
   // Overrides for multi-fork
 
-  encodeValue(value: allForks.SignedBeaconBlock): Uint8Array {
-    return this.config.getForkTypes(value.message.slot).SignedBeaconBlock.serialize(value);
+  encodeValue(value: allForks.FullOrBlindedSignedBeaconBlock): Uint8Array {
+    return serializeFullOrBlindedSignedBeaconBlock(this.config, value);
   }
 
-  decodeValue(data: Uint8Array): allForks.SignedBeaconBlock {
-    return getSignedBlockTypeFromBytes(this.config, data).deserialize(data);
+  decodeValue(data: Uint8Array): allForks.FullOrBlindedSignedBeaconBlock {
+    return deserializeFullOrBlindedSignedBeaconBlock(this.config, data);
   }
 
   // Handle key as slot
 
-  getId(value: allForks.SignedBeaconBlock): Slot {
+  getId(value: allForks.FullOrBlindedSignedBeaconBlock): Slot {
     return value.message.slot;
   }
 
@@ -50,8 +54,8 @@ export class BlockArchiveRepository extends Repository<Slot, allForks.SignedBeac
 
   // Overrides to index
 
-  async put(key: Slot, value: allForks.SignedBeaconBlock): Promise<void> {
-    const blockRoot = this.config.getForkTypes(value.message.slot).BeaconBlock.hashTreeRoot(value.message);
+  async put(key: Slot, value: allForks.FullOrBlindedSignedBeaconBlock): Promise<void> {
+    const blockRoot = blindedOrFullBlockHashTreeRoot(this.config, value.message);
     const slot = value.message.slot;
     await Promise.all([
       super.put(key, value),
@@ -60,12 +64,12 @@ export class BlockArchiveRepository extends Repository<Slot, allForks.SignedBeac
     ]);
   }
 
-  async batchPut(items: KeyValue<Slot, allForks.SignedBeaconBlock>[]): Promise<void> {
+  async batchPut(items: KeyValue<Slot, allForks.FullOrBlindedSignedBeaconBlock>[]): Promise<void> {
     await Promise.all([
       super.batchPut(items),
       Array.from(items).map((item) => {
         const slot = item.value.message.slot;
-        const blockRoot = this.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(item.value.message);
+        const blockRoot = blindedOrFullBlockHashTreeRoot(this.config, item.value.message);
         return storeRootIndex(this.db, slot, blockRoot);
       }),
       Array.from(items).map((item) => {
@@ -84,25 +88,25 @@ export class BlockArchiveRepository extends Repository<Slot, allForks.SignedBeac
     ]);
   }
 
-  async remove(value: allForks.SignedBeaconBlock): Promise<void> {
+  async remove(value: allForks.FullOrBlindedSignedBeaconBlock): Promise<void> {
     await Promise.all([
       super.remove(value),
-      deleteRootIndex(this.db, this.config.getForkTypes(value.message.slot).SignedBeaconBlock, value),
+      deleteRootIndex(this.db, this.config.getForkTypes(value.message.slot).BeaconBlock, value),
       deleteParentRootIndex(this.db, value),
     ]);
   }
 
-  async batchRemove(values: allForks.SignedBeaconBlock[]): Promise<void> {
+  async batchRemove(values: allForks.FullOrBlindedSignedBeaconBlock[]): Promise<void> {
     await Promise.all([
       super.batchRemove(values),
       Array.from(values).map((value) =>
-        deleteRootIndex(this.db, this.config.getForkTypes(value.message.slot).SignedBeaconBlock, value)
+        deleteRootIndex(this.db, this.config.getForkTypes(value.message.slot).BeaconBlock, value)
       ),
       Array.from(values).map((value) => deleteParentRootIndex(this.db, value)),
     ]);
   }
 
-  async *valuesStream(opts?: BlockFilterOptions): AsyncIterable<allForks.SignedBeaconBlock> {
+  async *valuesStream(opts?: BlockFilterOptions): AsyncIterable<allForks.FullOrBlindedSignedBeaconBlock> {
     const firstSlot = this.getFirstSlot(opts);
     const valuesStream = super.valuesStream(opts);
     const step = (opts && opts.step) ?? 1;
@@ -114,13 +118,13 @@ export class BlockArchiveRepository extends Repository<Slot, allForks.SignedBeac
     }
   }
 
-  async values(opts?: BlockFilterOptions): Promise<allForks.SignedBeaconBlock[]> {
+  async values(opts?: BlockFilterOptions): Promise<allForks.FullOrBlindedSignedBeaconBlock[]> {
     return all(this.valuesStream(opts));
   }
 
   // INDEX
 
-  async getByRoot(root: Root): Promise<allForks.SignedBeaconBlock | null> {
+  async getByRoot(root: Root): Promise<allForks.FullOrBlindedSignedBeaconBlock | null> {
     const slot = await this.getSlotByRoot(root);
     return slot !== null ? this.get(slot) : null;
   }
@@ -130,7 +134,7 @@ export class BlockArchiveRepository extends Repository<Slot, allForks.SignedBeac
     return slot !== null ? ({key: slot, value: await this.getBinary(slot)} as KeyValue<Slot, Buffer>) : null;
   }
 
-  async getByParentRoot(root: Root): Promise<allForks.SignedBeaconBlock | null> {
+  async getByParentRoot(root: Root): Promise<allForks.FullOrBlindedSignedBeaconBlock | null> {
     const slot = await this.getSlotByParentRoot(root);
     return slot !== null ? this.get(slot) : null;
   }

packages/beacon-node/src/db/repositories/blockArchiveIndex.ts

@@ -1,5 +1,5 @@
 import {Db, encodeKey} from "@lodestar/db";
-import {Slot, Root, allForks, ssz} from "@lodestar/types";
+import {Slot, Root, allForks} from "@lodestar/types";
 import {intToBytes} from "@lodestar/utils";
 import {Bucket} from "../buckets.js";
 
@@ -13,14 +13,13 @@ export async function storeParentRootIndex(db: Db, slot: Slot, parentRoot: Root)
 
 export async function deleteRootIndex(
   db: Db,
-  signedBeaconBlockType: allForks.AllForksSSZTypes["SignedBeaconBlock"],
-  block: allForks.SignedBeaconBlock
+  beaconBlockType: allForks.AllForksSSZTypes["BeaconBlock"],
+  block: allForks.FullOrBlindedSignedBeaconBlock
 ): Promise<void> {
-  const beaconBlockType = (signedBeaconBlockType as typeof ssz.phase0.SignedBeaconBlock).fields["message"];
   return db.delete(getRootIndexKey(beaconBlockType.hashTreeRoot(block.message)));
 }
 
-export async function deleteParentRootIndex(db: Db, block: allForks.SignedBeaconBlock): Promise<void> {
+export async function deleteParentRootIndex(db: Db, block: allForks.FullOrBlindedSignedBeaconBlock): Promise<void> {
   return db.delete(getParentRootIndexKey(block.message.parentRoot));
 }