Skip to content

Commit d101913

Browse files
authored
feat: generalize blobs to data for extension ready for ils and/or data columns (#6693)
rename types and fix references to them update forkchoice and availability flow propagate changes to the codebase lint and tsc some fixes cleanup test run fixes fix the tests fix apply feedback
1 parent 79a008f commit d101913

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+458
-229
lines changed

packages/beacon-node/src/api/impl/beacon/blocks/index.ts

+17-6
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,14 @@ import {computeEpochAtSlot, computeTimeAtSlot, reconstructFullBlockOrContents} f
44
import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
55
import {sleep, fromHex, toHex} from "@lodestar/utils";
66
import {allForks, deneb, isSignedBlockContents, ProducedBlockSource} from "@lodestar/types";
7-
import {BlockSource, getBlockInput, ImportBlockOpts, BlockInput, BlobsSource} from "../../../../chain/blocks/types.js";
7+
import {
8+
BlockSource,
9+
getBlockInput,
10+
ImportBlockOpts,
11+
BlockInput,
12+
BlobsSource,
13+
BlockInputDataBlobs,
14+
} from "../../../../chain/blocks/types.js";
815
import {promiseAllMaybeAsync} from "../../../../util/promises.js";
916
import {isOptimisticBlock} from "../../../../util/forkChoice.js";
1017
import {computeBlobSidecars} from "../../../../util/blobs.js";
@@ -51,20 +58,24 @@ export function getBeaconBlockApi({
5158
if (isSignedBlockContents(signedBlockOrContents)) {
5259
({signedBlock} = signedBlockOrContents);
5360
blobSidecars = computeBlobSidecars(config, signedBlock, signedBlockOrContents);
54-
blockForImport = getBlockInput.postDeneb(
61+
const blockData = {
62+
fork: config.getForkName(signedBlock.message.slot),
63+
blobs: blobSidecars,
64+
blobsSource: BlobsSource.api,
65+
blobsBytes: blobSidecars.map(() => null),
66+
} as BlockInputDataBlobs;
67+
blockForImport = getBlockInput.availableData(
5568
config,
5669
signedBlock,
5770
BlockSource.api,
58-
blobSidecars,
59-
BlobsSource.api,
6071
// don't bundle any bytes for block and blobs
6172
null,
62-
blobSidecars.map(() => null)
73+
blockData
6374
);
6475
} else {
6576
signedBlock = signedBlockOrContents;
6677
blobSidecars = [];
67-
blockForImport = getBlockInput.preDeneb(config, signedBlock, BlockSource.api, context?.sszBytes ?? null);
78+
blockForImport = getBlockInput.preData(config, signedBlock, BlockSource.api, context?.sszBytes ?? null);
6879
}
6980

7081
// check what validations have been requested before broadcasting and publishing the block

packages/beacon-node/src/api/impl/validator/index.ts

+20-2
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ import {
3838
phase0,
3939
Wei,
4040
} from "@lodestar/types";
41-
import {ExecutionStatus} from "@lodestar/fork-choice";
41+
import {ExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice";
4242
import {fromHex, toHex, resolveOrRacePromises, prettyWeiToEth} from "@lodestar/utils";
4343
import {
4444
AttestationError,
@@ -324,7 +324,7 @@ export function getValidatorApi({
324324
function notOnOptimisticBlockRoot(beaconBlockRoot: Root): void {
325325
const protoBeaconBlock = chain.forkChoice.getBlock(beaconBlockRoot);
326326
if (!protoBeaconBlock) {
327-
throw new ApiError(400, "Block not in forkChoice");
327+
throw new ApiError(400, `Block not in forkChoice, beaconBlockRoot=${toHex(beaconBlockRoot)}`);
328328
}
329329

330330
if (protoBeaconBlock.executionStatus === ExecutionStatus.Syncing)
@@ -333,6 +333,16 @@ export function getValidatorApi({
333333
);
334334
}
335335

336+
function notOnOutOfRangeData(beaconBlockRoot: Root): void {
337+
const protoBeaconBlock = chain.forkChoice.getBlock(beaconBlockRoot);
338+
if (!protoBeaconBlock) {
339+
throw new ApiError(400, `Block not in forkChoice, beaconBlockRoot=${toHex(beaconBlockRoot)}`);
340+
}
341+
342+
if (protoBeaconBlock.dataAvailabilityStatus === DataAvailabilityStatus.OutOfRange)
343+
throw new NodeIsSyncing("Block's data is out of range and not validated");
344+
}
345+
336346
async function produceBuilderBlindedBlock(
337347
slot: Slot,
338348
randaoReveal: BLSSignature,
@@ -385,6 +395,7 @@ export function getValidatorApi({
385395
} else {
386396
parentBlockRoot = inParentBlockRoot;
387397
}
398+
notOnOutOfRangeData(parentBlockRoot);
388399

389400
let timer;
390401
try {
@@ -452,6 +463,7 @@ export function getValidatorApi({
452463
} else {
453464
parentBlockRoot = inParentBlockRoot;
454465
}
466+
notOnOutOfRangeData(parentBlockRoot);
455467

456468
let timer;
457469
try {
@@ -522,6 +534,7 @@ export function getValidatorApi({
522534
// handler, in which case this should just return.
523535
chain.forkChoice.updateTime(slot);
524536
const parentBlockRoot = fromHex(chain.getProposerHead(slot).blockRoot);
537+
notOnOutOfRangeData(parentBlockRoot);
525538

526539
const fork = config.getForkName(slot);
527540
// set some sensible opts
@@ -825,11 +838,15 @@ export function getValidatorApi({
825838
// Check the execution status as validator shouldn't vote on an optimistic head
826839
// Check on target is sufficient as a valid target would imply a valid source
827840
notOnOptimisticBlockRoot(targetRoot);
841+
notOnOutOfRangeData(targetRoot);
828842

829843
// To get the correct source we must get a state in the same epoch as the attestation's epoch.
830844
// An epoch transition may change state.currentJustifiedCheckpoint
831845
const attEpochState = await chain.getHeadStateAtEpoch(attEpoch, RegenCaller.produceAttestationData);
832846

847+
// TODO confirm if the below is correct assertion
848+
// notOnOutOfRangeData(attEpochState.currentJustifiedCheckpoint.root);
849+
833850
return {
834851
data: {
835852
slot,
@@ -865,6 +882,7 @@ export function getValidatorApi({
865882

866883
// Check the execution status as validator shouldn't contribute on an optimistic head
867884
notOnOptimisticBlockRoot(beaconBlockRoot);
885+
notOnOutOfRangeData(beaconBlockRoot);
868886

869887
const contribution = chain.syncCommitteeMessagePool.getContribution(subcommitteeIndex, slot, beaconBlockRoot);
870888
if (!contribution) throw new ApiError(500, "No contribution available");

packages/beacon-node/src/chain/blocks/importBlock.ts

+8-7
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ export async function importBlock(
5858
fullyVerifiedBlock: FullyVerifiedBlock,
5959
opts: ImportBlockOpts
6060
): Promise<void> {
61-
const {blockInput, postState, parentBlockSlot, executionStatus} = fullyVerifiedBlock;
61+
const {blockInput, postState, parentBlockSlot, executionStatus, dataAvailabilityStatus} = fullyVerifiedBlock;
6262
const {block, source} = blockInput;
6363
const {slot: blockSlot} = block.message;
6464
const blockRoot = this.config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(block.message);
@@ -70,8 +70,8 @@ export async function importBlock(
7070
const blockDelaySec = (fullyVerifiedBlock.seenTimestampSec - postState.genesisTime) % this.config.SECONDS_PER_SLOT;
7171
const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000);
7272

73-
// this is just a type assertion since blockinput with blobsPromise type will not end up here
74-
if (blockInput.type === BlockInputType.blobsPromise) {
73+
// this is just a type assertion since blockinput with dataPromise type will not end up here
74+
if (blockInput.type === BlockInputType.dataPromise) {
7575
throw Error("Unavailable block can not be imported in forkchoice");
7676
}
7777

@@ -90,7 +90,8 @@ export async function importBlock(
9090
postState,
9191
blockDelaySec,
9292
this.clock.currentSlot,
93-
executionStatus
93+
executionStatus,
94+
dataAvailabilityStatus
9495
);
9596

9697
// This adds the state necessary to process the next block
@@ -108,11 +109,11 @@ export async function importBlock(
108109
executionOptimistic: blockSummary != null && isOptimisticBlock(blockSummary),
109110
});
110111

111-
// blobsPromise will not end up here, but preDeneb could. In future we might also allow syncing
112+
// dataPromise will not end up here, but preDeneb could. In future we might also allow syncing
112113
// out of data range blocks and import then in forkchoice although one would not be able to
113114
// attest and propose with such head similar to optimistic sync
114-
if (blockInput.type === BlockInputType.postDeneb) {
115-
const {blobsSource, blobs} = blockInput;
115+
if (blockInput.type === BlockInputType.availableData) {
116+
const {blobsSource, blobs} = blockInput.blockData;
116117

117118
this.metrics?.importBlock.blobsBySource.inc({blobsSource});
118119
for (const blobSidecar of blobs) {

packages/beacon-node/src/chain/blocks/index.ts

+1-2
Original file line numberDiff line numberDiff line change
@@ -87,9 +87,8 @@ export async function processBlocks(
8787
postState: postStates[i],
8888
parentBlockSlot: parentSlots[i],
8989
executionStatus: executionStatuses[i],
90-
// Currently dataAvailableStatus is not used upstream but that can change if we
9190
// start supporting optimistic syncing/processing
92-
dataAvailableStatus: dataAvailabilityStatuses[i],
91+
dataAvailabilityStatus: dataAvailabilityStatuses[i],
9392
proposerBalanceDelta: proposerBalanceDeltas[i],
9493
// TODO: Make this param mandatory and capture in gossip
9594
seenTimestampSec: opts.seenTimestampSec ?? Math.floor(Date.now() / 1000),

packages/beacon-node/src/chain/blocks/types.ts

+61-36
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,16 @@
1-
import {CachedBeaconStateAllForks, computeEpochAtSlot, DataAvailableStatus} from "@lodestar/state-transition";
2-
import {MaybeValidExecutionStatus} from "@lodestar/fork-choice";
1+
import {CachedBeaconStateAllForks, computeEpochAtSlot} from "@lodestar/state-transition";
2+
import {MaybeValidExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice";
33
import {allForks, deneb, Slot, RootHex} from "@lodestar/types";
4-
import {ForkSeq} from "@lodestar/params";
4+
import {ForkSeq, ForkName} from "@lodestar/params";
55
import {ChainForkConfig} from "@lodestar/config";
66

77
export enum BlockInputType {
8-
preDeneb = "preDeneb",
9-
postDeneb = "postDeneb",
10-
blobsPromise = "blobsPromise",
8+
// preData is preDeneb
9+
preData = "preData",
10+
// data is out of available window, can be used to sync forward and keep adding to forkchoice
11+
outOfRangeData = "outOfRangeData",
12+
availableData = "availableData",
13+
dataPromise = "dataPromise",
1114
}
1215

1316
/** Enum to represent where blocks come from */
@@ -31,21 +34,28 @@ export enum GossipedInputType {
3134
blob = "blob",
3235
}
3336

34-
export type BlobsCache = Map<number, {blobSidecar: deneb.BlobSidecar; blobBytes: Uint8Array | null}>;
37+
type BlobsCacheMap = Map<number, {blobSidecar: deneb.BlobSidecar; blobBytes: Uint8Array | null}>;
38+
39+
type ForkBlobsInfo = {fork: ForkName.deneb};
40+
type BlobsData = {blobs: deneb.BlobSidecars; blobsBytes: (Uint8Array | null)[]; blobsSource: BlobsSource};
41+
export type BlockInputDataBlobs = ForkBlobsInfo & BlobsData;
42+
export type BlockInputData = BlockInputDataBlobs;
43+
3544
export type BlockInputBlobs = {blobs: deneb.BlobSidecars; blobsBytes: (Uint8Array | null)[]; blobsSource: BlobsSource};
36-
type CachedBlobs = {
37-
blobsCache: BlobsCache;
38-
availabilityPromise: Promise<BlockInputBlobs>;
39-
resolveAvailability: (blobs: BlockInputBlobs) => void;
40-
};
45+
type Availability<T> = {availabilityPromise: Promise<T>; resolveAvailability: (data: T) => void};
46+
47+
type CachedBlobs = {blobsCache: BlobsCacheMap} & Availability<BlockInputDataBlobs>;
48+
export type CachedData = ForkBlobsInfo & CachedBlobs;
4149

4250
export type BlockInput = {block: allForks.SignedBeaconBlock; source: BlockSource; blockBytes: Uint8Array | null} & (
43-
| {type: BlockInputType.preDeneb}
44-
| ({type: BlockInputType.postDeneb} & BlockInputBlobs)
51+
| {type: BlockInputType.preData | BlockInputType.outOfRangeData}
52+
| ({type: BlockInputType.availableData} & {blockData: BlockInputData})
4553
// the blobsSource here is added to BlockInputBlobs when availability is resolved
46-
| ({type: BlockInputType.blobsPromise} & CachedBlobs)
54+
| ({type: BlockInputType.dataPromise} & {cachedData: CachedData})
4755
);
48-
export type NullBlockInput = {block: null; blockRootHex: RootHex; blockInputPromise: Promise<BlockInput>} & CachedBlobs;
56+
export type NullBlockInput = {block: null; blockRootHex: RootHex; blockInputPromise: Promise<BlockInput>} & {
57+
cachedData: CachedData;
58+
};
4959

5060
export function blockRequiresBlobs(config: ChainForkConfig, blockSlot: Slot, clockSlot: Slot): boolean {
5161
return (
@@ -56,7 +66,7 @@ export function blockRequiresBlobs(config: ChainForkConfig, blockSlot: Slot, clo
5666
}
5767

5868
export const getBlockInput = {
59-
preDeneb(
69+
preData(
6070
config: ChainForkConfig,
6171
block: allForks.SignedBeaconBlock,
6272
source: BlockSource,
@@ -66,61 +76,76 @@ export const getBlockInput = {
6676
throw Error(`Post Deneb block slot ${block.message.slot}`);
6777
}
6878
return {
69-
type: BlockInputType.preDeneb,
79+
type: BlockInputType.preData,
80+
block,
81+
source,
82+
blockBytes,
83+
};
84+
},
85+
86+
// This isn't used right now but we might enable importing blobs into forkchoice from a point
87+
// where data is not guaranteed to be available to hopefully reach a point where we have
88+
// available data. Hence the validator duties can't be performed on outOfRangeData
89+
//
90+
// This can help with some of the requests of syncing without data for some use cases for e.g.
91+
// building states or where importing data isn't important if valid child exists like ILs
92+
outOfRangeData(
93+
config: ChainForkConfig,
94+
block: allForks.SignedBeaconBlock,
95+
source: BlockSource,
96+
blockBytes: Uint8Array | null
97+
): BlockInput {
98+
if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) {
99+
throw Error(`Pre Deneb block slot ${block.message.slot}`);
100+
}
101+
return {
102+
type: BlockInputType.outOfRangeData,
70103
block,
71104
source,
72105
blockBytes,
73106
};
74107
},
75108

76-
postDeneb(
109+
availableData(
77110
config: ChainForkConfig,
78111
block: allForks.SignedBeaconBlock,
79112
source: BlockSource,
80-
blobs: deneb.BlobSidecars,
81-
blobsSource: BlobsSource,
82113
blockBytes: Uint8Array | null,
83-
blobsBytes: (Uint8Array | null)[]
114+
blockData: BlockInputData
84115
): BlockInput {
85116
if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) {
86117
throw Error(`Pre Deneb block slot ${block.message.slot}`);
87118
}
88119
return {
89-
type: BlockInputType.postDeneb,
120+
type: BlockInputType.availableData,
90121
block,
91122
source,
92-
blobs,
93-
blobsSource,
94123
blockBytes,
95-
blobsBytes,
124+
blockData,
96125
};
97126
},
98127

99-
blobsPromise(
128+
dataPromise(
100129
config: ChainForkConfig,
101130
block: allForks.SignedBeaconBlock,
102131
source: BlockSource,
103-
blobsCache: BlobsCache,
104132
blockBytes: Uint8Array | null,
105-
availabilityPromise: Promise<BlockInputBlobs>,
106-
resolveAvailability: (blobs: BlockInputBlobs) => void
133+
cachedData: CachedData
107134
): BlockInput {
108135
if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) {
109136
throw Error(`Pre Deneb block slot ${block.message.slot}`);
110137
}
111138
return {
112-
type: BlockInputType.blobsPromise,
139+
type: BlockInputType.dataPromise,
113140
block,
114141
source,
115-
blobsCache,
116142
blockBytes,
117-
availabilityPromise,
118-
resolveAvailability,
143+
cachedData,
119144
};
120145
},
121146
};
122147

123-
export function getBlockInputBlobs(blobsCache: BlobsCache): Omit<BlockInputBlobs, "blobsSource"> {
148+
export function getBlockInputBlobs(blobsCache: BlobsCacheMap): Omit<BlobsData, "blobsSource"> {
124149
const blobs = [];
125150
const blobsBytes = [];
126151

@@ -206,7 +231,7 @@ export type FullyVerifiedBlock = {
206231
* used in optimistic sync or for merge block
207232
*/
208233
executionStatus: MaybeValidExecutionStatus;
209-
dataAvailableStatus: DataAvailableStatus;
234+
dataAvailabilityStatus: DataAvailabilityStatus;
210235
/** Seen timestamp seconds */
211236
seenTimestampSec: number;
212237
};

packages/beacon-node/src/chain/blocks/verifyBlock.ts

+3-3
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import {
77
} from "@lodestar/state-transition";
88
import {bellatrix, deneb} from "@lodestar/types";
99
import {ForkName} from "@lodestar/params";
10-
import {ProtoBlock, ExecutionStatus} from "@lodestar/fork-choice";
10+
import {ProtoBlock, ExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice";
1111
import {ChainForkConfig} from "@lodestar/config";
1212
import {Logger} from "@lodestar/utils";
1313
import {BlockError, BlockErrorCode} from "../errors/index.js";
@@ -44,7 +44,7 @@ export async function verifyBlocksInEpoch(
4444
postStates: CachedBeaconStateAllForks[];
4545
proposerBalanceDeltas: number[];
4646
segmentExecStatus: SegmentExecStatus;
47-
dataAvailabilityStatuses: DataAvailableStatus[];
47+
dataAvailabilityStatuses: DataAvailabilityStatus[];
4848
availableBlockInputs: BlockInput[];
4949
}> {
5050
const blocks = blocksInput.map(({block}) => block);
@@ -169,7 +169,7 @@ export async function verifyBlocksInEpoch(
169169
blocksInput.length === 1 &&
170170
// gossip blocks have seenTimestampSec
171171
opts.seenTimestampSec !== undefined &&
172-
blocksInput[0].type !== BlockInputType.preDeneb &&
172+
blocksInput[0].type !== BlockInputType.preData &&
173173
executionStatuses[0] === ExecutionStatus.Valid
174174
) {
175175
// Find the max time when the block was actually verified

0 commit comments

Comments
 (0)