feat: load state from Uint8Array (#6057)

* fix: implement loadState api

* chore: benchmark findModifiedValidators()

* feat: implement deserializeContainerIgnoreFields()

* feat: implement loadValidator()

* fix: type the ignoreFields

* chore: benchmark loadState()

* fix: add '--max-old-space-size=4096' to github workflow

* fix: revert default vc count in perf test

* Revert "fix: add '--max-old-space-size=4096' to github workflow"

This reverts commit a420c54.

* chore: rename loadCachedBeaconState to loadUnfinalizedCachedBeaconState

* chore: loadValidator - reuse fields as much as possible

* chore: more benchmarks to compare Uint8Array

Author: twoeths

packages/api/package.json

@@ -70,7 +70,7 @@
   },
   "dependencies": {
     "@chainsafe/persistent-merkle-tree": "^0.6.1",
-    "@chainsafe/ssz": "^0.13.0",
+    "@chainsafe/ssz": "^0.14.0",
     "@lodestar/config": "^1.11.3",
     "@lodestar/params": "^1.11.3",
     "@lodestar/types": "^1.11.3",

packages/beacon-node/package.json

@@ -104,7 +104,7 @@
     "@chainsafe/libp2p-noise": "^13.0.1",
     "@chainsafe/persistent-merkle-tree": "^0.6.1",
     "@chainsafe/prometheus-gc-stats": "^1.0.0",
-    "@chainsafe/ssz": "^0.13.0",
+    "@chainsafe/ssz": "^0.14.0",
     "@chainsafe/threads": "^1.11.1",
     "@ethersproject/abi": "^5.7.0",
     "@fastify/bearer-auth": "^9.0.0",

packages/cli/package.json

@@ -59,7 +59,7 @@
     "@chainsafe/bls-keystore": "^2.0.0",
     "@chainsafe/blst": "^0.2.9",
     "@chainsafe/discv5": "^5.1.0",
-    "@chainsafe/ssz": "^0.13.0",
+    "@chainsafe/ssz": "^0.14.0",
     "@chainsafe/threads": "^1.11.1",
     "@libp2p/crypto": "^2.0.4",
     "@libp2p/peer-id": "^3.0.2",

packages/config/package.json

@@ -64,7 +64,7 @@
     "blockchain"
   ],
   "dependencies": {
-    "@chainsafe/ssz": "^0.13.0",
+    "@chainsafe/ssz": "^0.14.0",
     "@lodestar/params": "^1.11.3",
     "@lodestar/types": "^1.11.3"
   }

packages/db/package.json

@@ -37,7 +37,7 @@
     "check-readme": "typescript-docs-verifier"
   },
   "dependencies": {
-    "@chainsafe/ssz": "^0.13.0",
+    "@chainsafe/ssz": "^0.14.0",
     "@lodestar/config": "^1.11.3",
     "@lodestar/utils": "^1.11.3",
     "@types/levelup": "^4.3.3",

packages/fork-choice/package.json

@@ -38,7 +38,7 @@
     "check-readme": "typescript-docs-verifier"
   },
   "dependencies": {
-    "@chainsafe/ssz": "^0.13.0",
+    "@chainsafe/ssz": "^0.14.0",
     "@lodestar/config": "^1.11.3",
     "@lodestar/params": "^1.11.3",
     "@lodestar/state-transition": "^1.11.3",

packages/light-client/package.json

@@ -66,7 +66,7 @@
   "dependencies": {
     "@chainsafe/bls": "7.1.1",
     "@chainsafe/persistent-merkle-tree": "^0.6.1",
-    "@chainsafe/ssz": "^0.13.0",
+    "@chainsafe/ssz": "^0.14.0",
     "@lodestar/api": "^1.11.3",
     "@lodestar/config": "^1.11.3",
     "@lodestar/params": "^1.11.3",

packages/state-transition/package.json

@@ -59,9 +59,10 @@
   "dependencies": {
     "@chainsafe/as-sha256": "^0.3.1",
     "@chainsafe/bls": "7.1.1",
+    "@chainsafe/blst": "^0.2.9",
     "@chainsafe/persistent-merkle-tree": "^0.6.1",
     "@chainsafe/persistent-ts": "^0.19.1",
-    "@chainsafe/ssz": "^0.13.0",
+    "@chainsafe/ssz": "^0.14.0",
     "@lodestar/config": "^1.11.3",
     "@lodestar/params": "^1.11.3",
     "@lodestar/types": "^1.11.3",

packages/state-transition/src/cache/epochCache.ts

@@ -26,12 +26,12 @@ import {
   computeProposers,
   getActivationChurnLimit,
 } from "../util/index.js";
-import {computeEpochShuffling, EpochShuffling} from "../util/epochShuffling.js";
+import {computeEpochShuffling, EpochShuffling, getShufflingDecisionBlock} from "../util/epochShuffling.js";
 import {computeBaseRewardPerIncrement, computeSyncParticipantReward} from "../util/syncCommittee.js";
 import {sumTargetUnslashedBalanceIncrements} from "../util/targetUnslashedBalance.js";
 import {EffectiveBalanceIncrements, getEffectiveBalanceIncrementsWithLen} from "./effectiveBalanceIncrements.js";
 import {Index2PubkeyCache, PubkeyIndexMap, syncPubkeys} from "./pubkeyCache.js";
-import {BeaconStateAllForks, BeaconStateAltair} from "./types.js";
+import {BeaconStateAllForks, BeaconStateAltair, ShufflingGetter} from "./types.js";
 import {
   computeSyncCommitteeCache,
   getSyncCommitteeCache,
@@ -51,6 +51,7 @@ export type EpochCacheImmutableData = {
 export type EpochCacheOpts = {
   skipSyncCommitteeCache?: boolean;
   skipSyncPubkeys?: boolean;
+  shufflingGetter?: ShufflingGetter;
 };
 
 /** Defers computing proposers by persisting only the seed, and dropping it once indexes are computed */
@@ -280,21 +281,32 @@ export class EpochCache {
     const currentActiveIndices: ValidatorIndex[] = [];
     const nextActiveIndices: ValidatorIndex[] = [];
 
+    // BeaconChain could provide a shuffling cache to avoid re-computing shuffling every epoch
+    // in that case, we don't need to compute shufflings again
+    const previousShufflingDecisionBlock = getShufflingDecisionBlock(state, previousEpoch);
+    const cachedPreviousShuffling = opts?.shufflingGetter?.(previousEpoch, previousShufflingDecisionBlock);
+    const currentShufflingDecisionBlock = getShufflingDecisionBlock(state, currentEpoch);
+    const cachedCurrentShuffling = opts?.shufflingGetter?.(currentEpoch, currentShufflingDecisionBlock);
+    const nextShufflingDecisionBlock = getShufflingDecisionBlock(state, nextEpoch);
+    const cachedNextShuffling = opts?.shufflingGetter?.(nextEpoch, nextShufflingDecisionBlock);
+
     for (let i = 0; i < validatorCount; i++) {
       const validator = validators[i];
 
       // Note: Not usable for fork-choice balances since in-active validators are not zero'ed
       effectiveBalanceIncrements[i] = Math.floor(validator.effectiveBalance / EFFECTIVE_BALANCE_INCREMENT);
 
-      if (isActiveValidator(validator, previousEpoch)) {
+      // we only need to track active indices for previous, current and next epoch if we have to compute shufflings
+      // skip doing that if we already have cached shufflings
+      if (cachedPreviousShuffling == null && isActiveValidator(validator, previousEpoch)) {
         previousActiveIndices.push(i);
       }
-      if (isActiveValidator(validator, currentEpoch)) {
+      if (cachedCurrentShuffling == null && isActiveValidator(validator, currentEpoch)) {
         currentActiveIndices.push(i);
         // We track totalActiveBalanceIncrements as ETH to fit total network balance in a JS number (53 bits)
         totalActiveBalanceIncrements += effectiveBalanceIncrements[i];
       }
-      if (isActiveValidator(validator, nextEpoch)) {
+      if (cachedNextShuffling == null && isActiveValidator(validator, nextEpoch)) {
         nextActiveIndices.push(i);
       }
 
@@ -317,11 +329,11 @@ export class EpochCache {
       throw Error("totalActiveBalanceIncrements >= Number.MAX_SAFE_INTEGER. MAX_EFFECTIVE_BALANCE is too low.");
     }
 
-    const currentShuffling = computeEpochShuffling(state, currentActiveIndices, currentEpoch);
-    const previousShuffling = isGenesis
-      ? currentShuffling
-      : computeEpochShuffling(state, previousActiveIndices, previousEpoch);
-    const nextShuffling = computeEpochShuffling(state, nextActiveIndices, nextEpoch);
+    const currentShuffling = cachedCurrentShuffling ?? computeEpochShuffling(state, currentActiveIndices, currentEpoch);
+    const previousShuffling =
+      cachedPreviousShuffling ??
+      (isGenesis ? currentShuffling : computeEpochShuffling(state, previousActiveIndices, previousEpoch));
+    const nextShuffling = cachedNextShuffling ?? computeEpochShuffling(state, nextActiveIndices, nextEpoch);
 
     const currentProposerSeed = getSeed(state, currentEpoch, DOMAIN_BEACON_PROPOSER);
 

packages/state-transition/src/cache/stateCache.ts

@@ -1,4 +1,7 @@
+import bls from "@chainsafe/bls";
+import {CoordType} from "@chainsafe/blst";
 import {BeaconConfig} from "@lodestar/config";
+import {loadState} from "../util/loadState/loadState.js";
 import {EpochCache, EpochCacheImmutableData, EpochCacheOpts} from "./epochCache.js";
 import {
   BeaconStateAllForks,
@@ -137,13 +140,49 @@ export function createCachedBeaconState<T extends BeaconStateAllForks>(
   immutableData: EpochCacheImmutableData,
   opts?: EpochCacheOpts
 ): T & BeaconStateCache {
-  return getCachedBeaconState(state, {
+  const epochCache = EpochCache.createFromState(state, immutableData, opts);
+  const cachedState = getCachedBeaconState(state, {
     config: immutableData.config,
-    epochCtx: EpochCache.createFromState(state, immutableData, opts),
+    epochCtx: epochCache,
     clonedCount: 0,
     clonedCountWithTransferCache: 0,
     createdWithTransferCache: false,
   });
+
+  return cachedState;
+}
+
+/**
+ * Create a CachedBeaconState given a cached seed state and state bytes
+ * This guarantees that the returned state shares the same tree with the seed state
+ * Check loadState() api for more details
+ * TODO: after EIP-6110 need to provide a pivotValidatorIndex to decide which comes to finalized validators cache, which comes to unfinalized cache
+ */
+export function loadUnfinalizedCachedBeaconState<T extends BeaconStateAllForks & BeaconStateCache>(
+  cachedSeedState: T,
+  stateBytes: Uint8Array,
+  opts?: EpochCacheOpts
+): T {
+  const {state: migratedState, modifiedValidators} = loadState(cachedSeedState.config, cachedSeedState, stateBytes);
+  const {pubkey2index, index2pubkey} = cachedSeedState.epochCtx;
+  // Get the validators sub tree once for all the loop
+  const validators = migratedState.validators;
+  for (const validatorIndex of modifiedValidators) {
+    const validator = validators.getReadonly(validatorIndex);
+    const pubkey = validator.pubkey;
+    pubkey2index.set(pubkey, validatorIndex);
+    index2pubkey[validatorIndex] = bls.PublicKey.fromBytes(pubkey, CoordType.jacobian);
+  }
+
+  return createCachedBeaconState(
+    migratedState,
+    {
+      config: cachedSeedState.config,
+      pubkey2index,
+      index2pubkey,
+    },
+    {...(opts ?? {}), ...{skipSyncPubkeys: true}}
+  ) as T;
 }
 
 /**

packages/state-transition/src/cache/types.ts

@@ -1,5 +1,6 @@
 import {CompositeViewDU} from "@chainsafe/ssz";
-import {ssz} from "@lodestar/types";
+import {Epoch, RootHex, ssz} from "@lodestar/types";
+import {EpochShuffling} from "../util/epochShuffling.js";
 
 export type BeaconStatePhase0 = CompositeViewDU<typeof ssz.phase0.BeaconState>;
 export type BeaconStateAltair = CompositeViewDU<typeof ssz.altair.BeaconState>;
@@ -20,3 +21,5 @@ export type BeaconStateAllForks =
   | BeaconStateDeneb;
 
 export type BeaconStateExecutions = BeaconStateBellatrix | BeaconStateCapella | BeaconStateDeneb;
+
+export type ShufflingGetter = (shufflingEpoch: Epoch, dependentRoot: RootHex) => EpochShuffling | null;

packages/state-transition/src/index.ts

@@ -25,6 +25,7 @@ export type {
 // Main state caches
 export {
   createCachedBeaconState,
+  loadUnfinalizedCachedBeaconState,
   type BeaconStateCache,
   isCachedBeaconState,
   isStateBalancesNodesPopulated,

packages/state-transition/src/util/epochShuffling.ts

@@ -1,4 +1,5 @@
-import {Epoch, ValidatorIndex} from "@lodestar/types";
+import {toHexString} from "@chainsafe/ssz";
+import {Epoch, RootHex, ValidatorIndex} from "@lodestar/types";
 import {intDiv} from "@lodestar/utils";
 import {
   DOMAIN_BEACON_ATTESTER,
@@ -9,6 +10,8 @@ import {
 import {BeaconStateAllForks} from "../types.js";
 import {getSeed} from "./seed.js";
 import {unshuffleList} from "./shuffle.js";
+import {computeStartSlotAtEpoch} from "./epoch.js";
+import {getBlockRootAtSlot} from "./blockRoot.js";
 
 /**
  * Readonly interface for EpochShuffling.
@@ -95,3 +98,8 @@ export function computeEpochShuffling(
     committeesPerSlot,
   };
 }
+
+export function getShufflingDecisionBlock(state: BeaconStateAllForks, epoch: Epoch): RootHex {
+  const pivotSlot = computeStartSlotAtEpoch(epoch - 1) - 1;
+  return toHexString(getBlockRootAtSlot(state, pivotSlot));
+}

packages/state-transition/src/util/loadState/findModifiedInactivityScores.ts

@@ -0,0 +1,47 @@
+// UintNum64 = 8 bytes
+export const INACTIVITY_SCORE_SIZE = 8;
+
+/**
+ * As monitored on mainnet, inactivityScores are not changed much and they are mostly 0
+ * Using Buffer.compare is the fastest way as noted in `./findModifiedValidators.ts`
+ * @returns output parameter modifiedValidators: validator indices that are modified
+ */
+export function findModifiedInactivityScores(
+  inactivityScoresBytes: Uint8Array,
+  inactivityScoresBytes2: Uint8Array,
+  modifiedValidators: number[],
+  validatorOffset = 0
+): void {
+  if (inactivityScoresBytes.length !== inactivityScoresBytes2.length) {
+    throw new Error(
+      "inactivityScoresBytes.length !== inactivityScoresBytes2.length " +
+        inactivityScoresBytes.length +
+        " vs " +
+        inactivityScoresBytes2.length
+    );
+  }
+
+  if (Buffer.compare(inactivityScoresBytes, inactivityScoresBytes2) === 0) {
+    return;
+  }
+
+  if (inactivityScoresBytes.length === INACTIVITY_SCORE_SIZE) {
+    modifiedValidators.push(validatorOffset);
+    return;
+  }
+
+  const numValidator = Math.floor(inactivityScoresBytes.length / INACTIVITY_SCORE_SIZE);
+  const halfValidator = Math.floor(numValidator / 2);
+  findModifiedInactivityScores(
+    inactivityScoresBytes.subarray(0, halfValidator * INACTIVITY_SCORE_SIZE),
+    inactivityScoresBytes2.subarray(0, halfValidator * INACTIVITY_SCORE_SIZE),
+    modifiedValidators,
+    validatorOffset
+  );
+  findModifiedInactivityScores(
+    inactivityScoresBytes.subarray(halfValidator * INACTIVITY_SCORE_SIZE),
+    inactivityScoresBytes2.subarray(halfValidator * INACTIVITY_SCORE_SIZE),
+    modifiedValidators,
+    validatorOffset + halfValidator
+  );
+}

packages/state-transition/src/util/loadState/findModifiedValidators.ts

@@ -0,0 +1,46 @@
+import {VALIDATOR_BYTES_SIZE} from "../sszBytes.js";
+
+/**
+ * Find modified validators by comparing two validators bytes using Buffer.compare() recursively
+ * - As noted in packages/state-transition/test/perf/util/loadState/findModifiedValidators.test.ts, serializing validators and compare Uint8Array is the fastest way
+ * - The performance is quite stable and can afford a lot of difference in validators (the benchmark tested up to 10k but it's not likely we have that difference in mainnet)
+ * - Also packages/state-transition/test/perf/misc/byteArrayEquals.test.ts shows that Buffer.compare() is very efficient for large Uint8Array
+ *
+ * @returns output parameter modifiedValidators: validator indices that are modified
+ */
+export function findModifiedValidators(
+  validatorsBytes: Uint8Array,
+  validatorsBytes2: Uint8Array,
+  modifiedValidators: number[],
+  validatorOffset = 0
+): void {
+  if (validatorsBytes.length !== validatorsBytes2.length) {
+    throw new Error(
+      "validatorsBytes.length !== validatorsBytes2.length " + validatorsBytes.length + " vs " + validatorsBytes2.length
+    );
+  }
+
+  if (Buffer.compare(validatorsBytes, validatorsBytes2) === 0) {
+    return;
+  }
+
+  if (validatorsBytes.length === VALIDATOR_BYTES_SIZE) {
+    modifiedValidators.push(validatorOffset);
+    return;
+  }
+
+  const numValidator = Math.floor(validatorsBytes.length / VALIDATOR_BYTES_SIZE);
+  const halfValidator = Math.floor(numValidator / 2);
+  findModifiedValidators(
+    validatorsBytes.subarray(0, halfValidator * VALIDATOR_BYTES_SIZE),
+    validatorsBytes2.subarray(0, halfValidator * VALIDATOR_BYTES_SIZE),
+    modifiedValidators,
+    validatorOffset
+  );
+  findModifiedValidators(
+    validatorsBytes.subarray(halfValidator * VALIDATOR_BYTES_SIZE),
+    validatorsBytes2.subarray(halfValidator * VALIDATOR_BYTES_SIZE),
+    modifiedValidators,
+    validatorOffset + halfValidator
+  );
+}