chore: benchmark findModifiedValidators()

packages/state-transition/src/cache/stateCache.ts

@@ -1,7 +1,7 @@
 import bls from "@chainsafe/bls";
 import {CoordType} from "@chainsafe/blst";
 import {BeaconConfig} from "@lodestar/config";
-import {loadState} from "../util/loadState.js";
+import {loadState} from "../util/loadState/loadState.js";
 import {EpochCache, EpochCacheImmutableData, EpochCacheOpts} from "./epochCache.js";
 import {
   BeaconStateAllForks,

packages/state-transition/src/util/loadState/findModifiedInactivityScores.ts

@@ -0,0 +1,47 @@
+// UintNum64 = 8 bytes
+export const INACTIVITY_SCORE_SIZE = 8;
+
+/**
+ * As monitored on mainnet, inactivityScores are not changed much and they are mostly 0
+ * Using Buffer.compare is the fastest way as noted in `./findModifiedValidators.ts`
+ * @returns output parameter modifiedValidators: validator indices that are modified
+ */
+export function findModifiedInactivityScores(
+  inactivityScoresBytes: Uint8Array,
+  inactivityScoresBytes2: Uint8Array,
+  modifiedValidators: number[],
+  validatorOffset = 0
+): void {
+  if (inactivityScoresBytes.length !== inactivityScoresBytes2.length) {
+    throw new Error(
+      "inactivityScoresBytes.length !== inactivityScoresBytes2.length " +
+        inactivityScoresBytes.length +
+        " vs " +
+        inactivityScoresBytes2.length
+    );
+  }
+
+  if (Buffer.compare(inactivityScoresBytes, inactivityScoresBytes2) === 0) {
+    return;
+  }
+
+  if (inactivityScoresBytes.length === INACTIVITY_SCORE_SIZE) {
+    modifiedValidators.push(validatorOffset);
+    return;
+  }
+
+  const numValidator = Math.floor(inactivityScoresBytes.length / INACTIVITY_SCORE_SIZE);
+  const halfValidator = Math.floor(numValidator / 2);
+  findModifiedInactivityScores(
+    inactivityScoresBytes.subarray(0, halfValidator * INACTIVITY_SCORE_SIZE),
+    inactivityScoresBytes2.subarray(0, halfValidator * INACTIVITY_SCORE_SIZE),
+    modifiedValidators,
+    validatorOffset
+  );
+  findModifiedInactivityScores(
+    inactivityScoresBytes.subarray(halfValidator * INACTIVITY_SCORE_SIZE),
+    inactivityScoresBytes2.subarray(halfValidator * INACTIVITY_SCORE_SIZE),
+    modifiedValidators,
+    validatorOffset + halfValidator
+  );
+}

packages/state-transition/src/util/loadState/findModifiedValidators.ts

@@ -0,0 +1,46 @@
+import {VALIDATOR_BYTES_SIZE} from "../sszBytes.js";
+
+/**
+ * Find modified validators by comparing two validators bytes using Buffer.compare() recursively
+ * - As noted in packages/state-transition/test/perf/util/loadState/findModifiedValidators.test.ts, serializing validators and compare Uint8Array is the fastest way
+ * - The performance is quite stable and can afford a lot of difference in validators (the benchmark tested up to 10k but it's not likely we have that difference in mainnet)
+ * - Also packages/state-transition/test/perf/misc/byteArrayEquals.test.ts shows that Buffer.compare() is very efficient for large Uint8Array
+ *
+ * @returns output parameter modifiedValidators: validator indices that are modified
+ */
+export function findModifiedValidators(
+  validatorsBytes: Uint8Array,
+  validatorsBytes2: Uint8Array,
+  modifiedValidators: number[],
+  validatorOffset = 0
+): void {
+  if (validatorsBytes.length !== validatorsBytes2.length) {
+    throw new Error(
+      "validatorsBytes.length !== validatorsBytes2.length " + validatorsBytes.length + " vs " + validatorsBytes2.length
+    );
+  }
+
+  if (Buffer.compare(validatorsBytes, validatorsBytes2) === 0) {
+    return;
+  }
+
+  if (validatorsBytes.length === VALIDATOR_BYTES_SIZE) {
+    modifiedValidators.push(validatorOffset);
+    return;
+  }
+
+  const numValidator = Math.floor(validatorsBytes.length / VALIDATOR_BYTES_SIZE);
+  const halfValidator = Math.floor(numValidator / 2);
+  findModifiedValidators(
+    validatorsBytes.subarray(0, halfValidator * VALIDATOR_BYTES_SIZE),
+    validatorsBytes2.subarray(0, halfValidator * VALIDATOR_BYTES_SIZE),
+    modifiedValidators,
+    validatorOffset
+  );
+  findModifiedValidators(
+    validatorsBytes.subarray(halfValidator * VALIDATOR_BYTES_SIZE),
+    validatorsBytes2.subarray(halfValidator * VALIDATOR_BYTES_SIZE),
+    modifiedValidators,
+    validatorOffset + halfValidator
+  );
+}

packages/state-transition/src/util/loadState.ts → packages/state-transition/src/util/loadState/loadState.ts

@@ -2,8 +2,10 @@ import {CompositeTypeAny, Type} from "@chainsafe/ssz";
 import {ssz} from "@lodestar/types";
 import {ForkSeq} from "@lodestar/params";
 import {ChainForkConfig} from "@lodestar/config";
-import {BeaconStateAllForks, BeaconStateAltair, BeaconStatePhase0} from "../types.js";
-import {VALIDATOR_BYTES_SIZE, getForkFromStateBytes, getStateTypeFromBytes} from "./sszBytes.js";
+import {BeaconStateAllForks, BeaconStateAltair, BeaconStatePhase0} from "../../types.js";
+import {VALIDATOR_BYTES_SIZE, getForkFromStateBytes, getStateTypeFromBytes} from "../sszBytes.js";
+import {findModifiedValidators} from "./findModifiedValidators.js";
+import {findModifiedInactivityScores} from "./findModifiedInactivityScores.js";
 
 type BeaconStateType =
   | typeof ssz.phase0.BeaconState
@@ -80,8 +82,26 @@ export function loadState(
   return {state: migratedState, modifiedValidators};
 }
 
-// state store inactivity scores of old seed state, we need to update it
-// this value rarely changes even after 3 months of data as monitored on mainnet in Sep 2023
+/**
+ * This value is rarely changed as monitored 3 month state diffs on mainnet as of Sep 2023.
+ * Reusing this data helps save hashTreeRoot time of state ~500ms
+ *
+ * Given the below tree:
+ *
+ * seedState.inactivityScores ====>  ROOT
+ *                                 /     \
+ *                            Hash01       Hash23
+ *                           /    \       /    \
+ *                       Sco0    Sco1   Sco2   Sco3
+ *
+ * if score 3 is modified, the new tree looks like this:
+ *
+ * migratedState.inactivityScores ====> ROOTa
+ *                                     /      \
+ *                                Hash01      Hash23a
+ *                               /    \       /    \
+ *                           Sco0    Sco1  Sco2   Sco3a
+ */
 function loadInactivityScores(
   migratedState: BeaconStateAltair,
   seedState: BeaconStateAltair,
@@ -125,6 +145,35 @@ function loadInactivityScores(
   }
 }
 
+/**
+ * As of Sep 2021, common validators of 2 mainnet states are rarely changed. However, the benchmark shows that
+ * 10k modified validators is not an issue. (see packages/state-transition/test/perf/util/loadState/findModifiedValidators.test.ts)
+ *
+ * This method loads validators from bytes given a seed state so that they share the same base tree. This gives some benefits:
+ *  - Have single base tree across the application
+ *  - Faster to load state
+ *  - Less memory usage
+ *  - Ultilize the cached HashObjects in seed state due to a lot of validators are not changed
+ *
+ * Given the below tree:
+ *
+ * seedState.validators ====>  ROOT
+ *                            /     \
+ *                       Hash01       Hash23
+ *                      /    \       /    \
+ *                  Val0    Val1   Val2   Val3
+ *
+ * if validator 3 is modified, the new tree looks like this:
+ *
+ * migratedState.validators ====>  ROOTa
+ *                               /      \
+ *                          Hash01      Hash23a
+ *                         /    \       /    \
+ *                     Val0    Val1   Val2   Val3a
+ *
+ * @param migratedState state to be migrated, the validators are loaded to this state
+ * @returns modified validator indices
+ */
 function loadValidators(
   migratedState: BeaconStateAllForks,
   seedState: BeaconStateAllForks,
@@ -170,82 +219,3 @@ function loadValidators(
   }
   return modifiedValidators;
 }
-
-function findModifiedValidators(
-  validatorsBytes: Uint8Array,
-  validatorsBytes2: Uint8Array,
-  modifiedValidators: number[],
-  validatorOffset = 0
-): void {
-  if (validatorsBytes.length !== validatorsBytes2.length) {
-    throw new Error(
-      "validatorsBytes.length !== validatorsBytes2.length " + validatorsBytes.length + " vs " + validatorsBytes2.length
-    );
-  }
-
-  if (Buffer.compare(validatorsBytes, validatorsBytes2) === 0) {
-    return;
-  }
-
-  if (validatorsBytes.length === VALIDATOR_BYTES_SIZE) {
-    modifiedValidators.push(validatorOffset);
-    return;
-  }
-
-  const numValidator = Math.floor(validatorsBytes.length / VALIDATOR_BYTES_SIZE);
-  const halfValidator = Math.floor(numValidator / 2);
-  findModifiedValidators(
-    validatorsBytes.subarray(0, halfValidator * VALIDATOR_BYTES_SIZE),
-    validatorsBytes2.subarray(0, halfValidator * VALIDATOR_BYTES_SIZE),
-    modifiedValidators,
-    validatorOffset
-  );
-  findModifiedValidators(
-    validatorsBytes.subarray(halfValidator * VALIDATOR_BYTES_SIZE),
-    validatorsBytes2.subarray(halfValidator * VALIDATOR_BYTES_SIZE),
-    modifiedValidators,
-    validatorOffset + halfValidator
-  );
-}
-
-// as monitored on mainnet, inactivityScores are not changed much and they are mostly 0
-function findModifiedInactivityScores(
-  inactivityScoresBytes: Uint8Array,
-  inactivityScoresBytes2: Uint8Array,
-  modifiedValidators: number[],
-  validatorOffset = 0
-): void {
-  if (inactivityScoresBytes.length !== inactivityScoresBytes2.length) {
-    throw new Error(
-      "inactivityScoresBytes.length !== inactivityScoresBytes2.length " +
-        inactivityScoresBytes.length +
-        " vs " +
-        inactivityScoresBytes2.length
-    );
-  }
-
-  if (Buffer.compare(inactivityScoresBytes, inactivityScoresBytes2) === 0) {
-    return;
-  }
-
-  // UintNum64 = 8 bytes
-  if (inactivityScoresBytes.length === 8) {
-    modifiedValidators.push(validatorOffset);
-    return;
-  }
-
-  const numValidator = Math.floor(inactivityScoresBytes.length / 8);
-  const halfValidator = Math.floor(numValidator / 2);
-  findModifiedInactivityScores(
-    inactivityScoresBytes.subarray(0, halfValidator * 8),
-    inactivityScoresBytes2.subarray(0, halfValidator * 8),
-    modifiedValidators,
-    validatorOffset
-  );
-  findModifiedInactivityScores(
-    inactivityScoresBytes.subarray(halfValidator * 8),
-    inactivityScoresBytes2.subarray(halfValidator * 8),
-    modifiedValidators,
-    validatorOffset + halfValidator
-  );
-}

packages/state-transition/test/perf/misc/byteArrayEquals.test.ts

@@ -0,0 +1,56 @@
+import {itBench} from "@dapplion/benchmark";
+import {byteArrayEquals} from "@chainsafe/ssz";
+import {generateState} from "../../utils/state.js";
+import {generateValidators} from "../../utils/validator.js";
+
+/**
+ *   compare Uint8Array, the longer the array, the better performance Buffer.compare() is
+ *   - with 32 bytes, Buffer.compare() is 1.5x faster (rootEquals.test.ts showed > 2x faster)
+ *    ✔ byteArrayEquals 32                                               1.004480e+7 ops/s    99.55400 ns/op        -      19199 runs   2.08 s
+ *    ✔ Buffer.compare 32                                                1.553495e+7 ops/s    64.37100 ns/op        -       3634 runs  0.303 s
+ *
+ *   - with 1024 bytes, Buffer.compare() is 21.8x faster
+ *    ✔ byteArrayEquals 1024                                                379239.7 ops/s    2.636855 us/op        -        117 runs  0.811 s
+ *    ✔ Buffer.compare 1024                                                  8269999 ops/s    120.9190 ns/op        -       3330 runs  0.525 s
+ *
+ *   - with 16384 bytes, Buffer.compare() is 41x faster
+ *    ✔ byteArrayEquals 16384                                               23808.76 ops/s    42.00135 us/op        -         13 runs   1.05 s
+ *    ✔ Buffer.compare 16384                                                975058.0 ops/s    1.025580 us/op        -        297 runs  0.806 s
+ *
+ *   - with 123687377 bytes, Buffer.compare() is 38x faster
+ *    ✔ byteArrayEquals 123687377                                           3.077884 ops/s    324.8985 ms/op        -          1 runs   64.5 s
+ *    ✔ Buffer.compare 123687377                                            114.7834 ops/s    8.712061 ms/op        -         13 runs   12.1 s
+ */
+describe("compare Uint8Array", () => {
+  const numValidator = 1_000_000;
+  const validators = generateValidators(numValidator);
+  const state = generateState({validators: validators});
+  const stateBytes = state.serialize();
+
+  const lengths = [32, 1024, 16384, stateBytes.length];
+  // const lengths = [stateBytes.length];
+  for (const length of lengths) {
+    const runsFactor = length > 16384 ? 100 : 1000;
+    const bytes = stateBytes.subarray(0, length);
+    const bytes2 = bytes.slice();
+    itBench({
+      id: `byteArrayEquals ${length}`,
+      fn: () => {
+        for (let i = 0; i < runsFactor; i++) {
+          byteArrayEquals(bytes, bytes2);
+        }
+      },
+      runsFactor,
+    });
+
+    itBench({
+      id: `Buffer.compare ${length}`,
+      fn: () => {
+        for (let i = 0; i < runsFactor; i++) {
+          Buffer.compare(bytes, bytes2);
+        }
+      },
+      runsFactor,
+    });
+  }
+});

packages/state-transition/test/perf/misc/rootEquals.test.ts

@@ -2,12 +2,11 @@ import {itBench, setBenchOpts} from "@dapplion/benchmark";
 import {byteArrayEquals, fromHexString} from "@chainsafe/ssz";
 import {ssz} from "@lodestar/types";
 
-// As of Jun 17 2021
-// Compare state root
-// ================================================================
-// ssz.Root.equals                                                        891265.6 ops/s      1.122000 us/op 10017946 runs    15.66 s
-// ssz.Root.equals with valueOf()                                         692041.5 ops/s      1.445000 us/op 8179741 runs    15.28 s
-// byteArrayEquals with valueOf()                                         853971.0 ops/s      1.171000 us/op 9963051 runs    16.07 s
+// As of Sep 2023
+// root equals
+// ✔ ssz.Root.equals                                                  2.703872e+7 ops/s    36.98400 ns/op        -      74234 runs   2.83 s
+// ✔ byteArrayEquals                                                  2.773617e+7 ops/s    36.05400 ns/op        -      15649 runs  0.606 s
+// ✔ Buffer.compare                                                   7.099247e+7 ops/s    14.08600 ns/op        -      26965 runs  0.404 s
 
 describe("root equals", () => {
   setBenchOpts({noThreshold: true});
@@ -16,11 +15,34 @@ describe("root equals", () => {
   const rootTree = ssz.Root.toViewDU(stateRoot);
 
   // This benchmark is very unstable in CI. We already know that "ssz.Root.equals" is the fastest
-  itBench("ssz.Root.equals", () => {
-    ssz.Root.equals(rootTree, stateRoot);
+  const runsFactor = 1000;
+  itBench({
+    id: "ssz.Root.equals",
+    fn: () => {
+      for (let i = 0; i < runsFactor; i++) {
+        ssz.Root.equals(rootTree, stateRoot);
+      }
+    },
+    runsFactor,
   });
 
-  itBench("byteArrayEquals", () => {
-    byteArrayEquals(rootTree, stateRoot);
+  itBench({
+    id: "byteArrayEquals",
+    fn: () => {
+      for (let i = 0; i < runsFactor; i++) {
+        byteArrayEquals(rootTree, stateRoot);
+      }
+    },
+    runsFactor,
+  });
+
+  itBench({
+    id: "Buffer.compare",
+    fn: () => {
+      for (let i = 0; i < runsFactor; i++) {
+        Buffer.compare(rootTree, stateRoot);
+      }
+    },
+    runsFactor,
   });
 });