feat: switch bls to napi rebuild (#6616)

* chore: update package.json deps and run yarn

* feat: remove unused blst config from vitest.browser.*.config.ts

* test: move G2@INF test to base blst repo.  No longer exported.

* fix: add Uint8Array.from for Buffer return value in unit test so type matches for toEqual

* refactor: update imports of CoordType to use bls instead of blst

* feat: add blsAddVerificationRandomness cli option

* feat: add randomBytesNonZero to utils

* feat: implement verification randomness for same message

* feat: implement napi blst and switch to libuv worker pool

* chore: lint

* fix: update comments for availableParallelism

* feat: set UV_THREADPOOL_SIZE for perf tests

* feat: set UV_THREADPOOL_SIZE=$(nproc) in lodestar entrance script

* feat: remove case for serialized PublicKey in deserializeSet

* feat: add back workBusy count logic to canAcceptWork

* feat: automatically set threadpool size in beaconHandler

* refactor: rename flag to disableSameMessageVerificationRandomness

* fix: remove UV_THREADPOOL_SIZE from ./lodestar

* docs: add TODO about updating metrics

* fix: make benchmark multi threading cross-compatible

* chore: add bls@8.0.0

* feat: default to added randomness

* feat: add warning for no same message randomness

* fix: passing to updated bls functions

* feat: add blst peerDep

* fix: default to same message randomness

* chore: update yarn.lock

* chore: remove unused function to clear lint

* feat: break out setThreadPoolSize

* docs: update packages/cli/src/cmds/beacon/setThreadPoolSize.ts

Co-authored-by: Nico Flaig <nflaig@protonmail.com>

* chore: update location for blst dep

* chore: run lint again

* fix: remove peerDep from light-client

* test: fix browser based tests

* fix: bad relative imports

* docs: add note to readme about switchable bls

* docs: typo

* chore: lint readme

* fix: light-client unit tests

* fix: light-client e2e test

* feat: remove disableSameMessageVerificationRandomness

* refactor: remove unused type

* docs: update wordlist

* fix: e2e tests

* Lint package.json

* Revert "fix: e2e tests"

This reverts commit f71acd7.

* Revert "fix: light-client e2e test"

This reverts commit 23c11f0.

* Revert "fix: light-client unit tests"

This reverts commit fa7893e.

* Revert "test: fix browser based tests"

This reverts commit 98d04ab.

* docs: update readme to remove switchable bls argument

* fix: replace optimizeDeps in vitest.browser.config

* chore: updated to bls 8.1.0

* feat: move setting threadpool size and warnings to applyPreset.ts

* fix: remove setThreadpoolSize from beacon handler

* feat: move libuv size logging into handler to use logger

* refactor: rename applyPreset to preInitialization

* refactor: logging about pool size

* Update packages/cli/src/options/globalOptions.ts

Co-authored-by: Nico Flaig <nflaig@protonmail.com>

* refactor: rename prover applyPreset

* docs: change comment in cli/index

* fix: update logging for

* fix: throw error for invalid uvThreadpoolSize

* Update packages/cli/src/preInitialization.ts

Co-authored-by: Nico Flaig <nflaig@protonmail.com>

* Fix lint issues

* Remove unused import

* refactor: move bls pool logging back to multithread.ts

* fix: log of threadcount

---------

Co-authored-by: Nico Flaig <nflaig@protonmail.com>

Author: matthewkeil

.wordlist.txt

@@ -163,6 +163,7 @@ orchestrator
 osx
 overriden
 params
+peerDependency
 pid
 plaintext
 pre

package.json

@@ -36,8 +36,8 @@
     "test-coverage:e2e": "c8 --config .c8rc.json --report-dir coverage/e2e/ --all npm run test:e2e",
     "test-coverage:e2e-sim": "c8 --config .c8rc.json --report-dir coverage/e2e-sim/ --all npm run test:e2e:sim",
     "test-coverage:spec": "c8 --config .c8rc.json --report-dir coverage/spec/ --all npm run test:spec",
-    "benchmark": "yarn benchmark:files 'packages/*/test/perf/**/*.test.ts'",
-    "benchmark:files": "NODE_OPTIONS='--max-old-space-size=4096 --loader=ts-node/esm' benchmark --config .benchrc.yaml --defaultBranch unstable",
+    "benchmark": "UV_THREADPOOL_SIZE=$(node -e 'console.log(require(`os`).availableParallelism())') yarn benchmark:files 'packages/*/test/perf/**/*.test.ts'",
+    "benchmark:files": "UV_THREADPOOL_SIZE=$(node -e 'console.log(require(`os`).availableParallelism())') NODE_OPTIONS='--max-old-space-size=4096 --loader=ts-node/esm' benchmark --config .benchrc.yaml --defaultBranch unstable",
     "release:create-rc": "node scripts/release/create_rc.mjs",
     "release:tag-rc": "node scripts/release/tag_rc.mjs",
     "release:tag-stable": "node scripts/release/tag_stable.mjs",

packages/beacon-node/package.json

@@ -95,8 +95,8 @@
   },
   "dependencies": {
     "@chainsafe/as-sha256": "^0.4.1",
-    "@chainsafe/bls": "7.1.3",
-    "@chainsafe/blst": "^0.2.10",
+    "@chainsafe/bls": "^8.1.0",
+    "@chainsafe/blst": "^1.0.0",
     "@chainsafe/discv5": "^9.0.0",
     "@chainsafe/enr": "^3.0.0",
     "@chainsafe/libp2p-gossipsub": "^11.2.1",

packages/beacon-node/src/chain/bls/index.ts

@@ -1,4 +1,5 @@
 export type {IBlsVerifier} from "./interface.js";
-export type {BlsMultiThreadWorkerPoolModules, JobQueueItemType} from "./multithread/index.js";
-export {BlsMultiThreadWorkerPool} from "./multithread/index.js";
+export type {JobQueueItemType} from "./jobItem.js";
+export type {BlsMultiThreadWorkerPoolModules} from "./multiThread.js";
+export {BlsMultiThreadWorkerPool} from "./multiThread.js";
 export {BlsSingleThreadVerifier} from "./singleThread.js";

packages/beacon-node/src/chain/bls/multithread/jobItem.ts packages/beacon-node/src/chain/bls/jobItem.ts

@@ -1,11 +1,12 @@
 import bls from "@chainsafe/bls";
 import {CoordType, PointFormat, PublicKey} from "@chainsafe/bls/types";
 import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition";
-import {VerifySignatureOpts} from "../interface.js";
-import {getAggregatedPubkey} from "../utils.js";
-import {LinkedList} from "../../../util/array.js";
-import {Metrics} from "../../../metrics/metrics.js";
+import {LinkedList} from "../../util/array.js";
+import {Metrics} from "../../metrics/metrics.js";
+import {VerifySignatureOpts} from "./interface.js";
+import {getAggregatedPubkey} from "./utils.js";
 import {BlsWorkReq} from "./types.js";
+import {randomBytesNonZero} from "./utils.js";
 
 export type JobQueueItem = JobQueueItemDefault | JobQueueItemSameMessage;
 
@@ -56,7 +57,7 @@ export function jobItemWorkReq(job: JobQueueItem, format: PointFormat, metrics:
         opts: job.opts,
         sets: job.sets.map((set) => ({
           // this can throw, handled in the consumer code
-          publicKey: getAggregatedPubkey(set, metrics).toBytes(format),
+          publicKey: getAggregatedPubkey(set, metrics),
           signature: set.signature,
           message: set.signingRoot,
         })),
@@ -73,13 +74,19 @@ export function jobItemWorkReq(job: JobQueueItem, format: PointFormat, metrics:
       const signatures = job.sets.map((set) => bls.Signature.fromBytes(set.signature, CoordType.affine, true));
       timer?.();
 
+      // adding verification randomness is napi specific. must not attempt with herumi until
+      // @chainsafe/bls is updated to support it with herumi
+      const randomness: Uint8Array[] = [];
+      for (let i = 0; i < job.sets.length; i++) {
+        randomness.push(randomBytesNonZero(8));
+      }
       return {
         opts: job.opts,
         sets: [
           {
-            publicKey: bls.PublicKey.aggregate(job.sets.map((set) => set.publicKey)).toBytes(format),
-            signature: bls.Signature.aggregate(signatures).toBytes(format),
             message: job.message,
+            publicKey: bls.PublicKey.aggregate(job.sets.map((set, i) => set.publicKey.multiplyBy(randomness[i]))),
+            signature: bls.Signature.aggregate(signatures.map((sig, i) => sig.multiplyBy(randomness[i]))),
           },
         ],
       };