diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index d36091ee27c2..12a79b42f35d 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -146,7 +146,7 @@ "multiformats": "^11.0.1", "prom-client": "^15.1.0", "qs": "^6.11.1", - "@chainsafe/snappy-wasm": "^0.5.0", + "snappyjs": "^0.7.0", "strict-event-emitter-types": "^2.0.0", "systeminformation": "^5.22.9", "uint8arraylist": "^2.4.7", diff --git a/packages/beacon-node/src/network/gossip/encoding.ts b/packages/beacon-node/src/network/gossip/encoding.ts index 663de40c0a9d..f6121bf8af91 100644 --- a/packages/beacon-node/src/network/gossip/encoding.ts +++ b/packages/beacon-node/src/network/gossip/encoding.ts @@ -1,10 +1,10 @@ import {digest} from "@chainsafe/as-sha256"; import {RPC} from "@chainsafe/libp2p-gossipsub/message"; import {DataTransform} from "@chainsafe/libp2p-gossipsub/types"; -import snappyWasm from "@chainsafe/snappy-wasm"; import {Message} from "@libp2p/interface"; import {ForkName} from "@lodestar/params"; import {intToBytes} from "@lodestar/utils"; +import {compress, uncompress} from "snappyjs"; import xxhashFactory from "xxhash-wasm"; import {MESSAGE_DOMAIN_VALID_SNAPPY} from "./constants.js"; import {GossipTopicCache, getGossipSSZType} from "./topic.js"; @@ -15,10 +15,6 @@ const xxhash = await xxhashFactory(); // Use salt to prevent msgId from being mined for collisions const h64Seed = BigInt(Math.floor(Math.random() * 1e9)); -// create singleton snappy encoder + decoder -const encoder = new snappyWasm.Encoder(); -const decoder = new snappyWasm.Decoder(); - // Shared buffer to convert msgId to string const sharedMsgIdBuf = Buffer.alloc(20); @@ -83,12 +79,11 @@ export class DataTransformSnappy implements DataTransform { * - `outboundTransform()`: compress snappy payload */ inboundTransform(topicStr: string, data: Uint8Array): Uint8Array { - // check uncompressed data length before we actually decompress - const uncompressedDataLength = snappyWasm.decompress_len(data); - if (uncompressedDataLength > this.maxSizePerMessage) { - throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${this.maxSizePerMessage}`); - } + const uncompressedData = uncompress(data, this.maxSizePerMessage); + // check uncompressed data length before we extract beacon block root, slot or + // attestation data at later steps + const uncompressedDataLength = uncompressedData.length; const topic = this.gossipTopicCache.getTopic(topicStr); const sszType = getGossipSSZType(topic); @@ -99,9 +94,6 @@ export class DataTransformSnappy implements DataTransform { throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${sszType.maxSize}`); } - // Only after saniy length checks, we can decompress the data - const uncompressedData = Buffer.allocUnsafe(uncompressedDataLength); - decoder.decompress_into(data, uncompressedData); return uncompressedData; } @@ -109,14 +101,11 @@ export class DataTransformSnappy implements DataTransform { * Takes the data to be published (a topic and associated data) transforms the data. The * transformed data will then be used to create a `RawGossipsubMessage` to be sent to peers. */ - // No need to parse topic, everything is snappy compressed outboundTransform(_topicStr: string, data: Uint8Array): Uint8Array { if (data.length > this.maxSizePerMessage) { throw Error(`ssz_snappy encoded data length ${data.length} > ${this.maxSizePerMessage}`); } - - const compressedData = Buffer.allocUnsafe(snappyWasm.max_compress_len(data.length)); - const compressedLen = encoder.compress_into(data, compressedData); - return compressedData.subarray(0, compressedLen); + // No need to parse topic, everything is snappy compressed + return compress(data); } } diff --git a/packages/beacon-node/test/perf/network/gossip/snappy.test.ts b/packages/beacon-node/test/perf/network/gossip/snappy.test.ts deleted file mode 100644 index 8ee7e13ff690..000000000000 --- a/packages/beacon-node/test/perf/network/gossip/snappy.test.ts +++ /dev/null @@ -1,189 +0,0 @@ -import {randomBytes} from "node:crypto"; -import {bench, describe} from "@chainsafe/benchmark"; -import snappyWasm from "@chainsafe/snappy-wasm"; -import * as snappy from "snappy"; -import * as snappyjs from "snappyjs"; - -/* 2024-08-05 - Linux 5.15 x86_64 - Node.js v22.4.1 - - network / gossip / snappy - compress - ✔ 100 bytes - compress - snappyjs 335566.9 ops/s 2.980032 us/op - 685 runs 2.54 s - ✔ 100 bytes - compress - snappy 388610.3 ops/s 2.573272 us/op - 870 runs 2.74 s - ✔ 100 bytes - compress - snappy-wasm 583254.0 ops/s 1.714519 us/op - 476 runs 1.32 s - ✔ 100 bytes - compress - snappy-wasm - prealloc 1586695 ops/s 630.2410 ns/op - 481 runs 0.804 s - ✔ 200 bytes - compress - snappyjs 298272.8 ops/s 3.352636 us/op - 213 runs 1.22 s - ✔ 200 bytes - compress - snappy 419528.0 ops/s 2.383631 us/op - 926 runs 2.71 s - ✔ 200 bytes - compress - snappy-wasm 472468.5 ops/s 2.116543 us/op - 577 runs 1.72 s - ✔ 200 bytes - compress - snappy-wasm - prealloc 1430445 ops/s 699.0830 ns/op - 868 runs 1.11 s - ✔ 300 bytes - compress - snappyjs 265124.9 ops/s 3.771807 us/op - 137 runs 1.02 s - ✔ 300 bytes - compress - snappy 361683.9 ops/s 2.764845 us/op - 1332 runs 4.18 s - ✔ 300 bytes - compress - snappy-wasm 443688.4 ops/s 2.253834 us/op - 859 runs 2.44 s - ✔ 300 bytes - compress - snappy-wasm - prealloc 1213825 ops/s 823.8420 ns/op - 370 runs 0.807 s - ✔ 400 bytes - compress - snappyjs 262168.5 ops/s 3.814341 us/op - 358 runs 1.87 s - ✔ 400 bytes - compress - snappy 382494.9 ops/s 2.614414 us/op - 1562 runs 4.58 s - ✔ 400 bytes - compress - snappy-wasm 406373.2 ops/s 2.460792 us/op - 797 runs 2.46 s - ✔ 400 bytes - compress - snappy-wasm - prealloc 1111715 ops/s 899.5110 ns/op - 450 runs 0.906 s - ✔ 500 bytes - compress - snappyjs 229213.1 ops/s 4.362753 us/op - 359 runs 2.07 s - ✔ 500 bytes - compress - snappy 373695.8 ops/s 2.675973 us/op - 2050 runs 5.99 s - ✔ 500 bytes - compress - snappy-wasm 714917.4 ops/s 1.398763 us/op - 960 runs 1.84 s - ✔ 500 bytes - compress - snappy-wasm - prealloc 1054619 ops/s 948.2100 ns/op - 427 runs 0.907 s - ✔ 1000 bytes - compress - snappyjs 148702.3 ops/s 6.724847 us/op - 171 runs 1.65 s - ✔ 1000 bytes - compress - snappy 423688.1 ops/s 2.360227 us/op - 525 runs 1.74 s - ✔ 1000 bytes - compress - snappy-wasm 524350.6 ops/s 1.907121 us/op - 273 runs 1.03 s - ✔ 1000 bytes - compress - snappy-wasm - prealloc 685191.5 ops/s 1.459446 us/op - 349 runs 1.01 s - ✔ 10000 bytes - compress - snappyjs 21716.92 ops/s 46.04704 us/op - 16 runs 1.24 s - ✔ 10000 bytes - compress - snappy 98051.32 ops/s 10.19874 us/op - 184 runs 2.39 s - ✔ 10000 bytes - compress - snappy-wasm 114681.8 ops/s 8.719783 us/op - 49 runs 0.937 s - ✔ 10000 bytes - compress - snappy-wasm - prealloc 111203.6 ops/s 8.992518 us/op - 49 runs 0.953 s - ✔ 100000 bytes - compress - snappyjs 2947.313 ops/s 339.2921 us/op - 12 runs 4.74 s - ✔ 100000 bytes - compress - snappy 14963.78 ops/s 66.82801 us/op - 70 runs 5.19 s - ✔ 100000 bytes - compress - snappy-wasm 19868.33 ops/s 50.33136 us/op - 14 runs 1.21 s - ✔ 100000 bytes - compress - snappy-wasm - prealloc 24579.34 ops/s 40.68457 us/op - 13 runs 1.06 s - uncompress - ✔ 100 bytes - uncompress - snappyjs 589201.6 ops/s 1.697212 us/op - 242 runs 0.911 s - ✔ 100 bytes - uncompress - snappy 537424.1 ops/s 1.860728 us/op - 220 runs 0.910 s - ✔ 100 bytes - uncompress - snappy-wasm 634966.2 ops/s 1.574887 us/op - 194 runs 0.808 s - ✔ 100 bytes - uncompress - snappy-wasm - prealloc 1846964 ops/s 541.4290 ns/op - 559 runs 0.804 s - ✔ 200 bytes - uncompress - snappyjs 395141.8 ops/s 2.530737 us/op - 281 runs 1.22 s - ✔ 200 bytes - uncompress - snappy 536862.6 ops/s 1.862674 us/op - 274 runs 1.01 s - ✔ 200 bytes - uncompress - snappy-wasm 420251.6 ops/s 2.379527 us/op - 129 runs 0.810 s - ✔ 200 bytes - uncompress - snappy-wasm - prealloc 1746167 ops/s 572.6830 ns/op - 529 runs 0.804 s - ✔ 300 bytes - uncompress - snappyjs 441676.2 ops/s 2.264102 us/op - 898 runs 2.53 s - ✔ 300 bytes - uncompress - snappy 551313.2 ops/s 1.813851 us/op - 336 runs 1.11 s - ✔ 300 bytes - uncompress - snappy-wasm 494773.0 ops/s 2.021129 us/op - 203 runs 0.912 s - ✔ 300 bytes - uncompress - snappy-wasm - prealloc 1528680 ops/s 654.1590 ns/op - 465 runs 0.805 s - ✔ 400 bytes - uncompress - snappyjs 383746.1 ops/s 2.605890 us/op - 235 runs 1.11 s - ✔ 400 bytes - uncompress - snappy 515986.6 ops/s 1.938035 us/op - 158 runs 0.809 s - ✔ 400 bytes - uncompress - snappy-wasm 392947.8 ops/s 2.544867 us/op - 322 runs 1.32 s - ✔ 400 bytes - uncompress - snappy-wasm - prealloc 1425978 ops/s 701.2730 ns/op - 721 runs 1.01 s - ✔ 500 bytes - uncompress - snappyjs 330727.5 ops/s 3.023637 us/op - 173 runs 1.02 s - ✔ 500 bytes - uncompress - snappy 513874.1 ops/s 1.946002 us/op - 157 runs 0.806 s - ✔ 500 bytes - uncompress - snappy-wasm 389263.0 ops/s 2.568957 us/op - 161 runs 0.914 s - ✔ 500 bytes - uncompress - snappy-wasm - prealloc 1330936 ops/s 751.3510 ns/op - 672 runs 1.01 s - ✔ 1000 bytes - uncompress - snappyjs 241393.9 ops/s 4.142606 us/op - 126 runs 1.03 s - ✔ 1000 bytes - uncompress - snappy 491119.6 ops/s 2.036164 us/op - 201 runs 0.911 s - ✔ 1000 bytes - uncompress - snappy-wasm 361794.5 ops/s 2.764000 us/op - 148 runs 0.910 s - ✔ 1000 bytes - uncompress - snappy-wasm - prealloc 959026.5 ops/s 1.042724 us/op - 390 runs 0.909 s - ✔ 10000 bytes - uncompress - snappyjs 40519.03 ops/s 24.67976 us/op - 16 runs 0.913 s - ✔ 10000 bytes - uncompress - snappy 202537.6 ops/s 4.937355 us/op - 796 runs 4.43 s - ✔ 10000 bytes - uncompress - snappy-wasm 165017.6 ops/s 6.059960 us/op - 52 runs 0.822 s - ✔ 10000 bytes - uncompress - snappy-wasm - prealloc 175061.5 ops/s 5.712277 us/op - 130 runs 1.25 s - ✔ 100000 bytes - uncompress - snappyjs 4030.391 ops/s 248.1149 us/op - 12 runs 3.71 s - ✔ 100000 bytes - uncompress - snappy 35459.43 ops/s 28.20124 us/op - 41 runs 1.67 s - ✔ 100000 bytes - uncompress - snappy-wasm 22449.16 ops/s 44.54509 us/op - 13 runs 1.11 s - ✔ 100000 bytes - uncompress - snappy-wasm - prealloc 27169.50 ops/s 36.80598 us/op - 13 runs 0.997 s - -*/ - -describe("network / gossip / snappy", () => { - const msgLens = [ - // -> - 100, - 200, - 300, - 400, - 500, - 1000, - 10000, // 100000, - ]; - describe("compress", () => { - const encoder = new snappyWasm.Encoder(); - - for (const msgLen of msgLens) { - const uncompressed = randomBytes(msgLen); - const RUNS_FACTOR = 1000; - - bench({ - id: `${msgLen} bytes - compress - snappyjs`, - runsFactor: RUNS_FACTOR, - fn: () => { - for (let i = 0; i < RUNS_FACTOR; i++) { - snappyjs.compress(uncompressed); - } - }, - }); - - bench({ - id: `${msgLen} bytes - compress - snappy`, - runsFactor: RUNS_FACTOR, - fn: () => { - for (let i = 0; i < RUNS_FACTOR; i++) { - snappy.compressSync(uncompressed); - } - }, - }); - - bench({ - id: `${msgLen} bytes - compress - snappy-wasm`, - runsFactor: RUNS_FACTOR, - fn: () => { - for (let i = 0; i < RUNS_FACTOR; i++) { - encoder.compress(uncompressed); - } - }, - }); - - bench({ - id: `${msgLen} bytes - compress - snappy-wasm - prealloc`, - runsFactor: RUNS_FACTOR, - fn: () => { - for (let i = 0; i < RUNS_FACTOR; i++) { - let out = Buffer.allocUnsafe(snappyWasm.max_compress_len(uncompressed.length)); - const len = encoder.compress_into(uncompressed, out); - out = out.subarray(0, len); - } - }, - }); - } - }); - describe("uncompress", () => { - const decoder = new snappyWasm.Decoder(); - - for (const msgLen of msgLens) { - const uncompressed = randomBytes(msgLen); - const compressed = snappyjs.compress(uncompressed); - const RUNS_FACTOR = 1000; - - bench({ - id: `${msgLen} bytes - uncompress - snappyjs`, - runsFactor: RUNS_FACTOR, - fn: () => { - for (let i = 0; i < RUNS_FACTOR; i++) { - snappyjs.uncompress(compressed); - } - }, - }); - - bench({ - id: `${msgLen} bytes - uncompress - snappy`, - runsFactor: RUNS_FACTOR, - fn: () => { - for (let i = 0; i < RUNS_FACTOR; i++) { - snappy.uncompressSync(compressed); - } - }, - }); - - bench({ - id: `${msgLen} bytes - uncompress - snappy-wasm`, - runsFactor: RUNS_FACTOR, - fn: () => { - for (let i = 0; i < RUNS_FACTOR; i++) { - decoder.decompress(compressed); - } - }, - }); - - bench({ - id: `${msgLen} bytes - uncompress - snappy-wasm - prealloc`, - runsFactor: RUNS_FACTOR, - fn: () => { - for (let i = 0; i < RUNS_FACTOR; i++) { - decoder.decompress_into(compressed, Buffer.allocUnsafe(snappyWasm.decompress_len(compressed))); - } - }, - }); - } - }); -}); diff --git a/yarn.lock b/yarn.lock index 509ca922453e..5b07801cf692 100644 --- a/yarn.lock +++ b/yarn.lock @@ -750,11 +750,6 @@ "@chainsafe/pubkey-index-map-linux-x64-gnu" "2.0.0" "@chainsafe/pubkey-index-map-win32-x64-msvc" "2.0.0" -"@chainsafe/snappy-wasm@^0.5.0": - version "0.5.0" - resolved "https://registry.yarnpkg.com/@chainsafe/snappy-wasm/-/snappy-wasm-0.5.0.tgz#067e534341ef746706e2dbf255bd7604c849be73" - integrity sha512-ydXvhr9p+JjvzSSEyi6XExq8pHugFnrk70mk17T6mhDsklPvaXc+8K90G7TJF+u51lxI/fpv7MahrA5ayjFcSA== - "@chainsafe/ssz@^0.11.1": version "0.11.1" resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.11.1.tgz#d4aec883af2ec5196ae67b96242c467da20b2476"