feat: consume merkleizeBlockArray

Author: twoeths

packages/ssz/src/type/arrayComposite.ts

@@ -211,29 +211,29 @@ export function tree_deserializeFromBytesArrayComposite<ElementType extends Comp
   }
 }
 
-export function value_getChunkBytesArrayComposite<ElementType extends CompositeType<unknown, unknown, unknown>>(
+export function value_getBlocksBytesArrayComposite<ElementType extends CompositeType<unknown, unknown, unknown>>(
   elementType: ElementType,
   length: number,
   value: ValueOf<ElementType>[],
-  chunkBytesBuffer: Uint8Array
+  blocksBuffer: Uint8Array
 ): Uint8Array {
-  const isOddChunk = length % 2 === 1;
-  const chunkBytesLen = isOddChunk ? length * 32 + 32 : length * 32;
-  if (chunkBytesLen > chunkBytesBuffer.length) {
-    throw new Error(`chunkBytesBuffer is too small: ${chunkBytesBuffer.length} < ${chunkBytesLen}`);
+  const blockBytesLen = Math.ceil(length / 2) * 64;
+  if (blockBytesLen > blocksBuffer.length) {
+    throw new Error(`blocksBuffer is too small: ${blocksBuffer.length} < ${blockBytesLen}`);
   }
-  const chunkBytes = chunkBytesBuffer.subarray(0, chunkBytesLen);
+  const blocksBytes = blocksBuffer.subarray(0, blockBytesLen);
 
   for (let i = 0; i < length; i++) {
-    elementType.hashTreeRootInto(value[i], chunkBytes, i * 32);
+    elementType.hashTreeRootInto(value[i], blocksBytes, i * 32);
   }
 
+  const isOddChunk = length % 2 === 1;
   if (isOddChunk) {
     // similar to append zeroHash(0)
-    chunkBytes.subarray(length * 32, chunkBytesLen).fill(0);
+    blocksBytes.subarray(length * 32, blockBytesLen).fill(0);
   }
 
-  return chunkBytes;
+  return blocksBytes;
 }
 
 function readOffsetsArrayComposite(

packages/ssz/src/type/bitArray.ts

@@ -4,7 +4,7 @@ import {CompositeType, LENGTH_GINDEX} from "./composite";
 import {BitArray} from "../value/bitArray";
 import {BitArrayTreeView} from "../view/bitArray";
 import {BitArrayTreeViewDU} from "../viewDU/bitArray";
-import {getChunkBytes} from "./byteArray";
+import {getBlocksBytes} from "./byteArray";
 
 /* eslint-disable @typescript-eslint/member-ordering */
 
@@ -40,15 +40,13 @@ export abstract class BitArrayType extends CompositeType<BitArray, BitArrayTreeV
 
   // Merkleization
 
-  protected getChunkBytes(value: BitArray): Uint8Array {
-    // reallocate this.merkleBytes if needed
-    if (value.uint8Array.length > this.chunkBytesBuffer.length) {
+  protected getBlocksBytes(value: BitArray): Uint8Array {
+    // reallocate this.blocksBuffer if needed
+    if (value.uint8Array.length > this.blocksBuffer.length) {
       const chunkCount = Math.ceil(value.bitLen / 8 / 32);
-      const chunkBytes = chunkCount * 32;
-      // pad 1 chunk if maxChunkCount is not even
-      this.chunkBytesBuffer = chunkCount % 2 === 1 ? new Uint8Array(chunkBytes + 32) : new Uint8Array(chunkBytes);
+      this.blocksBuffer = new Uint8Array(Math.ceil(chunkCount / 2) * 64);
     }
-    return getChunkBytes(value.uint8Array, this.chunkBytesBuffer);
+    return getBlocksBytes(value.uint8Array, this.blocksBuffer);
   }
 
   // Proofs

packages/ssz/src/type/bitList.ts

@@ -1,7 +1,7 @@
 import {allocUnsafe} from "@chainsafe/as-sha256";
 import {
   getNodesAtDepth,
-  merkleizeInto,
+  merkleizeBlocksBytes,
   Node,
   packedNodeRootsToBytes,
   packedRootsBytesToNode,
@@ -36,11 +36,11 @@ export class BitListType extends BitArrayType {
   readonly maxSize: number;
   readonly maxChunkCount: number;
   readonly isList = true;
-  readonly mixInLengthChunkBytes = new Uint8Array(64);
+  readonly mixInLengthBlockBytes = new Uint8Array(64);
   readonly mixInLengthBuffer = Buffer.from(
-    this.mixInLengthChunkBytes.buffer,
-    this.mixInLengthChunkBytes.byteOffset,
-    this.mixInLengthChunkBytes.byteLength
+    this.mixInLengthBlockBytes.buffer,
+    this.mixInLengthBlockBytes.byteOffset,
+    this.mixInLengthBlockBytes.byteLength
   );
 
   constructor(readonly limitBits: number, opts?: BitListOptions) {
@@ -120,12 +120,12 @@ export class BitListType extends BitArrayType {
   }
 
   hashTreeRootInto(value: BitArray, output: Uint8Array, offset: number): void {
-    super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0);
+    super.hashTreeRootInto(value, this.mixInLengthBlockBytes, 0);
     // mixInLength
     this.mixInLengthBuffer.writeUIntLE(value.bitLen, 32, 6);
     // one for hashTreeRoot(value), one for length
     const chunkCount = 2;
-    merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset);
+    merkleizeBlocksBytes(this.mixInLengthBlockBytes, chunkCount, output, offset);
   }
 
   // Proofs: inherited from BitArrayType

packages/ssz/src/type/byteArray.ts

@@ -89,15 +89,13 @@ export abstract class ByteArrayType extends CompositeType<ByteArray, ByteArray,
 
   // Merkleization
 
-  protected getChunkBytes(value: ByteArray): Uint8Array {
-    // reallocate this.merkleBytes if needed
-    if (value.length > this.chunkBytesBuffer.length) {
+  protected getBlocksBytes(value: ByteArray): Uint8Array {
+    // reallocate this.blocksBuffer if needed
+    if (value.length > this.blocksBuffer.length) {
       const chunkCount = Math.ceil(value.length / 32);
-      const chunkBytes = chunkCount * 32;
-      // pad 1 chunk if maxChunkCount is not even
-      this.chunkBytesBuffer = chunkCount % 2 === 1 ? new Uint8Array(chunkBytes + 32) : new Uint8Array(chunkBytes);
+      this.blocksBuffer = new Uint8Array(Math.ceil(chunkCount / 2) * 64);
     }
-    return getChunkBytes(value, this.chunkBytesBuffer);
+    return getBlocksBytes(value, this.blocksBuffer);
   }
 
   // Proofs
@@ -162,15 +160,15 @@ export abstract class ByteArrayType extends CompositeType<ByteArray, ByteArray,
   protected abstract assertValidSize(size: number): void;
 }
 
-export function getChunkBytes(data: Uint8Array, merkleBytesBuffer: Uint8Array): Uint8Array {
-  if (data.length > merkleBytesBuffer.length) {
-    throw new Error(`data length ${data.length} exceeds merkleBytesBuffer length ${merkleBytesBuffer.length}`);
+export function getBlocksBytes(value: Uint8Array, blocksBuffer: Uint8Array): Uint8Array {
+  if (value.length > blocksBuffer.length) {
+    throw new Error(`data length ${value.length} exceeds blocksBuffer length ${blocksBuffer.length}`);
   }
 
-  merkleBytesBuffer.set(data);
-  const valueLen = data.length;
-  const chunkByteLen = Math.ceil(valueLen / 64) * 64;
+  blocksBuffer.set(value);
+  const valueLen = value.length;
+  const blockByteLen = Math.ceil(valueLen / 64) * 64;
   // all padding bytes must be zero, this is similar to set zeroHash(0)
-  merkleBytesBuffer.subarray(valueLen, chunkByteLen).fill(0);
-  return merkleBytesBuffer.subarray(0, chunkByteLen);
+  blocksBuffer.subarray(valueLen, blockByteLen).fill(0);
+  return blocksBuffer.subarray(0, blockByteLen);
 }

packages/ssz/src/type/byteList.ts

@@ -4,7 +4,8 @@ import {
   Node,
   packedNodeRootsToBytes,
   packedRootsBytesToNode,
-  merkleizeInto,
+  merkleizeBlocksBytes,
+  merkleizeBlockArray,
 } from "@chainsafe/persistent-merkle-tree";
 import {maxChunksToDepth} from "../util/merkleize";
 import {Require} from "../util/types";
@@ -40,11 +41,13 @@ export class ByteListType extends ByteArrayType {
   readonly maxSize: number;
   readonly maxChunkCount: number;
   readonly isList = true;
-  readonly mixInLengthChunkBytes = new Uint8Array(64);
+  readonly blockArray: Uint8Array[] = [];
+  private blockBytesLen = 0;
+  readonly mixInLengthBlockBytes = new Uint8Array(64);
   readonly mixInLengthBuffer = Buffer.from(
-    this.mixInLengthChunkBytes.buffer,
-    this.mixInLengthChunkBytes.byteOffset,
-    this.mixInLengthChunkBytes.byteLength
+    this.mixInLengthBlockBytes.buffer,
+    this.mixInLengthBlockBytes.byteOffset,
+    this.mixInLengthBlockBytes.byteLength
   );
 
   constructor(readonly limitBytes: number, opts?: ByteListOptions) {
@@ -106,13 +109,44 @@ export class ByteListType extends ByteArrayType {
     return root;
   }
 
+  /**
+   * Use  merkleizeBlockArray() instead of merkleizeBlocksBytes() to avoid big memory allocation
+   */
   hashTreeRootInto(value: Uint8Array, output: Uint8Array, offset: number): void {
-    super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0);
+    // should not call super.hashTreeRoot() here
+    // use  merkleizeBlockArray() instead of merkleizeBlocksBytes() to avoid big memory allocation
+    // reallocate this.blockArray if needed
+    if (value.length > this.blockBytesLen) {
+      const newBlockCount = Math.ceil(value.length / 64);
+      // this.blockBytesLen should be a multiple of 64
+      const oldBlockCount = Math.ceil(this.blockBytesLen / 64);
+      const blockDiff = newBlockCount - oldBlockCount;
+      const newBlocksBytes = new Uint8Array(blockDiff * 64);
+      for (let i = 0; i < blockDiff; i++) {
+        this.blockArray.push(newBlocksBytes.subarray(i * 64, (i + 1) * 64));
+        this.blockBytesLen += 64;
+      }
+    }
+
+    // populate this.blockArray
+    for (let i = 0; i < value.length; i += 64) {
+      const block = this.blockArray[i / 64];
+      // zero out the last block if it's over value.length
+      if (i + 64 > value.length) {
+        block.fill(0);
+      }
+      block.set(value.subarray(i, Math.min(i + 64, value.length)));
+    }
+
+    // compute hashTreeRoot
+    const blockLimit = Math.ceil(value.length / 64);
+    merkleizeBlockArray(this.blockArray, blockLimit, this.maxChunkCount, this.mixInLengthBlockBytes, 0);
+
     // mixInLength
     this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6);
     // one for hashTreeRoot(value), one for length
     const chunkCount = 2;
-    merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset);
+    merkleizeBlocksBytes(this.mixInLengthBlockBytes, chunkCount, output, offset);
   }
 
   // Proofs: inherited from BitArrayType

packages/ssz/src/type/composite.ts

@@ -8,7 +8,7 @@ import {
   Proof,
   ProofType,
   Tree,
-  merkleizeInto,
+  merkleizeBlocksBytes,
   HashComputationLevel,
 } from "@chainsafe/persistent-merkle-tree";
 import {byteArrayEquals} from "../util/byteArray";
@@ -61,7 +61,7 @@ export abstract class CompositeType<V, TV, TVDU> extends Type<V> {
    * Required for ContainerNodeStruct to ensure no dangerous types are constructed.
    */
   abstract readonly isViewMutable: boolean;
-  protected chunkBytesBuffer = new Uint8Array(0);
+  protected blocksBuffer = new Uint8Array(0);
 
   constructor(
     /**
@@ -238,8 +238,8 @@ export abstract class CompositeType<V, TV, TVDU> extends Type<V> {
       }
     }
 
-    const merkleBytes = this.getChunkBytes(value);
-    merkleizeInto(merkleBytes, this.maxChunkCount, output, offset);
+    const blocksBuffer = this.getBlocksBytes(value);
+    merkleizeBlocksBytes(blocksBuffer, this.maxChunkCount, output, offset);
     if (this.cachePermanentRootStruct) {
       cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache);
     }
@@ -258,10 +258,10 @@ export abstract class CompositeType<V, TV, TVDU> extends Type<V> {
   //   to hashObject and back.
 
   /**
-   * Get merkle bytes of each value, the returned Uint8Array should be multiple of 64 bytes.
+   * Get multiple SHA256 blocks, each is 64 bytes long.
    * If chunk count is not even, need to append zeroHash(0)
    */
-  protected abstract getChunkBytes(value: V): Uint8Array;
+  protected abstract getBlocksBytes(value: V): Uint8Array;
 
   // Proofs API
 

packages/ssz/src/type/container.ts

@@ -131,8 +131,7 @@ export class ContainerType<Fields extends Record<string, Type<unknown>>> extends
     this.TreeView = opts?.getContainerTreeViewClass?.(this) ?? getContainerTreeViewClass(this);
     this.TreeViewDU = opts?.getContainerTreeViewDUClass?.(this) ?? getContainerTreeViewDUClass(this);
     const fieldBytes = this.fieldsEntries.length * 32;
-    const chunkBytes = Math.ceil(fieldBytes / 64) * 64;
-    this.chunkBytesBuffer = new Uint8Array(chunkBytes);
+    this.blocksBuffer = new Uint8Array(Math.ceil(fieldBytes / 64) * 64);
   }
 
   static named<Fields extends Record<string, Type<unknown>>>(
@@ -275,13 +274,13 @@ export class ContainerType<Fields extends Record<string, Type<unknown>>> extends
 
   // Merkleization
 
-  protected getChunkBytes(struct: ValueOfFields<Fields>): Uint8Array {
+  protected getBlocksBytes(struct: ValueOfFields<Fields>): Uint8Array {
     for (let i = 0; i < this.fieldsEntries.length; i++) {
       const {fieldName, fieldType} = this.fieldsEntries[i];
-      fieldType.hashTreeRootInto(struct[fieldName], this.chunkBytesBuffer, i * 32);
+      fieldType.hashTreeRootInto(struct[fieldName], this.blocksBuffer, i * 32);
     }
     // remaining bytes are zeroed as we never write them
-    return this.chunkBytesBuffer;
+    return this.blocksBuffer;
   }
 
   // Proofs

packages/ssz/src/type/listBasic.ts

@@ -1,4 +1,4 @@
-import {HashComputationLevel, LeafNode, Node, Tree, merkleizeInto} from "@chainsafe/persistent-merkle-tree";
+import {HashComputationLevel, LeafNode, Node, Tree, merkleizeBlocksBytes} from "@chainsafe/persistent-merkle-tree";
 import {ValueOf} from "./abstract";
 import {BasicType} from "./basic";
 import {ByteViews} from "./composite";
@@ -47,11 +47,11 @@ export class ListBasicType<ElementType extends BasicType<unknown>>
   readonly maxSize: number;
   readonly isList = true;
   readonly isViewMutable = true;
-  readonly mixInLengthChunkBytes = new Uint8Array(64);
+  readonly mixInLengthBlockBytes = new Uint8Array(64);
   readonly mixInLengthBuffer = Buffer.from(
-    this.mixInLengthChunkBytes.buffer,
-    this.mixInLengthChunkBytes.byteOffset,
-    this.mixInLengthChunkBytes.byteLength
+    this.mixInLengthBlockBytes.buffer,
+    this.mixInLengthBlockBytes.byteOffset,
+    this.mixInLengthBlockBytes.byteLength
   );
   protected readonly defaultLen = 0;
 
@@ -193,34 +193,34 @@ export class ListBasicType<ElementType extends BasicType<unknown>>
       }
     }
 
-    super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0);
+    super.hashTreeRootInto(value, this.mixInLengthBlockBytes, 0);
     // mixInLength
     this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6);
     // one for hashTreeRoot(value), one for length
     const chunkCount = 2;
-    merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset);
+    merkleizeBlocksBytes(this.mixInLengthBlockBytes, chunkCount, output, offset);
 
     if (this.cachePermanentRootStruct) {
       cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache);
     }
   }
 
-  protected getChunkBytes(value: ValueOf<ElementType>[]): Uint8Array {
+  protected getBlocksBytes(value: ValueOf<ElementType>[]): Uint8Array {
     const byteLen = this.value_serializedSize(value);
-    const chunkByteLen = Math.ceil(byteLen / 64) * 64;
-    // reallocate this.verkleBytes if needed
-    if (byteLen > this.chunkBytesBuffer.length) {
+    const blockByteLen = Math.ceil(byteLen / 64) * 64;
+    // reallocate this.blocksBuffer if needed
+    if (byteLen > this.blocksBuffer.length) {
       // pad 1 chunk if maxChunkCount is not even
-      this.chunkBytesBuffer = new Uint8Array(chunkByteLen);
+      this.blocksBuffer = new Uint8Array(blockByteLen);
     }
-    const chunkBytes = this.chunkBytesBuffer.subarray(0, chunkByteLen);
-    const uint8Array = chunkBytes.subarray(0, byteLen);
+    const blockBytes = this.blocksBuffer.subarray(0, blockByteLen);
+    const uint8Array = blockBytes.subarray(0, byteLen);
     const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength);
     value_serializeToBytesArrayBasic(this.elementType, value.length, {uint8Array, dataView}, 0, value);
 
     // all padding bytes must be zero, this is similar to set zeroHash(0)
-    this.chunkBytesBuffer.subarray(byteLen, chunkByteLen).fill(0);
-    return chunkBytes;
+    this.blocksBuffer.subarray(byteLen, blockByteLen).fill(0);
+    return blockBytes;
   }
 
   // JSON: inherited from ArrayType