Revert "Implement more memory efficient deposit merkle tree (#5501)" (#5603)

This reverts commit 784799f.

Author: ajsutton

CHANGELOG.md

@@ -18,7 +18,6 @@ For information on changes in released versions of Teku, see the [releases page]
 
 ### Additions and Improvements
 - Upated ropsten testnet config to include extremely high TTD and enable proposer boost.
-- Reduced memory requirements for storing the deposit merkle tree.
 - Changed the default maximum peers count from 74 to 100 (`--p2p-peer-upper-bound 74` was old setting)
 - Update proposer boost weighting to 40%.
 - Update `BeaconBlocksByRange` to only return the first block if the step is greater than 1, in line with 1.20 spec.

beacon/validator/build.gradle

@@ -10,7 +10,6 @@ dependencies {
   implementation project(':ethereum:core')
   implementation project(':ethereum:events')
   implementation project(':ethereum:pow:api')
-  implementation project(':ethereum:pow:merkletree')
   implementation project(':ethereum:spec')
   implementation project(':networking:eth2')
   implementation project(':infrastructure:logging')

beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/DepositProvider.java

@@ -13,7 +13,6 @@
 
 package tech.pegasys.teku.validator.coordinator;
 
-import static java.util.Collections.emptyList;
 import static tech.pegasys.teku.infrastructure.unsigned.UInt64.ONE;
 
 import java.util.NavigableMap;
@@ -28,7 +27,6 @@
 import tech.pegasys.teku.ethereum.pow.api.DepositsFromBlockEvent;
 import tech.pegasys.teku.ethereum.pow.api.Eth1EventsChannel;
 import tech.pegasys.teku.ethereum.pow.api.MinGenesisTimeBlockEvent;
-import tech.pegasys.teku.ethereum.pow.merkletree.DepositTree;
 import tech.pegasys.teku.infrastructure.logging.EventLogger;
 import tech.pegasys.teku.infrastructure.metrics.TekuMetricCategory;
 import tech.pegasys.teku.infrastructure.ssz.SszList;
@@ -43,6 +41,8 @@
 import tech.pegasys.teku.spec.datastructures.state.Checkpoint;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
 import tech.pegasys.teku.spec.datastructures.util.DepositUtil;
+import tech.pegasys.teku.spec.datastructures.util.MerkleTree;
+import tech.pegasys.teku.spec.datastructures.util.OptimizedMerkleTree;
 import tech.pegasys.teku.storage.api.FinalizedCheckpointChannel;
 import tech.pegasys.teku.storage.client.RecentChainData;
 
@@ -55,7 +55,7 @@ public class DepositProvider
 
   private final RecentChainData recentChainData;
   private final Eth1DataCache eth1DataCache;
-  private final DepositTree depositMerkleTree;
+  private final MerkleTree depositMerkleTree;
 
   private final NavigableMap<UInt64, DepositWithIndex> depositNavigableMap = new TreeMap<>();
   private final Counter depositCounter;
@@ -77,7 +77,8 @@ public DepositProvider(
     this.eth1DataCache = eth1DataCache;
     this.spec = spec;
     depositUtil = new DepositUtil(spec);
-    depositMerkleTree = new DepositTree();
+    depositMerkleTree =
+        new OptimizedMerkleTree(spec.getGenesisSpecConfig().getDepositContractTreeDepth());
     depositCounter =
         metricsSystem.createCounter(
             TekuMetricCategory.BEACON,
@@ -97,30 +98,32 @@ public synchronized void onDepositsFromBlock(DepositsFromBlockEvent event) {
               }
 
               depositNavigableMap.put(deposit.getIndex(), deposit);
-              depositMerkleTree.pushLeaf(deposit.getData().hashTreeRoot());
+              depositMerkleTree.add(deposit.getData().hashTreeRoot());
             });
     depositCounter.inc(event.getDeposits().size());
     eth1DataCache.onBlockWithDeposit(
         event.getBlockTimestamp(),
         new Eth1Data(
             depositMerkleTree.getRoot(),
-            UInt64.valueOf(depositMerkleTree.getDepositCount()),
+            UInt64.valueOf(depositMerkleTree.getNumberOfLeaves()),
             event.getBlockHash()));
   }
 
   @Override
   public void onNewFinalizedCheckpoint(
       final Checkpoint checkpoint, final boolean fromOptimisticBlock) {
-    final BeaconState finalizedState = recentChainData.getStore().getLatestFinalized().getState();
-    final UInt64 depositIndex = finalizedState.getEth1DepositIndex();
-    pruneDeposits(depositIndex);
-    synchronized (this) {
-      if (depositIndex.isGreaterThanOrEqualTo(finalizedState.getEth1Data().getDepositCount())
-          && depositMerkleTree.getDepositCount()
-              >= finalizedState.getEth1Data().getDepositCount().longValue()) {
-        depositMerkleTree.finalize(finalizedState.getEth1Data());
-      }
-    }
+    recentChainData
+        .retrieveBlockState(checkpoint.getRoot())
+        .thenAccept(
+            finalizedState -> {
+              if (finalizedState.isEmpty()) {
+                LOG.error("Finalized checkpoint state not found.");
+                return;
+              }
+              final UInt64 depositIndex = finalizedState.get().getEth1DepositIndex();
+              pruneDeposits(depositIndex);
+            })
+        .reportExceptions();
   }
 
   private synchronized void pruneDeposits(final UInt64 toIndex) {
@@ -213,22 +216,13 @@ public synchronized int getDepositMapSize() {
    * @param fromDepositIndex inclusive
    * @param toDepositIndex exclusive
    * @param eth1DepositCount number of deposits in the merkle tree according to Eth1Data in state
+   * @return
    */
   private SszList<Deposit> getDepositsWithProof(
       UInt64 fromDepositIndex, UInt64 toDepositIndex, UInt64 eth1DepositCount, long maxDeposits) {
     final AtomicReference<UInt64> expectedDepositIndex = new AtomicReference<>(fromDepositIndex);
     final SszListSchema<Deposit, ?> depositsSchema = depositsSchemaCache.get(maxDeposits);
     final SszBytes32VectorSchema<?> depositProofSchema = Deposit.SSZ_SCHEMA.getProofSchema();
-    // No deposits to include so don't bother rewinding the merkle tree.
-    if (fromDepositIndex.equals(toDepositIndex)) {
-      return depositsSchema.createFromElements(emptyList());
-    }
-    if (depositMerkleTree.getDepositCount() < eth1DepositCount.intValue()) {
-      throw MissingDepositsException.missingRange(
-          UInt64.valueOf(depositMerkleTree.getDepositCount()), eth1DepositCount);
-    }
-    final DepositTree merkleTree =
-        depositMerkleTree.getTreeAtDepositIndex(eth1DepositCount.intValue());
     return depositNavigableMap
         .subMap(fromDepositIndex, true, toDepositIndex, false)
         .values()
@@ -241,7 +235,9 @@ private SszList<Deposit> getDepositsWithProof(
               }
               expectedDepositIndex.set(deposit.getIndex().plus(ONE));
               SszBytes32Vector proof =
-                  depositProofSchema.of(merkleTree.getProof(deposit.getIndex().intValue()));
+                  depositProofSchema.of(
+                      depositMerkleTree.getProofWithViewBoundary(
+                          deposit.getIndex().intValue(), eth1DepositCount.intValue()));
               return new DepositWithIndex(proof, deposit.getData(), deposit.getIndex());
             })
         .collect(depositsSchema.collector());

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/DepositProviderTest.java

@@ -15,6 +15,7 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoInteractions;
@@ -43,14 +44,13 @@
 import tech.pegasys.teku.spec.datastructures.operations.Deposit;
 import tech.pegasys.teku.spec.datastructures.operations.DepositData;
 import tech.pegasys.teku.spec.datastructures.operations.DepositWithIndex;
-import tech.pegasys.teku.spec.datastructures.state.AnchorPoint;
 import tech.pegasys.teku.spec.datastructures.state.Checkpoint;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
 import tech.pegasys.teku.spec.datastructures.util.DepositUtil;
 import tech.pegasys.teku.spec.datastructures.util.MerkleTree;
+import tech.pegasys.teku.spec.datastructures.util.OptimizedMerkleTree;
 import tech.pegasys.teku.spec.util.DataStructureUtil;
 import tech.pegasys.teku.storage.client.RecentChainData;
-import tech.pegasys.teku.storage.store.UpdatableStore;
 
 public class DepositProviderTest {
 
@@ -78,7 +78,8 @@ void setup(final int maxDeposits) {
         new DepositProvider(
             new StubMetricsSystem(), recentChainData, eth1DataCache, spec, eventLogger, true);
     depositProvider.onSyncingStatusChanged(true);
-    depositMerkleTree = new MerkleTree(spec.getGenesisSpecConfig().getDepositContractTreeDepth());
+    depositMerkleTree =
+        new OptimizedMerkleTree(spec.getGenesisSpecConfig().getDepositContractTreeDepth());
     mockStateEth1DataVotes();
     createDepositEvents(40);
     randomEth1Data = dataStructureUtil.randomEth1Data();
@@ -152,13 +153,9 @@ void depositsWithFinalizedIndicesGetPrunedFromMap() {
     setup(16);
     Bytes32 finalizedBlockRoot = Bytes32.fromHexString("0x01");
     mockStateEth1DepositIndex(10);
-    mockEth1DataDepositCount(10);
     mockDepositsFromEth1Block(0, 20);
-    final AnchorPoint anchorPoint = mock(AnchorPoint.class);
-    final UpdatableStore store = mock(UpdatableStore.class);
-    when(recentChainData.getStore()).thenReturn(store);
-    when(store.getLatestFinalized()).thenReturn(anchorPoint);
-    when(anchorPoint.getState()).thenReturn(state);
+    when(recentChainData.retrieveBlockState(eq(finalizedBlockRoot)))
+        .thenReturn(SafeFuture.completedFuture(Optional.ofNullable(state)));
 
     assertThat(depositProvider.getDepositMapSize()).isEqualTo(20);
 
@@ -301,7 +298,6 @@ private void checkThatDepositProofIsValid(SszList<Deposit> deposits) {
                             genesisSpec.getConfig().getDepositContractTreeDepth() + 1,
                             ((DepositWithIndex) deposit).getIndex().intValue(),
                             depositMerkleTree.getRoot()))
-                .withFailMessage("Expected proof to be valid but was not")
                 .isTrue());
   }
 
@@ -329,7 +325,6 @@ private void mockDepositsFromEth1Block(int startIndex, int n) {
   private void mockEth1DataDepositCount(int n) {
     Eth1Data eth1Data = mock(Eth1Data.class);
     when(state.getEth1Data()).thenReturn(eth1Data);
-    when(eth1Data.getBlockHash()).thenReturn(dataStructureUtil.randomBytes32());
     when(eth1Data.getDepositCount()).thenReturn(UInt64.valueOf(n));
   }