From e5f90a0299e0cdfc79e4a1f583b72c0caf58b4df Mon Sep 17 00:00:00 2001 From: garyschulte Date: Tue, 7 Jun 2022 10:07:33 -0700 Subject: [PATCH 01/29] block proposal temporary workaround to roll back executed block proposals until we get a forkchoice updated Signed-off-by: garyschulte --- .../consensus/merge/blockcreation/MergeCoordinator.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java index 9be549a522f..81e78d52d1f 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java @@ -175,6 +175,8 @@ public PayloadIdentifier preparePayload( Result result = executeBlock(emptyBlock); if (result.blockProcessingOutputs.isPresent()) { mergeContext.putPayloadById(payloadIdentifier, emptyBlock); + // TODO: temporary workaround for implicit head change by validateAndProcessBlock: + protocolContext.getBlockchain().rewindToBlock(emptyBlock.getHeader().getParentHash()); } else { LOG.warn( "failed to execute empty block proposal {}, reason {}", @@ -194,6 +196,10 @@ public PayloadIdentifier preparePayload( final var resultBest = executeBlock(bestBlock); if (resultBest.blockProcessingOutputs.isPresent()) { mergeContext.putPayloadById(payloadIdentifier, bestBlock); + // TODO: temporary workaround for implicit head change by validateAndProcessBlock: + protocolContext + .getBlockchain() + .rewindToBlock(emptyBlock.getHeader().getParentHash()); } else { LOG.warn( "failed to execute block proposal {}, reason {}", From ab920ddc8322561609c92ec1b997d5332011a4f9 Mon Sep 17 00:00:00 2001 From: Jiri Peinlich Date: Thu, 9 Jun 2022 11:38:41 +0100 Subject: [PATCH 02/29] Trying execute without save Signed-off-by: Jiri Peinlich --- .../merge/blockcreation/MergeCoordinator.java | 71 ++++++++----------- .../blockcreation/MergeMiningCoordinator.java | 2 + .../blockcreation/TransitionCoordinator.java | 5 ++ .../methods/engine/EngineNewPayload.java | 2 +- .../besu/ethereum/BlockValidator.java | 8 +++ .../besu/ethereum/MainnetBlockValidator.java | 13 +++- .../worldstate/WorldStateArchive.java | 1 + 7 files changed, 57 insertions(+), 45 deletions(-) diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java index 81e78d52d1f..58a77d0c6f4 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java @@ -169,20 +169,9 @@ public PayloadIdentifier preparePayload( this.mergeBlockCreator.forParams(parentHeader, Optional.ofNullable(feeRecipient)); // put the empty block in first - final Block emptyBlock = - mergeBlockCreator.createBlock(Optional.of(Collections.emptyList()), random, timestamp); - - Result result = executeBlock(emptyBlock); - if (result.blockProcessingOutputs.isPresent()) { - mergeContext.putPayloadById(payloadIdentifier, emptyBlock); - // TODO: temporary workaround for implicit head change by validateAndProcessBlock: - protocolContext.getBlockchain().rewindToBlock(emptyBlock.getHeader().getParentHash()); - } else { - LOG.warn( - "failed to execute empty block proposal {}, reason {}", - emptyBlock.getHash(), - result.errorMessage); - } + mergeContext.putPayloadById( + payloadIdentifier, + mergeBlockCreator.createBlock(Optional.of(Collections.emptyList()), random, timestamp)); // start working on a full block and update the payload value and candidate when it's ready CompletableFuture.supplyAsync( @@ -193,19 +182,7 @@ public PayloadIdentifier preparePayload( if (throwable != null) { LOG.warn("something went wrong creating block", throwable); } else { - final var resultBest = executeBlock(bestBlock); - if (resultBest.blockProcessingOutputs.isPresent()) { - mergeContext.putPayloadById(payloadIdentifier, bestBlock); - // TODO: temporary workaround for implicit head change by validateAndProcessBlock: - protocolContext - .getBlockchain() - .rewindToBlock(emptyBlock.getHeader().getParentHash()); - } else { - LOG.warn( - "failed to execute block proposal {}, reason {}", - bestBlock.getHash(), - resultBest.errorMessage); - } + mergeContext.putPayloadById(payloadIdentifier, bestBlock); } }); @@ -228,29 +205,37 @@ public Optional getOrSyncHeaderByHash(final Hash blockhash) { @Override public Result executeBlock(final Block block) { + return executeBlockWithoutSaving(block,true); + } + + @Override + public Result executeBlockWithoutSaving(final Block block) { + return executeBlockWithoutSaving(block,false); + } + public Result executeBlockWithoutSaving(final Block block, boolean shouldSave) { final var chain = protocolContext.getBlockchain(); chain - .getBlockHeader(block.getHeader().getParentHash()) - .ifPresentOrElse( - blockHeader -> - debugLambda(LOG, "Parent of block {} is already present", block::toLogString), - () -> backwardSyncContext.syncBackwardsUntil(block)); + .getBlockHeader(block.getHeader().getParentHash()) + .ifPresentOrElse( + blockHeader -> + debugLambda(LOG, "Parent of block {} is already present", block::toLogString), + () -> backwardSyncContext.syncBackwardsUntil(block)); final var validationResult = - protocolSchedule - .getByBlockNumber(block.getHeader().getNumber()) - .getBlockValidator() - .validateAndProcessBlock( - protocolContext, block, HeaderValidationMode.FULL, HeaderValidationMode.NONE); + protocolSchedule + .getByBlockNumber(block.getHeader().getNumber()) + .getBlockValidator() + .validateAndProcessBlock( + protocolContext, block, HeaderValidationMode.FULL, HeaderValidationMode.NONE,shouldSave); validationResult.blockProcessingOutputs.ifPresentOrElse( - result -> chain.appendBlock(block, result.receipts), - () -> - protocolSchedule - .getByBlockNumber(chain.getChainHeadBlockNumber()) - .getBadBlocksManager() - .addBadBlock(block)); + result -> chain.appendBlock(block, result.receipts), + () -> + protocolSchedule + .getByBlockNumber(chain.getChainHeadBlockNumber()) + .getBadBlocksManager() + .addBadBlock(block)); return validationResult; } diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeMiningCoordinator.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeMiningCoordinator.java index ac725da7eb1..2c89f02a2f4 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeMiningCoordinator.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeMiningCoordinator.java @@ -35,6 +35,8 @@ PayloadIdentifier preparePayload( Result executeBlock(final Block block); + Result executeBlockWithoutSaving(final Block block); + ForkchoiceResult updateForkChoice( final BlockHeader newHead, final Hash finalizedBlockHash, final Hash safeBlockHash); diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/TransitionCoordinator.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/TransitionCoordinator.java index f4b03326367..9856d930623 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/TransitionCoordinator.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/TransitionCoordinator.java @@ -137,6 +137,11 @@ public Result executeBlock(final Block block) { return mergeCoordinator.executeBlock(block); } + @Override + public Result executeBlockWithoutSaving(final Block block) { + return mergeCoordinator.executeBlockWithoutSaving(block); + } + @Override public ForkchoiceResult updateForkChoice( final BlockHeader newHead, final Hash finalizedBlockHash, final Hash safeBlockHash) { diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayload.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayload.java index 4e2502d28ae..1489fe80f6e 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayload.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayload.java @@ -177,7 +177,7 @@ public JsonRpcResponse syncResponse(final JsonRpcRequestContext requestContext) } // execute block and return result response - final BlockValidator.Result executionResult = mergeCoordinator.executeBlock(block); + final BlockValidator.Result executionResult = mergeCoordinator.executeBlockWithoutSaving(block); if (executionResult.errorMessage.isEmpty()) { return respondWith(reqId, newBlockHeader.getHash(), VALID); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/BlockValidator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/BlockValidator.java index de1d9eb8563..a8417a0d79b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/BlockValidator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/BlockValidator.java @@ -56,6 +56,14 @@ Result validateAndProcessBlock( final HeaderValidationMode headerValidationMode, final HeaderValidationMode ommerValidationMode); + + Result validateAndProcessBlock( + final ProtocolContext context, + final Block block, + final HeaderValidationMode headerValidationMode, + final HeaderValidationMode ommerValidationMode, + final boolean shouldPersist); + boolean fastBlockValidation( final ProtocolContext context, final Block block, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java index 2ef6c7606eb..03249cc108e 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java @@ -68,6 +68,16 @@ public BlockValidator.Result validateAndProcessBlock( final Block block, final HeaderValidationMode headerValidationMode, final HeaderValidationMode ommerValidationMode) { + return validateAndProcessBlock(context, block, headerValidationMode, ommerValidationMode,true); + } + + @Override + public BlockValidator.Result validateAndProcessBlock( + final ProtocolContext context, + final Block block, + final HeaderValidationMode headerValidationMode, + final HeaderValidationMode ommerValidationMode, + final boolean shouldPersist) { final BlockHeader header = block.getHeader(); @@ -87,7 +97,8 @@ public BlockValidator.Result validateAndProcessBlock( final Optional maybeWorldState = context .getWorldStateArchive() - .getMutable(parentHeader.getStateRoot(), parentHeader.getHash()); + .getMutable(parentHeader.getStateRoot(), parentHeader.getHash(),shouldPersist); + if (maybeWorldState.isEmpty()) { return handleAndReportFailure( block, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java index 17a33760e9c..ab2de6bbb59 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java @@ -35,6 +35,7 @@ public interface WorldStateArchive { boolean isWorldStateAvailable(Hash rootHash, Hash blockHash); + @Deprecated Optional getMutable(long blockNumber, boolean isPersistingState); Optional getMutable(Hash rootHash, Hash blockHash, boolean isPersistingState); From fb8f5917746a7e850d7b5936470f6ebfa9b24b9e Mon Sep 17 00:00:00 2001 From: Jiri Peinlich Date: Thu, 9 Jun 2022 11:41:23 +0100 Subject: [PATCH 03/29] custom version of build Signed-off-by: Jiri Peinlich --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index ccb71efba54..cf610604b9c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=22.4.3-SNAPSHOT +version=22.4.3-gary-jiri # Workaround for Java 16 and spotless bug 834 https://github.com/diffplug/spotless/issues/834 org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ From 8f660be38346e6393388ccde1f78a698a2f78e4a Mon Sep 17 00:00:00 2001 From: Jiri Peinlich Date: Thu, 9 Jun 2022 11:44:24 +0100 Subject: [PATCH 04/29] some fix Signed-off-by: Jiri Peinlich --- .../besu/consensus/merge/blockcreation/MergeCoordinator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java index 58a77d0c6f4..8f0debafd16 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java @@ -212,7 +212,7 @@ public Result executeBlock(final Block block) { public Result executeBlockWithoutSaving(final Block block) { return executeBlockWithoutSaving(block,false); } - public Result executeBlockWithoutSaving(final Block block, boolean shouldSave) { + public Result executeBlockWithoutSaving(final Block block, final boolean shouldSave) { final var chain = protocolContext.getBlockchain(); chain From 8a644608c7ff4bed666f97707edd951ff5e03016 Mon Sep 17 00:00:00 2001 From: Jiri Peinlich Date: Thu, 9 Jun 2022 13:31:55 +0100 Subject: [PATCH 05/29] do not append block when not needed Signed-off-by: Jiri Peinlich --- .../merge/blockcreation/MergeCoordinator.java | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java index 8f0debafd16..883da83bdcc 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java @@ -14,9 +14,15 @@ */ package org.hyperledger.besu.consensus.merge.blockcreation; -import static org.hyperledger.besu.consensus.merge.TransitionUtils.isTerminalProofOfWorkBlock; -import static org.hyperledger.besu.util.Slf4jLambdaHelper.debugLambda; - +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; import org.hyperledger.besu.consensus.merge.MergeContext; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; @@ -36,20 +42,12 @@ import org.hyperledger.besu.ethereum.mainnet.AbstractGasLimitSpecification; import org.hyperledger.besu.ethereum.mainnet.HeaderValidationMode; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; - -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.hyperledger.besu.consensus.merge.TransitionUtils.isTerminalProofOfWorkBlock; +import static org.hyperledger.besu.util.Slf4jLambdaHelper.debugLambda; + public class MergeCoordinator implements MergeMiningCoordinator { private static final Logger LOG = LoggerFactory.getLogger(MergeCoordinator.class); @@ -230,7 +228,11 @@ public Result executeBlockWithoutSaving(final Block block, final boolean shouldS protocolContext, block, HeaderValidationMode.FULL, HeaderValidationMode.NONE,shouldSave); validationResult.blockProcessingOutputs.ifPresentOrElse( - result -> chain.appendBlock(block, result.receipts), + result -> { + if (shouldSave) { + chain.appendBlock(block, result.receipts); + } + }, () -> protocolSchedule .getByBlockNumber(chain.getChainHeadBlockNumber()) From 4a13932f261f77a25323b2d5b372b00976feb808 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Mon, 20 Jun 2022 14:58:12 +0200 Subject: [PATCH 06/29] Do not update chain state and world state in MergeCoordinator.executeBlock When executing a newPayload or build a new block, we do not need to move the chain head and update the world state, since this will be done by a following forkchoice update call, but we still need to validate the block and doing so we can also prepare everything for the future call, so we do not need to re-execute everything, but only update the pointers, so that the response to the forkchoice update call is quick. Signed-off-by: Fabio Di Fabio --- .../merge/blockcreation/MergeCoordinator.java | 5 +- .../bonsai/BonsaiPersistedWorldState.java | 34 +++++++- .../besu/ethereum/chain/BlockAddedEvent.java | 14 +++- .../ethereum/chain/DefaultBlockchain.java | 78 +++++++++++++------ .../ethereum/chain/MutableBlockchain.java | 14 +++- .../besu/ethereum/core/MutableWorldState.java | 11 +++ 6 files changed, 127 insertions(+), 29 deletions(-) diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java index 04b04899ba0..624e933cd46 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java @@ -257,7 +257,10 @@ public Result executeBlock(final Block block) { protocolContext, block, HeaderValidationMode.FULL, HeaderValidationMode.NONE); validationResult.blockProcessingOutputs.ifPresentOrElse( - result -> chain.appendBlock(block, result.receipts), + result -> { + result.worldState.remember(block.getHeader()); + chain.storeBlock(block, result.receipts); + }, () -> protocolSchedule .getByBlockNumber(chain.getChainHeadBlockNumber()) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java index 04a383a0229..01d659f3820 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java @@ -16,6 +16,7 @@ package org.hyperledger.besu.ethereum.bonsai; +import static com.google.common.base.Preconditions.checkArgument; import static org.hyperledger.besu.ethereum.bonsai.BonsaiAccount.fromRLP; import static org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; import static org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; @@ -236,6 +237,35 @@ protected Hash calculateRootHash( return Hash.wrap(rootHash); } + @Override + public void remember(final BlockHeader blockHeader) { + checkArgument(blockHeader != null, "Block header must not be null"); + + final BonsaiWorldStateUpdater localUpdater = updater.copy(); + final BonsaiWorldStateKeyValueStorage.Updater stateUpdater = worldStateStorage.updater(); + + try { + worldStateRootHash = calculateRootHash(stateUpdater, localUpdater); + + if (!worldStateRootHash.equals(blockHeader.getStateRoot())) { + throw new RuntimeException( + "World State Root does not match expected value, header " + + blockHeader.getStateRoot().toHexString() + + " calculated " + + worldStateRootHash.toHexString()); + } + + final TrieLogLayer trieLog = + localUpdater.generateTrieLog(Hash.fromPlugin(blockHeader.getBlockHash())); + trieLog.freeze(); + archive.addLayeredWorldState(this, blockHeader, worldStateRootHash, trieLog); + } finally { + stateUpdater.rollback(); + updater.reset(); + } + archive.scrubLayeredCache(blockHeader.getNumber()); + } + @Override public void persist(final BlockHeader blockHeader) { boolean success = false; @@ -253,8 +283,8 @@ public void persist(final BlockHeader blockHeader) { .put(WORLD_ROOT_HASH_KEY, worldStateRootHash.toArrayUnsafe()); // if we are persisted with a block header, and the prior state is the parent - // then persist the TrieLog for that transition. If specified but not a direct - // descendant simply store the new block hash. + // then persist the TrieLog for that transition. + // If specified but not a direct descendant simply store the new block hash. if (blockHeader != null) { if (!worldStateRootHash.equals(blockHeader.getStateRoot())) { throw new RuntimeException( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/BlockAddedEvent.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/BlockAddedEvent.java index b9d0f91ab17..dbe54139ca8 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/BlockAddedEvent.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/BlockAddedEvent.java @@ -36,7 +36,8 @@ public class BlockAddedEvent { public enum EventType { HEAD_ADVANCED, FORK, - CHAIN_REORG + CHAIN_REORG, + STORED_ONLY } private BlockAddedEvent( @@ -98,6 +99,17 @@ public static BlockAddedEvent createForFork(final Block block) { block.getHeader().getParentHash()); } + public static BlockAddedEvent createForStoredOnly(final Block block) { + return new BlockAddedEvent( + EventType.STORED_ONLY, + block, + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), + block.getHeader().getParentHash()); + } + public Block getBlock() { return block; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java index f22307a1bb4..8934bca0157 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java @@ -21,6 +21,7 @@ import static java.util.stream.Collectors.toList; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.chain.BlockchainStorage.Updater; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockBody; import org.hyperledger.besu.ethereum.core.BlockHeader; @@ -50,6 +51,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.collect.Streams; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -282,20 +284,33 @@ public void setBlockChoiceRule(final Comparator blockChoiceRule) { @Override public synchronized void appendBlock(final Block block, final List receipts) { + appendBlockHelper(new BlockWithReceipts(block, receipts), false); + } + + @Override + public synchronized void storeBlock(final Block block, final List receipts) { + appendBlockHelper(new BlockWithReceipts(block, receipts), true); + } + + private boolean blockShouldBeProcessed( + final Block block, final List receipts) { checkArgument( block.getBody().getTransactions().size() == receipts.size(), "Supplied receipts do not match block transactions."); if (blockIsAlreadyTracked(block)) { - return; + return false; } checkArgument(blockIsConnected(block), "Attempt to append non-connected block."); - - final BlockAddedEvent blockAddedEvent = - appendBlockHelper(new BlockWithReceipts(block, receipts)); - blockAddedObservers.forEach(observer -> observer.onBlockAdded(blockAddedEvent)); + return true; } - private BlockAddedEvent appendBlockHelper(final BlockWithReceipts blockWithReceipts) { + private void appendBlockHelper( + final BlockWithReceipts blockWithReceipts, final boolean storeOnly) { + + if (!blockShouldBeProcessed(blockWithReceipts.getBlock(), blockWithReceipts.getReceipts())) { + return; + } + final Block block = blockWithReceipts.getBlock(); final List receipts = blockWithReceipts.getReceipts(); final Hash hash = block.getHash(); @@ -308,15 +323,18 @@ private BlockAddedEvent appendBlockHelper(final BlockWithReceipts blockWithRecei updater.putTransactionReceipts(hash, receipts); updater.putTotalDifficulty(hash, td); - // Update canonical chain data - final BlockAddedEvent blockAddedEvent = updateCanonicalChainData(updater, blockWithReceipts); - - updater.commit(); - if (blockAddedEvent.isNewCanonicalHead()) { - updateCacheForNewCanonicalHead(block, td); + final BlockAddedEvent blockAddedEvent; + if (storeOnly) { + blockAddedEvent = handleStoreOnly(blockWithReceipts); + } else { + blockAddedEvent = updateCanonicalChainData(updater, blockWithReceipts); + if (blockAddedEvent.isNewCanonicalHead()) { + updateCacheForNewCanonicalHead(block, td); + } } - return blockAddedEvent; + updater.commit(); + blockAddedObservers.forEach(observer -> observer.onBlockAdded(blockAddedEvent)); } @Override @@ -371,18 +389,9 @@ private BlockAddedEvent updateCanonicalChainData( throw new IllegalStateException("Blockchain is missing chain head."); } - final Hash newBlockHash = newBlock.getHash(); try { - if (chainHead == null || newBlock.getHeader().getParentHash().equals(chainHead)) { - // This block advances the chain, update the chain head - updater.putBlockHash(newBlock.getHeader().getNumber(), newBlockHash); - updater.setChainHead(newBlockHash); - indexTransactionForBlock(updater, newBlockHash, newBlock.getBody().getTransactions()); - return BlockAddedEvent.createForHeadAdvancement( - newBlock, - LogWithMetadata.generate( - blockWithReceipts.getBlock(), blockWithReceipts.getReceipts(), false), - blockWithReceipts.getReceipts()); + if (newBlock.getHeader().getParentHash().equals(chainHead) || chainHead == null) { + return handleNewHead(updater, blockWithReceipts); } else if (blockChoiceRule.compare(newBlock.getHeader(), chainHeader) > 0) { // New block represents a chain reorganization return handleChainReorg(updater, blockWithReceipts); @@ -398,6 +407,27 @@ private BlockAddedEvent updateCanonicalChainData( } } + private BlockAddedEvent handleStoreOnly(final BlockWithReceipts blockWithReceipts) { + return BlockAddedEvent.createForStoredOnly(blockWithReceipts.getBlock()); + } + + @NotNull + private BlockAddedEvent handleNewHead( + final Updater updater, final BlockWithReceipts blockWithReceipts) { + // This block advances the chain, update the chain head + final Hash newBlockHash = blockWithReceipts.getHash(); + + updater.putBlockHash(blockWithReceipts.getNumber(), newBlockHash); + updater.setChainHead(newBlockHash); + indexTransactionForBlock( + updater, newBlockHash, blockWithReceipts.getBlock().getBody().getTransactions()); + return BlockAddedEvent.createForHeadAdvancement( + blockWithReceipts.getBlock(), + LogWithMetadata.generate( + blockWithReceipts.getBlock(), blockWithReceipts.getReceipts(), false), + blockWithReceipts.getReceipts()); + } + private BlockAddedEvent handleFork(final BlockchainStorage.Updater updater, final Block fork) { final Collection forkHeads = blockchainStorage.getForkHeads(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/MutableBlockchain.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/MutableBlockchain.java index 79e4f76bdc8..13d61cd9b18 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/MutableBlockchain.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/MutableBlockchain.java @@ -37,10 +37,22 @@ public interface MutableBlockchain extends Blockchain { */ void appendBlock(Block block, List receipts); + /** + * Adds a block to the blockchain, without updating the chain state. + * + *

Block must be connected to the existing blockchain (its parent must already be stored), + * otherwise an {@link IllegalArgumentException} is thrown. Blocks representing forks are allowed + * as long as they are connected. + * + * @param block The block to append. + * @param receipts The list of receipts associated with this block's transactions. + */ + void storeBlock(Block block, List receipts); + void unsafeImportBlock( final Block block, final List receipts, - final Optional maybeTtalDifficulty); + final Optional maybeTotalDifficulty); void unsafeSetChainHead(final BlockHeader blockHeader, final Difficulty totalDifficulty); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java index f5af290aba3..d3647beeced 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java @@ -34,4 +34,15 @@ public interface MutableWorldState extends WorldState, MutableWorldView { * `null` should be passed in. */ void persist(BlockHeader blockHeader); + + /** + * Remember accumulated changes to underlying storage for future use, without making this the + * current world state. This is useful in the PoS where can precompute a future world state, that + * will be committed in a later step + * + * @param blockHeader The block hash of the world state this represents. Must not be null + */ + default void remember(final BlockHeader blockHeader) { + persist(blockHeader); + } } From b7e0e341805f66e47d4c7cd041af89d9194c1dcb Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Mon, 20 Jun 2022 16:03:24 +0200 Subject: [PATCH 07/29] When proposing a block, use a lightweight validation, without storing nothing Signed-off-by: Fabio Di Fabio --- .../merge/blockcreation/MergeCoordinator.java | 14 +++++++++++--- .../blockcreation/MergeMiningCoordinator.java | 2 ++ .../merge/blockcreation/TransitionCoordinator.java | 5 +++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java index 624e933cd46..a8da421e646 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java @@ -174,7 +174,7 @@ public PayloadIdentifier preparePayload( final Block emptyBlock = mergeBlockCreator.createBlock(Optional.of(Collections.emptyList()), random, timestamp); - Result result = executeBlock(emptyBlock); + Result result = validateBlock(emptyBlock); if (result.blockProcessingOutputs.isPresent()) { mergeContext.putPayloadById(payloadIdentifier, emptyBlock); } else { @@ -193,7 +193,7 @@ public PayloadIdentifier preparePayload( if (throwable != null) { LOG.warn("something went wrong creating block", throwable); } else { - final var resultBest = executeBlock(bestBlock); + final var resultBest = validateBlock(bestBlock); if (resultBest.blockProcessingOutputs.isPresent()) { mergeContext.putPayloadById(payloadIdentifier, bestBlock); } else { @@ -239,7 +239,7 @@ public Optional getOrSyncHeaderByHash( } @Override - public Result executeBlock(final Block block) { + public Result validateBlock(final Block block) { final var chain = protocolContext.getBlockchain(); chain @@ -256,6 +256,14 @@ public Result executeBlock(final Block block) { .validateAndProcessBlock( protocolContext, block, HeaderValidationMode.FULL, HeaderValidationMode.NONE); + return validationResult; + } + + @Override + public Result executeBlock(final Block block) { + final var chain = protocolContext.getBlockchain(); + + final var validationResult = validateBlock(block); validationResult.blockProcessingOutputs.ifPresentOrElse( result -> { result.worldState.remember(block.getHeader()); diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeMiningCoordinator.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeMiningCoordinator.java index 5be09c41ef7..9d9e531f327 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeMiningCoordinator.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeMiningCoordinator.java @@ -37,6 +37,8 @@ PayloadIdentifier preparePayload( Result executeBlock(final Block block); + Result validateBlock(final Block block); + ForkchoiceResult updateForkChoice( final BlockHeader newHead, final Hash finalizedBlockHash, diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/TransitionCoordinator.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/TransitionCoordinator.java index cd9658f0628..c2339d21fea 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/TransitionCoordinator.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/TransitionCoordinator.java @@ -137,6 +137,11 @@ public Result executeBlock(final Block block) { return mergeCoordinator.executeBlock(block); } + @Override + public Result validateBlock(final Block block) { + return mergeCoordinator.validateBlock(block); + } + @Override public ForkchoiceResult updateForkChoice( final BlockHeader newHead, From 430f3cc54deff65effdb836c375c2c202a2a5078 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Mon, 20 Jun 2022 16:30:17 +0200 Subject: [PATCH 08/29] Update unit test Signed-off-by: Fabio Di Fabio --- .../merge/blockcreation/MergeReorgTest.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeReorgTest.java b/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeReorgTest.java index 90128139dee..3ced3e74965 100644 --- a/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeReorgTest.java +++ b/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeReorgTest.java @@ -24,6 +24,7 @@ import org.hyperledger.besu.consensus.merge.PostMergeContext; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.BlockValidator.Result; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.GenesisState; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; @@ -101,20 +102,19 @@ public void setUp() { then you can and should be able to re-org to a different pre-TTD block say there is viable TTD block A and B, then we can have a PoS chain build on A for a while and then see another PoS chain build on B that has a higher fork choice weight and causes a re-org - once any post-merge PoS chain is finalied though, you'd never re-org any PoW blocks in the tree ever again */ + once any post-merge PoS chain is finalized though, you'd never re-org any PoW blocks in the tree ever again */ @Test public void reorgsAcrossTDDToDifferentTargetsWhenNotFinal() { // Add N blocks to chain from genesis, where total diff is < TTD Log4j2ConfiguratorUtil.setLevelDebug(BlockHeaderValidator.class.getName()); List endOfWork = subChain(genesisState.getBlock().getHeader(), 10, Difficulty.of(100L)); - endOfWork.stream().forEach(coordinator::executeBlock); + endOfWork.stream().forEach(this::appendBlock); assertThat(blockchain.getChainHead().getHeight()).isEqualTo(10L); BlockHeader tddPenultimate = this.blockchain.getChainHeadHeader(); // Add TTD block A to chain as child of N. Block ttdA = new Block(terminalPowBlock(tddPenultimate, Difficulty.ONE), BlockBody.empty()); - boolean worked = coordinator.executeBlock(ttdA).blockProcessingOutputs.isPresent(); - assertThat(worked).isTrue(); + appendBlock(ttdA); assertThat(blockchain.getChainHead().getHeight()).isEqualTo(11L); assertThat(blockchain.getTotalDifficultyByHash(ttdA.getHash())).isPresent(); Difficulty tdd = blockchain.getTotalDifficultyByHash(ttdA.getHash()).get(); @@ -127,16 +127,15 @@ public void reorgsAcrossTDDToDifferentTargetsWhenNotFinal() { .toBigInteger()); assertThat(mergeContext.isPostMerge()).isTrue(); List builtOnTTDA = subChain(ttdA.getHeader(), 5, Difficulty.of(0L)); - builtOnTTDA.stream().forEach(coordinator::executeBlock); + builtOnTTDA.stream().forEach(this::appendBlock); assertThat(blockchain.getChainHead().getHeight()).isEqualTo(16); assertThat(blockchain.getChainHead().getHash()) .isEqualTo(builtOnTTDA.get(builtOnTTDA.size() - 1).getHash()); Block ttdB = new Block(terminalPowBlock(tddPenultimate, Difficulty.of(2L)), BlockBody.empty()); - worked = coordinator.executeBlock(ttdB).blockProcessingOutputs.isPresent(); - assertThat(worked).isTrue(); + appendBlock(ttdB); List builtOnTTDB = subChain(ttdB.getHeader(), 10, Difficulty.of(0L)); - builtOnTTDB.stream().forEach(coordinator::executeBlock); + builtOnTTDB.stream().forEach(this::appendBlock); assertThat(blockchain.getChainHead().getHeight()).isEqualTo(21); assertThat(blockchain.getChainHead().getHash()) .isEqualTo(builtOnTTDB.get(builtOnTTDB.size() - 1).getHash()); @@ -145,6 +144,16 @@ public void reorgsAcrossTDDToDifferentTargetsWhenNotFinal() { } + private void appendBlock(final Block block) { + final Result result = coordinator.validateBlock(block); + + result.blockProcessingOutputs.ifPresentOrElse( + outputs -> blockchain.appendBlock(block, outputs.receipts), + () -> { + throw new RuntimeException(result.errorMessage.get()); + }); + } + private List subChain( final BlockHeader parentHeader, final long length, final Difficulty each) { BlockHeader newParent = parentHeader; From 116f16c684949726b1943d9e9477a7de9a42b7d6 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Mon, 20 Jun 2022 16:48:18 +0200 Subject: [PATCH 09/29] Update CHANGELOG Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29214fe652d..935798f0fbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ ## 22.7.1 ### Additions and Improvements +- Do not require a minimum block height when downloading headers or blocks [#3911](https://github.com/hyperledger/besu/pull/3911) +- When on PoS the head can be only be updated by ForkchoiceUpdate [#3994](https://github.com/hyperledger/besu/pull/3994) + ### Bug Fixes ## 22.4.3 @@ -12,7 +15,6 @@ - Support `finalized` and `safe` as tags for the block parameter in RPC APIs [#3950](https://github.com/hyperledger/besu/pull/3950) - Added verification of payload attributes in ForkchoiceUpdated [#3837](https://github.com/hyperledger/besu/pull/3837) - Add support for Gray Glacier hardfork [#3961](https://github.com/hyperledger/besu/issues/3961) -- Do not require a minimum block height when downloading headers or blocks [#3911](https://github.com/hyperledger/besu/pull/3911) ### Bug Fixes - alias engine-rpc-port parameter with the former rpc param name [#3958](https://github.com/hyperledger/besu/pull/3958) From b02de746dd560d68e227f108eebd8dbde214428b Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Wed, 22 Jun 2022 14:02:27 +0200 Subject: [PATCH 10/29] Introduce forwardToBlock It moves head to the block sent by a previous newPayload, and triggers the advanced head event that will also update the Bonsai world state. Signed-off-by: Fabio Di Fabio --- .../merge/blockcreation/MergeCoordinator.java | 28 +++++++++++++-- .../engine/EngineForkchoiceUpdated.java | 4 +-- .../bonsai/BonsaiWorldStateArchive.java | 36 +++++++++---------- .../besu/ethereum/chain/BlockAddedEvent.java | 22 +++++++++++- .../ethereum/chain/DefaultBlockchain.java | 27 ++++++++++++++ .../ethereum/chain/MutableBlockchain.java | 10 ++++++ 6 files changed, 103 insertions(+), 24 deletions(-) diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java index a8da421e646..a2ec2a15bcd 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java @@ -317,14 +317,16 @@ public ForkchoiceResult updateForkChoice( return ForkchoiceResult.withFailure(INVALID, descendantError.get(), latestValid); } - Optional parentOfNewHead = blockchain.getBlockHeader(newHead.getParentHash()); + final Optional parentOfNewHead = + blockchain.getBlockHeader(newHead.getParentHash()); + if (parentOfNewHead.isPresent() && parentOfNewHead.get().getTimestamp() >= newHead.getTimestamp()) { return ForkchoiceResult.withFailure( INVALID, "new head timestamp not greater than parent", latestValid); } - // set the new head - blockchain.rewindToBlock(newHead.getHash()); + + setNewHead(blockchain, newHead, newHead.getParentHash()); // set and persist the new finalized block if it is present newFinalized.ifPresent( @@ -349,6 +351,26 @@ public ForkchoiceResult updateForkChoice( return ForkchoiceResult.withResult(newFinalized, Optional.of(newHead)); } + private boolean setNewHead( + final MutableBlockchain blockchain, final BlockHeader newHead, final Hash parentHash) { + + if (newHead.getHash().equals(blockchain.getChainHeadHash())) { + debugLambda(LOG, "Nothing to do new head {} is already chain head", newHead::toLogString); + return true; + } + + if (parentHash.equals(blockchain.getChainHeadHash())) { + debugLambda( + LOG, + "Forwarding chain head to the block {} saved from a previous newPayload invocation", + newHead::toLogString); + return blockchain.forwardToBlock(newHead); + } + + debugLambda(LOG, "New head {} is a chain reorg, rewind chain head to it", newHead::toLogString); + return blockchain.rewindToBlock(newHead.getHash()); + } + @Override public boolean latestValidAncestorDescendsFromTerminal(final BlockHeader blockHeader) { if (blockHeader.getNumber() <= 1L) { diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdated.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdated.java index 41d4765d3d2..1dfb9cdd4c9 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdated.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdated.java @@ -87,9 +87,9 @@ public JsonRpcResponse syncResponse(final JsonRpcRequestContext requestContext) Optional.ofNullable(forkChoice.getHeadBlockHash()) .filter(hash -> !hash.equals(Hash.ZERO)) .ifPresent( - blockhash -> + blockHash -> mergeCoordinator.getOrSyncHeaderByHash( - blockhash, forkChoice.getFinalizedBlockHash())); + blockHash, forkChoice.getFinalizedBlockHash())); return syncingResponse(requestId); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java index 3f10fd96dbf..3aa43c65797 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.chain.BlockAddedEvent; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; @@ -73,24 +74,23 @@ public BonsaiWorldStateArchive( this.persistedState = new BonsaiPersistedWorldState(this, worldStateStorage); this.layeredWorldStatesByHash = layeredWorldStatesByHash; this.maxLayersToLoad = maxLayersToLoad; - blockchain.observeBlockAdded( - event -> { - if (event.isNewCanonicalHead()) { - final BlockHeader eventBlockHeader = event.getBlock().getHeader(); - layeredWorldStatesByHash.computeIfPresent( - eventBlockHeader.getParentHash(), - (hash, bonsaiLayeredWorldState) -> { - if (layeredWorldStatesByHash.containsKey( - fromPlugin(eventBlockHeader.getBlockHash()))) { - bonsaiLayeredWorldState.setNextWorldView( - Optional.of( - layeredWorldStatesByHash.get( - fromPlugin(eventBlockHeader.getBlockHash())))); - } - return bonsaiLayeredWorldState; - }); - } - }); + blockchain.observeBlockAdded(this::blockAddedHandler); + } + + private void blockAddedHandler(final BlockAddedEvent event) { + LOG.debug("New block add event {}", event); + if (event.isNewCanonicalHead()) { + final BlockHeader eventBlockHeader = event.getBlock().getHeader(); + layeredWorldStatesByHash.computeIfPresent( + eventBlockHeader.getParentHash(), + (hash, bonsaiLayeredWorldState) -> { + if (layeredWorldStatesByHash.containsKey(eventBlockHeader.getBlockHash())) { + bonsaiLayeredWorldState.setNextWorldView( + Optional.of(layeredWorldStatesByHash.get(eventBlockHeader.getBlockHash()))); + } + return bonsaiLayeredWorldState; + }); + } } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/BlockAddedEvent.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/BlockAddedEvent.java index dbe54139ca8..ae35fd7084e 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/BlockAddedEvent.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/BlockAddedEvent.java @@ -115,7 +115,7 @@ public Block getBlock() { } public boolean isNewCanonicalHead() { - return eventType != EventType.FORK; + return eventType == EventType.HEAD_ADVANCED || eventType == EventType.CHAIN_REORG; } public EventType getEventType() { @@ -141,4 +141,24 @@ public List getLogsWithMetadata() { public Hash getCommonAncestorHash() { return commonAncestorHash; } + + @Override + public String toString() { + return "BlockAddedEvent{" + + "eventType=" + + eventType + + ", block=" + + block.toLogString() + + ", commonAncestorHash=" + + commonAncestorHash + + ", addedTransactions count=" + + addedTransactions.size() + + ", removedTransactions count=" + + removedTransactions.size() + + ", transactionReceipts count =" + + transactionReceipts.size() + + ", logsWithMetadata count=" + + logsWithMetadata.size() + + '}'; + } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java index 8934bca0157..a66a1c96f89 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java @@ -383,8 +383,10 @@ private Difficulty calculateTotalDifficulty(final BlockHeader blockHeader) { private BlockAddedEvent updateCanonicalChainData( final BlockchainStorage.Updater updater, final BlockWithReceipts blockWithReceipts) { + final Block newBlock = blockWithReceipts.getBlock(); final Hash chainHead = blockchainStorage.getChainHead().orElse(null); + if (newBlock.getHeader().getNumber() != BlockHeader.GENESIS_BLOCK_NUMBER && chainHead == null) { throw new IllegalStateException("Blockchain is missing chain head."); } @@ -589,6 +591,31 @@ public boolean rewindToBlock(final Hash blockHash) { } } + @Override + public boolean forwardToBlock(final BlockHeader blockHeader) { + checkArgument( + chainHeader.getHash().equals(blockHeader.getParentHash()), + "Supplied block header is not a child of the current chain head."); + + final BlockchainStorage.Updater updater = blockchainStorage.updater(); + + try { + final BlockWithReceipts blockWithReceipts = getBlockWithReceipts(blockHeader).get(); + + BlockAddedEvent newHeadEvent = handleNewHead(updater, blockWithReceipts); + updateCacheForNewCanonicalHead( + blockWithReceipts.getBlock(), calculateTotalDifficulty(blockHeader)); + updater.commit(); + blockAddedObservers.forEach(observer -> observer.onBlockAdded(newHeadEvent)); + return true; + } catch (final NoSuchElementException e) { + // Any Optional.get() calls in this block should be present, missing data means data + // corruption or a bug. + updater.rollback(); + throw new IllegalStateException("Blockchain is missing data that should be present.", e); + } + } + @Override public void setFinalized(final Hash blockHash) { final var updater = blockchainStorage.updater(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/MutableBlockchain.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/MutableBlockchain.java index 13d61cd9b18..9ec7c8a9d9d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/MutableBlockchain.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/MutableBlockchain.java @@ -74,6 +74,16 @@ void unsafeImportBlock( */ boolean rewindToBlock(final Hash blockHash); + /** + * Forward the canonical chainhead to the specified block hash. The block hash must be a child of + * the current chainhead, that is already stored + * + * @param blockHeader The block header to forward to. + * @return {@code true} on success, {@code false} if the block is not a child of the current head + * {@code blockNumber} + */ + boolean forwardToBlock(final BlockHeader blockHeader); + /** * Set the hash of the last finalized block. * From 5153211aa8797726e6ded93426fa556240a59f07 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Wed, 22 Jun 2022 19:46:03 +0200 Subject: [PATCH 11/29] Do not persist prepared blocks Signed-off-by: Fabio Di Fabio --- .../merge/blockcreation/MergeCoordinator.java | 59 +++++++------------ .../blockcreation/MergeMiningCoordinator.java | 4 +- .../blockcreation/TransitionCoordinator.java | 9 +-- .../methods/engine/EngineNewPayload.java | 2 +- .../besu/ethereum/MainnetBlockValidator.java | 21 +++++-- .../bonsai/BonsaiLayeredWorldState.java | 5 ++ .../bonsai/BonsaiPersistedWorldState.java | 3 +- .../bonsai/BonsaiWorldStateArchive.java | 47 ++++++++++++--- .../besu/ethereum/core/MutableWorldState.java | 4 +- .../mainnet/AbstractBlockProcessor.java | 25 +++++--- .../besu/ethereum/mainnet/BlockProcessor.java | 35 +++++++++-- .../mainnet/MainnetProtocolSpecs.java | 11 +++- .../mainnet/PrivacyBlockProcessor.java | 11 +++- .../worldstate/DefaultMutableWorldState.java | 5 ++ 14 files changed, 161 insertions(+), 80 deletions(-) diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java index 6d9c098b2cf..6cd3d23183a 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java @@ -241,36 +241,6 @@ public Optional getOrSyncHeaderByHash( @Override public Result validateBlock(final Block block) { - final var chain = protocolContext.getBlockchain(); - chain - .getBlockHeader(block.getHeader().getParentHash()) - .ifPresentOrElse( - blockHeader -> - debugLambda(LOG, "Parent of block {} is already present", block::toLogString), - () -> backwardSyncContext.syncBackwardsUntil(block)); - - final var validationResult = - protocolSchedule - .getByBlockNumber(block.getHeader().getNumber()) - .getBlockValidator() - .validateAndProcessBlock( - protocolContext, block, HeaderValidationMode.FULL, HeaderValidationMode.NONE); - - return validationResult; - } - - @Override - public Result executeBlock(final Block block) { - return executeBlockWithoutSaving(block, true); - } - - @Override - public Result executeBlockWithoutSaving(final Block block) { - return executeBlockWithoutSaving(block, false); - } - - public Result executeBlockWithoutSaving(final Block block, final boolean shouldSave) { - final var chain = protocolContext.getBlockchain(); chain .getBlockHeader(block.getHeader().getParentHash()) @@ -288,15 +258,10 @@ public Result executeBlockWithoutSaving(final Block block, final boolean shouldS block, HeaderValidationMode.FULL, HeaderValidationMode.NONE, - shouldSave); + false); - validationResult.blockProcessingOutputs.ifPresentOrElse( - result -> { - if (shouldSave) { - chain.appendBlock(block, result.receipts); - } - }, - () -> + validationResult.errorMessage.ifPresent( + errMsg -> protocolSchedule .getByBlockNumber(chain.getChainHeadBlockNumber()) .getBadBlocksManager() @@ -305,6 +270,24 @@ public Result executeBlockWithoutSaving(final Block block, final boolean shouldS return validationResult; } + @Override + public Result rememberBlock(final Block block) { + final var chain = protocolContext.getBlockchain(); + LOG.error("rememberBlock {}", block.toLogString()); + final var validationResult = validateBlock(block); + LOG.error( + "validationResult.blockProcessingOutputs present? {}", + validationResult.blockProcessingOutputs.isPresent()); + validationResult.blockProcessingOutputs.ifPresent( + result -> { + LOG.error("result.worldState {}", result.worldState.getClass()); + + result.worldState.remember(block.getHeader()); + chain.storeBlock(block, result.receipts); + }); + return validationResult; + } + @Override public ForkchoiceResult updateForkChoice( final BlockHeader newHead, diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeMiningCoordinator.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeMiningCoordinator.java index cab04cea8ee..7a9232ac50a 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeMiningCoordinator.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeMiningCoordinator.java @@ -35,9 +35,7 @@ PayloadIdentifier preparePayload( final Bytes32 random, final Address feeRecipient); - Result executeBlock(final Block block); - - Result executeBlockWithoutSaving(final Block block); + Result rememberBlock(final Block block); Result validateBlock(final Block block); diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/TransitionCoordinator.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/TransitionCoordinator.java index d631c1431d5..f99c3fa71f4 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/TransitionCoordinator.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/TransitionCoordinator.java @@ -133,13 +133,8 @@ public PayloadIdentifier preparePayload( } @Override - public Result executeBlock(final Block block) { - return mergeCoordinator.executeBlock(block); - } - - @Override - public Result executeBlockWithoutSaving(final Block block) { - return mergeCoordinator.executeBlockWithoutSaving(block); + public Result rememberBlock(final Block block) { + return mergeCoordinator.rememberBlock(block); } @Override diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayload.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayload.java index 1489fe80f6e..17d5c621cff 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayload.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayload.java @@ -177,7 +177,7 @@ public JsonRpcResponse syncResponse(final JsonRpcRequestContext requestContext) } // execute block and return result response - final BlockValidator.Result executionResult = mergeCoordinator.executeBlockWithoutSaving(block); + final BlockValidator.Result executionResult = mergeCoordinator.rememberBlock(block); if (executionResult.errorMessage.isEmpty()) { return respondWith(reqId, newBlockHeader.getHash(), VALID); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java index 5b2e5b2cf3b..5b287f5d60f 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java @@ -107,8 +107,8 @@ public BlockValidator.Result validateAndProcessBlock( + " is not available"); } final MutableWorldState worldState = maybeWorldState.get(); - - final BlockProcessor.Result result = processBlock(context, worldState, block); + LOG.error("world state {}", worldState.rootHash()); + final BlockProcessor.Result result = processBlock(context, worldState, block, shouldPersist); if (result.isFailed()) { return handleAndReportFailure(block, "Error processing block"); } @@ -152,9 +152,22 @@ private Result handleAndReportFailure(final Block invalidBlock, final String rea * @return the result of processing the block */ protected BlockProcessor.Result processBlock( - final ProtocolContext context, final MutableWorldState worldState, final Block block) { + final ProtocolContext context, final MutableWorldState worldState, final Block block, final boolean shouldPersist) { + + return blockProcessor.processBlock(context.getBlockchain(), worldState, block, shouldPersist); + } - return blockProcessor.processBlock(context.getBlockchain(), worldState, block); + /** + * Processes a block, returning the result of the processing + * + * @param context the ProtocolContext + * @param worldState the world state for the parent block state root hash + * @param block the block to be processed + * @return the result of processing the block + */ + protected BlockProcessor.Result processBlock( + final ProtocolContext context, final MutableWorldState worldState, final Block block) { + return processBlock(context,worldState,block,true); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java index 47bcfd516f9..e7c0640d993 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java @@ -279,4 +279,9 @@ public void persist(final BlockHeader blockHeader) { public WorldUpdater updater() { return new BonsaiWorldStateUpdater(this); } + + @Override + public void remember(final BlockHeader blockHeader) { + archive.addLayeredWorldState(blockHeader, this); + } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java index 01d659f3820..f6deb02734c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java @@ -240,7 +240,7 @@ protected Hash calculateRootHash( @Override public void remember(final BlockHeader blockHeader) { checkArgument(blockHeader != null, "Block header must not be null"); - + LOG.error("remember {}", blockHeader.toLogString()); final BonsaiWorldStateUpdater localUpdater = updater.copy(); final BonsaiWorldStateKeyValueStorage.Updater stateUpdater = worldStateStorage.updater(); @@ -268,6 +268,7 @@ public void remember(final BlockHeader blockHeader) { @Override public void persist(final BlockHeader blockHeader) { + LOG.warn("persist {}", new Exception()); boolean success = false; final BonsaiWorldStateUpdater localUpdater = updater.copy(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java index 3aa43c65797..cf49c3e5dc3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java @@ -80,15 +80,41 @@ public BonsaiWorldStateArchive( private void blockAddedHandler(final BlockAddedEvent event) { LOG.debug("New block add event {}", event); if (event.isNewCanonicalHead()) { + LOG.error("layeredWorldStatesByHash {}", layeredWorldStatesByHash.toString()); final BlockHeader eventBlockHeader = event.getBlock().getHeader(); + LOG.error( + "layeredWorldStatesByHash parent hash {} is present? {}", + eventBlockHeader.getParentHash(), + layeredWorldStatesByHash.containsKey(eventBlockHeader.getParentHash())); + layeredWorldStatesByHash.computeIfPresent( eventBlockHeader.getParentHash(), - (hash, bonsaiLayeredWorldState) -> { - if (layeredWorldStatesByHash.containsKey(eventBlockHeader.getBlockHash())) { - bonsaiLayeredWorldState.setNextWorldView( - Optional.of(layeredWorldStatesByHash.get(eventBlockHeader.getBlockHash()))); - } - return bonsaiLayeredWorldState; + (parentHash, parentBonsaiLayeredWorldState) -> { + LOG.error( + "layeredWorldStatesByHash hash {} is present? {}", + eventBlockHeader.getBlockHash(), + layeredWorldStatesByHash.containsKey(eventBlockHeader.getBlockHash())); + layeredWorldStatesByHash.computeIfPresent( + eventBlockHeader.getBlockHash(), + (hash, bonsaiLayeredWorldState) -> { + LOG.error( + "worldStateStorage root hash {}, block hash {}, is present? {}", + bonsaiLayeredWorldState.rootHash(), + eventBlockHeader.getBlockHash(), + worldStateStorage.isWorldStateAvailable( + bonsaiLayeredWorldState.rootHash(), eventBlockHeader.getBlockHash())); + + if (!worldStateStorage.isWorldStateAvailable( + bonsaiLayeredWorldState.rootHash(), eventBlockHeader.getBlockHash())) { + LOG.error( + "saving root hash {}, block hash {}", + bonsaiLayeredWorldState.rootHash(), + eventBlockHeader.getBlockHash()); + } + return bonsaiLayeredWorldState; + }); + return layeredWorldStatesByHash.getOrDefault( + eventBlockHeader.getBlockHash(), parentBonsaiLayeredWorldState); }); } } @@ -117,7 +143,14 @@ public void addLayeredWorldState( blockHeader.getNumber(), worldStateRootHash, trieLog); - layeredWorldStatesByHash.put(bonsaiLayeredWorldState.blockHash(), bonsaiLayeredWorldState); + addLayeredWorldState(blockHeader, bonsaiLayeredWorldState); + } + + public void addLayeredWorldState( + final BlockHeader blockHeader, final BonsaiLayeredWorldState bonsaiLayeredWorldState) { + LOG.error("adding {}, block {}", bonsaiLayeredWorldState, blockHeader.toLogString()); + layeredWorldStatesByHash.put(blockHeader.getHash(), bonsaiLayeredWorldState); + LOG.error("layeredWorldStatesByHash {}", layeredWorldStatesByHash); } public Optional getTrieLogLayer(final Hash blockHash) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java index d3647beeced..3b843b551bd 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java @@ -42,7 +42,5 @@ public interface MutableWorldState extends WorldState, MutableWorldView { * * @param blockHeader The block hash of the world state this represents. Must not be null */ - default void remember(final BlockHeader blockHeader) { - persist(blockHeader); - } + void remember(final BlockHeader blockHeader); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java index 8ed852e5428..bd6d070affa 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java @@ -144,7 +144,9 @@ public AbstractBlockProcessor.Result processBlock( final BlockHeader blockHeader, final List transactions, final List ommers, - final PrivateMetadataUpdater privateMetadataUpdater) { + final PrivateMetadataUpdater privateMetadataUpdater, + final boolean shouldPersist) { + LOG.error("world state {}", worldState.rootHash()); final List receipts = new ArrayList<>(); long currentGasUsed = 0; for (final Transaction transaction : transactions) { @@ -169,6 +171,7 @@ public AbstractBlockProcessor.Result processBlock( true, TransactionValidationParams.processingBlock(), privateMetadataUpdater); + LOG.error("world state {}", worldState.rootHash()); if (result.isInvalid()) { LOG.info( "Block processing error: transaction invalid '{}'. Block {} Transaction {}", @@ -180,11 +183,11 @@ public AbstractBlockProcessor.Result processBlock( } return AbstractBlockProcessor.Result.failed(); } - + LOG.error("world state {}", worldState.rootHash()); worldStateUpdater.commit(); currentGasUsed += transaction.getGasLimit() - result.getGasRemaining(); - + LOG.error("world state {}", worldState.rootHash()); final TransactionReceipt transactionReceipt = transactionReceiptFactory.create( transaction.getType(), result, worldState, currentGasUsed); @@ -199,12 +202,18 @@ public AbstractBlockProcessor.Result processBlock( return AbstractBlockProcessor.Result.failed(); } - try { - worldState.persist(blockHeader); - } catch (Exception e) { - LOG.error("failed persisting block", e); - return AbstractBlockProcessor.Result.failed(); + LOG.error("world state {}", worldState.getClass()); + if (shouldPersist) { + try { + worldState.persist(blockHeader); + } catch (Exception e) { + LOG.error("failed persisting block", e); + return AbstractBlockProcessor.Result.failed(); + } + } else { + worldState.remember(blockHeader); } + LOG.error("world state {}", worldState.rootHash()); return AbstractBlockProcessor.Result.successful(receipts); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BlockProcessor.java index 94e63fcf843..0a6ebb097b3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BlockProcessor.java @@ -71,14 +71,18 @@ default boolean isFailed() { * @return the block processing result */ default Result processBlock( - final Blockchain blockchain, final MutableWorldState worldState, final Block block) { + final Blockchain blockchain, + final MutableWorldState worldState, + final Block block, + final boolean shouldPersist) { return processBlock( blockchain, worldState, block.getHeader(), block.getBody().getTransactions(), block.getBody().getOmmers(), - null); + null, + shouldPersist); } /** @@ -97,7 +101,29 @@ default Result processBlock( final BlockHeader blockHeader, final List transactions, final List ommers) { - return processBlock(blockchain, worldState, blockHeader, transactions, ommers, null); + return processBlock(blockchain, worldState, blockHeader, transactions, ommers, null, true); + } + + /** + * Processes the block and persist everything + * + * @param blockchain the blockchain to append the block to + * @param worldState the world state to apply changes to + * @param blockHeader the block header for the block + * @param transactions the transactions in the block + * @param ommers the block ommers + * @param privateMetadataUpdater the updater used to update the private metadata for the block + * @return the block processing result + */ + default Result processBlock( + final Blockchain blockchain, + final MutableWorldState worldState, + final BlockHeader blockHeader, + final List transactions, + final List ommers, + final PrivateMetadataUpdater privateMetadataUpdater) { + return processBlock( + blockchain, worldState, blockHeader, transactions, ommers, privateMetadataUpdater, true); } /** @@ -117,7 +143,8 @@ Result processBlock( BlockHeader blockHeader, List transactions, List ommers, - PrivateMetadataUpdater privateMetadataUpdater); + PrivateMetadataUpdater privateMetadataUpdater, + boolean shouldPersist); /** * Processes the block when running Besu in GoQuorum-compatible mode diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java index 3d98746c0f4..b0970cb56aa 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java @@ -706,10 +706,17 @@ public Result processBlock( final BlockHeader blockHeader, final List transactions, final List ommers, - final PrivateMetadataUpdater privateMetadataUpdater) { + final PrivateMetadataUpdater privateMetadataUpdater, + final boolean shouldPersist) { updateWorldStateForDao(worldState); return wrapped.processBlock( - blockchain, worldState, blockHeader, transactions, ommers, privateMetadataUpdater); + blockchain, + worldState, + blockHeader, + transactions, + ommers, + privateMetadataUpdater, + shouldPersist); } private static final Address DAO_REFUND_CONTRACT_ADDRESS = diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessor.java index bbd454ffbb9..d58727bf40c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessor.java @@ -86,7 +86,8 @@ public Result processBlock( final BlockHeader blockHeader, final List transactions, final List ommers, - final PrivateMetadataUpdater privateMetadataUpdater) { + final PrivateMetadataUpdater privateMetadataUpdater, + final boolean shouldPersist) { if (privateMetadataUpdater != null) { throw new IllegalArgumentException("PrivateMetadataUpdater passed in is not null."); @@ -99,7 +100,13 @@ public Result processBlock( final Result result = blockProcessor.processBlock( - blockchain, worldState, blockHeader, transactions, ommers, metadataUpdater); + blockchain, + worldState, + blockHeader, + transactions, + ommers, + metadataUpdater, + shouldPersist); metadataUpdater.commit(); return result; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java index 87e2a82aa6c..4f3e2f58d8d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java @@ -203,6 +203,11 @@ public void persist(final BlockHeader blockHeader) { stateUpdater.commit(); } + @Override + public void remember(final BlockHeader blockHeader) { + persist(blockHeader); + } + private Optional getStorageTrieKeyPreimage(final Bytes32 trieKey) { return Optional.ofNullable(newStorageKeyPreimages.get(trieKey)) .or(() -> preimageStorage.getStorageTrieKeyPreimage(trieKey)); From 5efca4b14f43c37908f779d2b384e93186e74c7e Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Thu, 23 Jun 2022 14:13:12 +0200 Subject: [PATCH 12/29] refactor how trie log is generated and persisted Signed-off-by: Fabio Di Fabio --- .../besu/ethereum/MainnetBlockValidator.java | 1 - .../bonsai/BonsaiPersistedWorldState.java | 121 +++++++++++------- .../bonsai/BonsaiWorldStateArchive.java | 30 +++-- .../BonsaiWorldStateKeyValueStorage.java | 12 +- .../mainnet/AbstractBlockProcessor.java | 6 - 5 files changed, 99 insertions(+), 71 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java index 5b287f5d60f..a1f6c5d0560 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java @@ -107,7 +107,6 @@ public BlockValidator.Result validateAndProcessBlock( + " is not available"); } final MutableWorldState worldState = maybeWorldState.get(); - LOG.error("world state {}", worldState.rootHash()); final BlockProcessor.Result result = processBlock(context, worldState, block, shouldPersist); if (result.isFailed()) { return handleAndReportFailure(block, "Error processing block"); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java index f6deb02734c..ecb21599dd7 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java @@ -20,9 +20,11 @@ import static org.hyperledger.besu.ethereum.bonsai.BonsaiAccount.fromRLP; import static org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; import static org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; +import static org.hyperledger.besu.util.Slf4jLambdaHelper.debugLambda; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.Updater; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; @@ -42,6 +44,7 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; +import org.hyperledger.besu.util.Slf4jLambdaHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -240,25 +243,13 @@ protected Hash calculateRootHash( @Override public void remember(final BlockHeader blockHeader) { checkArgument(blockHeader != null, "Block header must not be null"); - LOG.error("remember {}", blockHeader.toLogString()); + debugLambda(LOG, "Remember world state for block {}", blockHeader::toLogString); final BonsaiWorldStateUpdater localUpdater = updater.copy(); final BonsaiWorldStateKeyValueStorage.Updater stateUpdater = worldStateStorage.updater(); try { - worldStateRootHash = calculateRootHash(stateUpdater, localUpdater); - - if (!worldStateRootHash.equals(blockHeader.getStateRoot())) { - throw new RuntimeException( - "World State Root does not match expected value, header " - + blockHeader.getStateRoot().toHexString() - + " calculated " - + worldStateRootHash.toHexString()); - } - - final TrieLogLayer trieLog = - localUpdater.generateTrieLog(Hash.fromPlugin(blockHeader.getBlockHash())); - trieLog.freeze(); - archive.addLayeredWorldState(this, blockHeader, worldStateRootHash, trieLog); + final Hash newWorldStateRootHash = calculateRootHash(stateUpdater, localUpdater); + prepareTrieLog(blockHeader, localUpdater, newWorldStateRootHash, worldStateBlockHash); } finally { stateUpdater.rollback(); updater.reset(); @@ -268,53 +259,28 @@ public void remember(final BlockHeader blockHeader) { @Override public void persist(final BlockHeader blockHeader) { - LOG.warn("persist {}", new Exception()); + debugLambda(LOG, "Persist world state for block {}", blockHeader::toLogString); boolean success = false; final BonsaiWorldStateUpdater localUpdater = updater.copy(); - - final Hash originalBlockHash = worldStateBlockHash; - final Hash originalRootHash = worldStateRootHash; final BonsaiWorldStateKeyValueStorage.Updater stateUpdater = worldStateStorage.updater(); try { - worldStateRootHash = calculateRootHash(stateUpdater, localUpdater); - stateUpdater - .getTrieBranchStorageTransaction() - .put(WORLD_ROOT_HASH_KEY, worldStateRootHash.toArrayUnsafe()); - + final Hash newWorldStateRootHash = calculateRootHash(stateUpdater, localUpdater); // if we are persisted with a block header, and the prior state is the parent // then persist the TrieLog for that transition. // If specified but not a direct descendant simply store the new block hash. if (blockHeader != null) { - if (!worldStateRootHash.equals(blockHeader.getStateRoot())) { - throw new RuntimeException( - "World State Root does not match expected value, header " - + blockHeader.getStateRoot().toHexString() - + " calculated " - + worldStateRootHash.toHexString()); - } - worldStateBlockHash = Hash.fromPlugin(blockHeader.getBlockHash()); - stateUpdater - .getTrieBranchStorageTransaction() - .put(WORLD_BLOCK_HASH_KEY, worldStateBlockHash.toArrayUnsafe()); - if (originalBlockHash.equals(blockHeader.getParentHash())) { - LOG.debug("Writing Trie Log for {}", worldStateBlockHash); - final TrieLogLayer trieLog = localUpdater.generateTrieLog(worldStateBlockHash); - trieLog.freeze(); - archive.addLayeredWorldState(this, blockHeader, worldStateRootHash, trieLog); - final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput(); - trieLog.writeTo(rlpLog); - - stateUpdater - .getTrieLogStorageTransaction() - .put(worldStateBlockHash.toArrayUnsafe(), rlpLog.encoded().toArrayUnsafe()); - } + final TrieLogLayer trieLog = + prepareTrieLog(blockHeader, localUpdater, newWorldStateRootHash, worldStateBlockHash); + persistTrieLog(blockHeader, newWorldStateRootHash, trieLog, stateUpdater); + worldStateBlockHash = blockHeader.getHash(); } else { stateUpdater.getTrieBranchStorageTransaction().remove(WORLD_BLOCK_HASH_KEY); worldStateBlockHash = null; } + worldStateRootHash = newWorldStateRootHash; success = true; } finally { if (success) { @@ -323,8 +289,6 @@ public void persist(final BlockHeader blockHeader) { } else { stateUpdater.rollback(); updater.reset(); - worldStateBlockHash = originalBlockHash; - worldStateRootHash = originalRootHash; } } if (blockHeader != null) { @@ -332,6 +296,65 @@ public void persist(final BlockHeader blockHeader) { } } + private TrieLogLayer prepareTrieLog( + final BlockHeader blockHeader, + final BonsaiWorldStateUpdater localUpdater, + final Hash currentWorldStateRootHash, + final Hash previousBlockHash) { + + if (!currentWorldStateRootHash.equals(blockHeader.getStateRoot())) { + throw new RuntimeException( + "World State Root does not match expected value, header " + + blockHeader.getStateRoot().toHexString() + + " calculated " + + currentWorldStateRootHash.toHexString()); + } + + if (blockHeader.getNumber() > 0 && !previousBlockHash.equals(blockHeader.getParentHash())) { + throw new RuntimeException( + "Previous block hash " + + previousBlockHash.toHexString() + + " is not the parent of the current block " + + blockHeader.toLogString()); + } + + debugLambda(LOG, "Adding layered world state for {}", blockHeader::toLogString); + final TrieLogLayer trieLog = localUpdater.generateTrieLog(blockHeader.getBlockHash()); + trieLog.freeze(); + archive.addLayeredWorldState(this, blockHeader, currentWorldStateRootHash, trieLog); + return trieLog; + } + + void persistTrieLog( + final BlockHeader blockHeader, final Hash worldStateRootHash, final TrieLogLayer trieLog) { + final BonsaiWorldStateKeyValueStorage.Updater stateUpdater = worldStateStorage.updater(); + persistTrieLog(blockHeader, worldStateRootHash, trieLog, stateUpdater); + stateUpdater.commit(); + } + + private void persistTrieLog( + final BlockHeader blockHeader, + final Hash worldStateRootHash, + final TrieLogLayer trieLog, + final BonsaiWorldStateKeyValueStorage.Updater stateUpdater) { + debugLambda( + LOG, + "Persisting trie log for block hash {} and world state root {}", + blockHeader::toLogString, + worldStateRootHash::toHexString); + stateUpdater + .getTrieBranchStorageTransaction() + .put(WORLD_ROOT_HASH_KEY, worldStateRootHash.toArrayUnsafe()); + stateUpdater + .getTrieBranchStorageTransaction() + .put(WORLD_BLOCK_HASH_KEY, blockHeader.getHash().toArrayUnsafe()); + final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput(); + trieLog.writeTo(rlpLog); + stateUpdater + .getTrieLogStorageTransaction() + .put(blockHeader.getHash().toArrayUnsafe(), rlpLog.encoded().toArrayUnsafe()); + } + @Override public WorldUpdater updater() { return updater; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java index cf49c3e5dc3..9454e3803c2 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java @@ -25,6 +25,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.proof.WorldStateProof; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.evm.worldstate.WorldState; @@ -80,36 +81,37 @@ public BonsaiWorldStateArchive( private void blockAddedHandler(final BlockAddedEvent event) { LOG.debug("New block add event {}", event); if (event.isNewCanonicalHead()) { - LOG.error("layeredWorldStatesByHash {}", layeredWorldStatesByHash.toString()); final BlockHeader eventBlockHeader = event.getBlock().getHeader(); - LOG.error( - "layeredWorldStatesByHash parent hash {} is present? {}", - eventBlockHeader.getParentHash(), - layeredWorldStatesByHash.containsKey(eventBlockHeader.getParentHash())); - layeredWorldStatesByHash.computeIfPresent( eventBlockHeader.getParentHash(), (parentHash, parentBonsaiLayeredWorldState) -> { - LOG.error( - "layeredWorldStatesByHash hash {} is present? {}", - eventBlockHeader.getBlockHash(), - layeredWorldStatesByHash.containsKey(eventBlockHeader.getBlockHash())); layeredWorldStatesByHash.computeIfPresent( eventBlockHeader.getBlockHash(), (hash, bonsaiLayeredWorldState) -> { LOG.error( - "worldStateStorage root hash {}, block hash {}, is present? {}", + "worldStateStorage root hash {}, block hash {}, trieLog is present? {}, isWorldStateAvailable? {}", bonsaiLayeredWorldState.rootHash(), eventBlockHeader.getBlockHash(), + worldStateStorage.getTrieLog(eventBlockHeader.getBlockHash()), worldStateStorage.isWorldStateAvailable( bonsaiLayeredWorldState.rootHash(), eventBlockHeader.getBlockHash())); - if (!worldStateStorage.isWorldStateAvailable( - bonsaiLayeredWorldState.rootHash(), eventBlockHeader.getBlockHash())) { + if (worldStateStorage.getTrieLog(eventBlockHeader.getBlockHash()).isEmpty()) { LOG.error( - "saving root hash {}, block hash {}", + "saving trieLog now for root hash {}, block hash {}", bonsaiLayeredWorldState.rootHash(), eventBlockHeader.getBlockHash()); + persistedState.persistTrieLog( + eventBlockHeader, + bonsaiLayeredWorldState.rootHash(), + bonsaiLayeredWorldState.getTrieLog()); + LOG.error( + "worldStateStorage root hash {}, block hash {}, trieLog is present? {}, isWorldStateAvailable? {}", + bonsaiLayeredWorldState.rootHash(), + eventBlockHeader.getBlockHash(), + worldStateStorage.getTrieLog(eventBlockHeader.getBlockHash()), + worldStateStorage.isWorldStateAvailable( + bonsaiLayeredWorldState.rootHash(), eventBlockHeader.getBlockHash())); } return bonsaiLayeredWorldState; }); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java index eeab612798e..e8ea0bced9d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java @@ -33,9 +33,11 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.rlp.RLP; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage { - + private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateKeyValueStorage.class); public static final byte[] WORLD_ROOT_HASH_KEY = "worldRoot".getBytes(StandardCharsets.UTF_8); public static final byte[] WORLD_BLOCK_HASH_KEY = @@ -171,6 +173,14 @@ public Optional getNodeData(final Bytes location, final Bytes32 hash) { @Override public boolean isWorldStateAvailable(final Bytes32 rootHash, final Hash blockHash) { + LOG.error( + "trieBranchStorage {}, trieLogStorage {}", + trieBranchStorage + .get(WORLD_ROOT_HASH_KEY) + .map(Bytes32::wrap) + .filter(hash -> hash.equals(rootHash)) + .isPresent(), + trieLogStorage.containsKey(blockHash.toArrayUnsafe())); return trieBranchStorage .get(WORLD_ROOT_HASH_KEY) .map(Bytes32::wrap) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java index bd6d070affa..3a8d0709537 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java @@ -146,7 +146,6 @@ public AbstractBlockProcessor.Result processBlock( final List ommers, final PrivateMetadataUpdater privateMetadataUpdater, final boolean shouldPersist) { - LOG.error("world state {}", worldState.rootHash()); final List receipts = new ArrayList<>(); long currentGasUsed = 0; for (final Transaction transaction : transactions) { @@ -171,7 +170,6 @@ public AbstractBlockProcessor.Result processBlock( true, TransactionValidationParams.processingBlock(), privateMetadataUpdater); - LOG.error("world state {}", worldState.rootHash()); if (result.isInvalid()) { LOG.info( "Block processing error: transaction invalid '{}'. Block {} Transaction {}", @@ -183,11 +181,9 @@ public AbstractBlockProcessor.Result processBlock( } return AbstractBlockProcessor.Result.failed(); } - LOG.error("world state {}", worldState.rootHash()); worldStateUpdater.commit(); currentGasUsed += transaction.getGasLimit() - result.getGasRemaining(); - LOG.error("world state {}", worldState.rootHash()); final TransactionReceipt transactionReceipt = transactionReceiptFactory.create( transaction.getType(), result, worldState, currentGasUsed); @@ -202,7 +198,6 @@ public AbstractBlockProcessor.Result processBlock( return AbstractBlockProcessor.Result.failed(); } - LOG.error("world state {}", worldState.getClass()); if (shouldPersist) { try { worldState.persist(blockHeader); @@ -213,7 +208,6 @@ public AbstractBlockProcessor.Result processBlock( } else { worldState.remember(blockHeader); } - LOG.error("world state {}", worldState.rootHash()); return AbstractBlockProcessor.Result.successful(receipts); } From da0eeb23afb540a37beeca2d205f8fc271efad4d Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Thu, 23 Jun 2022 17:07:26 +0200 Subject: [PATCH 13/29] clean log Signed-off-by: Fabio Di Fabio --- .../merge/blockcreation/MergeCoordinator.java | 7 +--- .../besu/ethereum/MainnetBlockValidator.java | 7 ++-- .../bonsai/BonsaiLayeredWorldState.java | 1 - .../bonsai/BonsaiPersistedWorldState.java | 2 -- .../bonsai/BonsaiWorldStateArchive.java | 33 +++++-------------- .../BonsaiWorldStateKeyValueStorage.java | 11 ------- .../mainnet/AbstractBlockProcessor.java | 3 +- 7 files changed, 16 insertions(+), 48 deletions(-) diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java index b28275be0b4..8fd1aa53a99 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java @@ -272,16 +272,11 @@ public Result validateBlock(final Block block) { @Override public Result rememberBlock(final Block block) { + debugLambda(LOG, "Remember block {}", block::toLogString); final var chain = protocolContext.getBlockchain(); - LOG.error("rememberBlock {}", block.toLogString()); final var validationResult = validateBlock(block); - LOG.error( - "validationResult.blockProcessingOutputs present? {}", - validationResult.blockProcessingOutputs.isPresent()); validationResult.blockProcessingOutputs.ifPresent( result -> { - LOG.error("result.worldState {}", result.worldState.getClass()); - result.worldState.remember(block.getHeader()); chain.storeBlock(block, result.receipts); }); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java index a1f6c5d0560..7652337c36c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java @@ -151,7 +151,10 @@ private Result handleAndReportFailure(final Block invalidBlock, final String rea * @return the result of processing the block */ protected BlockProcessor.Result processBlock( - final ProtocolContext context, final MutableWorldState worldState, final Block block, final boolean shouldPersist) { + final ProtocolContext context, + final MutableWorldState worldState, + final Block block, + final boolean shouldPersist) { return blockProcessor.processBlock(context.getBlockchain(), worldState, block, shouldPersist); } @@ -166,7 +169,7 @@ protected BlockProcessor.Result processBlock( */ protected BlockProcessor.Result processBlock( final ProtocolContext context, final MutableWorldState worldState, final Block block) { - return processBlock(context,worldState,block,true); + return processBlock(context, worldState, block, true); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java index e7c0640d993..5e3c350578c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java @@ -37,7 +37,6 @@ /** A World State backed first by trie log layer and then by another world state. */ public class BonsaiLayeredWorldState implements MutableWorldState, BonsaiWorldView, WorldState { - private Optional nextWorldView; protected final long height; protected final TrieLogLayer trieLog; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java index ecb21599dd7..4c592372d24 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java @@ -24,7 +24,6 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.Updater; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; @@ -44,7 +43,6 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; -import org.hyperledger.besu.util.Slf4jLambdaHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java index 9454e3803c2..e5914aca0af 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java @@ -17,6 +17,7 @@ package org.hyperledger.besu.ethereum.bonsai; import static org.hyperledger.besu.datatypes.Hash.fromPlugin; +import static org.hyperledger.besu.util.Slf4jLambdaHelper.debugLambda; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; @@ -25,7 +26,6 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.proof.WorldStateProof; -import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.evm.worldstate.WorldState; @@ -88,35 +88,22 @@ private void blockAddedHandler(final BlockAddedEvent event) { layeredWorldStatesByHash.computeIfPresent( eventBlockHeader.getBlockHash(), (hash, bonsaiLayeredWorldState) -> { - LOG.error( - "worldStateStorage root hash {}, block hash {}, trieLog is present? {}, isWorldStateAvailable? {}", - bonsaiLayeredWorldState.rootHash(), - eventBlockHeader.getBlockHash(), - worldStateStorage.getTrieLog(eventBlockHeader.getBlockHash()), - worldStateStorage.isWorldStateAvailable( - bonsaiLayeredWorldState.rootHash(), eventBlockHeader.getBlockHash())); - if (worldStateStorage.getTrieLog(eventBlockHeader.getBlockHash()).isEmpty()) { - LOG.error( - "saving trieLog now for root hash {}, block hash {}", - bonsaiLayeredWorldState.rootHash(), - eventBlockHeader.getBlockHash()); + debugLambda( + LOG, + "Trie log not yet stored, doing it now for root hash {}, block hash {}", + () -> bonsaiLayeredWorldState.rootHash(), + () -> eventBlockHeader.getBlockHash()); persistedState.persistTrieLog( eventBlockHeader, bonsaiLayeredWorldState.rootHash(), bonsaiLayeredWorldState.getTrieLog()); - LOG.error( - "worldStateStorage root hash {}, block hash {}, trieLog is present? {}, isWorldStateAvailable? {}", - bonsaiLayeredWorldState.rootHash(), - eventBlockHeader.getBlockHash(), - worldStateStorage.getTrieLog(eventBlockHeader.getBlockHash()), - worldStateStorage.isWorldStateAvailable( - bonsaiLayeredWorldState.rootHash(), eventBlockHeader.getBlockHash())); } + parentBonsaiLayeredWorldState.setNextWorldView( + Optional.of(layeredWorldStatesByHash.get(eventBlockHeader.getBlockHash()))); return bonsaiLayeredWorldState; }); - return layeredWorldStatesByHash.getOrDefault( - eventBlockHeader.getBlockHash(), parentBonsaiLayeredWorldState); + return parentBonsaiLayeredWorldState; }); } } @@ -150,9 +137,7 @@ public void addLayeredWorldState( public void addLayeredWorldState( final BlockHeader blockHeader, final BonsaiLayeredWorldState bonsaiLayeredWorldState) { - LOG.error("adding {}, block {}", bonsaiLayeredWorldState, blockHeader.toLogString()); layeredWorldStatesByHash.put(blockHeader.getHash(), bonsaiLayeredWorldState); - LOG.error("layeredWorldStatesByHash {}", layeredWorldStatesByHash); } public Optional getTrieLogLayer(final Hash blockHash) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java index e8ea0bced9d..29b52265177 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java @@ -33,11 +33,8 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.rlp.RLP; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage { - private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateKeyValueStorage.class); public static final byte[] WORLD_ROOT_HASH_KEY = "worldRoot".getBytes(StandardCharsets.UTF_8); public static final byte[] WORLD_BLOCK_HASH_KEY = @@ -173,14 +170,6 @@ public Optional getNodeData(final Bytes location, final Bytes32 hash) { @Override public boolean isWorldStateAvailable(final Bytes32 rootHash, final Hash blockHash) { - LOG.error( - "trieBranchStorage {}, trieLogStorage {}", - trieBranchStorage - .get(WORLD_ROOT_HASH_KEY) - .map(Bytes32::wrap) - .filter(hash -> hash.equals(rootHash)) - .isPresent(), - trieLogStorage.containsKey(blockHash.toArrayUnsafe())); return trieBranchStorage .get(WORLD_ROOT_HASH_KEY) .map(Bytes32::wrap) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java index 3a8d0709537..5fae088b938 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java @@ -205,9 +205,8 @@ public AbstractBlockProcessor.Result processBlock( LOG.error("failed persisting block", e); return AbstractBlockProcessor.Result.failed(); } - } else { - worldState.remember(blockHeader); } + return AbstractBlockProcessor.Result.successful(receipts); } From c2a9f3b133bfa9e3c135ebcb134f368cc2c0a86a Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Fri, 24 Jun 2022 10:08:24 +0200 Subject: [PATCH 14/29] try safe copy Signed-off-by: Karim TAAM --- .../ethereum/bonsai/BonsaiLayeredWorldState.java | 3 ++- .../bonsai/BonsaiPersistedWorldState.java | 1 + .../ethereum/bonsai/BonsaiWorldStateArchive.java | 16 +++++++++++++--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java index 47bcfd516f9..9f5e9e8edd7 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java @@ -230,7 +230,8 @@ public Account get(final Address address) { } else if (currentLayer.getNextWorldView().get() instanceof BonsaiLayeredWorldState) { currentLayer = (BonsaiLayeredWorldState) currentLayer.getNextWorldView().get(); } else { - return currentLayer.getNextWorldView().get().get(address); + return new BonsaiAccount( + (BonsaiAccount) currentLayer.getNextWorldView().get().get(address), this, false); } } return null; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java index 04a383a0229..063aa3cf739 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java @@ -289,6 +289,7 @@ public void persist(final BlockHeader blockHeader) { if (success) { stateUpdater.commit(); updater.reset(); + archive.applyPersistState(this); } else { stateUpdater.rollback(); updater.reset(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java index 3f10fd96dbf..c9df5e36a9a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java @@ -48,7 +48,7 @@ public class BonsaiWorldStateArchive implements WorldStateArchive { private final Blockchain blockchain; - private final BonsaiPersistedWorldState persistedState; + private BonsaiPersistedWorldState persistedState; private final Map layeredWorldStatesByHash; private final BonsaiWorldStateKeyValueStorage worldStateStorage; private final long maxLayersToLoad; @@ -179,7 +179,7 @@ public Optional getMutable( @Override public Optional getMutable(final Hash rootHash, final Hash blockHash) { if (blockHash.equals(persistedState.blockHash())) { - return Optional.of(persistedState); + return Optional.of(safeCopy(persistedState)); } else { try { @@ -242,7 +242,7 @@ public Optional getMutable(final Hash rootHash, final Hash bl persistedState.persist(blockchain.getBlockHeader(blockHash).get()); LOG.debug("Archive rolling finished, now at {}", blockHash); - return Optional.of(persistedState); + return Optional.of(safeCopy(persistedState)); } catch (final Exception e) { // if we fail we must clean up the updater bonsaiUpdater.reset(); @@ -255,11 +255,21 @@ public Optional getMutable(final Hash rootHash, final Hash bl } } + private BonsaiPersistedWorldState safeCopy( + final BonsaiPersistedWorldState bonsaiPersistedWorldState) { + return new BonsaiPersistedWorldState( + bonsaiPersistedWorldState.getArchive(), bonsaiPersistedWorldState.getWorldStateStorage()); + } + @Override public MutableWorldState getMutable() { return persistedState; } + public void applyPersistState(BonsaiPersistedWorldState bonsaiPersistedWorldState) { + this.persistedState = bonsaiPersistedWorldState; + } + @Override public void setArchiveStateUnSafe(final BlockHeader blockHeader) { persistedState.setArchiveStateUnSafe(blockHeader); From 66a7116534f509f52000115b6802b152348925ca Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Fri, 24 Jun 2022 10:14:41 +0200 Subject: [PATCH 15/29] fix build Signed-off-by: Karim TAAM --- .../besu/ethereum/bonsai/BonsaiWorldStateArchive.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java index c9df5e36a9a..a3791a83ad4 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java @@ -266,7 +266,7 @@ public MutableWorldState getMutable() { return persistedState; } - public void applyPersistState(BonsaiPersistedWorldState bonsaiPersistedWorldState) { + public void applyPersistState(final BonsaiPersistedWorldState bonsaiPersistedWorldState) { this.persistedState = bonsaiPersistedWorldState; } From d4603fbc1769d8e84102fd611d8a9b27fa457c9c Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Fri, 24 Jun 2022 10:30:12 +0200 Subject: [PATCH 16/29] fix nullpointer Signed-off-by: Karim TAAM --- .../besu/ethereum/bonsai/BonsaiLayeredWorldState.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java index 9f5e9e8edd7..47bcfd516f9 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java @@ -230,8 +230,7 @@ public Account get(final Address address) { } else if (currentLayer.getNextWorldView().get() instanceof BonsaiLayeredWorldState) { currentLayer = (BonsaiLayeredWorldState) currentLayer.getNextWorldView().get(); } else { - return new BonsaiAccount( - (BonsaiAccount) currentLayer.getNextWorldView().get().get(address), this, false); + return currentLayer.getNextWorldView().get().get(address); } } return null; From b21e9373c28f350c7b8a00bdbd76cf8962205402 Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Fri, 24 Jun 2022 12:01:10 +0200 Subject: [PATCH 17/29] update code Signed-off-by: Karim TAAM --- .../bonsai/BonsaiPersistedWorldState.java | 6 --- .../bonsai/BonsaiWorldStateArchive.java | 37 +++++++------------ .../BonsaiWorldStateKeyValueStorage.java | 6 ++- .../keyvalue/WorldStateKeyValueStorage.java | 2 +- .../worldstate/DefaultWorldStateArchive.java | 6 --- .../worldstate/WorldStateArchive.java | 3 -- .../worldstate/WorldStateStorage.java | 2 +- .../BonsaiWorldStateKeyValueStorageTest.java | 3 +- .../eth/sync/DefaultSynchronizer.java | 5 --- .../worldstate/FastWorldDownloadState.java | 2 +- .../sync/snapsync/SnapWorldDownloadState.java | 3 +- 11 files changed, 24 insertions(+), 51 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java index 063aa3cf739..c5b1cfd491d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java @@ -90,11 +90,6 @@ public Optional getCode(@Nonnull final Address address) { return worldStateStorage.getCode(null, Hash.hash(address)); } - public void setArchiveStateUnSafe(final BlockHeader blockHeader) { - worldStateBlockHash = Hash.fromPlugin(blockHeader.getBlockHash()); - worldStateRootHash = Hash.fromPlugin(blockHeader.getStateRoot()); - } - public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { return worldStateStorage; } @@ -289,7 +284,6 @@ public void persist(final BlockHeader blockHeader) { if (success) { stateUpdater.commit(); updater.reset(); - archive.applyPersistState(this); } else { stateUpdater.rollback(); updater.reset(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java index a3791a83ad4..3d05c4c449a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java @@ -48,7 +48,6 @@ public class BonsaiWorldStateArchive implements WorldStateArchive { private final Blockchain blockchain; - private BonsaiPersistedWorldState persistedState; private final Map layeredWorldStatesByHash; private final BonsaiWorldStateKeyValueStorage worldStateStorage; private final long maxLayersToLoad; @@ -70,7 +69,6 @@ public BonsaiWorldStateArchive( this.blockchain = blockchain; this.worldStateStorage = new BonsaiWorldStateKeyValueStorage(provider); - this.persistedState = new BonsaiPersistedWorldState(this, worldStateStorage); this.layeredWorldStatesByHash = layeredWorldStatesByHash; this.maxLayersToLoad = maxLayersToLoad; blockchain.observeBlockAdded( @@ -95,6 +93,7 @@ public BonsaiWorldStateArchive( @Override public Optional get(final Hash rootHash, final Hash blockHash) { + final BonsaiPersistedWorldState persistedState = persistedState(); if (layeredWorldStatesByHash.containsKey(blockHash)) { return Optional.of(layeredWorldStatesByHash.get(blockHash)); } else if (rootHash.equals(persistedState.blockHash())) { @@ -131,7 +130,7 @@ public Optional getTrieLogLayer(final Hash blockHash) { @Override public boolean isWorldStateAvailable(final Hash rootHash, final Hash blockHash) { return layeredWorldStatesByHash.containsKey(blockHash) - || persistedState.blockHash().equals(blockHash) + || persistedState().blockHash().equals(blockHash) || worldStateStorage.isWorldStateAvailable(rootHash, blockHash); } @@ -178,18 +177,19 @@ public Optional getMutable( @Override public Optional getMutable(final Hash rootHash, final Hash blockHash) { - if (blockHash.equals(persistedState.blockHash())) { - return Optional.of(safeCopy(persistedState)); + final BonsaiPersistedWorldState persistedWorldState = persistedState(); + if (blockHash.equals(persistedWorldState.blockHash())) { + return Optional.of(persistedWorldState); } else { try { final Optional maybePersistedHeader = - blockchain.getBlockHeader(persistedState.blockHash()).map(BlockHeader.class::cast); + blockchain.getBlockHeader(persistedWorldState.blockHash()).map(BlockHeader.class::cast); final List rollBacks = new ArrayList<>(); final List rollForwards = new ArrayList<>(); if (maybePersistedHeader.isEmpty()) { - getTrieLogLayer(persistedState.blockHash()).ifPresent(rollBacks::add); + getTrieLogLayer(persistedWorldState.blockHash()).ifPresent(rollBacks::add); } else { BlockHeader targetHeader = blockchain.getBlockHeader(blockHash).get(); BlockHeader persistedHeader = maybePersistedHeader.get(); @@ -227,7 +227,7 @@ public Optional getMutable(final Hash rootHash, final Hash bl // attempt the state rolling final BonsaiWorldStateUpdater bonsaiUpdater = - (BonsaiWorldStateUpdater) persistedState.updater(); + (BonsaiWorldStateUpdater) persistedWorldState.updater(); try { for (final TrieLogLayer rollBack : rollBacks) { LOG.debug("Attempting Rollback of {}", rollBack.getBlockHash()); @@ -239,10 +239,10 @@ public Optional getMutable(final Hash rootHash, final Hash bl } bonsaiUpdater.commit(); - persistedState.persist(blockchain.getBlockHeader(blockHash).get()); + persistedWorldState.persist(blockchain.getBlockHeader(blockHash).get()); LOG.debug("Archive rolling finished, now at {}", blockHash); - return Optional.of(safeCopy(persistedState)); + return Optional.of(persistedWorldState); } catch (final Exception e) { // if we fail we must clean up the updater bonsaiUpdater.reset(); @@ -255,24 +255,13 @@ public Optional getMutable(final Hash rootHash, final Hash bl } } - private BonsaiPersistedWorldState safeCopy( - final BonsaiPersistedWorldState bonsaiPersistedWorldState) { - return new BonsaiPersistedWorldState( - bonsaiPersistedWorldState.getArchive(), bonsaiPersistedWorldState.getWorldStateStorage()); + private BonsaiPersistedWorldState persistedState() { + return new BonsaiPersistedWorldState(this, worldStateStorage); } @Override public MutableWorldState getMutable() { - return persistedState; - } - - public void applyPersistState(final BonsaiPersistedWorldState bonsaiPersistedWorldState) { - this.persistedState = bonsaiPersistedWorldState; - } - - @Override - public void setArchiveStateUnSafe(final BlockHeader blockHeader) { - persistedState.setArchiveStateUnSafe(blockHeader); + return persistedState(); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java index eeab612798e..1602e796e55 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java @@ -272,8 +272,10 @@ public Updater putAccountInfoState(final Hash accountHash, final Bytes accountVa @Override public WorldStateStorage.Updater saveWorldState( - final Bytes blockHash, final Bytes32 nodeHash, final Bytes node) { - trieBranchStorageTransaction.put(Bytes.EMPTY.toArrayUnsafe(), node.toArrayUnsafe()); + final Bytes blockHash, final Bytes32 nodeHash, final Optional node) { + node.ifPresent( + bytes -> + trieBranchStorageTransaction.put(Bytes.EMPTY.toArrayUnsafe(), bytes.toArrayUnsafe())); trieBranchStorageTransaction.put(WORLD_ROOT_HASH_KEY, nodeHash.toArrayUnsafe()); trieBranchStorageTransaction.put(WORLD_BLOCK_HASH_KEY, blockHash.toArrayUnsafe()); return this; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/WorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/WorldStateKeyValueStorage.java index c67db7f62e0..4c64a4dbb87 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/WorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/WorldStateKeyValueStorage.java @@ -168,7 +168,7 @@ public WorldStateStorage.Updater putCode( @Override public WorldStateStorage.Updater saveWorldState( - final Bytes blockHash, final Bytes32 nodeHash, final Bytes node) { + final Bytes blockHash, final Bytes32 nodeHash, final Optional node) { return putAccountStateTrieNode(null, nodeHash, node); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java index ae93b31ba3d..3bd97ae4166 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java @@ -17,7 +17,6 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.proof.WorldStateProof; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; @@ -80,11 +79,6 @@ public MutableWorldState getMutable() { return getMutable(EMPTY_ROOT_HASH, null).get(); } - @Override - public void setArchiveStateUnSafe(final BlockHeader blockHeader) { - // ignore for forest - } - @Override public Optional getNodeData(final Hash hash) { // query by location is not supported, only query by content diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java index 17a33760e9c..a23251ec27f 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java @@ -16,7 +16,6 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.proof.WorldStateProof; import org.hyperledger.besu.ethereum.trie.MerklePatriciaTrie; @@ -43,8 +42,6 @@ public interface WorldStateArchive { MutableWorldState getMutable(); - void setArchiveStateUnSafe(BlockHeader blockHeader); - Optional getNodeData(Hash hash); Optional getAccountProof( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java index 5411829e186..97d6db6eddb 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java @@ -64,7 +64,7 @@ default Updater putCode(final Hash accountHash, final Bytes code) { return putCode(accountHash, codeHash, code); } - Updater saveWorldState(Bytes blockHash, Bytes32 nodeHash, Bytes node); + Updater saveWorldState(Bytes blockHash, Bytes32 nodeHash, Optional node); Updater putAccountStateTrieNode(Bytes location, Bytes32 nodeHash, Bytes node); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java index ea7804c8bbc..ea273128efb 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java @@ -32,6 +32,7 @@ import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; +import java.util.Optional; import java.util.TreeMap; import org.apache.tuweni.bytes.Bytes; @@ -285,7 +286,7 @@ public void isWorldStateAvailable_afterCallingSaveWorldstate() { assertThat(storage.isWorldStateAvailable(Bytes32.wrap(nodeHashKey), Hash.EMPTY)).isFalse(); - updater.saveWorldState(blockHash, nodeHashKey, nodeValue); + updater.saveWorldState(blockHash, nodeHashKey, Optional.of(nodeValue)); updater.commit(); assertThat(storage.isWorldStateAvailable(Bytes32.wrap(nodeHashKey), Hash.EMPTY)).isTrue(); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java index dd1e19e2c70..886f2077dc9 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java @@ -218,11 +218,6 @@ private CompletableFuture handleSyncResult(final FastSyncState result) { return CompletableFuture.completedFuture(null); } fastSyncDownloader.ifPresent(FastSyncDownloader::deleteFastSyncState); - result - .getPivotBlockHeader() - .ifPresent( - blockHeader -> - protocolContext.getWorldStateArchive().setArchiveStateUnSafe(blockHeader)); LOG.info( "Sync completed successfully with pivot block {}", result.getPivotBlockNumber().getAsLong()); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadState.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadState.java index 0703c93ef89..3d19906b956 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadState.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadState.java @@ -54,7 +54,7 @@ public synchronized boolean checkCompletion(final BlockHeader header) { return false; } final Updater updater = worldStateStorage.updater(); - updater.saveWorldState(header.getHash(), header.getStateRoot(), rootNodeData); + updater.saveWorldState(header.getHash(), header.getStateRoot(), Optional.of(rootNodeData)); updater.commit(); internalFuture.complete(null); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java index 5d73e655f61..c93d8fd6d5c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java @@ -33,6 +33,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.function.Predicate; import java.util.stream.Stream; @@ -139,7 +140,7 @@ public synchronized boolean checkCompletion(final BlockHeader header) { startHeal(); } else { final WorldStateStorage.Updater updater = worldStateStorage.updater(); - updater.saveWorldState(header.getHash(), header.getStateRoot(), rootNodeData); + updater.saveWorldState(header.getHash(), header.getStateRoot(), Optional.of(rootNodeData)); updater.commit(); metricsManager.notifySnapSyncCompleted(); internalFuture.complete(null); From 0feeedd3eefbbac7535c6402ff526eaf542a245f Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Fri, 24 Jun 2022 12:20:01 +0200 Subject: [PATCH 18/29] Version with the last modification from Karim and the use of BonsaiInMemoryWorldState Signed-off-by: Fabio Di Fabio --- .../merge/blockcreation/MergeCoordinator.java | 6 ++-- .../besu/ethereum/MainnetBlockValidator.java | 8 ++++-- .../bonsai/BonsaiInMemoryWorldState.java | 7 ++++- .../bonsai/BonsaiPersistedWorldState.java | 10 +++---- .../bonsai/BonsaiWorldStateArchive.java | 28 +++++++++++++------ 5 files changed, 37 insertions(+), 22 deletions(-) diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java index 8fd1aa53a99..e0a7eccd876 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java @@ -276,10 +276,7 @@ public Result rememberBlock(final Block block) { final var chain = protocolContext.getBlockchain(); final var validationResult = validateBlock(block); validationResult.blockProcessingOutputs.ifPresent( - result -> { - result.worldState.remember(block.getHeader()); - chain.storeBlock(block, result.receipts); - }); + result -> chain.storeBlock(block, result.receipts)); return validationResult; } @@ -369,6 +366,7 @@ private boolean setNewHead( LOG, "Forwarding chain head to the block {} saved from a previous newPayload invocation", newHead::toLogString); + return blockchain.forwardToBlock(newHead); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java index 7652337c36c..bc35cf9e4b8 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java @@ -97,7 +97,7 @@ public BlockValidator.Result validateAndProcessBlock( final Optional maybeWorldState = context .getWorldStateArchive() - .getMutable(parentHeader.getStateRoot(), parentHeader.getHash(), shouldPersist); + .getMutable(parentHeader.getStateRoot(), parentHeader.getHash()); if (maybeWorldState.isEmpty()) { return handleAndReportFailure( @@ -106,8 +106,10 @@ public BlockValidator.Result validateAndProcessBlock( + parentHeader.getStateRoot() + " is not available"); } - final MutableWorldState worldState = maybeWorldState.get(); - final BlockProcessor.Result result = processBlock(context, worldState, block, shouldPersist); + final MutableWorldState worldState = + shouldPersist ? maybeWorldState.get() : maybeWorldState.get().copy(); + + final BlockProcessor.Result result = processBlock(context, worldState, block); if (result.isFailed()) { return handleAndReportFailure(block, "Error processing block"); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java index 3d2cebf9781..b3b165b36ae 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java @@ -37,6 +37,11 @@ public Hash rootHash() { @Override public void persist(final BlockHeader blockHeader) { - throw new UnsupportedOperationException("In Memory worldState can not be persisted."); + final BonsaiWorldStateUpdater localUpdater = updater.copy(); + worldStateBlockHash = blockHeader.getHash(); + worldStateRootHash = blockHeader.getStateRoot(); + final TrieLogLayer trieLog = localUpdater.generateTrieLog(worldStateBlockHash); + trieLog.freeze(); + archive.addLayeredWorldState(this, blockHeader, worldStateRootHash, trieLog); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java index 330668ffe13..bbe9cfddac8 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java @@ -52,11 +52,11 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld protected final BonsaiWorldStateKeyValueStorage worldStateStorage; - private final BonsaiWorldStateArchive archive; - private final BonsaiWorldStateUpdater updater; + protected final BonsaiWorldStateArchive archive; + protected final BonsaiWorldStateUpdater updater; - private Hash worldStateRootHash; - private Hash worldStateBlockHash; + protected Hash worldStateRootHash; + protected Hash worldStateBlockHash; public BonsaiPersistedWorldState( final BonsaiWorldStateArchive archive, @@ -295,7 +295,7 @@ public void persist(final BlockHeader blockHeader) { } } - private TrieLogLayer prepareTrieLog( + protected TrieLogLayer prepareTrieLog( final BlockHeader blockHeader, final BonsaiWorldStateUpdater localUpdater, final Hash currentWorldStateRootHash, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java index 2ef8278ceab..3e9deacb537 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java @@ -86,18 +86,28 @@ private void blockAddedHandler(final BlockAddedEvent event) { eventBlockHeader.getParentHash(), (parentHash, parentBonsaiLayeredWorldState) -> { layeredWorldStatesByHash.computeIfPresent( - eventBlockHeader.getBlockHash(), + eventBlockHeader.getHash(), (hash, bonsaiLayeredWorldState) -> { - if (worldStateStorage.getTrieLog(eventBlockHeader.getBlockHash()).isEmpty()) { + if (worldStateStorage.getTrieLog(eventBlockHeader.getHash()).isEmpty()) { debugLambda( LOG, - "Trie log not yet stored, doing it now for root hash {}, block hash {}", - () -> bonsaiLayeredWorldState.rootHash(), - () -> eventBlockHeader.getBlockHash()); - persistedState.persistTrieLog( - eventBlockHeader, - bonsaiLayeredWorldState.rootHash(), - bonsaiLayeredWorldState.getTrieLog()); + "World state not yet persisted, doing it now for root hash {}, block hash {}", + eventBlockHeader::getStateRoot, + eventBlockHeader::getHash); + + getMutable(eventBlockHeader.getStateRoot(), eventBlockHeader.getHash()) + .ifPresentOrElse( + mutableWorldState -> + debugLambda( + LOG, + "World state for state root hash {} and block hash {} persisted successfully", + mutableWorldState::rootHash, + eventBlockHeader::getHash), + () -> + LOG.error( + "Could not persist world for root hash {} and block hash {}", + eventBlockHeader.getStateRoot(), + eventBlockHeader.getHash())); } parentBonsaiLayeredWorldState.setNextWorldView( Optional.of(layeredWorldStatesByHash.get(eventBlockHeader.getBlockHash()))); From f2572828647ec6f02c35cfa889ccf13f07555281 Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Fri, 24 Jun 2022 14:27:03 +0200 Subject: [PATCH 19/29] fix build Signed-off-by: Karim TAAM --- .../ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java | 6 ++---- .../storage/keyvalue/WorldStateKeyValueStorage.java | 2 +- .../besu/ethereum/worldstate/WorldStateStorage.java | 2 +- .../bonsai/BonsaiWorldStateKeyValueStorageTest.java | 3 +-- .../sync/fastsync/worldstate/FastWorldDownloadState.java | 2 +- .../ethereum/eth/sync/snapsync/SnapWorldDownloadState.java | 3 +-- 6 files changed, 7 insertions(+), 11 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java index 1602e796e55..eeab612798e 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java @@ -272,10 +272,8 @@ public Updater putAccountInfoState(final Hash accountHash, final Bytes accountVa @Override public WorldStateStorage.Updater saveWorldState( - final Bytes blockHash, final Bytes32 nodeHash, final Optional node) { - node.ifPresent( - bytes -> - trieBranchStorageTransaction.put(Bytes.EMPTY.toArrayUnsafe(), bytes.toArrayUnsafe())); + final Bytes blockHash, final Bytes32 nodeHash, final Bytes node) { + trieBranchStorageTransaction.put(Bytes.EMPTY.toArrayUnsafe(), node.toArrayUnsafe()); trieBranchStorageTransaction.put(WORLD_ROOT_HASH_KEY, nodeHash.toArrayUnsafe()); trieBranchStorageTransaction.put(WORLD_BLOCK_HASH_KEY, blockHash.toArrayUnsafe()); return this; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/WorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/WorldStateKeyValueStorage.java index 4c64a4dbb87..c67db7f62e0 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/WorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/WorldStateKeyValueStorage.java @@ -168,7 +168,7 @@ public WorldStateStorage.Updater putCode( @Override public WorldStateStorage.Updater saveWorldState( - final Bytes blockHash, final Bytes32 nodeHash, final Optional node) { + final Bytes blockHash, final Bytes32 nodeHash, final Bytes node) { return putAccountStateTrieNode(null, nodeHash, node); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java index 97d6db6eddb..5411829e186 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java @@ -64,7 +64,7 @@ default Updater putCode(final Hash accountHash, final Bytes code) { return putCode(accountHash, codeHash, code); } - Updater saveWorldState(Bytes blockHash, Bytes32 nodeHash, Optional node); + Updater saveWorldState(Bytes blockHash, Bytes32 nodeHash, Bytes node); Updater putAccountStateTrieNode(Bytes location, Bytes32 nodeHash, Bytes node); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java index ea273128efb..ea7804c8bbc 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java @@ -32,7 +32,6 @@ import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import java.util.Optional; import java.util.TreeMap; import org.apache.tuweni.bytes.Bytes; @@ -286,7 +285,7 @@ public void isWorldStateAvailable_afterCallingSaveWorldstate() { assertThat(storage.isWorldStateAvailable(Bytes32.wrap(nodeHashKey), Hash.EMPTY)).isFalse(); - updater.saveWorldState(blockHash, nodeHashKey, Optional.of(nodeValue)); + updater.saveWorldState(blockHash, nodeHashKey, nodeValue); updater.commit(); assertThat(storage.isWorldStateAvailable(Bytes32.wrap(nodeHashKey), Hash.EMPTY)).isTrue(); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadState.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadState.java index 3d19906b956..0703c93ef89 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadState.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadState.java @@ -54,7 +54,7 @@ public synchronized boolean checkCompletion(final BlockHeader header) { return false; } final Updater updater = worldStateStorage.updater(); - updater.saveWorldState(header.getHash(), header.getStateRoot(), Optional.of(rootNodeData)); + updater.saveWorldState(header.getHash(), header.getStateRoot(), rootNodeData); updater.commit(); internalFuture.complete(null); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java index c93d8fd6d5c..5d73e655f61 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java @@ -33,7 +33,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Optional; import java.util.function.Predicate; import java.util.stream.Stream; @@ -140,7 +139,7 @@ public synchronized boolean checkCompletion(final BlockHeader header) { startHeal(); } else { final WorldStateStorage.Updater updater = worldStateStorage.updater(); - updater.saveWorldState(header.getHash(), header.getStateRoot(), Optional.of(rootNodeData)); + updater.saveWorldState(header.getHash(), header.getStateRoot(), rootNodeData); updater.commit(); metricsManager.notifySnapSyncCompleted(); internalFuture.complete(null); From 9fb37d680dad9a6ffc0388e7e1c3295c4ea66d50 Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Fri, 24 Jun 2022 15:10:31 +0200 Subject: [PATCH 20/29] fix build Signed-off-by: Karim TAAM --- .../hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java index 886f2077dc9..04e9a6de9f9 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java @@ -55,7 +55,6 @@ public class DefaultSynchronizer implements Synchronizer { private final Optional blockPropagationManager; private final Optional> fastSyncDownloader; private final Optional fullSyncDownloader; - private final ProtocolContext protocolContext; private final PivotBlockSelector pivotBlockSelector; private final SyncTerminationCondition terminationCondition; @@ -76,7 +75,6 @@ public DefaultSynchronizer( this.maybePruner = maybePruner; this.syncState = syncState; this.pivotBlockSelector = pivotBlockSelector; - this.protocolContext = protocolContext; this.terminationCondition = terminationCondition; ChainHeadTracker.trackChainHeadForPeers( From a6c09a9771fccff7df2433b65c08692bf2d7bc8f Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Fri, 24 Jun 2022 15:15:30 +0200 Subject: [PATCH 21/29] Cleanup and fixes Signed-off-by: Fabio Di Fabio --- .../merge/blockcreation/MergeCoordinator.java | 20 +++++++- .../besu/ethereum/MainnetBlockValidator.java | 18 +------- .../bonsai/BonsaiInMemoryWorldState.java | 22 +++++---- .../bonsai/BonsaiLayeredWorldState.java | 5 -- .../bonsai/BonsaiPersistedWorldState.java | 32 ------------- .../bonsai/BonsaiWorldStateArchive.java | 46 +++++-------------- .../besu/ethereum/core/MutableWorldState.java | 9 ---- .../mainnet/AbstractBlockProcessor.java | 15 +++--- .../besu/ethereum/mainnet/BlockProcessor.java | 35 ++------------ .../mainnet/MainnetProtocolSpecs.java | 11 +---- .../mainnet/PrivacyBlockProcessor.java | 11 +---- .../worldstate/DefaultMutableWorldState.java | 5 -- 12 files changed, 60 insertions(+), 169 deletions(-) diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java index e0a7eccd876..58f0b4e0095 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java @@ -366,7 +366,7 @@ private boolean setNewHead( LOG, "Forwarding chain head to the block {} saved from a previous newPayload invocation", newHead::toLogString); - + forwardWorldStateTo(newHead); return blockchain.forwardToBlock(newHead); } @@ -374,6 +374,24 @@ private boolean setNewHead( return blockchain.rewindToBlock(newHead.getHash()); } + private void forwardWorldStateTo(final BlockHeader newHead) { + protocolContext + .getWorldStateArchive() + .getMutable(newHead.getStateRoot(), newHead.getHash()) + .ifPresentOrElse( + mutableWorldState -> + debugLambda( + LOG, + "World state for state root hash {} and block hash {} persisted successfully", + mutableWorldState::rootHash, + newHead::getHash), + () -> + LOG.error( + "Could not persist world for root hash {} and block hash {}", + newHead.getStateRoot(), + newHead.getHash())); + } + @Override public boolean latestValidAncestorDescendsFromTerminal(final BlockHeader blockHeader) { if (blockHeader.getNumber() <= 1L) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java index bc35cf9e4b8..e446d12a633 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java @@ -155,23 +155,9 @@ private Result handleAndReportFailure(final Block invalidBlock, final String rea protected BlockProcessor.Result processBlock( final ProtocolContext context, final MutableWorldState worldState, - final Block block, - final boolean shouldPersist) { - - return blockProcessor.processBlock(context.getBlockchain(), worldState, block, shouldPersist); - } + final Block block) { - /** - * Processes a block, returning the result of the processing - * - * @param context the ProtocolContext - * @param worldState the world state for the parent block state root hash - * @param block the block to be processed - * @return the result of processing the block - */ - protected BlockProcessor.Result processBlock( - final ProtocolContext context, final MutableWorldState worldState, final Block block) { - return processBlock(context, worldState, block, true); + return blockProcessor.processBlock(context.getBlockchain(), worldState, block); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java index b3b165b36ae..8a052e6c759 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java @@ -30,18 +30,24 @@ public BonsaiInMemoryWorldState( @Override public Hash rootHash() { final BonsaiWorldStateKeyValueStorage.Updater updater = worldStateStorage.updater(); - final Hash calculatedRootHash = calculateRootHash(updater); - updater.rollback(); - return Hash.wrap(calculatedRootHash); + try { + final Hash calculatedRootHash = calculateRootHash(updater); + return Hash.wrap(calculatedRootHash); + } finally { + updater.rollback(); + } } @Override public void persist(final BlockHeader blockHeader) { final BonsaiWorldStateUpdater localUpdater = updater.copy(); - worldStateBlockHash = blockHeader.getHash(); - worldStateRootHash = blockHeader.getStateRoot(); - final TrieLogLayer trieLog = localUpdater.generateTrieLog(worldStateBlockHash); - trieLog.freeze(); - archive.addLayeredWorldState(this, blockHeader, worldStateRootHash, trieLog); + try { + final Hash newWorldStateRootHash = rootHash(); + prepareTrieLog(blockHeader, localUpdater, newWorldStateRootHash, worldStateBlockHash); + worldStateBlockHash = blockHeader.getHash(); + worldStateRootHash = newWorldStateRootHash; + } finally { + localUpdater.reset(); + } } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java index 5e3c350578c..4d76ff4aefa 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java @@ -278,9 +278,4 @@ public void persist(final BlockHeader blockHeader) { public WorldUpdater updater() { return new BonsaiWorldStateUpdater(this); } - - @Override - public void remember(final BlockHeader blockHeader) { - archive.addLayeredWorldState(blockHeader, this); - } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java index bbe9cfddac8..f0b63f60674 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java @@ -238,23 +238,6 @@ protected Hash calculateRootHash( return Hash.wrap(rootHash); } - @Override - public void remember(final BlockHeader blockHeader) { - checkArgument(blockHeader != null, "Block header must not be null"); - debugLambda(LOG, "Remember world state for block {}", blockHeader::toLogString); - final BonsaiWorldStateUpdater localUpdater = updater.copy(); - final BonsaiWorldStateKeyValueStorage.Updater stateUpdater = worldStateStorage.updater(); - - try { - final Hash newWorldStateRootHash = calculateRootHash(stateUpdater, localUpdater); - prepareTrieLog(blockHeader, localUpdater, newWorldStateRootHash, worldStateBlockHash); - } finally { - stateUpdater.rollback(); - updater.reset(); - } - archive.scrubLayeredCache(blockHeader.getNumber()); - } - @Override public void persist(final BlockHeader blockHeader) { debugLambda(LOG, "Persist world state for block {}", blockHeader::toLogString); @@ -309,14 +292,6 @@ protected TrieLogLayer prepareTrieLog( + currentWorldStateRootHash.toHexString()); } - if (blockHeader.getNumber() > 0 && !previousBlockHash.equals(blockHeader.getParentHash())) { - throw new RuntimeException( - "Previous block hash " - + previousBlockHash.toHexString() - + " is not the parent of the current block " - + blockHeader.toLogString()); - } - debugLambda(LOG, "Adding layered world state for {}", blockHeader::toLogString); final TrieLogLayer trieLog = localUpdater.generateTrieLog(blockHeader.getBlockHash()); trieLog.freeze(); @@ -324,13 +299,6 @@ protected TrieLogLayer prepareTrieLog( return trieLog; } - void persistTrieLog( - final BlockHeader blockHeader, final Hash worldStateRootHash, final TrieLogLayer trieLog) { - final BonsaiWorldStateKeyValueStorage.Updater stateUpdater = worldStateStorage.updater(); - persistTrieLog(blockHeader, worldStateRootHash, trieLog, stateUpdater); - stateUpdater.commit(); - } - private void persistTrieLog( final BlockHeader blockHeader, final Hash worldStateRootHash, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java index 3e9deacb537..f16cf31d32e 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java @@ -84,36 +84,12 @@ private void blockAddedHandler(final BlockAddedEvent event) { final BlockHeader eventBlockHeader = event.getBlock().getHeader(); layeredWorldStatesByHash.computeIfPresent( eventBlockHeader.getParentHash(), - (parentHash, parentBonsaiLayeredWorldState) -> { - layeredWorldStatesByHash.computeIfPresent( - eventBlockHeader.getHash(), - (hash, bonsaiLayeredWorldState) -> { - if (worldStateStorage.getTrieLog(eventBlockHeader.getHash()).isEmpty()) { - debugLambda( - LOG, - "World state not yet persisted, doing it now for root hash {}, block hash {}", - eventBlockHeader::getStateRoot, - eventBlockHeader::getHash); - - getMutable(eventBlockHeader.getStateRoot(), eventBlockHeader.getHash()) - .ifPresentOrElse( - mutableWorldState -> - debugLambda( - LOG, - "World state for state root hash {} and block hash {} persisted successfully", - mutableWorldState::rootHash, - eventBlockHeader::getHash), - () -> - LOG.error( - "Could not persist world for root hash {} and block hash {}", - eventBlockHeader.getStateRoot(), - eventBlockHeader.getHash())); - } - parentBonsaiLayeredWorldState.setNextWorldView( - Optional.of(layeredWorldStatesByHash.get(eventBlockHeader.getBlockHash()))); - return bonsaiLayeredWorldState; - }); - return parentBonsaiLayeredWorldState; + (parentHash, bonsaiLayeredWorldState) -> { + if (layeredWorldStatesByHash.containsKey(eventBlockHeader.getHash())) { + bonsaiLayeredWorldState.setNextWorldView( + Optional.of(layeredWorldStatesByHash.get(eventBlockHeader.getHash()))); + } + return bonsaiLayeredWorldState; }); } } @@ -142,11 +118,11 @@ public void addLayeredWorldState( blockHeader.getNumber(), worldStateRootHash, trieLog); - addLayeredWorldState(blockHeader, bonsaiLayeredWorldState); - } - - public void addLayeredWorldState( - final BlockHeader blockHeader, final BonsaiLayeredWorldState bonsaiLayeredWorldState) { + debugLambda( + LOG, + "adding layered world state for block {}, state root hash {}", + blockHeader::toLogString, + worldStateRootHash::toHexString); layeredWorldStatesByHash.put(blockHeader.getHash(), bonsaiLayeredWorldState); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java index 3b843b551bd..f5af290aba3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java @@ -34,13 +34,4 @@ public interface MutableWorldState extends WorldState, MutableWorldView { * `null` should be passed in. */ void persist(BlockHeader blockHeader); - - /** - * Remember accumulated changes to underlying storage for future use, without making this the - * current world state. This is useful in the PoS where can precompute a future world state, that - * will be committed in a later step - * - * @param blockHeader The block hash of the world state this represents. Must not be null - */ - void remember(final BlockHeader blockHeader); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java index 5fae088b938..0e09be2f647 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java @@ -144,8 +144,7 @@ public AbstractBlockProcessor.Result processBlock( final BlockHeader blockHeader, final List transactions, final List ommers, - final PrivateMetadataUpdater privateMetadataUpdater, - final boolean shouldPersist) { + final PrivateMetadataUpdater privateMetadataUpdater) { final List receipts = new ArrayList<>(); long currentGasUsed = 0; for (final Transaction transaction : transactions) { @@ -198,13 +197,11 @@ public AbstractBlockProcessor.Result processBlock( return AbstractBlockProcessor.Result.failed(); } - if (shouldPersist) { - try { - worldState.persist(blockHeader); - } catch (Exception e) { - LOG.error("failed persisting block", e); - return AbstractBlockProcessor.Result.failed(); - } + try { + worldState.persist(blockHeader); + } catch (Exception e) { + LOG.error("failed persisting block", e); + return AbstractBlockProcessor.Result.failed(); } return AbstractBlockProcessor.Result.successful(receipts); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BlockProcessor.java index 0a6ebb097b3..94e63fcf843 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BlockProcessor.java @@ -71,18 +71,14 @@ default boolean isFailed() { * @return the block processing result */ default Result processBlock( - final Blockchain blockchain, - final MutableWorldState worldState, - final Block block, - final boolean shouldPersist) { + final Blockchain blockchain, final MutableWorldState worldState, final Block block) { return processBlock( blockchain, worldState, block.getHeader(), block.getBody().getTransactions(), block.getBody().getOmmers(), - null, - shouldPersist); + null); } /** @@ -101,29 +97,7 @@ default Result processBlock( final BlockHeader blockHeader, final List transactions, final List ommers) { - return processBlock(blockchain, worldState, blockHeader, transactions, ommers, null, true); - } - - /** - * Processes the block and persist everything - * - * @param blockchain the blockchain to append the block to - * @param worldState the world state to apply changes to - * @param blockHeader the block header for the block - * @param transactions the transactions in the block - * @param ommers the block ommers - * @param privateMetadataUpdater the updater used to update the private metadata for the block - * @return the block processing result - */ - default Result processBlock( - final Blockchain blockchain, - final MutableWorldState worldState, - final BlockHeader blockHeader, - final List transactions, - final List ommers, - final PrivateMetadataUpdater privateMetadataUpdater) { - return processBlock( - blockchain, worldState, blockHeader, transactions, ommers, privateMetadataUpdater, true); + return processBlock(blockchain, worldState, blockHeader, transactions, ommers, null); } /** @@ -143,8 +117,7 @@ Result processBlock( BlockHeader blockHeader, List transactions, List ommers, - PrivateMetadataUpdater privateMetadataUpdater, - boolean shouldPersist); + PrivateMetadataUpdater privateMetadataUpdater); /** * Processes the block when running Besu in GoQuorum-compatible mode diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java index b0970cb56aa..3d98746c0f4 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java @@ -706,17 +706,10 @@ public Result processBlock( final BlockHeader blockHeader, final List transactions, final List ommers, - final PrivateMetadataUpdater privateMetadataUpdater, - final boolean shouldPersist) { + final PrivateMetadataUpdater privateMetadataUpdater) { updateWorldStateForDao(worldState); return wrapped.processBlock( - blockchain, - worldState, - blockHeader, - transactions, - ommers, - privateMetadataUpdater, - shouldPersist); + blockchain, worldState, blockHeader, transactions, ommers, privateMetadataUpdater); } private static final Address DAO_REFUND_CONTRACT_ADDRESS = diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessor.java index d58727bf40c..bbd454ffbb9 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessor.java @@ -86,8 +86,7 @@ public Result processBlock( final BlockHeader blockHeader, final List transactions, final List ommers, - final PrivateMetadataUpdater privateMetadataUpdater, - final boolean shouldPersist) { + final PrivateMetadataUpdater privateMetadataUpdater) { if (privateMetadataUpdater != null) { throw new IllegalArgumentException("PrivateMetadataUpdater passed in is not null."); @@ -100,13 +99,7 @@ public Result processBlock( final Result result = blockProcessor.processBlock( - blockchain, - worldState, - blockHeader, - transactions, - ommers, - metadataUpdater, - shouldPersist); + blockchain, worldState, blockHeader, transactions, ommers, metadataUpdater); metadataUpdater.commit(); return result; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java index 4f3e2f58d8d..87e2a82aa6c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java @@ -203,11 +203,6 @@ public void persist(final BlockHeader blockHeader) { stateUpdater.commit(); } - @Override - public void remember(final BlockHeader blockHeader) { - persist(blockHeader); - } - private Optional getStorageTrieKeyPreimage(final Bytes32 trieKey) { return Optional.ofNullable(newStorageKeyPreimages.get(trieKey)) .or(() -> preimageStorage.getStorageTrieKeyPreimage(trieKey)); From 0acf581f6227d274c6a5cb65f7b2b86737f387a0 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Fri, 24 Jun 2022 16:55:26 +0200 Subject: [PATCH 22/29] spotless apply Signed-off-by: Fabio Di Fabio --- .../org/hyperledger/besu/ethereum/MainnetBlockValidator.java | 4 +--- .../besu/ethereum/bonsai/BonsaiPersistedWorldState.java | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java index e446d12a633..c1cd7d952a0 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java @@ -153,9 +153,7 @@ private Result handleAndReportFailure(final Block invalidBlock, final String rea * @return the result of processing the block */ protected BlockProcessor.Result processBlock( - final ProtocolContext context, - final MutableWorldState worldState, - final Block block) { + final ProtocolContext context, final MutableWorldState worldState, final Block block) { return blockProcessor.processBlock(context.getBlockchain(), worldState, block); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java index 76ace8f9abc..88285208863 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java @@ -16,7 +16,6 @@ package org.hyperledger.besu.ethereum.bonsai; -import static com.google.common.base.Preconditions.checkArgument; import static org.hyperledger.besu.ethereum.bonsai.BonsaiAccount.fromRLP; import static org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; import static org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; From aa0ae0f422f47d2cd52b5f577ad110f2ed625c71 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Fri, 24 Jun 2022 19:02:31 +0200 Subject: [PATCH 23/29] Adapt unit tests to the new newPayload FcU flow Signed-off-by: Fabio Di Fabio --- .../merge/blockcreation/MergeCoordinator.java | 7 +- .../blockcreation/MergeCoordinatorTest.java | 102 +++++++++--------- .../methods/engine/EngineNewPayloadTest.java | 4 +- 3 files changed, 59 insertions(+), 54 deletions(-) diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java index b71a671accb..93108be4aea 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java @@ -298,7 +298,7 @@ public ForkchoiceResult updateForkChoice( INVALID, "new head timestamp not greater than parent", latestValid); } - setNewHead(blockchain, newHead, newHead.getParentHash()); + setNewHead(blockchain, newHead); // set and persist the new finalized block if it is present newFinalized.ifPresent( @@ -323,15 +323,14 @@ public ForkchoiceResult updateForkChoice( return ForkchoiceResult.withResult(newFinalized, Optional.of(newHead)); } - private boolean setNewHead( - final MutableBlockchain blockchain, final BlockHeader newHead, final Hash parentHash) { + private boolean setNewHead(final MutableBlockchain blockchain, final BlockHeader newHead) { if (newHead.getHash().equals(blockchain.getChainHeadHash())) { debugLambda(LOG, "Nothing to do new head {} is already chain head", newHead::toLogString); return true; } - if (parentHash.equals(blockchain.getChainHeadHash())) { + if (newHead.getParentHash().equals(blockchain.getChainHeadHash())) { debugLambda( LOG, "Forwarding chain head to the block {} saved from a previous newPayload invocation", diff --git a/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinatorTest.java b/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinatorTest.java index a0d1bab2ba2..5d876b6ae32 100644 --- a/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinatorTest.java +++ b/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinatorTest.java @@ -128,18 +128,17 @@ public void coinbaseShouldMatchSuggestedFeeRecipient() { @Test public void childTimestampExceedsParentsFails() { - BlockHeader terminalHeader = terminalPowBlock(); - coordinator.executeBlock(new Block(terminalHeader, BlockBody.empty())); + sendNewPayloadAndForkchoiceUpdate( + new Block(terminalHeader, BlockBody.empty()), Optional.empty(), Hash.ZERO); BlockHeader parentHeader = nextBlockHeader(terminalHeader); - Block parent = new Block(parentHeader, BlockBody.empty()); - coordinator.executeBlock(parent); + sendNewPayloadAndForkchoiceUpdate(parent, Optional.empty(), terminalHeader.getHash()); BlockHeader childHeader = nextBlockHeader(parentHeader, parentHeader.getTimestamp()); Block child = new Block(childHeader, BlockBody.empty()); - coordinator.executeBlock(child); + coordinator.rememberBlock(child); ForkchoiceResult result = coordinator.updateForkChoice( @@ -161,42 +160,39 @@ public void childTimestampExceedsParentsFails() { @Test public void latestValidAncestorDescendsFromTerminal() { - BlockHeader terminalHeader = terminalPowBlock(); - coordinator.executeBlock(new Block(terminalHeader, BlockBody.empty())); + sendNewPayloadAndForkchoiceUpdate( + new Block(terminalHeader, BlockBody.empty()), Optional.empty(), Hash.ZERO); BlockHeader parentHeader = nextBlockHeader(terminalHeader); - Block parent = new Block(parentHeader, BlockBody.empty()); - coordinator.executeBlock(parent); + sendNewPayloadAndForkchoiceUpdate(parent, Optional.empty(), terminalHeader.getHash()); BlockHeader childHeader = nextBlockHeader(parentHeader); Block child = new Block(childHeader, BlockBody.empty()); - coordinator.executeBlock(child); + coordinator.validateBlock(child); assertThat(this.coordinator.latestValidAncestorDescendsFromTerminal(child.getHeader())) .isTrue(); } @Test public void latestValidAncestorDescendsFromFinalizedBlock() { - BlockHeader terminalHeader = terminalPowBlock(); - coordinator.executeBlock(new Block(terminalHeader, BlockBody.empty())); + sendNewPayloadAndForkchoiceUpdate( + new Block(terminalHeader, BlockBody.empty()), Optional.empty(), Hash.ZERO); BlockHeader grandParentHeader = nextBlockHeader(terminalHeader); - Block grandParent = new Block(grandParentHeader, BlockBody.empty()); - coordinator.executeBlock(grandParent); - when(mergeContext.getFinalized()).thenReturn(Optional.of(grandParentHeader)); + sendNewPayloadAndForkchoiceUpdate(grandParent, Optional.empty(), terminalHeader.getHash()); BlockHeader parentHeader = nextBlockHeader(grandParentHeader); - Block parent = new Block(parentHeader, BlockBody.empty()); - coordinator.executeBlock(parent); + sendNewPayloadAndForkchoiceUpdate( + parent, Optional.of(grandParentHeader), grandParentHeader.getHash()); BlockHeader childHeader = nextBlockHeader(parentHeader); Block child = new Block(childHeader, BlockBody.empty()); - coordinator.executeBlock(child); + coordinator.validateBlock(child); assertThat(this.coordinator.latestValidAncestorDescendsFromTerminal(child.getHeader())) .isTrue(); @@ -205,25 +201,19 @@ public void latestValidAncestorDescendsFromFinalizedBlock() { @Test public void updateForkChoiceShouldPersistFirstFinalizedBlockHash() { - - when(mergeContext.getFinalized()).thenReturn(Optional.empty()); - BlockHeader terminalHeader = terminalPowBlock(); - coordinator.executeBlock(new Block(terminalHeader, BlockBody.empty())); + sendNewPayloadAndForkchoiceUpdate( + new Block(terminalHeader, BlockBody.empty()), Optional.empty(), Hash.ZERO); BlockHeader firstFinalizedHeader = nextBlockHeader(terminalHeader); Block firstFinalizedBlock = new Block(firstFinalizedHeader, BlockBody.empty()); - coordinator.executeBlock(firstFinalizedBlock); + sendNewPayloadAndForkchoiceUpdate( + firstFinalizedBlock, Optional.empty(), terminalHeader.getHash()); BlockHeader headBlockHeader = nextBlockHeader(firstFinalizedHeader); Block headBlock = new Block(headBlockHeader, BlockBody.empty()); - coordinator.executeBlock(headBlock); - - coordinator.updateForkChoice( - headBlockHeader, - firstFinalizedBlock.getHash(), - firstFinalizedBlock.getHash(), - Optional.empty()); + sendNewPayloadAndForkchoiceUpdate( + headBlock, Optional.of(firstFinalizedHeader), firstFinalizedHeader.getHash()); verify(blockchain).setFinalized(firstFinalizedBlock.getHash()); verify(mergeContext).setFinalized(firstFinalizedHeader); @@ -234,27 +224,23 @@ public void updateForkChoiceShouldPersistFirstFinalizedBlockHash() { @Test public void updateForkChoiceShouldPersistLastFinalizedBlockHash() { BlockHeader terminalHeader = terminalPowBlock(); - coordinator.executeBlock(new Block(terminalHeader, BlockBody.empty())); + sendNewPayloadAndForkchoiceUpdate( + new Block(terminalHeader, BlockBody.empty()), Optional.empty(), Hash.ZERO); BlockHeader prevFinalizedHeader = nextBlockHeader(terminalHeader); Block prevFinalizedBlock = new Block(prevFinalizedHeader, BlockBody.empty()); - coordinator.executeBlock(prevFinalizedBlock); - - when(mergeContext.getFinalized()).thenReturn(Optional.of(prevFinalizedHeader)); + sendNewPayloadAndForkchoiceUpdate( + prevFinalizedBlock, Optional.empty(), terminalHeader.getHash()); BlockHeader lastFinalizedHeader = nextBlockHeader(prevFinalizedHeader); Block lastFinalizedBlock = new Block(lastFinalizedHeader, BlockBody.empty()); - coordinator.executeBlock(lastFinalizedBlock); + sendNewPayloadAndForkchoiceUpdate( + lastFinalizedBlock, Optional.of(prevFinalizedHeader), prevFinalizedHeader.getHash()); BlockHeader headBlockHeader = nextBlockHeader(lastFinalizedHeader); Block headBlock = new Block(headBlockHeader, BlockBody.empty()); - coordinator.executeBlock(headBlock); - - coordinator.updateForkChoice( - headBlockHeader, - lastFinalizedBlock.getHash(), - lastFinalizedBlock.getHash(), - Optional.empty()); + sendNewPayloadAndForkchoiceUpdate( + headBlock, Optional.of(lastFinalizedHeader), lastFinalizedHeader.getHash()); verify(blockchain).setFinalized(lastFinalizedBlock.getHash()); verify(mergeContext).setFinalized(lastFinalizedHeader); @@ -427,21 +413,23 @@ public void assertMergeAtGenesisSatisifiesTerminalPoW() { @Test public void invalidPayloadShouldReturnErrorAndUpdateForkchoiceState() { BlockHeader terminalHeader = terminalPowBlock(); - coordinator.executeBlock(new Block(terminalHeader, BlockBody.empty())); + sendNewPayloadAndForkchoiceUpdate( + new Block(terminalHeader, BlockBody.empty()), Optional.empty(), Hash.ZERO); BlockHeader prevFinalizedHeader = nextBlockHeader(terminalHeader); Block prevFinalizedBlock = new Block(prevFinalizedHeader, BlockBody.empty()); - coordinator.executeBlock(prevFinalizedBlock); - - when(mergeContext.getFinalized()).thenReturn(Optional.of(prevFinalizedHeader)); + sendNewPayloadAndForkchoiceUpdate( + prevFinalizedBlock, Optional.empty(), terminalHeader.getHash()); BlockHeader lastFinalizedHeader = nextBlockHeader(prevFinalizedHeader); Block lastFinalizedBlock = new Block(lastFinalizedHeader, BlockBody.empty()); - coordinator.executeBlock(lastFinalizedBlock); + + sendNewPayloadAndForkchoiceUpdate( + lastFinalizedBlock, Optional.of(prevFinalizedHeader), prevFinalizedHeader.getHash()); BlockHeader headBlockHeader = nextBlockHeader(lastFinalizedHeader); Block headBlock = new Block(headBlockHeader, BlockBody.empty()); - coordinator.executeBlock(headBlock); + assertThat(coordinator.rememberBlock(headBlock).blockProcessingOutputs).isPresent(); var res = coordinator.updateForkChoice( @@ -461,6 +449,23 @@ public void invalidPayloadShouldReturnErrorAndUpdateForkchoiceState() { verify(mergeContext).setSafeBlock(lastFinalizedHeader); } + private void sendNewPayloadAndForkchoiceUpdate( + final Block block, final Optional finalizedHeader, final Hash safeHash) { + + assertThat(coordinator.rememberBlock(block).blockProcessingOutputs).isPresent(); + assertThat( + coordinator + .updateForkChoice( + block.getHeader(), + finalizedHeader.map(BlockHeader::getHash).orElse(Hash.ZERO), + safeHash, + Optional.empty()) + .isValid()) + .isTrue(); + + when(mergeContext.getFinalized()).thenReturn(finalizedHeader); + } + private BlockHeader terminalPowBlock() { return headerGenerator .difficulty(Difficulty.MAX_VALUE) @@ -472,6 +477,7 @@ private BlockHeader terminalPowBlock() { genesisState.getBlock().getHeader().getBaseFee().orElse(Wei.of(0x3b9aca00)), 0, 15000000l)) + .timestamp(1) .gasLimit(genesisState.getBlock().getHeader().getGasLimit()) .stateRoot(genesisState.getBlock().getHeader().getStateRoot()) .buildHeader(); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadTest.java index 5da7a52db1f..fbda0c4646c 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadTest.java @@ -96,7 +96,7 @@ public void shouldReturnValid() { .thenReturn(true); when(mergeCoordinator.getOrSyncHeaderByHash(any(Hash.class))) .thenReturn(Optional.of(mockHeader)); - when(mergeCoordinator.executeBlock(any())) + when(mergeCoordinator.rememberBlock(any())) .thenReturn(new Result(new BlockProcessingOutputs(null, List.of()))); var resp = resp(mockPayload(mockHeader, Collections.emptyList())); @@ -117,7 +117,7 @@ public void shouldReturnInvalidOnBlockExecutionError() { .thenReturn(true); when(mergeCoordinator.getOrSyncHeaderByHash(any(Hash.class))) .thenReturn(Optional.of(mockHeader)); - when(mergeCoordinator.executeBlock(any())).thenReturn(new Result("error 42")); + when(mergeCoordinator.rememberBlock(any())).thenReturn(new Result("error 42")); var resp = resp(mockPayload(mockHeader, Collections.emptyList())); From b92e3cf53739b89ca2fdf68ffefd4237a9937b19 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Fri, 24 Jun 2022 19:05:24 +0200 Subject: [PATCH 24/29] Remove unused parameter Signed-off-by: Fabio Di Fabio --- .../besu/ethereum/bonsai/BonsaiInMemoryWorldState.java | 2 +- .../besu/ethereum/bonsai/BonsaiPersistedWorldState.java | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java index 8a052e6c759..43e36e00810 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java @@ -43,7 +43,7 @@ public void persist(final BlockHeader blockHeader) { final BonsaiWorldStateUpdater localUpdater = updater.copy(); try { final Hash newWorldStateRootHash = rootHash(); - prepareTrieLog(blockHeader, localUpdater, newWorldStateRootHash, worldStateBlockHash); + prepareTrieLog(blockHeader, localUpdater, newWorldStateRootHash); worldStateBlockHash = blockHeader.getHash(); worldStateRootHash = newWorldStateRootHash; } finally { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java index 88285208863..a1aed500c2a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java @@ -247,7 +247,7 @@ public void persist(final BlockHeader blockHeader) { // If specified but not a direct descendant simply store the new block hash. if (blockHeader != null) { final TrieLogLayer trieLog = - prepareTrieLog(blockHeader, localUpdater, newWorldStateRootHash, worldStateBlockHash); + prepareTrieLog(blockHeader, localUpdater, newWorldStateRootHash); persistTrieLog(blockHeader, newWorldStateRootHash, trieLog, stateUpdater); worldStateBlockHash = blockHeader.getHash(); } else { @@ -274,8 +274,7 @@ public void persist(final BlockHeader blockHeader) { protected TrieLogLayer prepareTrieLog( final BlockHeader blockHeader, final BonsaiWorldStateUpdater localUpdater, - final Hash currentWorldStateRootHash, - final Hash previousBlockHash) { + final Hash currentWorldStateRootHash) { if (!currentWorldStateRootHash.equals(blockHeader.getStateRoot())) { throw new RuntimeException( From c4a527f0478d2d95441eb13d6a4f0598acd572af Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 28 Jun 2022 17:19:10 +0200 Subject: [PATCH 25/29] Fix null pointer exception Signed-off-by: Fabio Di Fabio --- .../besu/ethereum/bonsai/BonsaiPersistedWorldState.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java index a1aed500c2a..943912423db 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java @@ -234,7 +234,10 @@ protected Hash calculateRootHash( @Override public void persist(final BlockHeader blockHeader) { - debugLambda(LOG, "Persist world state for block {}", blockHeader::toLogString); + debugLambda( + LOG, + "Persist world state for block {}", + () -> (blockHeader == null ? "null" : blockHeader.toLogString())); boolean success = false; final BonsaiWorldStateUpdater localUpdater = updater.copy(); From 42db3ce375f400113efd265f6870fc8749abf1a5 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 28 Jun 2022 17:44:19 +0200 Subject: [PATCH 26/29] Fix refactoring Signed-off-by: Fabio Di Fabio --- .../ethereum/bonsai/BonsaiPersistedWorldState.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java index 943912423db..f79ed9ae1d5 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java @@ -234,10 +234,8 @@ protected Hash calculateRootHash( @Override public void persist(final BlockHeader blockHeader) { - debugLambda( - LOG, - "Persist world state for block {}", - () -> (blockHeader == null ? "null" : blockHeader.toLogString())); + final Optional maybeBlockHeader = Optional.ofNullable(blockHeader); + debugLambda(LOG, "Persist world state for block {}", maybeBlockHeader::toString); boolean success = false; final BonsaiWorldStateUpdater localUpdater = updater.copy(); @@ -258,6 +256,9 @@ public void persist(final BlockHeader blockHeader) { worldStateBlockHash = null; } + stateUpdater + .getTrieBranchStorageTransaction() + .put(WORLD_ROOT_HASH_KEY, newWorldStateRootHash.toArrayUnsafe()); worldStateRootHash = newWorldStateRootHash; success = true; } finally { @@ -304,9 +305,6 @@ private void persistTrieLog( "Persisting trie log for block hash {} and world state root {}", blockHeader::toLogString, worldStateRootHash::toHexString); - stateUpdater - .getTrieBranchStorageTransaction() - .put(WORLD_ROOT_HASH_KEY, worldStateRootHash.toArrayUnsafe()); stateUpdater .getTrieBranchStorageTransaction() .put(WORLD_BLOCK_HASH_KEY, blockHeader.getHash().toArrayUnsafe()); From f77977c21ac4fc5e171e90c67ef76ac1cc706e87 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Tue, 28 Jun 2022 12:06:28 -0700 Subject: [PATCH 27/29] revert updater protection pr Signed-off-by: garyschulte --- .../bonsai/BonsaiPersistedWorldState.java | 5 ++++ .../bonsai/BonsaiWorldStateArchive.java | 29 ++++++++++--------- .../worldstate/DefaultWorldStateArchive.java | 6 ++++ .../worldstate/WorldStateArchive.java | 3 ++ .../eth/sync/DefaultSynchronizer.java | 7 +++++ 5 files changed, 36 insertions(+), 14 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java index f79ed9ae1d5..7ab3df82ece 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java @@ -91,6 +91,11 @@ public Optional getCode(@Nonnull final Address address) { return worldStateStorage.getCode(null, Hash.hash(address)); } + public void setArchiveStateUnSafe(final BlockHeader blockHeader) { + worldStateBlockHash = Hash.fromPlugin(blockHeader.getBlockHash()); + worldStateRootHash = Hash.fromPlugin(blockHeader.getStateRoot()); + } + public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { return worldStateStorage; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java index 0d37dc14c0f..86c9d2bfaab 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java @@ -50,6 +50,7 @@ public class BonsaiWorldStateArchive implements WorldStateArchive { private final Blockchain blockchain; + private final BonsaiPersistedWorldState persistedState; private final Map layeredWorldStatesByHash; private final BonsaiWorldStateKeyValueStorage worldStateStorage; private final long maxLayersToLoad; @@ -71,6 +72,7 @@ public BonsaiWorldStateArchive( this.blockchain = blockchain; this.worldStateStorage = new BonsaiWorldStateKeyValueStorage(provider); + this.persistedState = new BonsaiPersistedWorldState(this, worldStateStorage); this.layeredWorldStatesByHash = layeredWorldStatesByHash; this.maxLayersToLoad = maxLayersToLoad; blockchain.observeBlockAdded(this::blockAddedHandler); @@ -94,7 +96,6 @@ private void blockAddedHandler(final BlockAddedEvent event) { @Override public Optional get(final Hash rootHash, final Hash blockHash) { - final BonsaiPersistedWorldState persistedState = persistedState(); if (layeredWorldStatesByHash.containsKey(blockHash)) { return Optional.of(layeredWorldStatesByHash.get(blockHash)); } else if (rootHash.equals(persistedState.blockHash())) { @@ -136,7 +137,7 @@ public Optional getTrieLogLayer(final Hash blockHash) { @Override public boolean isWorldStateAvailable(final Hash rootHash, final Hash blockHash) { return layeredWorldStatesByHash.containsKey(blockHash) - || persistedState().blockHash().equals(blockHash) + || persistedState.blockHash().equals(blockHash) || worldStateStorage.isWorldStateAvailable(rootHash, blockHash); } @@ -183,19 +184,18 @@ public Optional getMutable( @Override public Optional getMutable(final Hash rootHash, final Hash blockHash) { - final BonsaiPersistedWorldState persistedWorldState = persistedState(); - if (blockHash.equals(persistedWorldState.blockHash())) { - return Optional.of(persistedWorldState); + if (blockHash.equals(persistedState.blockHash())) { + return Optional.of(persistedState); } else { try { final Optional maybePersistedHeader = - blockchain.getBlockHeader(persistedWorldState.blockHash()).map(BlockHeader.class::cast); + blockchain.getBlockHeader(persistedState.blockHash()).map(BlockHeader.class::cast); final List rollBacks = new ArrayList<>(); final List rollForwards = new ArrayList<>(); if (maybePersistedHeader.isEmpty()) { - getTrieLogLayer(persistedWorldState.blockHash()).ifPresent(rollBacks::add); + getTrieLogLayer(persistedState.blockHash()).ifPresent(rollBacks::add); } else { BlockHeader targetHeader = blockchain.getBlockHeader(blockHash).get(); BlockHeader persistedHeader = maybePersistedHeader.get(); @@ -233,7 +233,7 @@ public Optional getMutable(final Hash rootHash, final Hash bl // attempt the state rolling final BonsaiWorldStateUpdater bonsaiUpdater = - (BonsaiWorldStateUpdater) persistedWorldState.updater(); + (BonsaiWorldStateUpdater) persistedState.updater(); try { for (final TrieLogLayer rollBack : rollBacks) { LOG.debug("Attempting Rollback of {}", rollBack.getBlockHash()); @@ -245,10 +245,10 @@ public Optional getMutable(final Hash rootHash, final Hash bl } bonsaiUpdater.commit(); - persistedWorldState.persist(blockchain.getBlockHeader(blockHash).get()); + persistedState.persist(blockchain.getBlockHeader(blockHash).get()); LOG.debug("Archive rolling finished, now at {}", blockHash); - return Optional.of(persistedWorldState); + return Optional.of(persistedState); } catch (final Exception e) { // if we fail we must clean up the updater bonsaiUpdater.reset(); @@ -261,13 +261,14 @@ public Optional getMutable(final Hash rootHash, final Hash bl } } - private BonsaiPersistedWorldState persistedState() { - return new BonsaiPersistedWorldState(this, worldStateStorage); + @Override + public MutableWorldState getMutable() { + return persistedState; } @Override - public MutableWorldState getMutable() { - return persistedState(); + public void setArchiveStateUnSafe(final BlockHeader blockHeader) { + persistedState.setArchiveStateUnSafe(blockHeader); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java index 3bd97ae4166..ae93b31ba3d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java @@ -17,6 +17,7 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.proof.WorldStateProof; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; @@ -79,6 +80,11 @@ public MutableWorldState getMutable() { return getMutable(EMPTY_ROOT_HASH, null).get(); } + @Override + public void setArchiveStateUnSafe(final BlockHeader blockHeader) { + // ignore for forest + } + @Override public Optional getNodeData(final Hash hash) { // query by location is not supported, only query by content diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java index 2252981cbf2..ab2de6bbb59 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java @@ -16,6 +16,7 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.proof.WorldStateProof; import org.hyperledger.besu.ethereum.trie.MerklePatriciaTrie; @@ -43,6 +44,8 @@ public interface WorldStateArchive { MutableWorldState getMutable(); + void setArchiveStateUnSafe(BlockHeader blockHeader); + Optional getNodeData(Hash hash); Optional getAccountProof( diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java index 04e9a6de9f9..dd1e19e2c70 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java @@ -55,6 +55,7 @@ public class DefaultSynchronizer implements Synchronizer { private final Optional blockPropagationManager; private final Optional> fastSyncDownloader; private final Optional fullSyncDownloader; + private final ProtocolContext protocolContext; private final PivotBlockSelector pivotBlockSelector; private final SyncTerminationCondition terminationCondition; @@ -75,6 +76,7 @@ public DefaultSynchronizer( this.maybePruner = maybePruner; this.syncState = syncState; this.pivotBlockSelector = pivotBlockSelector; + this.protocolContext = protocolContext; this.terminationCondition = terminationCondition; ChainHeadTracker.trackChainHeadForPeers( @@ -216,6 +218,11 @@ private CompletableFuture handleSyncResult(final FastSyncState result) { return CompletableFuture.completedFuture(null); } fastSyncDownloader.ifPresent(FastSyncDownloader::deleteFastSyncState); + result + .getPivotBlockHeader() + .ifPresent( + blockHeader -> + protocolContext.getWorldStateArchive().setArchiveStateUnSafe(blockHeader)); LOG.info( "Sync completed successfully with pivot block {}", result.getPivotBlockNumber().getAsLong()); From bb858769e7238ab73dc12ea0bb20ffe91383d667 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Tue, 28 Jun 2022 15:28:35 -0700 Subject: [PATCH 28/29] update bonsai worldstate archive tests Signed-off-by: garyschulte --- .../bonsai/BonsaiWorldStateArchive.java | 7 +++-- .../bonsai/BonsaiWorldStateArchiveTest.java | 26 +++++++++---------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java index 86c9d2bfaab..ed69ab02f3e 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java @@ -232,8 +232,7 @@ public Optional getMutable(final Hash rootHash, final Hash bl } // attempt the state rolling - final BonsaiWorldStateUpdater bonsaiUpdater = - (BonsaiWorldStateUpdater) persistedState.updater(); + final BonsaiWorldStateUpdater bonsaiUpdater = getUpdater(); try { for (final TrieLogLayer rollBack : rollBacks) { LOG.debug("Attempting Rollback of {}", rollBack.getBlockHash()); @@ -261,6 +260,10 @@ public Optional getMutable(final Hash rootHash, final Hash bl } } + BonsaiWorldStateUpdater getUpdater() { + return (BonsaiWorldStateUpdater) persistedState.updater(); + } + @Override public MutableWorldState getMutable() { return persistedState; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java index 86092747a5d..57da55cda1b 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java @@ -21,8 +21,9 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Hash; @@ -123,10 +124,6 @@ public void testGetMutableWithStorageInconsistencyRollbackTheState() { bonsaiWorldStateArchive = new BonsaiWorldStateArchive(storageProvider, blockchain, 12, layeredWorldStatesByHash); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); - final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput(); - final TrieLogLayer trieLogLayer = new TrieLogLayer(); - trieLogLayer.setBlockHash(blockHeader.getHash()); - trieLogLayer.writeTo(rlpLog); when(blockchain.getBlockHeader(eq(blockHeader.getHash()))).thenReturn(Optional.of(blockHeader)); @@ -145,12 +142,11 @@ public void testGetMutableWithStorageConsistencyNotRollbackTheState() { final Map layeredWorldStatesByHash = mock(HashMap.class); bonsaiWorldStateArchive = - new BonsaiWorldStateArchive(storageProvider, blockchain, 12, layeredWorldStatesByHash); + spy(new BonsaiWorldStateArchive(storageProvider, blockchain, 12, layeredWorldStatesByHash)); + var updater = spy(bonsaiWorldStateArchive.getUpdater()); + when(bonsaiWorldStateArchive.getUpdater()).thenReturn(updater); + final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); - final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput(); - final TrieLogLayer trieLogLayer = new TrieLogLayer(); - trieLogLayer.setBlockHash(blockHeader.getHash()); - trieLogLayer.writeTo(rlpLog); when(blockchain.getBlockHeader(eq(blockHeader.getHash()))).thenReturn(Optional.of(blockHeader)); when(blockchain.getBlockHeader(eq(Hash.ZERO))).thenReturn(Optional.of(blockHeader)); @@ -160,7 +156,8 @@ public void testGetMutableWithStorageConsistencyNotRollbackTheState() { // verify is not trying to get the trie log layer to rollback when block is present verify(layeredWorldStatesByHash).entrySet(); - verifyNoMoreInteractions(layeredWorldStatesByHash); + verify(updater, times(0)).rollBack(any()); + verify(updater, times(0)).rollForward(any()); } @SuppressWarnings({"unchecked"}) @@ -181,7 +178,9 @@ public void testGetMutableWithStorageConsistencyToRollbackAndRollForwardTheState .thenReturn(mock(BonsaiLayeredWorldState.class, Answers.RETURNS_MOCKS)); bonsaiWorldStateArchive = - new BonsaiWorldStateArchive(storageProvider, blockchain, 12, layeredWorldStatesByHash); + spy(new BonsaiWorldStateArchive(storageProvider, blockchain, 12, layeredWorldStatesByHash)); + var updater = spy(bonsaiWorldStateArchive.getUpdater()); + when(bonsaiWorldStateArchive.getUpdater()).thenReturn(updater); // initial persisted state hash key when(blockchain.getBlockHeader(eq(Hash.ZERO))).thenReturn(Optional.of(blockHeaderChainA)); @@ -198,6 +197,7 @@ public void testGetMutableWithStorageConsistencyToRollbackAndRollForwardTheState verify(layeredWorldStatesByHash).containsKey(eq(blockHeaderChainB.getHash())); verify(layeredWorldStatesByHash).get(eq(blockHeaderChainB.getHash())); verify(layeredWorldStatesByHash).entrySet(); - verifyNoMoreInteractions(layeredWorldStatesByHash); + verify(updater, times(1)).rollBack(any()); + verify(updater, times(1)).rollForward(any()); } } From 4f5d203008162b67504b72ed00412c12f15f9c4f Mon Sep 17 00:00:00 2001 From: garyschulte Date: Fri, 1 Jul 2022 11:15:44 -0700 Subject: [PATCH 29/29] remove jetbrains static analysis annotation Signed-off-by: garyschulte --- .../org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java index a66a1c96f89..893ddb8a8f0 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java @@ -51,7 +51,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.collect.Streams; -import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -413,7 +412,6 @@ private BlockAddedEvent handleStoreOnly(final BlockWithReceipts blockWithReceipt return BlockAddedEvent.createForStoredOnly(blockWithReceipts.getBlock()); } - @NotNull private BlockAddedEvent handleNewHead( final Updater updater, final BlockWithReceipts blockWithReceipts) { // This block advances the chain, update the chain head