From 3538c55a37ffae6ea4ea87e90c107c538fc919dd Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Mon, 26 Feb 2024 12:10:45 +0100 Subject: [PATCH 1/8] Extend Blockchain service (#6592) Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 11 ++++ .../org/hyperledger/besu/cli/BesuCommand.java | 17 ++++--- .../besu/services/BlockchainServiceImpl.java | 51 ++++++++++++++++--- .../besu/cli/CommandTestAbstract.java | 4 +- plugin-api/build.gradle | 2 +- .../plugin/services/BlockchainService.java | 24 +++++++++ 6 files changed, 95 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a9a2a4eadb..6122cd3bbb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 24.2.1-SNAPSHOT + +### Breaking Changes + +### Deprecations + +### Additions and Improvements +- Extend `Blockchain` service [#6592](https://github.com/hyperledger/besu/pull/6592) + +### Bug fixes + ## 24.2.0-SNAPSHOT ### Breaking Changes diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index f4b08e4f688..bcb0ce68576 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -373,6 +373,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable { private final TransactionSelectionServiceImpl transactionSelectionServiceImpl; private final PluginTransactionValidatorServiceImpl transactionValidatorServiceImpl; + private final BlockchainServiceImpl blockchainServiceImpl; static class P2PDiscoveryOptionGroup { @@ -957,7 +958,8 @@ public BesuCommand( new PkiBlockCreationConfigurationProvider(), new RpcEndpointServiceImpl(), new TransactionSelectionServiceImpl(), - new PluginTransactionValidatorServiceImpl()); + new PluginTransactionValidatorServiceImpl(), + new BlockchainServiceImpl()); } /** @@ -979,6 +981,7 @@ public BesuCommand( * @param rpcEndpointServiceImpl instance of RpcEndpointServiceImpl * @param transactionSelectionServiceImpl instance of TransactionSelectionServiceImpl * @param transactionValidatorServiceImpl instance of TransactionValidatorServiceImpl + * @param blockchainServiceImpl instance of BlockchainServiceImpl */ @VisibleForTesting protected BesuCommand( @@ -997,7 +1000,8 @@ protected BesuCommand( final PkiBlockCreationConfigurationProvider pkiBlockCreationConfigProvider, final RpcEndpointServiceImpl rpcEndpointServiceImpl, final TransactionSelectionServiceImpl transactionSelectionServiceImpl, - final PluginTransactionValidatorServiceImpl transactionValidatorServiceImpl) { + final PluginTransactionValidatorServiceImpl transactionValidatorServiceImpl, + final BlockchainServiceImpl blockchainServiceImpl) { this.besuComponent = besuComponent; this.logger = besuComponent.getBesuCommandLogger(); this.rlpBlockImporter = rlpBlockImporter; @@ -1017,6 +1021,7 @@ protected BesuCommand( this.rpcEndpointServiceImpl = rpcEndpointServiceImpl; this.transactionSelectionServiceImpl = transactionSelectionServiceImpl; this.transactionValidatorServiceImpl = transactionValidatorServiceImpl; + this.blockchainServiceImpl = blockchainServiceImpl; } /** @@ -1208,6 +1213,7 @@ private void preparePlugins() { TransactionSelectionService.class, transactionSelectionServiceImpl); besuPluginContext.addService( PluginTransactionValidatorService.class, transactionValidatorServiceImpl); + besuPluginContext.addService(BlockchainService.class, blockchainServiceImpl); // register built-in plugins rocksDBPlugin = new RocksDBPlugin(); @@ -1288,6 +1294,9 @@ private Runner buildRunner() { } private void startPlugins() { + blockchainServiceImpl.init( + besuController.getProtocolContext(), besuController.getProtocolSchedule()); + besuPluginContext.addService( BesuEvents.class, new BesuEventsImpl( @@ -1297,10 +1306,6 @@ private void startPlugins() { besuController.getSyncState())); besuPluginContext.addService(MetricsSystem.class, getMetricsSystem()); - besuPluginContext.addService( - BlockchainService.class, - new BlockchainServiceImpl(besuController.getProtocolContext().getBlockchain())); - besuPluginContext.addService( TraceService.class, new TraceServiceImpl( diff --git a/besu/src/main/java/org/hyperledger/besu/services/BlockchainServiceImpl.java b/besu/src/main/java/org/hyperledger/besu/services/BlockchainServiceImpl.java index a3b37a6665a..aa1c6a2128b 100644 --- a/besu/src/main/java/org/hyperledger/besu/services/BlockchainServiceImpl.java +++ b/besu/src/main/java/org/hyperledger/besu/services/BlockchainServiceImpl.java @@ -15,8 +15,13 @@ package org.hyperledger.besu.services; -import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.core.BlockBody; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.feemarket.BaseFeeMarket; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; import org.hyperledger.besu.plugin.Unstable; import org.hyperledger.besu.plugin.data.BlockContext; import org.hyperledger.besu.plugin.data.BlockHeader; @@ -29,15 +34,21 @@ @Unstable public class BlockchainServiceImpl implements BlockchainService { - private final Blockchain blockchain; + private ProtocolContext protocolContext; + private ProtocolSchedule protocolSchedule; + + /** Create a new instance */ + public BlockchainServiceImpl() {} /** * Instantiates a new Blockchain service. * - * @param blockchain the blockchain + * @param protocolContext the protocol context + * @param protocolSchedule the protocol schedule */ - public BlockchainServiceImpl(final Blockchain blockchain) { - this.blockchain = blockchain; + public void init(final ProtocolContext protocolContext, final ProtocolSchedule protocolSchedule) { + this.protocolContext = protocolContext; + this.protocolSchedule = protocolSchedule; } /** @@ -48,11 +59,39 @@ public BlockchainServiceImpl(final Blockchain blockchain) { */ @Override public Optional getBlockByNumber(final long number) { - return blockchain + return protocolContext + .getBlockchain() .getBlockByNumber(number) .map(block -> blockContext(block::getHeader, block::getBody)); } + @Override + public Hash getChainHeadHash() { + return protocolContext.getBlockchain().getChainHeadHash(); + } + + @Override + public BlockHeader getChainHeadHeader() { + return protocolContext.getBlockchain().getChainHeadHeader(); + } + + @Override + public Optional getNextBlockBaseFee() { + final var chainHeadHeader = protocolContext.getBlockchain().getChainHeadHeader(); + final var protocolSpec = + protocolSchedule.getForNextBlockHeader(chainHeadHeader, System.currentTimeMillis()); + return Optional.of(protocolSpec.getFeeMarket()) + .filter(FeeMarket::implementsBaseFee) + .map(BaseFeeMarket.class::cast) + .map( + feeMarket -> + feeMarket.computeBaseFee( + chainHeadHeader.getNumber() + 1, + chainHeadHeader.getBaseFee().orElse(Wei.ZERO), + chainHeadHeader.getGasUsed(), + feeMarket.targetGasUsed(chainHeadHeader))); + } + private static BlockContext blockContext( final Supplier blockHeaderSupplier, final Supplier blockBodySupplier) { diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java index 34c9d44e0fa..331f7d54a98 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java @@ -83,6 +83,7 @@ import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.services.BesuConfigurationImpl; import org.hyperledger.besu.services.BesuPluginContextImpl; +import org.hyperledger.besu.services.BlockchainServiceImpl; import org.hyperledger.besu.services.PermissioningServiceImpl; import org.hyperledger.besu.services.PluginTransactionValidatorServiceImpl; import org.hyperledger.besu.services.PrivacyPluginServiceImpl; @@ -564,7 +565,8 @@ public static class TestBesuCommand extends BesuCommand { pkiBlockCreationConfigProvider, rpcEndpointServiceImpl, new TransactionSelectionServiceImpl(), - new PluginTransactionValidatorServiceImpl()); + new PluginTransactionValidatorServiceImpl(), + new BlockchainServiceImpl()); } @Override diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 9184bb57060..46611fc29d0 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -69,7 +69,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = 'f6P3+XG9GjPYEg7zrXHlujoE2/4axgd+EjKGDDJVVp8=' + knownHash = 'Pi7Veo9W9kmjDJJNB89UTUXbYyRmoN6osK/tD163h3E=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BlockchainService.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BlockchainService.java index e9356ae76ef..76f66aba6ca 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BlockchainService.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BlockchainService.java @@ -14,8 +14,11 @@ */ package org.hyperledger.besu.plugin.services; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.plugin.Unstable; import org.hyperledger.besu.plugin.data.BlockContext; +import org.hyperledger.besu.plugin.data.BlockHeader; import java.util.Optional; @@ -29,4 +32,25 @@ public interface BlockchainService extends BesuService { * @return the BlockContext */ Optional getBlockByNumber(final long number); + + /** + * Get the hash of the chain head + * + * @return chain head hash + */ + Hash getChainHeadHash(); + + /** + * Get the block header of the chain head + * + * @return chain head block header + */ + BlockHeader getChainHeadHeader(); + + /** + * Return the base fee for the next block + * + * @return base fee of the next block or empty if the fee market does not support base fee + */ + Optional getNextBlockBaseFee(); } From e0bedff962710023124f51e963f86292754951fe Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Mon, 26 Feb 2024 13:58:54 +0100 Subject: [PATCH 2/8] Database metadata refactor (#6555) Signed-off-by: Fabio Di Fabio Co-authored-by: Gabriel Fukushima --- CHANGELOG.md | 2 + .../org/hyperledger/besu/cli/BesuCommand.java | 2 +- .../options/stable/DataStorageOptions.java | 2 +- .../storage/RevertMetadataSubCommand.java | 135 +++++ .../storage/StorageSubCommand.java | 3 +- .../storage/TrieLogSubCommand.java | 2 +- .../controller/BesuControllerBuilder.java | 2 +- .../besu/services/BesuConfigurationImpl.java | 5 +- .../hyperledger/besu/cli/BesuCommandTest.java | 2 +- .../stable/DataStorageOptionsTest.java | 2 +- .../storage/TrieLogHelperTest.java | 2 +- .../controller/BesuControllerBuilderTest.java | 2 +- .../besu/services/TraceServiceImplTest.java | 2 +- .../AbstractEthGraphQLHttpServiceTest.java | 2 +- .../AbstractJsonRpcHttpServiceTest.java | 2 +- .../DebugTraceJsonRpcHttpBySpecTest.java | 2 +- .../EthByzantiumJsonRpcHttpBySpecTest.java | 2 +- .../bonsai/TraceJsonRpcHttpBySpecTest.java | 2 +- .../DebugTraceJsonRpcHttpBySpecTest.java | 2 +- .../EthByzantiumJsonRpcHttpBySpecTest.java | 2 +- .../forest/TraceJsonRpcHttpBySpecTest.java | 2 +- .../filter/EthJsonRpcHttpServiceTest.java | 2 +- .../keyvalue/KeyValueSegmentIdentifier.java | 44 +- .../keyvalue/KeyValueStorageProvider.java | 2 +- .../BonsaiWorldStateKeyValueStorage.java | 2 +- .../common/GenesisWorldStateProvider.java | 2 +- .../ForestWorldStateKeyValueStorage.java | 2 +- .../worldstate/DataStorageConfiguration.java | 2 + .../worldstate/DataStorageFormat.java | 41 -- .../worldstate/WorldStateStorage.java | 1 + .../ethereum/core/BlockchainSetupUtil.java | 2 +- .../trie/bonsai/AbstractIsolationTests.java | 6 +- .../BonsaiWorldStateKeyValueStorageTest.java | 2 +- .../flat/FlatDbStrategyProviderTest.java | 2 +- .../bonsai/trielog/TrieLogFactoryTests.java | 2 +- .../eth/sync/fastsync/FastSyncDownloader.java | 2 +- .../snapsync/SnapWorldStateDownloader.java | 2 +- .../eth/manager/EthProtocolManagerTest.java | 2 +- .../manager/EthProtocolManagerTestUtil.java | 2 +- .../ethtaskutils/AbstractMessageTaskTest.java | 2 +- .../task/SnapProtocolManagerTestUtil.java | 2 +- .../AbstractBlockPropagationManagerTest.java | 2 +- .../BonsaiBlockPropagationManagerTest.java | 2 +- .../eth/sync/ChainHeadTrackerTest.java | 2 +- .../eth/sync/DownloadHeadersStepTest.java | 2 +- .../ForestBlockPropagationManagerTest.java | 2 +- .../eth/sync/RangeHeadersFetcherTest.java | 2 +- .../CheckPointSyncChainDownloaderTest.java | 2 +- .../fastsync/DownloadReceiptsStepTest.java | 2 +- .../sync/fastsync/FastSyncActionsTest.java | 2 +- .../fastsync/FastSyncChainDownloaderTest.java | 2 +- .../sync/fastsync/FastSyncDownloaderTest.java | 2 +- .../fastsync/PivotBlockConfirmerTest.java | 2 +- .../fastsync/PivotBlockRetrieverTest.java | 2 +- .../FastWorldDownloadStateTest.java | 2 +- .../fullsync/FullSyncChainDownloaderTest.java | 2 +- ...DownloaderTotalTerminalDifficultyTest.java | 2 +- .../sync/fullsync/FullSyncDownloaderTest.java | 2 +- .../fullsync/FullSyncTargetManagerTest.java | 2 +- .../snapsync/SnapWorldDownloadStateTest.java | 2 +- .../StorageTrieNodeHealingRequestTest.java | 2 +- .../eth/sync/tasks/PersistBlockTaskTest.java | 2 +- plugin-api/build.gradle | 2 +- .../plugin/services/BesuConfiguration.java | 7 +- .../services/storage/DataStorageFormat.java | 23 + .../services/storage/SegmentIdentifier.java | 8 +- plugins/rocksdb/build.gradle | 1 + .../RocksDBKeyValuePrivacyStorageFactory.java | 132 ++++- .../RocksDBKeyValueStorageFactory.java | 281 +++++----- .../BaseVersionedStorageFormat.java | 78 +++ .../configuration/DatabaseMetadata.java | 248 ++++++--- .../PrivacyVersionedStorageFormat.java | 84 +++ .../configuration/VersionedStorageFormat.java | 43 ++ ...ksDBKeyValuePrivacyStorageFactoryTest.java | 70 ++- .../RocksDBKeyValueStorageFactoryTest.java | 162 +++--- .../services/storage/rocksdb/Utils.java | 91 ++++ .../configuration/DatabaseMetadataTest.java | 55 +- .../RocksDBColumnarKeyValueStorageTest.java | 97 ++-- .../kvstore/AbstractKeyValueStorageTest.java | 507 +++++++++--------- 79 files changed, 1461 insertions(+), 773 deletions(-) create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RevertMetadataSubCommand.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageFormat.java create mode 100644 plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/DataStorageFormat.java create mode 100644 plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/BaseVersionedStorageFormat.java create mode 100644 plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/PrivacyVersionedStorageFormat.java create mode 100644 plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/VersionedStorageFormat.java create mode 100644 plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/Utils.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 6122cd3bbb4..f36ad7a14f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,11 +3,13 @@ ## 24.2.1-SNAPSHOT ### Breaking Changes +- RocksDB database metadata format has changed to be more expressive, the migration of an existing metadata file to the new format is automatic at startup. Before performing a downgrade to a previous version it is mandatory to revert to the original format using the subcommand `besu --data-path=/path/to/besu/datadir storage revert-metadata v2-to-v1`. ### Deprecations ### Additions and Improvements - Extend `Blockchain` service [#6592](https://github.com/hyperledger/besu/pull/6592) +- RocksDB database metadata refactoring [#6555](https://github.com/hyperledger/besu/pull/6555) ### Bug fixes diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index bcb0ce68576..6429339dbb4 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -141,7 +141,6 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.evm.precompile.AbstractAltBnPrecompiledContract; import org.hyperledger.besu.evm.precompile.BigIntegerModularExponentiationPrecompiledContract; import org.hyperledger.besu.evm.precompile.KZGPointEvalPrecompiledContract; @@ -172,6 +171,7 @@ import org.hyperledger.besu.plugin.services.metrics.MetricCategory; import org.hyperledger.besu.plugin.services.metrics.MetricCategoryRegistry; import org.hyperledger.besu.plugin.services.securitymodule.SecurityModule; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.storage.PrivacyKeyValueStorageFactory; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBPlugin; import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java index 53f98fc660c..043f41cee6a 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java @@ -25,8 +25,8 @@ import org.hyperledger.besu.cli.options.CLIOptions; import org.hyperledger.besu.cli.util.CommandLineUtils; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.List; diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RevertMetadataSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RevertMetadataSubCommand.java new file mode 100644 index 00000000000..ee239d4aef1 --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RevertMetadataSubCommand.java @@ -0,0 +1,135 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.subcommands.storage; + +import org.hyperledger.besu.cli.util.VersionProvider; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.OptionalInt; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.ParentCommand; + +/** The revert metadata to v1 subcommand. */ +@Command( + name = "revert-metadata", + description = "Revert database metadata to previous format", + mixinStandardHelpOptions = true, + versionProvider = VersionProvider.class, + subcommands = RevertMetadataSubCommand.v2ToV1.class) +public class RevertMetadataSubCommand implements Runnable { + private static final Logger LOG = LoggerFactory.getLogger(RevertMetadataSubCommand.class); + private static final String METADATA_FILENAME = "DATABASE_METADATA.json"; + private static final ObjectMapper MAPPER = + new ObjectMapper() + .registerModule(new Jdk8Module()) + .setSerializationInclusion(JsonInclude.Include.NON_ABSENT) + .enable(SerializationFeature.INDENT_OUTPUT); + + @SuppressWarnings("unused") + @ParentCommand + private StorageSubCommand parentCommand; + + @SuppressWarnings("unused") + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public void run() { + spec.commandLine().usage(System.out); + } + + @Command( + name = "v2-to-v1", + description = "Revert a database metadata v2 format to v1 format", + mixinStandardHelpOptions = true, + versionProvider = VersionProvider.class) + static class v2ToV1 implements Runnable { + + @SuppressWarnings("unused") + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @SuppressWarnings("unused") + @ParentCommand + private RevertMetadataSubCommand parentCommand; + + @Override + public void run() { + + final Path dataDir = parentCommand.parentCommand.besuCommand.dataDir(); + + final File dbMetadata = dataDir.resolve(METADATA_FILENAME).toFile(); + if (!dbMetadata.exists()) { + String errMsg = + String.format( + "Could not find database metadata file %s, check your data dir %s", + dbMetadata, dataDir); + LOG.error(errMsg); + throw new IllegalArgumentException(errMsg); + } + try { + final var root = MAPPER.readTree(dbMetadata); + if (!root.has("v2")) { + String errMsg = + String.format("Database metadata file %s is not in v2 format", dbMetadata); + LOG.error(errMsg); + throw new IllegalArgumentException(errMsg); + } + + final var v2Obj = root.get("v2"); + if (!v2Obj.has("format")) { + String errMsg = + String.format( + "Database metadata file %s is malformed, \"format\" field not found", dbMetadata); + LOG.error(errMsg); + throw new IllegalArgumentException(errMsg); + } + + final var formatField = v2Obj.get("format").asText(); + final OptionalInt maybePrivacyVersion = + v2Obj.has("privacyVersion") + ? OptionalInt.of(v2Obj.get("privacyVersion").asInt()) + : OptionalInt.empty(); + + final DataStorageFormat dataStorageFormat = DataStorageFormat.valueOf(formatField); + final int v1Version = + switch (dataStorageFormat) { + case FOREST -> 1; + case BONSAI -> 2; + }; + + @JsonSerialize + record V1(int version, OptionalInt privacyVersion) {} + + MAPPER.writeValue(dbMetadata, new V1(v1Version, maybePrivacyVersion)); + LOG.info("Successfully reverted database metadata from v2 to v1 in {}", dbMetadata); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java index 57eb50846d8..b377ae1e7a2 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java @@ -48,7 +48,8 @@ subcommands = { StorageSubCommand.RevertVariablesStorage.class, RocksDbSubCommand.class, - TrieLogSubCommand.class + TrieLogSubCommand.class, + RevertMetadataSubCommand.class }) public class StorageSubCommand implements Runnable { diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index 47bad2292e4..a47271e247b 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -28,7 +28,7 @@ import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.io.IOException; import java.io.PrintWriter; diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 9529b3d365d..2ca4a1d12ec 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -91,7 +91,6 @@ import org.hyperledger.besu.ethereum.trie.forest.pruner.Pruner; import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; @@ -99,6 +98,7 @@ import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.permissioning.NodeMessagePermissioningProvider; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; import org.hyperledger.besu.plugin.services.txvalidator.PluginTransactionValidatorFactory; diff --git a/besu/src/main/java/org/hyperledger/besu/services/BesuConfigurationImpl.java b/besu/src/main/java/org/hyperledger/besu/services/BesuConfigurationImpl.java index ae907e0a381..e7fb8202b17 100644 --- a/besu/src/main/java/org/hyperledger/besu/services/BesuConfigurationImpl.java +++ b/besu/src/main/java/org/hyperledger/besu/services/BesuConfigurationImpl.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.plugin.services.BesuConfiguration; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.nio.file.Path; @@ -58,8 +59,8 @@ public Path getDataPath() { } @Override - public int getDatabaseVersion() { - return dataStorageConfiguration.getDataStorageFormat().getDatabaseVersion(); + public DataStorageFormat getDatabaseFormat() { + return dataStorageConfiguration.getDataStorageFormat(); } @Override diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 09171e012e5..252e30003d8 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -32,8 +32,8 @@ import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.GOERLI_DISCOVERY_URL; import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.MAINNET_BOOTSTRAP_NODES; import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.MAINNET_DISCOVERY_URL; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageFormat.BONSAI; import static org.hyperledger.besu.nat.kubernetes.KubernetesNatManager.DEFAULT_BESU_SERVICE_NAME_FILTER; +import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.BONSAI; import static org.junit.Assume.assumeThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.contains; diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java index 8dc2ef87803..bd6985df577 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java @@ -20,8 +20,8 @@ import org.hyperledger.besu.cli.options.AbstractCLIOptionsTest; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.junit.jupiter.api.Test; diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index c1e6f43ae86..c2789c68085 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -19,7 +19,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageFormat.BONSAI; +import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.BONSAI; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; diff --git a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java index 225b4ea62d3..df6a427dc25 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java @@ -48,7 +48,6 @@ import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; @@ -56,6 +55,7 @@ import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.math.BigInteger; diff --git a/besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java b/besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java index 3df1e8e24d8..6909d985864 100644 --- a/besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java +++ b/besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java @@ -31,7 +31,6 @@ import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.evm.log.Log; import org.hyperledger.besu.evm.worldstate.WorldView; @@ -40,6 +39,7 @@ import org.hyperledger.besu.plugin.data.BlockTraceResult; import org.hyperledger.besu.plugin.data.TransactionTraceResult; import org.hyperledger.besu.plugin.services.TraceService; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.tracer.BlockAwareOperationTracer; import java.util.HashSet; diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java index fd95832a741..771385af3c3 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java @@ -35,8 +35,8 @@ import org.hyperledger.besu.ethereum.mainnet.ValidationResult; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.Capability; import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.plugin.data.SyncStatus; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.nio.file.Path; import java.util.Collections; diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/AbstractJsonRpcHttpServiceTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/AbstractJsonRpcHttpServiceTest.java index e43c39db4df..9b1163d1e0f 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/AbstractJsonRpcHttpServiceTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/AbstractJsonRpcHttpServiceTest.java @@ -45,10 +45,10 @@ import org.hyperledger.besu.ethereum.p2p.network.P2PNetwork; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.Capability; import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration; import org.hyperledger.besu.nat.NatService; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.testutil.BlockTestUtil.ChainResources; import java.math.BigInteger; diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/bonsai/DebugTraceJsonRpcHttpBySpecTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/bonsai/DebugTraceJsonRpcHttpBySpecTest.java index acfbcbaf492..37b2cedf7cf 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/bonsai/DebugTraceJsonRpcHttpBySpecTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/bonsai/DebugTraceJsonRpcHttpBySpecTest.java @@ -16,7 +16,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.AbstractJsonRpcHttpBySpecTest; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.junit.jupiter.api.BeforeEach; diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/bonsai/EthByzantiumJsonRpcHttpBySpecTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/bonsai/EthByzantiumJsonRpcHttpBySpecTest.java index a5f3ae0ab0c..358aeb15542 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/bonsai/EthByzantiumJsonRpcHttpBySpecTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/bonsai/EthByzantiumJsonRpcHttpBySpecTest.java @@ -16,7 +16,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.AbstractJsonRpcHttpBySpecTest; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.junit.jupiter.api.BeforeEach; diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/bonsai/TraceJsonRpcHttpBySpecTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/bonsai/TraceJsonRpcHttpBySpecTest.java index 127d7d3b696..a3bf6fc51db 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/bonsai/TraceJsonRpcHttpBySpecTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/bonsai/TraceJsonRpcHttpBySpecTest.java @@ -16,7 +16,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.AbstractJsonRpcHttpBySpecTest; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.junit.jupiter.api.BeforeEach; diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/forest/DebugTraceJsonRpcHttpBySpecTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/forest/DebugTraceJsonRpcHttpBySpecTest.java index 412a4958c3d..89b79c2af31 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/forest/DebugTraceJsonRpcHttpBySpecTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/forest/DebugTraceJsonRpcHttpBySpecTest.java @@ -16,7 +16,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.AbstractJsonRpcHttpBySpecTest; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.junit.jupiter.api.BeforeEach; diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/forest/EthByzantiumJsonRpcHttpBySpecTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/forest/EthByzantiumJsonRpcHttpBySpecTest.java index a98aa9b45e5..16392979923 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/forest/EthByzantiumJsonRpcHttpBySpecTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/forest/EthByzantiumJsonRpcHttpBySpecTest.java @@ -16,7 +16,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.AbstractJsonRpcHttpBySpecTest; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.junit.jupiter.api.BeforeEach; diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/forest/TraceJsonRpcHttpBySpecTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/forest/TraceJsonRpcHttpBySpecTest.java index 36b5b6f17e1..d71c28a2234 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/forest/TraceJsonRpcHttpBySpecTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/forest/TraceJsonRpcHttpBySpecTest.java @@ -16,7 +16,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.AbstractJsonRpcHttpBySpecTest; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.junit.jupiter.api.BeforeEach; diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/filter/EthJsonRpcHttpServiceTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/filter/EthJsonRpcHttpServiceTest.java index bdef98bb039..38dc9decc57 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/filter/EthJsonRpcHttpServiceTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/filter/EthJsonRpcHttpServiceTest.java @@ -21,7 +21,7 @@ import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; import org.hyperledger.besu.ethereum.core.Transaction; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.io.IOException; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java index 9326c97b32e..32e372cdecf 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java @@ -14,24 +14,27 @@ */ package org.hyperledger.besu.ethereum.storage.keyvalue; +import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.BONSAI; +import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.FOREST; + +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import java.nio.charset.StandardCharsets; - -import org.bouncycastle.util.Arrays; +import java.util.EnumSet; public enum KeyValueSegmentIdentifier implements SegmentIdentifier { DEFAULT("default".getBytes(StandardCharsets.UTF_8)), BLOCKCHAIN(new byte[] {1}, true, true), - WORLD_STATE(new byte[] {2}, new int[] {0, 1}, false, true, false), + WORLD_STATE(new byte[] {2}, EnumSet.of(FOREST), false, true, false), PRIVATE_TRANSACTIONS(new byte[] {3}), PRIVATE_STATE(new byte[] {4}), - PRUNING_STATE(new byte[] {5}, new int[] {0, 1}), - ACCOUNT_INFO_STATE(new byte[] {6}, new int[] {2}, false, true, false), - CODE_STORAGE(new byte[] {7}, new int[] {2}), - ACCOUNT_STORAGE_STORAGE(new byte[] {8}, new int[] {2}, false, true, false), - TRIE_BRANCH_STORAGE(new byte[] {9}, new int[] {2}, false, true, false), - TRIE_LOG_STORAGE(new byte[] {10}, new int[] {2}, true, false, true), + PRUNING_STATE(new byte[] {5}, EnumSet.of(FOREST)), + ACCOUNT_INFO_STATE(new byte[] {6}, EnumSet.of(BONSAI), false, true, false), + CODE_STORAGE(new byte[] {7}, EnumSet.of(BONSAI)), + ACCOUNT_STORAGE_STORAGE(new byte[] {8}, EnumSet.of(BONSAI), false, true, false), + TRIE_BRANCH_STORAGE(new byte[] {9}, EnumSet.of(BONSAI), false, true, false), + TRIE_LOG_STORAGE(new byte[] {10}, EnumSet.of(BONSAI), true, false, true), VARIABLES(new byte[] {11}), // formerly GOQUORUM_PRIVATE_WORLD_STATE // previously supported GoQuorum private states @@ -46,32 +49,37 @@ public enum KeyValueSegmentIdentifier implements SegmentIdentifier { CHAIN_PRUNER_STATE(new byte[] {18}); private final byte[] id; - private final int[] versionList; + private final EnumSet formats; private final boolean containsStaticData; private final boolean eligibleToHighSpecFlag; private final boolean staticDataGarbageCollectionEnabled; KeyValueSegmentIdentifier(final byte[] id) { - this(id, new int[] {0, 1, 2}); + this(id, EnumSet.allOf(DataStorageFormat.class)); } KeyValueSegmentIdentifier( final byte[] id, final boolean containsStaticData, final boolean eligibleToHighSpecFlag) { - this(id, new int[] {0, 1, 2}, containsStaticData, eligibleToHighSpecFlag, false); + this( + id, + EnumSet.allOf(DataStorageFormat.class), + containsStaticData, + eligibleToHighSpecFlag, + false); } - KeyValueSegmentIdentifier(final byte[] id, final int[] versionList) { - this(id, versionList, false, false, false); + KeyValueSegmentIdentifier(final byte[] id, final EnumSet formats) { + this(id, formats, false, false, false); } KeyValueSegmentIdentifier( final byte[] id, - final int[] versionList, + final EnumSet formats, final boolean containsStaticData, final boolean eligibleToHighSpecFlag, final boolean staticDataGarbageCollectionEnabled) { this.id = id; - this.versionList = versionList; + this.formats = formats; this.containsStaticData = containsStaticData; this.eligibleToHighSpecFlag = eligibleToHighSpecFlag; this.staticDataGarbageCollectionEnabled = staticDataGarbageCollectionEnabled; @@ -103,7 +111,7 @@ public boolean isStaticDataGarbageCollectionEnabled() { } @Override - public boolean includeInDatabaseVersion(final int version) { - return Arrays.contains(versionList, version); + public boolean includeInDatabaseFormat(final DataStorageFormat format) { + return formats.contains(format); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java index 44c15e81502..08a1ba23a83 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java @@ -22,10 +22,10 @@ import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.ObservableMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index f950e7c9163..4a4dcf44e1a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -27,12 +27,12 @@ import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.FlatDbStrategy; import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.FlatDbStrategyProvider; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.evm.account.AccountStorageEntry; import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java index 08bc1b2eda9..121fec3f4dc 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java @@ -27,9 +27,9 @@ import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.hyperledger.besu.services.kvstore.SegmentedInMemoryKeyValueStorage; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestWorldStateKeyValueStorage.java index 294164e3d9b..9df312b0e4f 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestWorldStateKeyValueStorage.java @@ -16,9 +16,9 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.trie.MerkleTrie; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.util.Subscribers; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java index 33304d53a98..064f5a961c0 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java @@ -16,6 +16,8 @@ package org.hyperledger.besu.ethereum.worldstate; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; + import org.immutables.value.Value; @Value.Immutable diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageFormat.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageFormat.java deleted file mode 100644 index aee8b87aacd..00000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageFormat.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ - -package org.hyperledger.besu.ethereum.worldstate; - -public enum DataStorageFormat { - FOREST(1), // Original format. Store all tries - BONSAI(2); // New format. Store one trie, and trie logs to roll forward and backward. - - private final int databaseVersion; - - DataStorageFormat(final int databaseVersion) { - this.databaseVersion = databaseVersion; - } - - public int getDatabaseVersion() { - return databaseVersion; - } - - public static String getName(final int databaseVersion) { - for (DataStorageFormat format : DataStorageFormat.values()) { - if (format.getDatabaseVersion() == databaseVersion) { - return format.name(); - } - } - return "Unknown"; - } -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java index c37b6334b7e..a70a4982e51 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.worldstate; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.Collection; import java.util.Collections; diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockchainSetupUtil.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockchainSetupUtil.java index 49a32b1ce37..349c819ddac 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockchainSetupUtil.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockchainSetupUtil.java @@ -36,10 +36,10 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; import org.hyperledger.besu.ethereum.util.RawBlockIterator; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.testutil.BlockTestUtil; import org.hyperledger.besu.testutil.BlockTestUtil.ChainResources; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java index 707327b1995..e14ab5969a5 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java @@ -72,6 +72,7 @@ import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.BesuConfiguration; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBKeyValueStorageFactory; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBMetricsFactory; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBFactoryConfiguration; @@ -189,7 +190,6 @@ protected StorageProvider createKeyValueStorageProvider() { 8388608 /*CACHE_CAPACITY*/, false), Arrays.asList(KeyValueSegmentIdentifier.values()), - 2, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS)) .withCommonConfiguration( new BesuConfiguration() { @@ -205,8 +205,8 @@ public Path getDataPath() { } @Override - public int getDatabaseVersion() { - return 2; + public DataStorageFormat getDatabaseFormat() { + return DataStorageFormat.BONSAI; } @Override diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java index fbdfa7724f7..eae7ecc7f20 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java @@ -38,11 +38,11 @@ import org.hyperledger.besu.ethereum.trie.StorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java index 1a115defd75..2340a6e8b05 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java @@ -21,10 +21,10 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; import org.hyperledger.besu.services.kvstore.SegmentedInMemoryKeyValueStorage; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java index c74fc7e1805..64f96d8b734 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java @@ -24,8 +24,8 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java index d7c2707af08..dc56cac5397 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java @@ -21,8 +21,8 @@ import org.hyperledger.besu.ethereum.eth.sync.TrailingPeerRequirements; import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.services.tasks.TaskCollection; import org.hyperledger.besu.util.ExceptionUtils; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java index 5b2f51d87ba..877559f6b7e 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java @@ -28,10 +28,10 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues; import java.time.Clock; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java index 1a80d2f742d..c05bdcb1865 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java @@ -71,10 +71,10 @@ import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.RawMessage; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.messages.DisconnectMessage.DisconnectReason; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.testutil.TestClock; import java.math.BigInteger; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java index 8fc4e614c76..63ada130408 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTestUtil.java @@ -35,9 +35,9 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.DefaultMessage; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.testutil.DeterministicEthScheduler; import org.hyperledger.besu.testutil.DeterministicEthScheduler.TimeoutPolicy; import org.hyperledger.besu.testutil.TestClock; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/ethtaskutils/AbstractMessageTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/ethtaskutils/AbstractMessageTaskTest.java index 868030334c8..3ea2dc27f3c 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/ethtaskutils/AbstractMessageTaskTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/ethtaskutils/AbstractMessageTaskTest.java @@ -42,9 +42,9 @@ import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolFactory; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.testutil.DeterministicEthScheduler; import org.hyperledger.besu.testutil.TestClock; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/task/SnapProtocolManagerTestUtil.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/task/SnapProtocolManagerTestUtil.java index 68bd4e4e5a1..4fd88a0f3ed 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/task/SnapProtocolManagerTestUtil.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/task/SnapProtocolManagerTestUtil.java @@ -20,8 +20,8 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.RespondingEthPeer; import org.hyperledger.besu.ethereum.eth.manager.snap.SnapProtocolManager; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.Collections; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/AbstractBlockPropagationManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/AbstractBlockPropagationManagerTest.java index 8a3dd6d22d1..4406d9a7ee5 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/AbstractBlockPropagationManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/AbstractBlockPropagationManagerTest.java @@ -58,9 +58,9 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.testutil.TestClock; import java.util.Collections; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/BonsaiBlockPropagationManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/BonsaiBlockPropagationManagerTest.java index 7bd4d2bbbc4..eda0473147a 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/BonsaiBlockPropagationManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/BonsaiBlockPropagationManagerTest.java @@ -16,7 +16,7 @@ import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/ChainHeadTrackerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/ChainHeadTrackerTest.java index 443cc017f3f..808005f858d 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/ChainHeadTrackerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/ChainHeadTrackerTest.java @@ -29,9 +29,9 @@ import org.hyperledger.besu.ethereum.eth.manager.RespondingEthPeer; import org.hyperledger.besu.ethereum.eth.manager.RespondingEthPeer.Responder; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.stream.Stream; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/DownloadHeadersStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/DownloadHeadersStepTest.java index b9d8eec0216..3dba7d1cf10 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/DownloadHeadersStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/DownloadHeadersStepTest.java @@ -30,8 +30,8 @@ import org.hyperledger.besu.ethereum.eth.sync.range.SyncTargetRange; import org.hyperledger.besu.ethereum.mainnet.HeaderValidationMode; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.ArrayList; import java.util.Collections; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/ForestBlockPropagationManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/ForestBlockPropagationManagerTest.java index 000958b1fac..9b2bd896203 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/ForestBlockPropagationManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/ForestBlockPropagationManagerTest.java @@ -16,7 +16,7 @@ import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/RangeHeadersFetcherTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/RangeHeadersFetcherTest.java index 0b18a0f64c6..39cad8407e7 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/RangeHeadersFetcherTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/RangeHeadersFetcherTest.java @@ -33,9 +33,9 @@ import org.hyperledger.besu.ethereum.eth.sync.range.RangeHeadersFetcher; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.List; import java.util.concurrent.CompletableFuture; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java index 62ef446503b..19b738d7c4d 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java @@ -36,9 +36,9 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.checkpoint.ImmutableCheckpoint; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.Optional; import java.util.concurrent.CompletableFuture; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java index 0a775b130fa..c9cfeda1191 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/DownloadReceiptsStepTest.java @@ -31,8 +31,8 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManagerTestUtil; import org.hyperledger.besu.ethereum.eth.manager.RespondingEthPeer; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.List; import java.util.concurrent.CompletableFuture; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java index a707228e92d..9813dc132fe 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java @@ -40,10 +40,10 @@ import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.ArrayList; import java.util.List; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java index d2c63a18a15..9312ca832eb 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java @@ -35,9 +35,9 @@ import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.concurrent.CompletableFuture; import java.util.concurrent.locks.LockSupport; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java index c05770e6d72..e8eadea3ade 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java @@ -34,8 +34,8 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate.NodeDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.services.tasks.TaskCollection; import java.nio.file.Path; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockConfirmerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockConfirmerTest.java index 08ce7e9d37f..050724ef55a 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockConfirmerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockConfirmerTest.java @@ -32,9 +32,9 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.PivotBlockConfirmer.ContestedPivotBlockException; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.Optional; import java.util.concurrent.CompletableFuture; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetrieverTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetrieverTest.java index 723c5afe65f..847ef535192 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetrieverTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetrieverTest.java @@ -34,9 +34,9 @@ import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.util.ExceptionUtils; import java.util.Optional; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java index 803a3e0b48e..04b3d7136ce 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java @@ -29,9 +29,9 @@ import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues; import org.hyperledger.besu.testutil.TestClock; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTest.java index b7588d1b4f9..f44c8e2da34 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTest.java @@ -42,9 +42,9 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.messages.DisconnectMessage.DisconnectReason; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.ArrayList; import java.util.List; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTotalTerminalDifficultyTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTotalTerminalDifficultyTest.java index c86be6d6407..c9f22ec5f01 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTotalTerminalDifficultyTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTotalTerminalDifficultyTest.java @@ -31,9 +31,9 @@ import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.concurrent.CompletableFuture; import java.util.stream.Stream; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncDownloaderTest.java index aff0195db2d..cb0f399ff81 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncDownloaderTest.java @@ -29,9 +29,9 @@ import org.hyperledger.besu.ethereum.eth.sync.TrailingPeerRequirements; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.stream.Stream; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManagerTest.java index f4b0708bfcb..b4a2c8eaa69 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManagerTest.java @@ -35,9 +35,9 @@ import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.state.SyncTarget; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.Optional; import java.util.concurrent.CompletableFuture; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java index 03249c790f9..b0995c57a54 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java @@ -41,9 +41,9 @@ import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues; import org.hyperledger.besu.testutil.TestClock; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java index 2b6de0e0254..847e624ca0c 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java @@ -25,10 +25,10 @@ import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.List; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/PersistBlockTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/PersistBlockTaskTest.java index 5d02f398470..d789405763c 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/PersistBlockTaskTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/PersistBlockTaskTest.java @@ -29,9 +29,9 @@ import org.hyperledger.besu.ethereum.eth.sync.tasks.exceptions.InvalidBlockException; import org.hyperledger.besu.ethereum.mainnet.HeaderValidationMode; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.Arrays; import java.util.Collections; diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 46611fc29d0..5e09c0cfa4f 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -69,7 +69,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = 'Pi7Veo9W9kmjDJJNB89UTUXbYyRmoN6osK/tD163h3E=' + knownHash = 'jvIsInEUQ/NaxWCGkQIb72tuZsx5288Ownn2F6rsvjQ=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BesuConfiguration.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BesuConfiguration.java index bf1b1ab58e9..c28995cc6ce 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BesuConfiguration.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BesuConfiguration.java @@ -16,6 +16,7 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.plugin.Unstable; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.nio.file.Path; @@ -37,12 +38,12 @@ public interface BesuConfiguration extends BesuService { Path getDataPath(); /** - * Database version. This sets the list of segmentIdentifiers that should be initialized. + * Database format. This sets the list of segmentIdentifiers that should be initialized. * - * @return Database version. + * @return Database format. */ @Unstable - int getDatabaseVersion(); + DataStorageFormat getDatabaseFormat(); /** * The runtime value of the min gas price diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/DataStorageFormat.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/DataStorageFormat.java new file mode 100644 index 00000000000..69df2717b3b --- /dev/null +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/DataStorageFormat.java @@ -0,0 +1,23 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.plugin.services.storage; + +/** Supported database storage format */ +public enum DataStorageFormat { + /** Original format. Store all tries */ + FOREST, + /** New format. Store one trie, and trie logs to roll forward and backward */ + BONSAI; +} diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java index 66335943705..efcac957dda 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java @@ -39,12 +39,12 @@ public interface SegmentIdentifier { /** * Not all segments are in all DB versions. This queries the segment to see if it is in the DB - * version. + * format. * - * @param version Version of the DB - * @return true if the segment is in that DB version + * @param format Version of the DB + * @return true if the segment is in that DB format */ - default boolean includeInDatabaseVersion(final int version) { + default boolean includeInDatabaseFormat(final DataStorageFormat format) { return true; } diff --git a/plugins/rocksdb/build.gradle b/plugins/rocksdb/build.gradle index f76e8d80ae0..6af582bf3b2 100644 --- a/plugins/rocksdb/build.gradle +++ b/plugins/rocksdb/build.gradle @@ -41,6 +41,7 @@ dependencies { implementation project(':util') implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8' implementation 'com.google.guava:guava' implementation 'info.picocli:picocli' implementation 'io.opentelemetry:opentelemetry-api' diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java index 355cc056924..bf5f92165f7 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java @@ -22,11 +22,15 @@ import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.DatabaseMetadata; +import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.PrivacyVersionedStorageFormat; +import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.VersionedStorageFormat; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.EnumSet; import java.util.List; +import java.util.Optional; import java.util.Set; import org.slf4j.Logger; @@ -39,12 +43,13 @@ public class RocksDBKeyValuePrivacyStorageFactory implements PrivacyKeyValueStorageFactory { private static final Logger LOG = LoggerFactory.getLogger(RocksDBKeyValuePrivacyStorageFactory.class); - private static final int DEFAULT_VERSION = 1; - private static final Set SUPPORTED_VERSIONS = Set.of(1); - + private static final Set SUPPORTED_VERSIONS = + EnumSet.of( + PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES, + PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES); private static final String PRIVATE_DATABASE_PATH = "private"; private final RocksDBKeyValueStorageFactory publicFactory; - private Integer databaseVersion; + private DatabaseMetadata databaseMetadata; /** * Instantiates a new RocksDb key value privacy storage factory. @@ -66,9 +71,9 @@ public KeyValueStorage create( final BesuConfiguration commonConfiguration, final MetricsSystem metricsSystem) throws StorageException { - if (databaseVersion == null) { + if (databaseMetadata == null) { try { - databaseVersion = readDatabaseVersion(commonConfiguration); + databaseMetadata = readDatabaseMetadata(commonConfiguration); } catch (final IOException e) { throw new StorageException("Failed to retrieve the RocksDB database meta version", e); } @@ -83,9 +88,9 @@ public SegmentedKeyValueStorage create( final BesuConfiguration commonConfiguration, final MetricsSystem metricsSystem) throws StorageException { - if (databaseVersion == null) { + if (databaseMetadata == null) { try { - databaseVersion = readDatabaseVersion(commonConfiguration); + databaseMetadata = readDatabaseMetadata(commonConfiguration); } catch (final IOException e) { throw new StorageException("Failed to retrieve the RocksDB database meta version", e); } @@ -114,37 +119,118 @@ public void close() throws IOException { * private database exists there may be a "privacyVersion" field in the metadata file otherwise * use the default version */ - private int readDatabaseVersion(final BesuConfiguration commonConfiguration) throws IOException { + private DatabaseMetadata readDatabaseMetadata(final BesuConfiguration commonConfiguration) + throws IOException { final Path dataDir = commonConfiguration.getDataPath(); final boolean privacyDatabaseExists = commonConfiguration.getStoragePath().resolve(PRIVATE_DATABASE_PATH).toFile().exists(); - final int privacyDatabaseVersion; - if (privacyDatabaseExists) { - privacyDatabaseVersion = DatabaseMetadata.lookUpFrom(dataDir).maybePrivacyVersion().orElse(1); - LOG.info( - "Existing private database detected at {}. Version {}", dataDir, privacyDatabaseVersion); + final boolean privacyMetadataExists = DatabaseMetadata.isPresent(dataDir); + DatabaseMetadata privacyMetadata; + if (privacyDatabaseExists && !privacyMetadataExists) { + throw new StorageException( + "Privacy database exists but metadata file not found, without it there is no safe way to open the database"); + } + if (privacyMetadataExists) { + final var existingPrivacyMetadata = DatabaseMetadata.lookUpFrom(dataDir); + final var maybeExistingPrivacyVersion = + existingPrivacyMetadata.getVersionedStorageFormat().getPrivacyVersion(); + if (maybeExistingPrivacyVersion.isEmpty()) { + privacyMetadata = existingPrivacyMetadata.upgradeToPrivacy(); + privacyMetadata.writeToDirectory(dataDir); + LOG.info( + "Upgraded existing database at {} to privacy database. Metadata {}", + dataDir, + existingPrivacyMetadata); + } else { + privacyMetadata = existingPrivacyMetadata; + final int existingPrivacyVersion = maybeExistingPrivacyVersion.getAsInt(); + final var runtimeVersion = + PrivacyVersionedStorageFormat.defaultForNewDB(commonConfiguration.getDatabaseFormat()); + + if (existingPrivacyVersion > runtimeVersion.getPrivacyVersion().getAsInt()) { + final var maybeDowngradedMetadata = + handleVersionDowngrade(dataDir, privacyMetadata, runtimeVersion); + if (maybeDowngradedMetadata.isPresent()) { + privacyMetadata = maybeDowngradedMetadata.get(); + privacyMetadata.writeToDirectory(dataDir); + } + } else if (existingPrivacyVersion < runtimeVersion.getPrivacyVersion().getAsInt()) { + final var maybeUpgradedMetadata = + handleVersionUpgrade(dataDir, privacyMetadata, runtimeVersion); + if (maybeUpgradedMetadata.isPresent()) { + privacyMetadata = maybeUpgradedMetadata.get(); + privacyMetadata.writeToDirectory(dataDir); + } + } else { + LOG.info("Existing privacy database at {}. Metadata {}", dataDir, privacyMetadata); + } + } } else { - privacyDatabaseVersion = DEFAULT_VERSION; + privacyMetadata = DatabaseMetadata.defaultForNewPrivateDb(); LOG.info( - "No existing private database detected at {}. Using version {}", + "No existing private database at {}. Using default metadata for new db {}", dataDir, - privacyDatabaseVersion); + privacyMetadata); Files.createDirectories(dataDir); - new DatabaseMetadata(publicFactory.getDefaultVersion(), privacyDatabaseVersion) - .writeToDirectory(dataDir); + privacyMetadata.writeToDirectory(dataDir); } - if (!SUPPORTED_VERSIONS.contains(privacyDatabaseVersion)) { - final String message = "Unsupported RocksDB Metadata version of: " + privacyDatabaseVersion; + if (!SUPPORTED_VERSIONS.contains(privacyMetadata.getVersionedStorageFormat())) { + final String message = "Unsupported RocksDB Metadata version of: " + privacyMetadata; LOG.error(message); throw new StorageException(message); } - return privacyDatabaseVersion; + return privacyMetadata; + } + + private Optional handleVersionDowngrade( + final Path dataDir, + final DatabaseMetadata existingPrivacyMetadata, + final VersionedStorageFormat runtimeVersion) { + // here we put the code, or the messages, to perform an automated, or manual, downgrade of the + // database, if supported, otherwise we just prevent Besu from starting since it will not + // recognize the newer version. + // In case we do an automated downgrade, then we also need to update the metadata on disk to + // reflect the change to the runtime version, and return it. + + // for the moment there are supported automated downgrades, so we just fail. + String error = + String.format( + "Database unsafe downgrade detect: DB at %s is %s with version %s but version %s is expected. " + + "Please check your config and review release notes for supported downgrade procedures.", + dataDir, + existingPrivacyMetadata.getVersionedStorageFormat().getFormat().name(), + existingPrivacyMetadata.getVersionedStorageFormat().getVersion(), + runtimeVersion.getVersion()); + + throw new StorageException(error); + } + + private Optional handleVersionUpgrade( + final Path dataDir, + final DatabaseMetadata existingPrivacyMetadata, + final VersionedStorageFormat runtimeVersion) { + // here we put the code, or the messages, to perform an automated, or manual, upgrade of the + // database. + // In case we do an automated upgrade, then we also need to update the metadata on disk to + // reflect the change to the runtime version, and return it. + + // for the moment there are no planned automated upgrades, so we just fail. + String error = + String.format( + "Database unsafe upgrade detect: DB at %s is %s with version %s but version %s is expected. " + + "Please check your config and review release notes for supported upgrade procedures.", + dataDir, + existingPrivacyMetadata.getVersionedStorageFormat().getFormat().name(), + existingPrivacyMetadata.getVersionedStorageFormat().getVersion(), + runtimeVersion.getVersion()); + + throw new StorageException(error); } @Override public int getVersion() { - return databaseVersion; + return databaseMetadata.getVersionedStorageFormat().getPrivacyVersion().getAsInt(); } } diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java index 1abd2481741..eaf95e82a1e 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java @@ -14,7 +14,9 @@ */ package org.hyperledger.besu.plugin.services.storage.rocksdb; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES; +import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat.FOREST_WITH_VARIABLES; + import org.hyperledger.besu.plugin.services.BesuConfiguration; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.exception.StorageException; @@ -22,10 +24,12 @@ import org.hyperledger.besu.plugin.services.storage.KeyValueStorageFactory; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.DatabaseMetadata; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBConfiguration; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBConfigurationBuilder; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBFactoryConfiguration; +import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.VersionedStorageFormat; import org.hyperledger.besu.plugin.services.storage.rocksdb.segmented.OptimisticRocksDBColumnarKeyValueStorage; import org.hyperledger.besu.plugin.services.storage.rocksdb.segmented.RocksDBColumnarKeyValueStorage; import org.hyperledger.besu.plugin.services.storage.rocksdb.segmented.TransactionDBRocksDBColumnarKeyValueStorage; @@ -34,8 +38,9 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.EnumSet; import java.util.List; -import java.util.Set; +import java.util.Optional; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -49,13 +54,11 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory { private static final Logger LOG = LoggerFactory.getLogger(RocksDBKeyValueStorageFactory.class); - private static final int DEFAULT_VERSION = 1; - private static final Set SUPPORTED_VERSIONS = Set.of(1, 2); + private static final EnumSet SUPPORTED_VERSIONED_FORMATS = + EnumSet.of(FOREST_WITH_VARIABLES, BONSAI_WITH_VARIABLES); private static final String NAME = "rocksdb"; private final RocksDBMetricsFactory rocksDBMetricsFactory; - - private final int defaultVersion; - private Integer databaseVersion; + private DatabaseMetadata databaseMetadata; private RocksDBColumnarKeyValueStorage segmentedStorage; private RocksDBConfiguration rocksDBConfiguration; @@ -69,19 +72,16 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory { * @param configuration the configuration * @param configuredSegments the segments * @param ignorableSegments the ignorable segments - * @param defaultVersion the default version * @param rocksDBMetricsFactory the rocks db metrics factory */ public RocksDBKeyValueStorageFactory( final Supplier configuration, final List configuredSegments, final List ignorableSegments, - final int defaultVersion, final RocksDBMetricsFactory rocksDBMetricsFactory) { this.configuration = configuration; this.configuredSegments = configuredSegments; this.ignorableSegments = ignorableSegments; - this.defaultVersion = defaultVersion; this.rocksDBMetricsFactory = rocksDBMetricsFactory; } @@ -90,59 +90,13 @@ public RocksDBKeyValueStorageFactory( * * @param configuration the configuration * @param configuredSegments the segments - * @param defaultVersion the default version * @param rocksDBMetricsFactory the rocks db metrics factory */ public RocksDBKeyValueStorageFactory( final Supplier configuration, final List configuredSegments, - final int defaultVersion, final RocksDBMetricsFactory rocksDBMetricsFactory) { - this(configuration, configuredSegments, List.of(), defaultVersion, rocksDBMetricsFactory); - } - - /** - * Instantiates a new Rocks db key value storage factory. - * - * @param configuration the configuration - * @param configuredSegments the segments - * @param ignorableSegments the ignorable segments - * @param rocksDBMetricsFactory the rocks db metrics factory - */ - public RocksDBKeyValueStorageFactory( - final Supplier configuration, - final List configuredSegments, - final List ignorableSegments, - final RocksDBMetricsFactory rocksDBMetricsFactory) { - this( - configuration, - configuredSegments, - ignorableSegments, - DEFAULT_VERSION, - rocksDBMetricsFactory); - } - - /** - * Instantiates a new Rocks db key value storage factory. - * - * @param configuration the configuration - * @param configuredSegments the segments - * @param rocksDBMetricsFactory the rocks db metrics factory - */ - public RocksDBKeyValueStorageFactory( - final Supplier configuration, - final List configuredSegments, - final RocksDBMetricsFactory rocksDBMetricsFactory) { - this(configuration, configuredSegments, List.of(), DEFAULT_VERSION, rocksDBMetricsFactory); - } - - /** - * Gets default version. - * - * @return the default version - */ - int getDefaultVersion() { - return defaultVersion; + this(configuration, configuredSegments, List.of(), rocksDBMetricsFactory); } @Override @@ -166,8 +120,6 @@ public SegmentedKeyValueStorage create( final BesuConfiguration commonConfiguration, final MetricsSystem metricsSystem) throws StorageException { - final boolean isForestStorageFormat = - DataStorageFormat.FOREST.getDatabaseVersion() == commonConfiguration.getDatabaseVersion(); if (requiresInit()) { init(commonConfiguration); } @@ -182,43 +134,43 @@ public SegmentedKeyValueStorage create( .collect(Collectors.joining(", "))); } - // It's probably a good idea for the creation logic to be entirely dependent on the database - // version. Introducing intermediate booleans that represent database properties and dispatching - // creation logic based on them is error-prone. - switch (databaseVersion) { - case 1, 2 -> { - if (segmentedStorage == null) { - final List segmentsForVersion = - configuredSegments.stream() - .filter(segmentId -> segmentId.includeInDatabaseVersion(databaseVersion)) - .collect(Collectors.toList()); - if (isForestStorageFormat) { - LOG.debug("FOREST mode detected, using TransactionDB."); - segmentedStorage = - new TransactionDBRocksDBColumnarKeyValueStorage( - rocksDBConfiguration, - segmentsForVersion, - ignorableSegments, - metricsSystem, - rocksDBMetricsFactory); - } else { - LOG.debug("Using OptimisticTransactionDB."); - segmentedStorage = - new OptimisticRocksDBColumnarKeyValueStorage( - rocksDBConfiguration, - segmentsForVersion, - ignorableSegments, - metricsSystem, - rocksDBMetricsFactory); - } + if (segmentedStorage == null) { + final List segmentsForFormat = + configuredSegments.stream() + .filter( + segmentId -> + segmentId.includeInDatabaseFormat( + databaseMetadata.getVersionedStorageFormat().getFormat())) + .toList(); + + // It's probably a good idea for the creation logic to be entirely dependent on the database + // version. Introducing intermediate booleans that represent database properties and + // dispatching + // creation logic based on them is error-prone. + switch (databaseMetadata.getVersionedStorageFormat().getFormat()) { + case FOREST -> { + LOG.debug("FOREST mode detected, using TransactionDB."); + segmentedStorage = + new TransactionDBRocksDBColumnarKeyValueStorage( + rocksDBConfiguration, + segmentsForFormat, + ignorableSegments, + metricsSystem, + rocksDBMetricsFactory); + } + case BONSAI -> { + LOG.debug("BONSAI mode detected, Using OptimisticTransactionDB."); + segmentedStorage = + new OptimisticRocksDBColumnarKeyValueStorage( + rocksDBConfiguration, + segmentsForFormat, + ignorableSegments, + metricsSystem, + rocksDBMetricsFactory); } - return segmentedStorage; } - default -> throw new IllegalStateException( - String.format( - "Developer error: A supported database version (%d) was detected but there is no associated creation logic.", - databaseVersion)); } + return segmentedStorage; } /** @@ -233,7 +185,7 @@ protected Path storagePath(final BesuConfiguration commonConfiguration) { private void init(final BesuConfiguration commonConfiguration) { try { - databaseVersion = readDatabaseVersion(commonConfiguration); + databaseMetadata = readDatabaseMetadata(commonConfiguration); } catch (final IOException e) { final String message = "Failed to retrieve the RocksDB database meta version: " @@ -251,45 +203,134 @@ private boolean requiresInit() { return segmentedStorage == null; } - private int readDatabaseVersion(final BesuConfiguration commonConfiguration) throws IOException { + private DatabaseMetadata readDatabaseMetadata(final BesuConfiguration commonConfiguration) + throws IOException { final Path dataDir = commonConfiguration.getDataPath(); - final boolean databaseExists = commonConfiguration.getStoragePath().toFile().exists(); final boolean dataDirExists = dataDir.toFile().exists(); - final int databaseVersion; - if (databaseExists) { - databaseVersion = DatabaseMetadata.lookUpFrom(dataDir).getVersion(); - if (databaseVersion != commonConfiguration.getDatabaseVersion()) { - String error = - String.format( - "Mismatch: DB at %s is %s (Version %s) but config expects %s (Version %s). Please check your config.", - dataDir, - DataStorageFormat.getName(databaseVersion), - databaseVersion, - DataStorageFormat.getName(commonConfiguration.getDatabaseVersion()), - commonConfiguration.getDatabaseVersion()); - - throw new StorageException(error); + final boolean databaseExists = commonConfiguration.getStoragePath().toFile().exists(); + final boolean metadataExists = DatabaseMetadata.isPresent(dataDir); + DatabaseMetadata metadata; + if (databaseExists && !metadataExists) { + throw new StorageException( + "Database exists but metadata file not found, without it there is no safe way to open the database"); + } + if (metadataExists) { + metadata = DatabaseMetadata.lookUpFrom(dataDir); + + if (!metadata + .getVersionedStorageFormat() + .getFormat() + .equals(commonConfiguration.getDatabaseFormat())) { + handleFormatMismatch(commonConfiguration, dataDir, metadata); } - LOG.info( - "Existing database detected at {}. Version {}. Compacting database...", - dataDir, - databaseVersion); + + final var runtimeVersion = + BaseVersionedStorageFormat.defaultForNewDB(commonConfiguration.getDatabaseFormat()); + + if (metadata.getVersionedStorageFormat().getVersion() > runtimeVersion.getVersion()) { + final var maybeDowngradedMetadata = + handleVersionDowngrade(dataDir, metadata, runtimeVersion); + if (maybeDowngradedMetadata.isPresent()) { + metadata = maybeDowngradedMetadata.get(); + metadata.writeToDirectory(dataDir); + } + } + + if (metadata.getVersionedStorageFormat().getVersion() < runtimeVersion.getVersion()) { + final var maybeUpgradedMetadata = handleVersionUpgrade(dataDir, metadata, runtimeVersion); + if (maybeUpgradedMetadata.isPresent()) { + metadata = maybeUpgradedMetadata.get(); + metadata.writeToDirectory(dataDir); + } + } + + LOG.info("Existing database at {}. Metadata {}. Processing WAL...", dataDir, metadata); } else { - databaseVersion = commonConfiguration.getDatabaseVersion(); - LOG.info("No existing database detected at {}. Using version {}", dataDir, databaseVersion); + + metadata = DatabaseMetadata.defaultForNewDb(commonConfiguration.getDatabaseFormat()); + LOG.info( + "No existing database at {}. Using default metadata for new db {}", dataDir, metadata); if (!dataDirExists) { Files.createDirectories(dataDir); } - new DatabaseMetadata(databaseVersion).writeToDirectory(dataDir); + metadata.writeToDirectory(dataDir); } - if (!SUPPORTED_VERSIONS.contains(databaseVersion)) { - final String message = "Unsupported RocksDB Metadata version of: " + databaseVersion; + if (!isSupportedVersionedFormat(metadata.getVersionedStorageFormat())) { + final String message = "Unsupported RocksDB metadata: " + metadata; LOG.error(message); throw new StorageException(message); } - return databaseVersion; + return metadata; + } + + private static void handleFormatMismatch( + final BesuConfiguration commonConfiguration, + final Path dataDir, + final DatabaseMetadata existingMetadata) { + String error = + String.format( + "Database format mismatch: DB at %s is %s but config expects %s. " + + "Please check your config.", + dataDir, + existingMetadata.getVersionedStorageFormat().getFormat().name(), + commonConfiguration.getDatabaseFormat()); + + throw new StorageException(error); + } + + private Optional handleVersionDowngrade( + final Path dataDir, + final DatabaseMetadata existingMetadata, + final BaseVersionedStorageFormat runtimeVersion) { + // here we put the code, or the messages, to perform an automated, or manual, downgrade of the + // database, if supported, otherwise we just prevent Besu from starting since it will not + // recognize the newer version. + // In case we do an automated downgrade, then we also need to update the metadata on disk to + // reflect the change to the runtime version, and return it. + + // for the moment there are supported automated downgrades, so we just fail. + String error = + String.format( + "Database unsafe downgrade detect: DB at %s is %s with version %s but version %s is expected. " + + "Please check your config and review release notes for supported downgrade procedures.", + dataDir, + existingMetadata.getVersionedStorageFormat().getFormat().name(), + existingMetadata.getVersionedStorageFormat().getVersion(), + runtimeVersion.getVersion()); + + throw new StorageException(error); + } + + private Optional handleVersionUpgrade( + final Path dataDir, + final DatabaseMetadata existingMetadata, + final BaseVersionedStorageFormat runtimeVersion) { + // here we put the code, or the messages, to perform an automated, or manual, upgrade of the + // database. + // In case we do an automated upgrade, then we also need to update the metadata on disk to + // reflect the change to the runtime version, and return it. + + // for the moment there are no planned automated upgrades, so we just fail. + String error = + String.format( + "Database unsafe downgrade detect: DB at %s is %s with version %s but version %s is expected. " + + "Please check your config and review release notes for supported downgrade procedures.", + dataDir, + existingMetadata.getVersionedStorageFormat().getFormat().name(), + existingMetadata.getVersionedStorageFormat().getVersion(), + runtimeVersion.getVersion()); + + throw new StorageException(error); + } + + private boolean isSupportedVersionedFormat(final VersionedStorageFormat versionedStorageFormat) { + return SUPPORTED_VERSIONED_FORMATS.stream() + .anyMatch( + vsf -> + vsf.getFormat().equals(versionedStorageFormat.getFormat()) + && vsf.getVersion() == versionedStorageFormat.getVersion()); } @Override diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/BaseVersionedStorageFormat.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/BaseVersionedStorageFormat.java new file mode 100644 index 00000000000..a18b907a6e7 --- /dev/null +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/BaseVersionedStorageFormat.java @@ -0,0 +1,78 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.plugin.services.storage.rocksdb.configuration; + +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; + +import java.util.OptionalInt; + +/** Base versioned data storage format */ +public enum BaseVersionedStorageFormat implements VersionedStorageFormat { + /** Original Forest version, not used since replace by FOREST_WITH_VARIABLES */ + FOREST_ORIGINAL(DataStorageFormat.FOREST, 1), + /** + * Current Forest version, with blockchain variables in a dedicated column family, in order to + * make BlobDB more effective + */ + FOREST_WITH_VARIABLES(DataStorageFormat.FOREST, 2), + /** Original Bonsai version, not used since replace by BONSAI_WITH_VARIABLES */ + BONSAI_ORIGINAL(DataStorageFormat.BONSAI, 1), + /** + * Current Bonsai version, with blockchain variables in a dedicated column family, in order to + * make BlobDB more effective + */ + BONSAI_WITH_VARIABLES(DataStorageFormat.BONSAI, 2); + + private final DataStorageFormat format; + private final int version; + + BaseVersionedStorageFormat(final DataStorageFormat format, final int version) { + this.format = format; + this.version = version; + } + + /** + * Return the default version for new db for a specific format + * + * @param format data storage format + * @return the version to use for new db + */ + public static BaseVersionedStorageFormat defaultForNewDB(final DataStorageFormat format) { + return switch (format) { + case FOREST -> FOREST_WITH_VARIABLES; + case BONSAI -> BONSAI_WITH_VARIABLES; + }; + } + + @Override + public DataStorageFormat getFormat() { + return format; + } + + @Override + public int getVersion() { + return version; + } + + @Override + public OptionalInt getPrivacyVersion() { + return OptionalInt.empty(); + } + + @Override + public String toString() { + return "BaseVersionedStorageFormat{" + "format=" + format + ", version=" + version + '}'; + } +} diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadata.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadata.java index 1faf610cdc9..4e18e35029c 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadata.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadata.java @@ -14,19 +14,23 @@ */ package org.hyperledger.besu.plugin.services.storage.rocksdb.configuration; +import org.hyperledger.besu.plugin.services.exception.StorageException; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; + import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Path; -import java.util.Optional; +import java.util.Arrays; +import java.util.OptionalInt; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DatabindException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,79 +39,43 @@ public class DatabaseMetadata { private static final Logger LOG = LoggerFactory.getLogger(DatabaseMetadata.class); private static final String METADATA_FILENAME = "DATABASE_METADATA.json"; - private static final ObjectMapper MAPPER = new ObjectMapper(); - private final int version; - - private Optional privacyVersion; - - /** - * Instantiates a new Database metadata. - * - * @param version the version - */ - @JsonCreator - public DatabaseMetadata(@JsonProperty("version") final int version) { - this(version, Optional.empty()); - } - - /** - * Instantiates a new Database metadata. - * - * @param version the version - * @param privacyVersion the privacy version - */ - public DatabaseMetadata(final int version, final Optional privacyVersion) { - this.version = version; - this.privacyVersion = privacyVersion; - } - - /** - * Instantiates a new Database metadata. - * - * @param version the version - * @param privacyVersion the privacy version - */ - public DatabaseMetadata(final int version, final int privacyVersion) { - this(version, Optional.of(privacyVersion)); + private static final ObjectMapper MAPPER = + new ObjectMapper() + .registerModule(new Jdk8Module()) + .setSerializationInclusion(JsonInclude.Include.NON_ABSENT) + .enable(SerializationFeature.INDENT_OUTPUT); + private final VersionedStorageFormat versionedStorageFormat; + + private DatabaseMetadata(final VersionedStorageFormat versionedStorageFormat) { + this.versionedStorageFormat = versionedStorageFormat; } /** - * Gets version. + * Return the default metadata for new db for a specific format * - * @return the version + * @param dataStorageFormat data storage format + * @return the metadata to use for new db */ - public int getVersion() { - return version; + public static DatabaseMetadata defaultForNewDb(final DataStorageFormat dataStorageFormat) { + return new DatabaseMetadata(BaseVersionedStorageFormat.defaultForNewDB(dataStorageFormat)); } /** - * Sets privacy version. + * Return the default metadata for new db when privacy feature is enabled * - * @param privacyVersion the privacy version + * @return the metadata to use for new db */ - @JsonSetter("privacyVersion") - public void setPrivacyVersion(final int privacyVersion) { - this.privacyVersion = Optional.of(privacyVersion); + public static DatabaseMetadata defaultForNewPrivateDb() { + return new DatabaseMetadata(PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES); } /** - * Gets privacy version. + * Return the version storage format contained in this metadata * - * @return the privacy version + * @return version storage format */ - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonGetter("privacyVersion") - public Integer getPrivacyVersion() { - return privacyVersion.orElse(null); - } - - /** - * Maybe privacy version. - * - * @return the optional - */ - public Optional maybePrivacyVersion() { - return privacyVersion; + public VersionedStorageFormat getVersionedStorageFormat() { + return versionedStorageFormat; } /** @@ -122,6 +90,17 @@ public static DatabaseMetadata lookUpFrom(final Path dataDir) throws IOException return resolveDatabaseMetadata(getDefaultMetadataFile(dataDir)); } + /** + * Is the metadata file present in the specified data dir? + * + * @param dataDir the dir to search for the metadata file + * @return true is the metadata file exists, false otherwise + * @throws IOException if there is an error trying to access the metadata file + */ + public static boolean isPresent(final Path dataDir) throws IOException { + return getDefaultMetadataFile(dataDir).exists(); + } + /** * Write to directory. * @@ -129,16 +108,17 @@ public static DatabaseMetadata lookUpFrom(final Path dataDir) throws IOException * @throws IOException the io exception */ public void writeToDirectory(final Path dataDir) throws IOException { - try { - final DatabaseMetadata currentMetadata = - MAPPER.readValue(getDefaultMetadataFile(dataDir), DatabaseMetadata.class); - if (currentMetadata.maybePrivacyVersion().isPresent()) { - setPrivacyVersion(currentMetadata.getPrivacyVersion()); - } - MAPPER.writeValue(getDefaultMetadataFile(dataDir), this); - } catch (FileNotFoundException fnfe) { - MAPPER.writeValue(getDefaultMetadataFile(dataDir), this); - } + writeToFile(getDefaultMetadataFile(dataDir)); + } + + private void writeToFile(final File file) throws IOException { + MAPPER.writeValue( + file, + new V2( + new MetadataV2( + versionedStorageFormat.getFormat(), + versionedStorageFormat.getVersion(), + versionedStorageFormat.getPrivacyVersion()))); } private static File getDefaultMetadataFile(final Path dataDir) { @@ -147,15 +127,129 @@ private static File getDefaultMetadataFile(final Path dataDir) { private static DatabaseMetadata resolveDatabaseMetadata(final File metadataFile) throws IOException { - DatabaseMetadata databaseMetadata; try { - databaseMetadata = MAPPER.readValue(metadataFile, DatabaseMetadata.class); + try { + return tryReadAndMigrateV1(metadataFile); + } catch (DatabindException dbe) { + return tryReadV2(metadataFile); + } } catch (FileNotFoundException fnfe) { - databaseMetadata = new DatabaseMetadata(1, 1); + throw new StorageException( + "Database exists but metadata file " + + metadataFile.toString() + + " not found, without it there is no safe way to open the database", + fnfe); } catch (JsonProcessingException jpe) { throw new IllegalStateException( String.format("Invalid metadata file %s", metadataFile.getAbsolutePath()), jpe); } - return databaseMetadata; } + + private static DatabaseMetadata tryReadAndMigrateV1(final File metadataFile) throws IOException { + final V1 v1 = MAPPER.readValue(metadataFile, V1.class); + // when migrating from v1, this version will automatically migrate the db to the variables + // storage, so we use the `_WITH_VARIABLES` variants + final VersionedStorageFormat versionedStorageFormat; + if (v1.privacyVersion().isEmpty()) { + versionedStorageFormat = + switch (v1.version()) { + case 1 -> BaseVersionedStorageFormat.FOREST_WITH_VARIABLES; + case 2 -> BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES; + default -> throw new StorageException("Unsupported db version: " + v1.version()); + }; + } else { + versionedStorageFormat = + switch (v1.privacyVersion().getAsInt()) { + case 1 -> switch (v1.version()) { + case 1 -> PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES; + case 2 -> PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES; + default -> throw new StorageException("Unsupported db version: " + v1.version()); + }; + default -> throw new StorageException( + "Unsupported db privacy version: " + v1.privacyVersion().getAsInt()); + }; + } + + final DatabaseMetadata metadataV2 = new DatabaseMetadata(versionedStorageFormat); + // writing the metadata will migrate to v2 + metadataV2.writeToFile(metadataFile); + return metadataV2; + } + + private static DatabaseMetadata tryReadV2(final File metadataFile) throws IOException { + final V2 v2 = MAPPER.readValue(metadataFile, V2.class); + return new DatabaseMetadata(fromV2(v2.v2)); + } + + private static VersionedStorageFormat fromV2(final MetadataV2 metadataV2) { + if (metadataV2.privacyVersion().isEmpty()) { + return Arrays.stream(BaseVersionedStorageFormat.values()) + .filter( + vsf -> + vsf.getFormat().equals(metadataV2.format()) + && vsf.getVersion() == metadataV2.version()) + .findFirst() + .orElseThrow( + () -> { + final String message = "Unsupported RocksDB metadata: " + metadataV2; + LOG.error(message); + throw new StorageException(message); + }); + } + return Arrays.stream(PrivacyVersionedStorageFormat.values()) + .filter( + vsf -> + vsf.getFormat().equals(metadataV2.format()) + && vsf.getVersion() == metadataV2.version() + && vsf.getPrivacyVersion().equals(metadataV2.privacyVersion())) + .findFirst() + .orElseThrow( + () -> { + final String message = "Unsupported RocksDB metadata: " + metadataV2; + LOG.error(message); + throw new StorageException(message); + }); + } + + /** + * Update an existing base storage to support privacy feature + * + * @return the update metadata with the privacy support + */ + public DatabaseMetadata upgradeToPrivacy() { + return new DatabaseMetadata( + switch (versionedStorageFormat.getFormat()) { + case FOREST -> switch (versionedStorageFormat.getVersion()) { + case 1 -> PrivacyVersionedStorageFormat.FOREST_ORIGINAL; + case 2 -> PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES; + default -> throw new StorageException( + "Unsupported database with format FOREST and version " + + versionedStorageFormat.getVersion()); + }; + case BONSAI -> switch (versionedStorageFormat.getVersion()) { + case 1 -> PrivacyVersionedStorageFormat.BONSAI_ORIGINAL; + case 2 -> PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES; + default -> throw new StorageException( + "Unsupported database with format BONSAI and version " + + versionedStorageFormat.getVersion()); + }; + }); + } + + @Override + public String toString() { + return "versionedStorageFormat=" + versionedStorageFormat; + } + + @JsonSerialize + @SuppressWarnings("unused") + private record V1(int version, OptionalInt privacyVersion) {} + + @JsonSerialize + @SuppressWarnings("unused") + private record V2(MetadataV2 v2) {} + + @JsonSerialize + @SuppressWarnings("unused") + private record MetadataV2(DataStorageFormat format, int version, OptionalInt privacyVersion) {} } diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/PrivacyVersionedStorageFormat.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/PrivacyVersionedStorageFormat.java new file mode 100644 index 00000000000..9a86d6ec15d --- /dev/null +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/PrivacyVersionedStorageFormat.java @@ -0,0 +1,84 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.plugin.services.storage.rocksdb.configuration; + +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; + +import java.util.OptionalInt; + +/** Privacy enabled versioned data storage format */ +public enum PrivacyVersionedStorageFormat implements VersionedStorageFormat { + /** Original Forest version, not used since replace by FOREST_WITH_VARIABLES */ + FOREST_ORIGINAL(BaseVersionedStorageFormat.FOREST_ORIGINAL, 1), + /** + * Current Forest version, with blockchain variables in a dedicated column family, in order to + * make BlobDB more effective + */ + FOREST_WITH_VARIABLES(BaseVersionedStorageFormat.FOREST_WITH_VARIABLES, 1), + /** Original Bonsai version, not used since replace by BONSAI_WITH_VARIABLES */ + BONSAI_ORIGINAL(BaseVersionedStorageFormat.BONSAI_ORIGINAL, 1), + /** + * Current Bonsai version, with blockchain variables in a dedicated column family, in order to + * make BlobDB more effective + */ + BONSAI_WITH_VARIABLES(BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES, 1); + + private final VersionedStorageFormat baseVersionedStorageFormat; + private final OptionalInt privacyVersion; + + PrivacyVersionedStorageFormat( + final VersionedStorageFormat baseVersionedStorageFormat, final int privacyVersion) { + this.baseVersionedStorageFormat = baseVersionedStorageFormat; + this.privacyVersion = OptionalInt.of(privacyVersion); + } + + /** + * Return the default version for new db for a specific format + * + * @param format data storage format + * @return the version to use for new db + */ + public static VersionedStorageFormat defaultForNewDB(final DataStorageFormat format) { + return switch (format) { + case FOREST -> FOREST_WITH_VARIABLES; + case BONSAI -> BONSAI_WITH_VARIABLES; + }; + } + + @Override + public DataStorageFormat getFormat() { + return baseVersionedStorageFormat.getFormat(); + } + + @Override + public int getVersion() { + return baseVersionedStorageFormat.getVersion(); + } + + @Override + public OptionalInt getPrivacyVersion() { + return privacyVersion; + } + + @Override + public String toString() { + return "PrivateVersionedStorageFormat{" + + "versionedStorageFormat=" + + baseVersionedStorageFormat + + ", privacyVersion=" + + privacyVersion + + '}'; + } +} diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/VersionedStorageFormat.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/VersionedStorageFormat.java new file mode 100644 index 00000000000..896438cc5ca --- /dev/null +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/VersionedStorageFormat.java @@ -0,0 +1,43 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.plugin.services.storage.rocksdb.configuration; + +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; + +import java.util.OptionalInt; + +/** Represent a specific version of a data storage format */ +public interface VersionedStorageFormat { + /** + * Get the data storage format + * + * @return the data storage format + */ + DataStorageFormat getFormat(); + + /** + * Get the version of the data storage format + * + * @return the version of the data storage format + */ + int getVersion(); + + /** + * Get the version of the privacy db, in case the privacy feature is enabled, or empty otherwise + * + * @return the optional privacy version + */ + OptionalInt getPrivacyVersion(); +} diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java index 26707073bb4..003d640be8d 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.plugin.services.storage.rocksdb; import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.FOREST; import static org.hyperledger.besu.plugin.services.storage.rocksdb.segmented.RocksDBColumnarKeyValueStorageTest.TestSegment; import static org.mockito.Mockito.when; @@ -22,7 +23,9 @@ import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.BesuConfiguration; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; +import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.DatabaseMetadata; +import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.PrivacyVersionedStorageFormat; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBFactoryConfiguration; import java.nio.file.Files; @@ -37,9 +40,6 @@ @ExtendWith(MockitoExtension.class) public class RocksDBKeyValuePrivacyStorageFactoryTest { - private static final int DEFAULT_VERSION = 1; - private static final int DEFAULT_PRIVACY_VERSION = 1; - @Mock private RocksDBFactoryConfiguration rocksDbConfiguration; @Mock private BesuConfiguration commonConfiguration; @TempDir private Path temporaryFolder; @@ -48,15 +48,16 @@ public class RocksDBKeyValuePrivacyStorageFactoryTest { private final List segments = List.of(TestSegment.DEFAULT, segment); @Test - public void shouldDetectVersion1DatabaseIfNoMetadataFileFound() throws Exception { + public void shouldDetectVersion1MetadataIfPresent() throws Exception { final Path tempDataDir = temporaryFolder.resolve("data"); final Path tempDatabaseDir = temporaryFolder.resolve("db"); final Path tempPrivateDatabaseDir = tempDatabaseDir.resolve("private"); Files.createDirectories(tempPrivateDatabaseDir); Files.createDirectories(tempDataDir); - when(commonConfiguration.getStoragePath()).thenReturn(tempDatabaseDir); - when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); - when(commonConfiguration.getDatabaseVersion()).thenReturn(DEFAULT_VERSION); + mockCommonConfiguration(tempDataDir, tempDatabaseDir); + + Utils.createDatabaseMetadataV1Privacy( + tempDataDir, PrivacyVersionedStorageFormat.FOREST_ORIGINAL); final RocksDBKeyValuePrivacyStorageFactory storageFactory = new RocksDBKeyValuePrivacyStorageFactory( @@ -66,23 +67,18 @@ public void shouldDetectVersion1DatabaseIfNoMetadataFileFound() throws Exception RocksDBMetricsFactory.PRIVATE_ROCKS_DB_METRICS)); // Side effect is creation of the Metadata version file - storageFactory.create(segment, commonConfiguration, metricsSystem); - - assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).maybePrivacyVersion()).isNotEmpty(); + try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) { - assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersion()).isEqualTo(DEFAULT_VERSION); - - assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).maybePrivacyVersion().get()) - .isEqualTo(DEFAULT_VERSION); + assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) + .isEqualTo(PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES); + } } @Test public void shouldCreateCorrectMetadataFileForLatestVersion() throws Exception { final Path tempDataDir = temporaryFolder.resolve("data"); final Path tempDatabaseDir = temporaryFolder.resolve("db"); - when(commonConfiguration.getStoragePath()).thenReturn(tempDatabaseDir); - when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); - when(commonConfiguration.getDatabaseVersion()).thenReturn(DEFAULT_VERSION); + mockCommonConfiguration(tempDataDir, tempDatabaseDir); final RocksDBKeyValuePrivacyStorageFactory storageFactory = new RocksDBKeyValuePrivacyStorageFactory( @@ -92,42 +88,44 @@ public void shouldCreateCorrectMetadataFileForLatestVersion() throws Exception { RocksDBMetricsFactory.PRIVATE_ROCKS_DB_METRICS)); // Side effect is creation of the Metadata version file - storageFactory.create(segment, commonConfiguration, metricsSystem); - - assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).maybePrivacyVersion()).isNotEmpty(); - - assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersion()).isEqualTo(DEFAULT_VERSION); - - assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).maybePrivacyVersion().get()) - .isEqualTo(DEFAULT_PRIVACY_VERSION); + try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) { + assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) + .isEqualTo(PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES); + } } @Test public void shouldUpdateCorrectMetadataFileForLatestVersion() throws Exception { final Path tempDataDir = temporaryFolder.resolve("data"); final Path tempDatabaseDir = temporaryFolder.resolve("db"); - when(commonConfiguration.getStoragePath()).thenReturn(tempDatabaseDir); - when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); - when(commonConfiguration.getDatabaseVersion()).thenReturn(DEFAULT_VERSION); + mockCommonConfiguration(tempDataDir, tempDatabaseDir); final RocksDBKeyValueStorageFactory storageFactory = new RocksDBKeyValueStorageFactory( () -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PRIVATE_ROCKS_DB_METRICS); - storageFactory.create(segment, commonConfiguration, metricsSystem); + try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) { - assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).maybePrivacyVersion()).isEmpty(); - - assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersion()).isEqualTo(DEFAULT_VERSION); + assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) + .isEqualTo(BaseVersionedStorageFormat.FOREST_WITH_VARIABLES); + } + storageFactory.close(); final RocksDBKeyValuePrivacyStorageFactory privacyStorageFactory = new RocksDBKeyValuePrivacyStorageFactory(storageFactory); - privacyStorageFactory.create(segment, commonConfiguration, metricsSystem); + try (final var storage = + privacyStorageFactory.create(segment, commonConfiguration, metricsSystem)) { - assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).maybePrivacyVersion()).isNotEmpty(); + assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) + .isEqualTo(PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES); + } + privacyStorageFactory.close(); + } - assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).maybePrivacyVersion().get()) - .isEqualTo(DEFAULT_PRIVACY_VERSION); + private void mockCommonConfiguration(final Path tempDataDir, final Path tempDatabaseDir) { + when(commonConfiguration.getStoragePath()).thenReturn(tempDatabaseDir); + when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); + when(commonConfiguration.getDatabaseFormat()).thenReturn(FOREST); } } diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java index 469c105459f..e2c56f0ac50 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java @@ -17,24 +17,30 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.fail; +import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.BONSAI; +import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.FOREST; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.BesuConfiguration; import org.hyperledger.besu.plugin.services.exception.StorageException; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; +import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.DatabaseMetadata; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBFactoryConfiguration; import org.hyperledger.besu.plugin.services.storage.rocksdb.segmented.RocksDBColumnarKeyValueStorageTest.TestSegment; -import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; import org.mockito.Mock; @@ -43,9 +49,6 @@ @ExtendWith(MockitoExtension.class) public class RocksDBKeyValueStorageFactoryTest { - private static final String METADATA_FILENAME = "DATABASE_METADATA.json"; - private static final int DEFAULT_VERSION = 1; - @Mock private RocksDBFactoryConfiguration rocksDbConfiguration; @Mock private BesuConfiguration commonConfiguration; @TempDir public Path temporaryFolder; @@ -54,80 +57,89 @@ public class RocksDBKeyValueStorageFactoryTest { private final List segments = List.of(TestSegment.DEFAULT, segment); @Test - public void shouldCreateCorrectMetadataFileForLatestVersion() throws Exception { + public void shouldCreateCorrectMetadataFileForLatestVersionForNewDb() throws Exception { final Path tempDataDir = temporaryFolder.resolve("data"); final Path tempDatabaseDir = temporaryFolder.resolve("db"); - mockCommonConfiguration(tempDataDir, tempDatabaseDir); + mockCommonConfiguration(tempDataDir, tempDatabaseDir, FOREST); final RocksDBKeyValueStorageFactory storageFactory = new RocksDBKeyValueStorageFactory( () -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); - // Side effect is creation of the Metadata version file - storageFactory.create(segment, commonConfiguration, metricsSystem); - - assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersion()).isEqualTo(DEFAULT_VERSION); - } - - private void mockCommonConfiguration(final Path tempDataDir, final Path tempDatabaseDir) { - when(commonConfiguration.getStoragePath()).thenReturn(tempDatabaseDir); - when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); - when(commonConfiguration.getDatabaseVersion()).thenReturn(DEFAULT_VERSION); + try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) { + // Side effect is creation of the Metadata version file + assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) + .isEqualTo(BaseVersionedStorageFormat.FOREST_WITH_VARIABLES); + } } @Test - public void shouldDetectVersion1DatabaseIfNoMetadataFileFound() throws Exception { + public void shouldFailIfDbExistsAndNoMetadataFileFound() throws Exception { final Path tempDataDir = temporaryFolder.resolve("data"); final Path tempDatabaseDir = temporaryFolder.resolve("db"); Files.createDirectories(tempDatabaseDir); Files.createDirectories(tempDataDir); - mockCommonConfiguration(tempDataDir, tempDatabaseDir); + mockCommonConfiguration(tempDataDir, tempDatabaseDir, FOREST); final RocksDBKeyValueStorageFactory storageFactory = new RocksDBKeyValueStorageFactory( () -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); - storageFactory.create(segment, commonConfiguration, metricsSystem); - - assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersion()).isEqualTo(DEFAULT_VERSION); + try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) { + fail("Must fail if db is present but metadata is not"); + } catch (StorageException se) { + assertThat(se) + .hasMessage( + "Database exists but metadata file not found, without it there is no safe way to open the database"); + } } @Test - public void shouldDetectCorrectVersionIfMetadataFileExists() throws Exception { + public void shouldDetectCorrectMetadataV1() throws Exception { final Path tempDataDir = temporaryFolder.resolve("data"); final Path tempDatabaseDir = temporaryFolder.resolve("db"); Files.createDirectories(tempDataDir); - mockCommonConfiguration(tempDataDir, tempDatabaseDir); + mockCommonConfiguration(tempDataDir, tempDatabaseDir, BONSAI); + + Utils.createDatabaseMetadataV1(tempDataDir, BONSAI); final RocksDBKeyValueStorageFactory storageFactory = new RocksDBKeyValueStorageFactory( () -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); - storageFactory.create(segment, commonConfiguration, metricsSystem); - - assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersion()).isEqualTo(DEFAULT_VERSION); - assertThat(storageFactory.isSegmentIsolationSupported()).isTrue(); + try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) { + assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) + .isEqualTo(BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES); + assertThat(storageFactory.isSegmentIsolationSupported()).isTrue(); + } } @Test - public void shouldDetectCorrectVersionInCaseOfRollback() throws Exception { + public void shouldFailInCaseOfUnmanagedRollback() throws Exception { final Path tempDataDir = temporaryFolder.resolve("data"); final Path tempDatabaseDir = temporaryFolder.resolve("db"); Files.createDirectories(tempDatabaseDir); Files.createDirectories(tempDataDir); - mockCommonConfiguration(tempDataDir, tempDatabaseDir); + mockCommonConfiguration(tempDataDir, tempDatabaseDir, BONSAI); + + Utils.createDatabaseMetadataV1(tempDataDir, BONSAI); final RocksDBKeyValueStorageFactory storageFactory = new RocksDBKeyValueStorageFactory( - () -> rocksDbConfiguration, segments, 2, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); + () -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); storageFactory.create(segment, commonConfiguration, metricsSystem); storageFactory.close(); + Utils.createDatabaseMetadataV2(tempDataDir, BONSAI, 1); + final RocksDBKeyValueStorageFactory rolledbackStorageFactory = new RocksDBKeyValueStorageFactory( - () -> rocksDbConfiguration, segments, 1, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); - rolledbackStorageFactory.create(segment, commonConfiguration, metricsSystem); + () -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); + assertThatThrownBy( + () -> rolledbackStorageFactory.create(segment, commonConfiguration, metricsSystem)) + .isInstanceOf(StorageException.class) + .hasMessageStartingWith("Database unsafe downgrade detect"); } @Test @@ -136,9 +148,9 @@ public void shouldThrowExceptionWhenVersionNumberIsInvalid() throws Exception { final Path tempDatabaseDir = temporaryFolder.resolve("db"); Files.createDirectories(tempDatabaseDir); Files.createDirectories(tempDataDir); - mockCommonConfiguration(tempDataDir, tempDatabaseDir); + mockCommonConfiguration(tempDataDir, tempDatabaseDir, FOREST); - new DatabaseMetadata(-1).writeToDirectory(tempDataDir); + Utils.createDatabaseMetadataV1(tempDataDir, 99); assertThatThrownBy( () -> new RocksDBKeyValueStorageFactory( @@ -146,33 +158,23 @@ public void shouldThrowExceptionWhenVersionNumberIsInvalid() throws Exception { segments, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS) .create(segment, commonConfiguration, metricsSystem)) - .isInstanceOf(StorageException.class); + .isInstanceOf(StorageException.class) + .hasMessageStartingWith("Unsupported db version"); } @Test - public void shouldThrowExceptionWhenExistingDatabaseVersionDifferentFromConfig() - throws Exception { + public void shouldThrowExceptionWhenExistingDatabaseFormatDiffersFromConfig() throws Exception { - final int actualDatabaseVersion = DataStorageFormat.FOREST.getDatabaseVersion(); - final int expectedDatabaseVersion = DataStorageFormat.BONSAI.getDatabaseVersion(); + final DataStorageFormat actualDatabaseFormat = FOREST; + final DataStorageFormat expectedDatabaseFormat = BONSAI; final Path tempDataDir = temporaryFolder.resolve("data"); final Path tempDatabaseDir = temporaryFolder.resolve("db"); Files.createDirectories(tempDatabaseDir); Files.createDirectories(tempDataDir); - mockCommonConfiguration(tempDataDir, tempDatabaseDir); - when(commonConfiguration.getDatabaseVersion()).thenReturn(expectedDatabaseVersion); - - new DatabaseMetadata(actualDatabaseVersion).writeToDirectory(tempDataDir); + mockCommonConfiguration(tempDataDir, tempDatabaseDir, BONSAI); - String exceptionMessage = - String.format( - "Mismatch: DB at %s is %s (Version %s) but config expects %s (Version %s). Please check your config.", - tempDataDir.toAbsolutePath(), - DataStorageFormat.getName(actualDatabaseVersion), - actualDatabaseVersion, - DataStorageFormat.getName(expectedDatabaseVersion), - expectedDatabaseVersion); + Utils.createDatabaseMetadataV2(tempDataDir, FOREST, 2); assertThatThrownBy( () -> @@ -182,22 +184,31 @@ public void shouldThrowExceptionWhenExistingDatabaseVersionDifferentFromConfig() RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS) .create(segment, commonConfiguration, metricsSystem)) .isInstanceOf(StorageException.class) - .hasMessage(exceptionMessage); + .hasMessage( + "Database format mismatch: DB at %s is %s but config expects %s. " + + "Please check your config.", + tempDataDir.toAbsolutePath(), actualDatabaseFormat, expectedDatabaseFormat); } @Test - public void shouldSetSegmentationFieldDuringCreation() throws Exception { + public void shouldDetectCorrectMetadataV2AndSetSegmentationFieldDuringCreation() + throws Exception { final Path tempDataDir = temporaryFolder.resolve("data"); final Path tempDatabaseDir = temporaryFolder.resolve("db"); Files.createDirectories(tempDatabaseDir); Files.createDirectories(tempDataDir); - mockCommonConfiguration(tempDataDir, tempDatabaseDir); + mockCommonConfiguration(tempDataDir, tempDatabaseDir, FOREST); + + Utils.createDatabaseMetadataV2(tempDataDir, FOREST, 2); final RocksDBKeyValueStorageFactory storageFactory = new RocksDBKeyValueStorageFactory( () -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); - storageFactory.create(segment, commonConfiguration, metricsSystem); - assertThatCode(storageFactory::isSegmentIsolationSupported).doesNotThrowAnyException(); + try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) { + assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) + .isEqualTo(BaseVersionedStorageFormat.FOREST_WITH_VARIABLES); + assertThatCode(storageFactory::isSegmentIsolationSupported).doesNotThrowAnyException(); + } } @Test @@ -206,11 +217,9 @@ public void shouldThrowExceptionWhenMetaDataFileIsCorrupted() throws Exception { final Path tempDatabaseDir = temporaryFolder.resolve("db"); Files.createDirectories(tempDatabaseDir); Files.createDirectories(tempDataDir); - mockCommonConfiguration(tempDataDir, tempDatabaseDir); + mockCommonConfiguration(tempDataDir, tempDatabaseDir, FOREST); - final String badVersion = "{\"🦄\":1}"; - Files.write( - tempDataDir.resolve(METADATA_FILENAME), badVersion.getBytes(Charset.defaultCharset())); + Utils.createDatabaseMetadataRaw(tempDataDir, "{\"🦄\":1}"); assertThatThrownBy( () -> @@ -219,11 +228,11 @@ public void shouldThrowExceptionWhenMetaDataFileIsCorrupted() throws Exception { segments, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS) .create(segment, commonConfiguration, metricsSystem)) - .isInstanceOf(IllegalStateException.class); + .isInstanceOf(IllegalStateException.class) + .hasMessageStartingWith("Invalid metadata file"); + ; - final String badValue = "{\"version\":\"iomedae\"}"; - Files.write( - tempDatabaseDir.resolve(METADATA_FILENAME), badValue.getBytes(Charset.defaultCharset())); + Utils.createDatabaseMetadataRaw(tempDataDir, "{\"version\"=1}"); assertThatThrownBy( () -> @@ -232,16 +241,21 @@ public void shouldThrowExceptionWhenMetaDataFileIsCorrupted() throws Exception { segments, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS) .create(segment, commonConfiguration, metricsSystem)) - .isInstanceOf(IllegalStateException.class); + .isInstanceOf(IllegalStateException.class) + .hasMessageStartingWith("Invalid metadata file"); + ; } @Test + @DisabledOnOs(OS.WINDOWS) public void shouldCreateDBCorrectlyIfSymlink() throws Exception { final Path tempRealDataDir = Files.createDirectories(temporaryFolder.resolve("real-data-dir")); final Path tempSymLinkDataDir = Files.createSymbolicLink(temporaryFolder.resolve("symlink-data-dir"), tempRealDataDir); final Path tempDatabaseDir = temporaryFolder.resolve("db"); - mockCommonConfiguration(tempSymLinkDataDir, tempDatabaseDir); + mockCommonConfiguration(tempSymLinkDataDir, tempDatabaseDir, FOREST); + + Utils.createDatabaseMetadataV2(tempSymLinkDataDir, FOREST, 2); final RocksDBKeyValueStorageFactory storageFactory = new RocksDBKeyValueStorageFactory( @@ -249,8 +263,16 @@ public void shouldCreateDBCorrectlyIfSymlink() throws Exception { // Ensure that having created everything via a symlink data dir the DB meta-data has been // created correctly - storageFactory.create(segment, commonConfiguration, metricsSystem); - assertThat(DatabaseMetadata.lookUpFrom(tempRealDataDir).getVersion()) - .isEqualTo(DEFAULT_VERSION); + try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) { + assertThat(DatabaseMetadata.lookUpFrom(tempRealDataDir).getVersionedStorageFormat()) + .isEqualTo(BaseVersionedStorageFormat.FOREST_WITH_VARIABLES); + } + } + + private void mockCommonConfiguration( + final Path tempDataDir, final Path tempDatabaseDir, final DataStorageFormat format) { + when(commonConfiguration.getStoragePath()).thenReturn(tempDatabaseDir); + when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); + lenient().when(commonConfiguration.getDatabaseFormat()).thenReturn(format); } } diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/Utils.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/Utils.java new file mode 100644 index 00000000000..4b18047413e --- /dev/null +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/Utils.java @@ -0,0 +1,91 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.plugin.services.storage.rocksdb; + +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; +import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.PrivacyVersionedStorageFormat; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; + +public class Utils { + public static final String METADATA_FILENAME = "DATABASE_METADATA.json"; + + public static void createDatabaseMetadataV1( + final Path tempDataDir, final DataStorageFormat dataStorageFormat) throws IOException { + createDatabaseMetadataV1(tempDataDir, dataStorageFormatToV1(dataStorageFormat)); + } + + public static void createDatabaseMetadataV1(final Path tempDataDir, final int version) + throws IOException { + final String content = "{\"version\":" + version + "}"; + Files.write(tempDataDir.resolve(METADATA_FILENAME), content.getBytes(Charset.defaultCharset())); + } + + public static void createDatabaseMetadataV1Privacy( + final Path tempDataDir, final PrivacyVersionedStorageFormat privacyVersionedStorageFormat) + throws IOException { + createDatabaseMetadataV1Privacy( + tempDataDir, + dataStorageFormatToV1(privacyVersionedStorageFormat.getFormat()), + privacyVersionedStorageFormat.getPrivacyVersion().getAsInt()); + } + + public static void createDatabaseMetadataV1Privacy( + final Path tempDataDir, final int version, final int privacyVersion) throws IOException { + final String content = + "{\"version\":" + version + ",\"privacyVersion\":" + privacyVersion + "}"; + Files.write(tempDataDir.resolve(METADATA_FILENAME), content.getBytes(Charset.defaultCharset())); + } + + public static void createDatabaseMetadataV2( + final Path tempDataDir, final DataStorageFormat dataStorageFormat, final int version) + throws IOException { + final String content = + "{\"v2\":{\"format\":\"" + dataStorageFormat + "\",\"version\":" + version + "}}"; + Files.write(tempDataDir.resolve(METADATA_FILENAME), content.getBytes(Charset.defaultCharset())); + } + + public static void createDatabaseMetadataV2Privacy( + final Path tempDataDir, + final DataStorageFormat dataStorageFormat, + final int version, + final int privacyVersion) + throws IOException { + final String content = + "{\"v2\":{\"format\":\"" + + dataStorageFormat + + "\",\"version\":" + + version + + ",\"privacyVersion\":" + + privacyVersion + + "}}"; + Files.write(tempDataDir.resolve(METADATA_FILENAME), content.getBytes(Charset.defaultCharset())); + } + + public static void createDatabaseMetadataRaw(final Path tempDataDir, final String content) + throws IOException { + Files.write(tempDataDir.resolve(METADATA_FILENAME), content.getBytes(Charset.defaultCharset())); + } + + private static int dataStorageFormatToV1(final DataStorageFormat dataStorageFormat) { + return switch (dataStorageFormat) { + case FOREST -> 1; + case BONSAI -> 2; + }; + } +} diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadataTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadataTest.java index 569819f8312..8ceea5b329f 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadataTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadataTest.java @@ -17,6 +17,8 @@ import static org.assertj.core.api.Assertions.assertThat; +import org.hyperledger.besu.plugin.services.exception.StorageException; + import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -28,31 +30,56 @@ class DatabaseMetadataTest { @TempDir public Path temporaryFolder; @Test - void getVersion() { - final DatabaseMetadata databaseMetadata = new DatabaseMetadata(42); - assertThat(databaseMetadata).isNotNull(); - assertThat(databaseMetadata.getVersion()).isEqualTo(42); + void readingMetadataV1() throws Exception { + final Path tempDataDir = createAndWrite("data", "DATABASE_METADATA.json", "{\"version\":2}"); + + final DatabaseMetadata databaseMetadata = DatabaseMetadata.lookUpFrom(tempDataDir); + assertThat(databaseMetadata.getVersionedStorageFormat()) + .isEqualTo(BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES); + } + + @Test + void readingMetadataV1Privacy() throws Exception { + final Path tempDataDir = + createAndWrite("data", "DATABASE_METADATA.json", "{\"version\":1,\"privacyVersion\":1}"); + + final DatabaseMetadata databaseMetadata = DatabaseMetadata.lookUpFrom(tempDataDir); + assertThat(databaseMetadata.getVersionedStorageFormat()) + .isEqualTo(PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES); } @Test - void metaFileShouldMayContain() throws Exception { + void readingMetadataV2() throws Exception { final Path tempDataDir = createAndWrite( - "data", "DATABASE_METADATA.json", "{\"version\":42 , \"privacyVersion\":55}"); + "data", "DATABASE_METADATA.json", "{\"v2\":{\"format\":\"FOREST\",\"version\":2}}"); final DatabaseMetadata databaseMetadata = DatabaseMetadata.lookUpFrom(tempDataDir); - assertThat(databaseMetadata).isNotNull(); - assertThat(databaseMetadata.getVersion()).isEqualTo(42); - assertThat(databaseMetadata.maybePrivacyVersion()).isNotEmpty(); - assertThat(databaseMetadata.maybePrivacyVersion().get()).isEqualTo(55); + assertThat(databaseMetadata.getVersionedStorageFormat()) + .isEqualTo(BaseVersionedStorageFormat.FOREST_WITH_VARIABLES); } @Test - void metaFileShouldBeSoughtIntoDataDirFirst() throws Exception { - final Path tempDataDir = createAndWrite("data", "DATABASE_METADATA.json", "{\"version\":42}"); + void readingMetadataV2Privacy() throws Exception { + final Path tempDataDir = + createAndWrite( + "data", + "DATABASE_METADATA.json", + "{\"v2\":{\"format\":\"FOREST\",\"version\":2,\"privacyVersion\":1}}"); + final DatabaseMetadata databaseMetadata = DatabaseMetadata.lookUpFrom(tempDataDir); - assertThat(databaseMetadata).isNotNull(); - assertThat(databaseMetadata.getVersion()).isEqualTo(42); + assertThat(databaseMetadata.getVersionedStorageFormat()) + .isEqualTo(PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES); + } + + @Test + void unsupportedMetadata() throws Exception { + final Path tempDataDir = createAndWrite("data", "DATABASE_METADATA.json", "{\"version\":42}"); + try { + DatabaseMetadata.lookUpFrom(tempDataDir); + } catch (final StorageException se) { + assertThat(se).hasMessage("Unsupported db version: 42"); + } } private Path createAndWrite(final String dir, final String file, final String content) diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java index 80b9685b792..6f01dbce63d 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java @@ -308,58 +308,59 @@ public void createStoreMustCreateMetrics() throws Exception { // Actual call - final SegmentedKeyValueStorage store = + try (final SegmentedKeyValueStorage store = createSegmentedStore( folder, metricsSystemMock, List.of(TestSegment.DEFAULT, TestSegment.FOO), - List.of(TestSegment.EXPERIMENTAL)); - - KeyValueStorage keyValueStorage = new SegmentedKeyValueStorageAdapter(TestSegment.FOO, store); - - // Assertions - assertThat(keyValueStorage).isNotNull(); - verify(metricsSystemMock, times(4)) - .createLabelledTimer( - eq(BesuMetricCategory.KVSTORE_ROCKSDB), - labelledTimersMetricsNameArgs.capture(), - labelledTimersHelpArgs.capture(), - any()); - assertThat(labelledTimersMetricsNameArgs.getAllValues()) - .containsExactly( - "read_latency_seconds", - "remove_latency_seconds", - "write_latency_seconds", - "commit_latency_seconds"); - assertThat(labelledTimersHelpArgs.getAllValues()) - .containsExactly( - "Latency for read from RocksDB.", - "Latency of remove requests from RocksDB.", - "Latency for write to RocksDB.", - "Latency for commits to RocksDB."); - - verify(metricsSystemMock, times(2)) - .createLongGauge( - eq(BesuMetricCategory.KVSTORE_ROCKSDB), - longGaugesMetricsNameArgs.capture(), - longGaugesHelpArgs.capture(), - any(LongSupplier.class)); - assertThat(longGaugesMetricsNameArgs.getAllValues()) - .containsExactly("rocks_db_table_readers_memory_bytes", "rocks_db_files_size_bytes"); - assertThat(longGaugesHelpArgs.getAllValues()) - .containsExactly( - "Estimated memory used for RocksDB index and filter blocks in bytes", - "Estimated database size in bytes"); - - verify(metricsSystemMock) - .createLabelledCounter( - eq(BesuMetricCategory.KVSTORE_ROCKSDB), - labelledCountersMetricsNameArgs.capture(), - labelledCountersHelpArgs.capture(), - any()); - assertThat(labelledCountersMetricsNameArgs.getValue()).isEqualTo("rollback_count"); - assertThat(labelledCountersHelpArgs.getValue()) - .isEqualTo("Number of RocksDB transactions rolled back."); + List.of(TestSegment.EXPERIMENTAL))) { + + KeyValueStorage keyValueStorage = new SegmentedKeyValueStorageAdapter(TestSegment.FOO, store); + + // Assertions + assertThat(keyValueStorage).isNotNull(); + verify(metricsSystemMock, times(4)) + .createLabelledTimer( + eq(BesuMetricCategory.KVSTORE_ROCKSDB), + labelledTimersMetricsNameArgs.capture(), + labelledTimersHelpArgs.capture(), + any()); + assertThat(labelledTimersMetricsNameArgs.getAllValues()) + .containsExactly( + "read_latency_seconds", + "remove_latency_seconds", + "write_latency_seconds", + "commit_latency_seconds"); + assertThat(labelledTimersHelpArgs.getAllValues()) + .containsExactly( + "Latency for read from RocksDB.", + "Latency of remove requests from RocksDB.", + "Latency for write to RocksDB.", + "Latency for commits to RocksDB."); + + verify(metricsSystemMock, times(2)) + .createLongGauge( + eq(BesuMetricCategory.KVSTORE_ROCKSDB), + longGaugesMetricsNameArgs.capture(), + longGaugesHelpArgs.capture(), + any(LongSupplier.class)); + assertThat(longGaugesMetricsNameArgs.getAllValues()) + .containsExactly("rocks_db_table_readers_memory_bytes", "rocks_db_files_size_bytes"); + assertThat(longGaugesHelpArgs.getAllValues()) + .containsExactly( + "Estimated memory used for RocksDB index and filter blocks in bytes", + "Estimated database size in bytes"); + + verify(metricsSystemMock) + .createLabelledCounter( + eq(BesuMetricCategory.KVSTORE_ROCKSDB), + labelledCountersMetricsNameArgs.capture(), + labelledCountersHelpArgs.capture(), + any()); + assertThat(labelledCountersMetricsNameArgs.getValue()).isEqualTo("rollback_count"); + assertThat(labelledCountersHelpArgs.getValue()) + .isEqualTo("Number of RocksDB transactions rolled back."); + } } public enum TestSegment implements SegmentIdentifier { diff --git a/testutil/src/main/java/org/hyperledger/besu/kvstore/AbstractKeyValueStorageTest.java b/testutil/src/main/java/org/hyperledger/besu/kvstore/AbstractKeyValueStorageTest.java index 60a66336765..2e0d9195e86 100644 --- a/testutil/src/main/java/org/hyperledger/besu/kvstore/AbstractKeyValueStorageTest.java +++ b/testutil/src/main/java/org/hyperledger/besu/kvstore/AbstractKeyValueStorageTest.java @@ -56,18 +56,20 @@ public abstract class AbstractKeyValueStorageTest { */ @Test public void twoStoresAreIndependent() throws Exception { - final KeyValueStorage store1 = createStore(); - final KeyValueStorage store2 = createStore(); + try (final KeyValueStorage store1 = createStore()) { + try (final KeyValueStorage store2 = createStore()) { - final KeyValueStorageTransaction tx = store1.startTransaction(); - final byte[] key = bytesFromHexString("0001"); - final byte[] value = bytesFromHexString("0FFF"); + final KeyValueStorageTransaction tx = store1.startTransaction(); + final byte[] key = bytesFromHexString("0001"); + final byte[] value = bytesFromHexString("0FFF"); - tx.put(key, value); - tx.commit(); + tx.put(key, value); + tx.commit(); - final Optional result = store2.get(key); - assertThat(result).isEmpty(); + final Optional result = store2.get(key); + assertThat(result).isEmpty(); + } + } } /** @@ -77,20 +79,21 @@ public void twoStoresAreIndependent() throws Exception { */ @Test public void put() throws Exception { - final KeyValueStorage store = createStore(); - final byte[] key = bytesFromHexString("0F"); - final byte[] firstValue = bytesFromHexString("0ABC"); - final byte[] secondValue = bytesFromHexString("0DEF"); - - KeyValueStorageTransaction tx = store.startTransaction(); - tx.put(key, firstValue); - tx.commit(); - assertThat(store.get(key)).contains(firstValue); - - tx = store.startTransaction(); - tx.put(key, secondValue); - tx.commit(); - assertThat(store.get(key)).contains(secondValue); + try (final KeyValueStorage store = createStore()) { + final byte[] key = bytesFromHexString("0F"); + final byte[] firstValue = bytesFromHexString("0ABC"); + final byte[] secondValue = bytesFromHexString("0DEF"); + + KeyValueStorageTransaction tx = store.startTransaction(); + tx.put(key, firstValue); + tx.commit(); + assertThat(store.get(key)).contains(firstValue); + + tx = store.startTransaction(); + tx.put(key, secondValue); + tx.commit(); + assertThat(store.get(key)).contains(secondValue); + } } /** @@ -100,16 +103,17 @@ public void put() throws Exception { */ @Test public void streamKeys() throws Exception { - final KeyValueStorage store = createStore(); - final KeyValueStorageTransaction tx = store.startTransaction(); - final List keys = - Stream.of("0F", "10", "11", "12") - .map(this::bytesFromHexString) - .collect(toUnmodifiableList()); - keys.forEach(key -> tx.put(key, bytesFromHexString("0ABC"))); - tx.commit(); - assertThat(store.stream().map(Pair::getKey).collect(toUnmodifiableSet())) - .containsExactlyInAnyOrder(keys.toArray(new byte[][] {})); + try (final KeyValueStorage store = createStore()) { + final KeyValueStorageTransaction tx = store.startTransaction(); + final List keys = + Stream.of("0F", "10", "11", "12") + .map(this::bytesFromHexString) + .collect(toUnmodifiableList()); + keys.forEach(key -> tx.put(key, bytesFromHexString("0ABC"))); + tx.commit(); + assertThat(store.stream().map(Pair::getKey).collect(toUnmodifiableSet())) + .containsExactlyInAnyOrder(keys.toArray(new byte[][] {})); + } } /** @@ -119,18 +123,19 @@ public void streamKeys() throws Exception { */ @Test public void getAllKeysThat() throws Exception { - final KeyValueStorage store = createStore(); - final KeyValueStorageTransaction tx = store.startTransaction(); - tx.put(bytesFromHexString("0F"), bytesFromHexString("0ABC")); - tx.put(bytesFromHexString("10"), bytesFromHexString("0ABC")); - tx.put(bytesFromHexString("11"), bytesFromHexString("0ABC")); - tx.put(bytesFromHexString("12"), bytesFromHexString("0ABC")); - tx.commit(); - Set keys = store.getAllKeysThat(bv -> Bytes.wrap(bv).toString().contains("1")); - assertThat(keys.size()).isEqualTo(3); - assertThat(keys) - .containsExactlyInAnyOrder( - bytesFromHexString("10"), bytesFromHexString("11"), bytesFromHexString("12")); + try (final KeyValueStorage store = createStore()) { + final KeyValueStorageTransaction tx = store.startTransaction(); + tx.put(bytesFromHexString("0F"), bytesFromHexString("0ABC")); + tx.put(bytesFromHexString("10"), bytesFromHexString("0ABC")); + tx.put(bytesFromHexString("11"), bytesFromHexString("0ABC")); + tx.put(bytesFromHexString("12"), bytesFromHexString("0ABC")); + tx.commit(); + Set keys = store.getAllKeysThat(bv -> Bytes.wrap(bv).toString().contains("1")); + assertThat(keys.size()).isEqualTo(3); + assertThat(keys) + .containsExactlyInAnyOrder( + bytesFromHexString("10"), bytesFromHexString("11"), bytesFromHexString("12")); + } } /** @@ -140,17 +145,18 @@ public void getAllKeysThat() throws Exception { */ @Test public void containsKey() throws Exception { - final KeyValueStorage store = createStore(); - final byte[] key = bytesFromHexString("ABCD"); - final byte[] value = bytesFromHexString("DEFF"); + try (final KeyValueStorage store = createStore()) { + final byte[] key = bytesFromHexString("ABCD"); + final byte[] value = bytesFromHexString("DEFF"); - assertThat(store.containsKey(key)).isFalse(); + assertThat(store.containsKey(key)).isFalse(); - final KeyValueStorageTransaction transaction = store.startTransaction(); - transaction.put(key, value); - transaction.commit(); + final KeyValueStorageTransaction transaction = store.startTransaction(); + transaction.put(key, value); + transaction.commit(); - assertThat(store.containsKey(key)).isTrue(); + assertThat(store.containsKey(key)).isTrue(); + } } /** @@ -160,18 +166,19 @@ public void containsKey() throws Exception { */ @Test public void removeExisting() throws Exception { - final KeyValueStorage store = createStore(); - final byte[] key = bytesFromHexString("0F"); - final byte[] value = bytesFromHexString("0ABC"); - - KeyValueStorageTransaction tx = store.startTransaction(); - tx.put(key, value); - tx.commit(); - - tx = store.startTransaction(); - tx.remove(key); - tx.commit(); - assertThat(store.get(key)).isEmpty(); + try (final KeyValueStorage store = createStore()) { + final byte[] key = bytesFromHexString("0F"); + final byte[] value = bytesFromHexString("0ABC"); + + KeyValueStorageTransaction tx = store.startTransaction(); + tx.put(key, value); + tx.commit(); + + tx = store.startTransaction(); + tx.remove(key); + tx.commit(); + assertThat(store.get(key)).isEmpty(); + } } /** @@ -181,15 +188,16 @@ public void removeExisting() throws Exception { */ @Test public void removeExistingSameTransaction() throws Exception { - final KeyValueStorage store = createStore(); - final byte[] key = bytesFromHexString("0F"); - final byte[] value = bytesFromHexString("0ABC"); - - KeyValueStorageTransaction tx = store.startTransaction(); - tx.put(key, value); - tx.remove(key); - tx.commit(); - assertThat(store.get(key)).isEmpty(); + try (final KeyValueStorage store = createStore()) { + final byte[] key = bytesFromHexString("0F"); + final byte[] value = bytesFromHexString("0ABC"); + + KeyValueStorageTransaction tx = store.startTransaction(); + tx.put(key, value); + tx.remove(key); + tx.commit(); + assertThat(store.get(key)).isEmpty(); + } } /** @@ -199,13 +207,14 @@ public void removeExistingSameTransaction() throws Exception { */ @Test public void removeNonExistent() throws Exception { - final KeyValueStorage store = createStore(); - final byte[] key = bytesFromHexString("0F"); + try (final KeyValueStorage store = createStore()) { + final byte[] key = bytesFromHexString("0F"); - KeyValueStorageTransaction tx = store.startTransaction(); - tx.remove(key); - tx.commit(); - assertThat(store.get(key)).isEmpty(); + KeyValueStorageTransaction tx = store.startTransaction(); + tx.remove(key); + tx.commit(); + assertThat(store.get(key)).isEmpty(); + } } /** @@ -216,39 +225,38 @@ public void removeNonExistent() throws Exception { @Test public void concurrentUpdate() throws Exception { final int keyCount = 1000; - final KeyValueStorage store = createStore(); - - final CountDownLatch finishedLatch = new CountDownLatch(2); - final Function updater = - (value) -> - new Thread( - () -> { - try { - for (int i = 0; i < keyCount; i++) { - KeyValueStorageTransaction tx = store.startTransaction(); - tx.put(Bytes.minimalBytes(i).toArrayUnsafe(), value); - tx.commit(); + try (final KeyValueStorage store = createStore()) { + + final CountDownLatch finishedLatch = new CountDownLatch(2); + final Function updater = + (value) -> + new Thread( + () -> { + try { + for (int i = 0; i < keyCount; i++) { + KeyValueStorageTransaction tx = store.startTransaction(); + tx.put(Bytes.minimalBytes(i).toArrayUnsafe(), value); + tx.commit(); + } + } finally { + finishedLatch.countDown(); } - } finally { - finishedLatch.countDown(); - } - }); + }); - // Run 2 concurrent transactions that write a bunch of values to the same keys - final byte[] a = Bytes.of(10).toArrayUnsafe(); - final byte[] b = Bytes.of(20).toArrayUnsafe(); - updater.apply(a).start(); - updater.apply(b).start(); + // Run 2 concurrent transactions that write a bunch of values to the same keys + final byte[] a = Bytes.of(10).toArrayUnsafe(); + final byte[] b = Bytes.of(20).toArrayUnsafe(); + updater.apply(a).start(); + updater.apply(b).start(); - finishedLatch.await(); + finishedLatch.await(); - for (int i = 0; i < keyCount; i++) { - final byte[] key = Bytes.minimalBytes(i).toArrayUnsafe(); - final byte[] actual = store.get(key).get(); - assertThat(Arrays.equals(actual, a) || Arrays.equals(actual, b)).isTrue(); + for (int i = 0; i < keyCount; i++) { + final byte[] key = Bytes.minimalBytes(i).toArrayUnsafe(); + final byte[] actual = store.get(key).get(); + assertThat(Arrays.equals(actual, a) || Arrays.equals(actual, b)).isTrue(); + } } - - store.close(); } /** @@ -258,34 +266,35 @@ public void concurrentUpdate() throws Exception { */ @Test public void transactionCommit() throws Exception { - final KeyValueStorage store = createStore(); - // Add some values - KeyValueStorageTransaction tx = store.startTransaction(); - tx.put(bytesOf(1), bytesOf(1)); - tx.put(bytesOf(2), bytesOf(2)); - tx.put(bytesOf(3), bytesOf(3)); - tx.commit(); - - // Start transaction that adds, modifies, and removes some values - tx = store.startTransaction(); - tx.put(bytesOf(2), bytesOf(3)); - tx.put(bytesOf(2), bytesOf(4)); - tx.remove(bytesOf(3)); - tx.put(bytesOf(4), bytesOf(8)); - - // Check values before committing have not changed - assertThat(store.get(bytesOf(1))).contains(bytesOf(1)); - assertThat(store.get(bytesOf(2))).contains(bytesOf(2)); - assertThat(store.get(bytesOf(3))).contains(bytesOf(3)); - assertThat(store.get(bytesOf(4))).isEmpty(); - - tx.commit(); - - // Check that values have been updated after commit - assertThat(store.get(bytesOf(1))).contains(bytesOf(1)); - assertThat(store.get(bytesOf(2))).contains(bytesOf(4)); - assertThat(store.get(bytesOf(3))).isEmpty(); - assertThat(store.get(bytesOf(4))).contains(bytesOf(8)); + try (final KeyValueStorage store = createStore()) { + // Add some values + KeyValueStorageTransaction tx = store.startTransaction(); + tx.put(bytesOf(1), bytesOf(1)); + tx.put(bytesOf(2), bytesOf(2)); + tx.put(bytesOf(3), bytesOf(3)); + tx.commit(); + + // Start transaction that adds, modifies, and removes some values + tx = store.startTransaction(); + tx.put(bytesOf(2), bytesOf(3)); + tx.put(bytesOf(2), bytesOf(4)); + tx.remove(bytesOf(3)); + tx.put(bytesOf(4), bytesOf(8)); + + // Check values before committing have not changed + assertThat(store.get(bytesOf(1))).contains(bytesOf(1)); + assertThat(store.get(bytesOf(2))).contains(bytesOf(2)); + assertThat(store.get(bytesOf(3))).contains(bytesOf(3)); + assertThat(store.get(bytesOf(4))).isEmpty(); + + tx.commit(); + + // Check that values have been updated after commit + assertThat(store.get(bytesOf(1))).contains(bytesOf(1)); + assertThat(store.get(bytesOf(2))).contains(bytesOf(4)); + assertThat(store.get(bytesOf(3))).isEmpty(); + assertThat(store.get(bytesOf(4))).contains(bytesOf(8)); + } } /** @@ -295,34 +304,35 @@ public void transactionCommit() throws Exception { */ @Test public void transactionRollback() throws Exception { - final KeyValueStorage store = createStore(); - // Add some values - KeyValueStorageTransaction tx = store.startTransaction(); - tx.put(bytesOf(1), bytesOf(1)); - tx.put(bytesOf(2), bytesOf(2)); - tx.put(bytesOf(3), bytesOf(3)); - tx.commit(); - - // Start transaction that adds, modifies, and removes some values - tx = store.startTransaction(); - tx.put(bytesOf(2), bytesOf(3)); - tx.put(bytesOf(2), bytesOf(4)); - tx.remove(bytesOf(3)); - tx.put(bytesOf(4), bytesOf(8)); - - // Check values before committing have not changed - assertThat(store.get(bytesOf(1))).contains(bytesOf(1)); - assertThat(store.get(bytesOf(2))).contains(bytesOf(2)); - assertThat(store.get(bytesOf(3))).contains(bytesOf(3)); - assertThat(store.get(bytesOf(4))).isEmpty(); - - tx.rollback(); - - // Check that values have not changed after rollback - assertThat(store.get(bytesOf(1))).contains(bytesOf(1)); - assertThat(store.get(bytesOf(2))).contains(bytesOf(2)); - assertThat(store.get(bytesOf(3))).contains(bytesOf(3)); - assertThat(store.get(bytesOf(4))).isEmpty(); + try (final KeyValueStorage store = createStore()) { + // Add some values + KeyValueStorageTransaction tx = store.startTransaction(); + tx.put(bytesOf(1), bytesOf(1)); + tx.put(bytesOf(2), bytesOf(2)); + tx.put(bytesOf(3), bytesOf(3)); + tx.commit(); + + // Start transaction that adds, modifies, and removes some values + tx = store.startTransaction(); + tx.put(bytesOf(2), bytesOf(3)); + tx.put(bytesOf(2), bytesOf(4)); + tx.remove(bytesOf(3)); + tx.put(bytesOf(4), bytesOf(8)); + + // Check values before committing have not changed + assertThat(store.get(bytesOf(1))).contains(bytesOf(1)); + assertThat(store.get(bytesOf(2))).contains(bytesOf(2)); + assertThat(store.get(bytesOf(3))).contains(bytesOf(3)); + assertThat(store.get(bytesOf(4))).isEmpty(); + + tx.rollback(); + + // Check that values have not changed after rollback + assertThat(store.get(bytesOf(1))).contains(bytesOf(1)); + assertThat(store.get(bytesOf(2))).contains(bytesOf(2)); + assertThat(store.get(bytesOf(3))).contains(bytesOf(3)); + assertThat(store.get(bytesOf(4))).isEmpty(); + } } /** @@ -332,9 +342,10 @@ public void transactionRollback() throws Exception { */ @Test public void transactionCommitEmpty() throws Exception { - final KeyValueStorage store = createStore(); - final KeyValueStorageTransaction tx = store.startTransaction(); - tx.commit(); + try (final KeyValueStorage store = createStore()) { + final KeyValueStorageTransaction tx = store.startTransaction(); + tx.commit(); + } } /** @@ -344,143 +355,120 @@ public void transactionCommitEmpty() throws Exception { */ @Test public void transactionRollbackEmpty() throws Exception { - final KeyValueStorage store = createStore(); - final KeyValueStorageTransaction tx = store.startTransaction(); - tx.rollback(); + try (final KeyValueStorage store = createStore()) { + final KeyValueStorageTransaction tx = store.startTransaction(); + tx.rollback(); + } } - /** - * Transaction put after commit. - * - * @throws Exception the exception - */ + /** Transaction put after commit. */ @Test - public void transactionPutAfterCommit() throws Exception { + public void transactionPutAfterCommit() { Assertions.assertThatThrownBy( () -> { - final KeyValueStorage store = createStore(); - final KeyValueStorageTransaction tx = store.startTransaction(); - tx.commit(); - tx.put(bytesOf(1), bytesOf(1)); + try (final KeyValueStorage store = createStore()) { + final KeyValueStorageTransaction tx = store.startTransaction(); + tx.commit(); + tx.put(bytesOf(1), bytesOf(1)); + } }) .isInstanceOf(IllegalStateException.class); } - /** - * Transaction remove after commit. - * - * @throws Exception the exception - */ + /** Transaction remove after commit. */ @Test - public void transactionRemoveAfterCommit() throws Exception { + public void transactionRemoveAfterCommit() { Assertions.assertThatThrownBy( () -> { - final KeyValueStorage store = createStore(); - final KeyValueStorageTransaction tx = store.startTransaction(); - tx.commit(); - tx.remove(bytesOf(1)); + try (final KeyValueStorage store = createStore()) { + final KeyValueStorageTransaction tx = store.startTransaction(); + tx.commit(); + tx.remove(bytesOf(1)); + } }) .isInstanceOf(IllegalStateException.class); } - /** - * Transaction put after rollback. - * - * @throws Exception the exception - */ + /** Transaction put after rollback. */ @Test - public void transactionPutAfterRollback() throws Exception { + public void transactionPutAfterRollback() { Assertions.assertThatThrownBy( () -> { - final KeyValueStorage store = createStore(); - final KeyValueStorageTransaction tx = store.startTransaction(); - tx.rollback(); - tx.put(bytesOf(1), bytesOf(1)); + try (final KeyValueStorage store = createStore()) { + final KeyValueStorageTransaction tx = store.startTransaction(); + tx.rollback(); + tx.put(bytesOf(1), bytesOf(1)); + } }) .isInstanceOf(IllegalStateException.class); } - /** - * Transaction remove after rollback. - * - * @throws Exception the exception - */ + /** Transaction remove after rollback. */ @Test - public void transactionRemoveAfterRollback() throws Exception { + public void transactionRemoveAfterRollback() { Assertions.assertThatThrownBy( () -> { - final KeyValueStorage store = createStore(); - final KeyValueStorageTransaction tx = store.startTransaction(); - tx.rollback(); - tx.remove(bytesOf(1)); + try (final KeyValueStorage store = createStore()) { + final KeyValueStorageTransaction tx = store.startTransaction(); + tx.rollback(); + tx.remove(bytesOf(1)); + } }) .isInstanceOf(IllegalStateException.class); } - /** - * Transaction commit after rollback. - * - * @throws Exception the exception - */ + /** Transaction commit after rollback. */ @Test - public void transactionCommitAfterRollback() throws Exception { + public void transactionCommitAfterRollback() { Assertions.assertThatThrownBy( () -> { - final KeyValueStorage store = createStore(); - final KeyValueStorageTransaction tx = store.startTransaction(); - tx.rollback(); - tx.commit(); + try (final KeyValueStorage store = createStore()) { + final KeyValueStorageTransaction tx = store.startTransaction(); + tx.rollback(); + tx.commit(); + } }) .isInstanceOf(IllegalStateException.class); } - /** - * Transaction commit twice. - * - * @throws Exception the exception - */ + /** Transaction commit twice. */ @Test - public void transactionCommitTwice() throws Exception { + public void transactionCommitTwice() { Assertions.assertThatThrownBy( () -> { - final KeyValueStorage store = createStore(); - final KeyValueStorageTransaction tx = store.startTransaction(); - tx.commit(); - tx.commit(); + try (final KeyValueStorage store = createStore()) { + final KeyValueStorageTransaction tx = store.startTransaction(); + tx.commit(); + tx.commit(); + } }) .isInstanceOf(IllegalStateException.class); } - /** - * Transaction rollback after commit. - * - * @throws Exception the exception - */ + /** Transaction rollback after commit. */ @Test - public void transactionRollbackAfterCommit() throws Exception { + public void transactionRollbackAfterCommit() { Assertions.assertThatThrownBy( () -> { - final KeyValueStorage store = createStore(); - final KeyValueStorageTransaction tx = store.startTransaction(); - tx.commit(); - tx.rollback(); + try (final KeyValueStorage store = createStore()) { + final KeyValueStorageTransaction tx = store.startTransaction(); + tx.commit(); + tx.rollback(); + } }) .isInstanceOf(IllegalStateException.class); } - /** - * Transaction rollback twice. - * - * @throws Exception the exception - */ + /** Transaction rollback twice. */ @Test - public void transactionRollbackTwice() throws Exception { + public void transactionRollbackTwice() { Assertions.assertThatThrownBy( () -> { - final KeyValueStorage store = createStore(); - final KeyValueStorageTransaction tx = store.startTransaction(); - tx.rollback(); - tx.rollback(); + try (final KeyValueStorage store = createStore()) { + final KeyValueStorageTransaction tx = store.startTransaction(); + tx.rollback(); + tx.rollback(); + } }) .isInstanceOf(IllegalStateException.class); } @@ -492,19 +480,20 @@ public void transactionRollbackTwice() throws Exception { */ @Test public void twoTransactions() throws Exception { - final KeyValueStorage store = createStore(); + try (final KeyValueStorage store = createStore()) { - final KeyValueStorageTransaction tx1 = store.startTransaction(); - final KeyValueStorageTransaction tx2 = store.startTransaction(); + final KeyValueStorageTransaction tx1 = store.startTransaction(); + final KeyValueStorageTransaction tx2 = store.startTransaction(); - tx1.put(bytesOf(1), bytesOf(1)); - tx2.put(bytesOf(2), bytesOf(2)); + tx1.put(bytesOf(1), bytesOf(1)); + tx2.put(bytesOf(2), bytesOf(2)); - tx1.commit(); - tx2.commit(); + tx1.commit(); + tx2.commit(); - assertThat(store.get(bytesOf(1))).contains(bytesOf(1)); - assertThat(store.get(bytesOf(2))).contains(bytesOf(2)); + assertThat(store.get(bytesOf(1))).contains(bytesOf(1)); + assertThat(store.get(bytesOf(2))).contains(bytesOf(2)); + } } /** From 56bc85205ad83808cb66c626ba1ea2e1ba5ecc7f Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Mon, 26 Feb 2024 17:03:28 -0500 Subject: [PATCH 3/8] Abstracts out which container registry (#6606) * repo owner didn't include repo name * switches back to docker.io * specify registry and login consistently * artifacts workflow can be manually executed --------- Signed-off-by: Justin Florentine --- .github/workflows/artifacts.yml | 1 + .github/workflows/docker.yml | 22 +++++++++++----------- .github/workflows/nightly.yml | 24 ++++++++++++------------ .github/workflows/release.yml | 14 ++++++++++---- 4 files changed, 34 insertions(+), 27 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 8ad8b4ecfc8..6923a43a94e 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -2,6 +2,7 @@ name: artifacts on: + workflow_dispatch: release: types: - prereleased diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0e20b51647b..8889b0dc7ad 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,7 +4,7 @@ on: types: - prereleased env: - registry: ghcr.io + registry: docker.io jobs: hadolint: @@ -70,22 +70,22 @@ jobs: run: | mkdir -p docker/reports curl -L https://github.com/aelsabbahy/goss/releases/download/v0.4.4/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} -o ./docker/tests/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} - - name: login to ghcr + - name: login to ${{ env.registry }} uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d with: registry: ${{ env.registry }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ secrets.DOCKER_USER_RW }} + password: ${{ secrets.DOCKER_PASSWORD_RW }} - name: build and test docker uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 env: architecture: ${{ steps.prep.outputs.ARCH }} with: - arguments: testDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pversion=${{github.ref_name}} -Prelease.releaseVersion=${{ github.ref_name }} + arguments: testDocker -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pversion=${{github.ref_name}} -Prelease.releaseVersion=${{ github.ref_name }} - name: publish env: architecture: ${{ steps.prep.outputs.ARCH }} - run: ./gradlew --no-daemon dockerUpload -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pversion=${{github.ref_name}} -Prelease.releaseVersion=${{ github.ref_name }} + run: ./gradlew --no-daemon dockerUpload -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pversion=${{github.ref_name}} -Prelease.releaseVersion=${{ github.ref_name }} multiArch: needs: buildDocker runs-on: ubuntu-22.04 @@ -102,14 +102,14 @@ jobs: java-version: 17 - name: setup gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 - - name: login to ghcr + - name: login to ${{ env.registry }} uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d with: registry: ${{ env.registry }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ secrets.DOCKER_USER_RW }} + password: ${{ secrets.DOCKER_PASSWORD_RW }} - name: multi-arch docker - run: ./gradlew manifestDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pversion=${{github.ref_name}} -Prelease.releaseVersion=${{ github.ref_name }} + run: ./gradlew manifestDocker -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pversion=${{github.ref_name}} -Prelease.releaseVersion=${{ github.ref_name }} amendNotes: needs: multiArch runs-on: ubuntu-22.04 @@ -121,4 +121,4 @@ jobs: with: append_body: true body: | - `docker pull ${{env.registry}}/${{github.repository_owner}}:${{github.ref_name}}` + `docker pull ${{env.registry}}/${{secrets.DOCKER_ORG}}/besu:${{github.ref_name}}` diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 798a9b25198..3defa71ab5a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -9,7 +9,7 @@ on: env: nightly-tag: develop - registry: ghcr.io + registry: docker.io jobs: hadolint: @@ -68,12 +68,12 @@ jobs: with: distribution: temurin java-version: 17 - - name: login to ghcr + - name: login to ${{ env.registry }} uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d with: registry: ${{ env.registry }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ secrets.DOCKER_USER_RW }} + password: ${{ secrets.DOCKER_PASSWORD_RW }} - name: build image uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 with: @@ -88,16 +88,16 @@ jobs: architecture: ${{ steps.prep.outputs.ARCH }} with: arguments: testDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main - - name: login to ghcr + - name: login to $ {{ env.registry }} uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d with: registry: ${{ env.registry }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ secrets.DOCKER_USER_RW }} + password: ${{ secrets.DOCKER_PASSWORD_RW }} - name: publish env: architecture: ${{ steps.prep.outputs.ARCH }} - run: ./gradlew --no-daemon dockerUpload -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main + run: ./gradlew --no-daemon dockerUpload -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pbranch=main multiArch: permissions: contents: read @@ -114,12 +114,12 @@ jobs: java-version: 17 - name: setup gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 - - name: Login to DockerHub + - name: Login to ${{ env.registry }} uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d with: registry: ${{ env.registry }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ secrets.DOCKER_USER_RW }} + password: ${{ secrets.DOCKER_PASSWORD_RW }} - name: multi-arch docker - run: ./gradlew manifestDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main + run: ./gradlew manifestDocker -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pbranch=main diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1be195fb9ba..ceb8ba602cb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,6 +3,8 @@ on: workflow_dispatch: release: types: [released] +env: + registry: docker.io jobs: dockerPromoteX64: runs-on: ubuntu-22.04 @@ -13,11 +15,15 @@ jobs: distribution: 'temurin' # See 'Supported distributions' for available options java-version: '17' cache: gradle - - name: Login to DockerHub - run: echo '${{ secrets.DOCKER_PASSWORD_RW }}' | docker login -u '${{ secrets.DOCKER_USER_RW }}' --password-stdin + - name: login to ${{ env.registry }} + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d + with: + registry: ${{ env.registry }} + username: ${{ secrets.DOCKER_USER_RW }} + password: ${{ secrets.DOCKER_PASSWORD_RW }} - name: Setup Gradle uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 - name: Docker upload - run: ./gradlew "-Prelease.releaseVersion=${{ github.ref_name }}" "-PdockerOrgName=${{ secrets.DOCKER_ORG }}" dockerUploadRelease + run: ./gradlew "-Prelease.releaseVersion=${{ github.ref_name }}" "-PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }}" dockerUploadRelease - name: Docker manifest - run: ./gradlew "-Prelease.releaseVersion=${{ github.ref_name }}" "-PdockerOrgName=${{ secrets.DOCKER_ORG }}" manifestDockerRelease + run: ./gradlew "-Prelease.releaseVersion=${{ github.ref_name }}" "-PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }}" manifestDockerRelease From 3e36b2cc6b3bca6ff8c5cec9da1cbf42953d7eb7 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Tue, 27 Feb 2024 09:32:37 +1000 Subject: [PATCH 4/8] remove unnecessary PoS checkpoint methods and variables (#6603) Signed-off-by: Gabriel Fukushima --- .../controller/MergeBesuControllerBuilder.java | 1 - .../besu/consensus/merge/MergeContext.java | 7 ------- .../besu/consensus/merge/PostMergeContext.java | 18 ------------------ .../consensus/merge/TransitionContext.java | 5 ----- 4 files changed, 31 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java index 50abcedbfd2..62bce6ac69b 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java @@ -198,7 +198,6 @@ protected MergeContext createConsensusContext( .getTerminalTotalDifficulty() .map(Difficulty::of) .orElse(Difficulty.ZERO)) - .setCheckpointPostMergeSync(syncConfig.isCheckpointPostMergeEnabled()) .setPostMergeAtGenesis(isPostMergeAtGenesis); blockchain diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/MergeContext.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/MergeContext.java index 9db6a97a0ca..b875a2d7c73 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/MergeContext.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/MergeContext.java @@ -185,13 +185,6 @@ default boolean isChainPruningEnabled() { return false; } - /** - * Is checkpoint post merge sync. - * - * @return the boolean - */ - boolean isCheckpointPostMergeSync(); - /** * Is configured for a post-merge from genesis. * diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/PostMergeContext.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/PostMergeContext.java index 08985ed326b..5cf4cc44038 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/PostMergeContext.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/PostMergeContext.java @@ -70,7 +70,6 @@ public class PostMergeContext implements MergeContext { private final AtomicReference> terminalPoWBlock = new AtomicReference<>(Optional.empty()); private final BlockValueCalculator blockValueCalculator = new BlockValueCalculator(); - private boolean isCheckpointPostMergeSync; private boolean isPostMergeAtGenesis; // TODO: cleanup - isChainPruningEnabled will not be required after @@ -92,7 +91,6 @@ public class PostMergeContext implements MergeContext { PostMergeContext(final Difficulty difficulty) { this.terminalTotalDifficulty = new AtomicReference<>(difficulty); this.syncState = new AtomicReference<>(); - this.isCheckpointPostMergeSync = false; } /** @@ -314,22 +312,6 @@ public boolean isChainPruningEnabled() { return isChainPruningEnabled; } - /** - * Sets checkpoint post merge sync. - * - * @param isCheckpointPostMergeSync the is checkpoint post merge sync - * @return the checkpoint post merge sync - */ - public PostMergeContext setCheckpointPostMergeSync(final boolean isCheckpointPostMergeSync) { - this.isCheckpointPostMergeSync = isCheckpointPostMergeSync; - return this; - } - - @Override - public boolean isCheckpointPostMergeSync() { - return this.isCheckpointPostMergeSync; - } - @Override public boolean isPostMergeAtGenesis() { return this.isPostMergeAtGenesis; diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/TransitionContext.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/TransitionContext.java index 35e51db563e..71353edd0df 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/TransitionContext.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/TransitionContext.java @@ -149,11 +149,6 @@ public Optional retrieveBlockById(final PayloadIdentifier pay return postMergeContext.retrieveBlockById(payloadId); } - @Override - public boolean isCheckpointPostMergeSync() { - return false; - } - @Override public boolean isPostMergeAtGenesis() { return postMergeContext.isPostMergeAtGenesis(); From 3499485cdccd670c44f552112bd7926c800d7f12 Mon Sep 17 00:00:00 2001 From: Simon Dudley Date: Tue, 27 Feb 2024 14:02:58 +1000 Subject: [PATCH 5/8] [MINOR] Rotate changelog since 24.2.0-RC has been cut (#6604) Signed-off-by: Simon Dudley --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f36ad7a14f1..225b1efc382 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,12 @@ # Changelog -## 24.2.1-SNAPSHOT +## Next Release ### Breaking Changes - RocksDB database metadata format has changed to be more expressive, the migration of an existing metadata file to the new format is automatic at startup. Before performing a downgrade to a previous version it is mandatory to revert to the original format using the subcommand `besu --data-path=/path/to/besu/datadir storage revert-metadata v2-to-v1`. +### Upcoming Breaking Changes + ### Deprecations ### Additions and Improvements @@ -13,6 +15,8 @@ ### Bug fixes +### Download Links + ## 24.2.0-SNAPSHOT ### Breaking Changes From b8ba3ee0805c59eb53c5e4d7401df2d57e689950 Mon Sep 17 00:00:00 2001 From: Jason Frame Date: Tue, 27 Feb 2024 15:59:13 +1000 Subject: [PATCH 6/8] Fix NPE when running revert variable subcommand (#6616) Signed-off-by: Jason Frame Co-authored-by: Sally MacFarlane --- .../storage/StorageSubCommand.java | 11 ++++---- .../besu/cli/CommandTestAbstract.java | 2 ++ .../storage/StorageSubCommandTest.java | 25 ++++++++----------- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java index b377ae1e7a2..e26a097d1fd 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java @@ -24,6 +24,7 @@ import org.hyperledger.besu.cli.BesuCommand; import org.hyperledger.besu.cli.util.VersionProvider; +import org.hyperledger.besu.controller.BesuController; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; @@ -98,21 +99,19 @@ static class RevertVariablesStorage implements Runnable { public void run() { checkNotNull(parentCommand); - final var storageProvider = getStorageProvider(); + final var storageProvider = createBesuController().getStorageProvider(); revert(storageProvider); } - private StorageProvider getStorageProvider() { - // init collection of ignorable segments - parentCommand.besuCommand.setIgnorableStorageSegments(); - return parentCommand.besuCommand.getStorageProvider(); + private BesuController createBesuController() { + return parentCommand.besuCommand.buildController(); } private void revert(final StorageProvider storageProvider) { final var variablesStorage = storageProvider.createVariablesStorage(); final var blockchainStorage = - getStorageProvider().getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.BLOCKCHAIN); + storageProvider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.BLOCKCHAIN); final var blockchainUpdater = blockchainStorage.startTransaction(); final var variablesUpdater = variablesStorage.updater(); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java index 331f7d54a98..c5e34295a5b 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java @@ -218,6 +218,7 @@ public abstract class CommandTestAbstract { @Mock protected WorldStateArchive mockWorldStateArchive; @Mock protected TransactionPool mockTransactionPool; @Mock protected PrivacyPluginServiceImpl privacyPluginService; + @Mock protected StorageProvider storageProvider; @SuppressWarnings("PrivateStaticFinalLoggers") // @Mocks are inited by JUnit @Mock @@ -313,6 +314,7 @@ public void initMocks() throws Exception { when(mockProtocolContext.getBlockchain()).thenReturn(mockMutableBlockchain); lenient().when(mockProtocolContext.getWorldStateArchive()).thenReturn(mockWorldStateArchive); when(mockController.getTransactionPool()).thenReturn(mockTransactionPool); + when(mockController.getStorageProvider()).thenReturn(storageProvider); when(mockRunnerBuilder.vertx(any())).thenReturn(mockRunnerBuilder); when(mockRunnerBuilder.besuController(any())).thenReturn(mockRunnerBuilder); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommandTest.java index 3ca4f41aaf8..17344e667e6 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommandTest.java @@ -25,17 +25,14 @@ import static org.hyperledger.besu.ethereum.core.VariablesStorageHelper.populateVariablesStorage; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.BLOCKCHAIN; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.VARIABLES; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; import org.hyperledger.besu.cli.CommandTestAbstract; +import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.hyperledger.besu.services.kvstore.SegmentedInMemoryKeyValueStorage; import org.hyperledger.besu.services.kvstore.SegmentedKeyValueStorageAdapter; -import java.util.List; - import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; @@ -67,10 +64,10 @@ public void revertVariables() { final var kvVariables = new SegmentedKeyValueStorageAdapter(VARIABLES, kvVariablesSeg); final var kvBlockchainSeg = new SegmentedInMemoryKeyValueStorage(); final var kvBlockchain = new SegmentedKeyValueStorageAdapter(BLOCKCHAIN, kvBlockchainSeg); - when(rocksDBStorageFactory.create(eq(List.of(VARIABLES)), any(), any())) - .thenReturn(kvVariablesSeg); - when(rocksDBStorageFactory.create(eq(List.of(BLOCKCHAIN)), any(), any())) - .thenReturn(kvBlockchainSeg); + when(storageProvider.createVariablesStorage()) + .thenReturn(new VariablesKeyValueStorage(kvVariables)); + when(storageProvider.getStorageBySegmentIdentifier(BLOCKCHAIN)).thenReturn(kvBlockchain); + final var variableValues = getSampleVariableValues(); assertNoVariablesInStorage(kvBlockchain); populateVariablesStorage(kvVariables, variableValues); @@ -87,10 +84,9 @@ public void revertVariablesWhenSomeVariablesDoNotExist() { final var kvVariables = new SegmentedKeyValueStorageAdapter(VARIABLES, kvVariablesSeg); final var kvBlockchainSeg = new SegmentedInMemoryKeyValueStorage(); final var kvBlockchain = new SegmentedKeyValueStorageAdapter(BLOCKCHAIN, kvBlockchainSeg); - when(rocksDBStorageFactory.create(eq(List.of(VARIABLES)), any(), any())) - .thenReturn(kvVariablesSeg); - when(rocksDBStorageFactory.create(eq(List.of(BLOCKCHAIN)), any(), any())) - .thenReturn(kvBlockchainSeg); + when(storageProvider.createVariablesStorage()) + .thenReturn(new VariablesKeyValueStorage(kvVariables)); + when(storageProvider.getStorageBySegmentIdentifier(BLOCKCHAIN)).thenReturn(kvBlockchain); final var variableValues = getSampleVariableValues(); variableValues.remove(FINALIZED_BLOCK_HASH); @@ -108,8 +104,9 @@ public void revertVariablesWhenSomeVariablesDoNotExist() { public void doesNothingWhenVariablesAlreadyReverted() { final var kvVariables = new InMemoryKeyValueStorage(); final var kvBlockchain = new InMemoryKeyValueStorage(); - when(rocksDBStorageFactory.create(eq(VARIABLES), any(), any())).thenReturn(kvVariables); - when(rocksDBStorageFactory.create(eq(BLOCKCHAIN), any(), any())).thenReturn(kvBlockchain); + when(storageProvider.createVariablesStorage()) + .thenReturn(new VariablesKeyValueStorage(kvVariables)); + when(storageProvider.getStorageBySegmentIdentifier(BLOCKCHAIN)).thenReturn(kvBlockchain); final var variableValues = getSampleVariableValues(); assertNoVariablesInStorage(kvVariables); From 2c1733c8f7d49b73b708fca073a8d78588e79d8e Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 27 Feb 2024 10:20:13 +0100 Subject: [PATCH 7/8] Refactor `TransactionSelectionService` (#6595) Signed-off-by: Fabio Di Fabio --- .../dsl/node/ThreadBesuNodeRunner.java | 24 ++-- .../org/hyperledger/besu/cli/BesuCommand.java | 12 +- .../besu/cli/options/MiningOptions.java | 25 ++++- .../controller/BesuControllerBuilder.java | 29 ++--- ...onsensusScheduleBesuControllerBuilder.java | 11 +- .../TransitionBesuControllerBuilder.java | 10 +- .../TransactionSelectionServiceImpl.java | 13 ++- .../cli/CascadingDefaultProviderTest.java | 28 ++--- .../besu/cli/CommandTestAbstract.java | 7 +- .../besu/cli/options/MiningOptionsTest.java | 2 +- .../CliqueDifficultyCalculatorTest.java | 4 +- .../clique/NodeCanProduceNextBlockTest.java | 22 ++-- .../blockcreation/CliqueBlockCreatorTest.java | 3 +- .../blockcreation/CliqueBlockMinerTest.java | 4 +- .../CliqueMinerExecutorTest.java | 3 +- .../CliqueDifficultyValidationRuleTest.java | 4 +- .../CliqueExtraDataValidationRuleTest.java | 4 +- .../common/MigratingProtocolContext.java | 10 +- .../common/MigratingProtocolContextTest.java | 7 +- .../BftCoinbaseValidationRuleTest.java | 13 +-- .../BftCommitSealsValidationRuleTest.java | 11 +- .../BftValidatorsValidationRuleTest.java | 4 - .../BftVanityDataValidationRuleTest.java | 3 - .../ibft/support/TestContextBuilder.java | 1 - .../tests/round/IbftRoundIntegrationTest.java | 1 - ...ockHeaderValidationRulesetFactoryTest.java | 1 - .../ibft/IbftProtocolScheduleTest.java | 2 - .../blockcreation/BftBlockCreatorTest.java | 1 - .../IbftBlockHeightManagerTest.java | 6 +- .../ibft/statemachine/IbftRoundTest.java | 1 - .../ibft/validation/MessageValidatorTest.java | 1 - .../blockcreation/MergeCoordinatorTest.java | 3 +- .../merge/blockcreation/MergeReorgTest.java | 3 +- .../qbft/support/TestContextBuilder.java | 1 - .../test/round/QbftRoundIntegrationTest.java | 2 - ...ockHeaderValidationRulesetFactoryTest.java | 1 - .../qbft/QbftProtocolScheduleTest.java | 1 - .../QbftValidatorsValidationRuleTest.java | 4 - .../QbftBlockHeightManagerTest.java | 1 - .../qbft/statemachine/QbftRoundTest.java | 1 - .../ProposalPayloadValidatorTest.java | 7 +- .../validation/ProposalValidatorTest.java | 1 - .../RoundChangeMessageValidatorTest.java | 1 - .../jsonrpc/JsonRpcTestMethodsFactory.java | 4 +- ...ckByNumberLatestDesyncIntegrationTest.java | 3 +- .../AbstractEthGraphQLHttpServiceTest.java | 6 +- ethereum/blockcreation/build.gradle | 1 + .../blockcreation/AbstractBlockCreator.java | 7 +- .../AllAcceptingTransactionSelector.java | 55 --------- .../AbstractBlockTransactionSelectorTest.java | 105 ++++++++++++------ .../blockcreation/BlockMinerTest.java | 6 +- ...FeeMarketBlockTransactionSelectorTest.java | 25 +++-- .../besu/ethereum/ProtocolContext.java | 18 --- .../besu/ethereum/core/MiningParameters.java | 17 +++ .../ethereum/core/BlockchainSetupUtil.java | 2 - .../core/ExecutionContextTestFixture.java | 4 +- .../trie/bonsai/AbstractIsolationTests.java | 3 +- .../AbstractBlockPropagationManagerTest.java | 2 - .../fullsync/FullSyncTargetManagerTest.java | 4 +- ...neCommonAncestorTaskParameterizedTest.java | 4 +- .../DetermineCommonAncestorTaskTest.java | 4 +- .../ethereum/eth/transactions/TestNode.java | 3 +- .../BlockchainReferenceTestCaseSpec.java | 3 +- .../ethereum/retesteth/RetestethContext.java | 3 +- plugin-api/build.gradle | 2 +- .../services/TransactionSelectionService.java | 11 +- .../PluginTransactionSelector.java | 18 +++ 67 files changed, 250 insertions(+), 353 deletions(-) delete mode 100644 ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/AllAcceptingTransactionSelector.java diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java index bd94a4a3451..2727c25e259 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java @@ -50,7 +50,6 @@ import org.hyperledger.besu.plugin.services.StorageService; import org.hyperledger.besu.plugin.services.TransactionSelectionService; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBPlugin; -import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; import org.hyperledger.besu.plugin.services.txvalidator.PluginTransactionValidatorFactory; import org.hyperledger.besu.services.BesuConfigurationImpl; import org.hyperledger.besu.services.BesuEventsImpl; @@ -94,6 +93,7 @@ private BesuPluginContextImpl buildPluginContext( final BesuNode node, final StorageServiceImpl storageService, final SecurityModuleServiceImpl securityModuleService, + final TransactionSelectionServiceImpl transactionSelectionServiceImpl, final BesuConfiguration commonPluginConfiguration) { final CommandLine commandLine = new CommandLine(CommandSpec.create()); final BesuPluginContextImpl besuPluginContext = new BesuPluginContextImpl(); @@ -102,7 +102,7 @@ private BesuPluginContextImpl buildPluginContext( besuPluginContext.addService(PicoCLIOptions.class, new PicoCLIOptionsImpl(commandLine)); besuPluginContext.addService(RpcEndpointService.class, new RpcEndpointServiceImpl()); besuPluginContext.addService( - TransactionSelectionService.class, new TransactionSelectionServiceImpl()); + TransactionSelectionService.class, transactionSelectionServiceImpl); besuPluginContext.addService( PluginTransactionValidatorService.class, new PluginTransactionValidatorServiceImpl()); final Path pluginsPath; @@ -144,6 +144,8 @@ public void startNode(final BesuNode node) { final StorageServiceImpl storageService = new StorageServiceImpl(); final SecurityModuleServiceImpl securityModuleService = new SecurityModuleServiceImpl(); + final TransactionSelectionServiceImpl transactionSelectionServiceImpl = + new TransactionSelectionServiceImpl(); final Path dataDir = node.homeDirectory(); final BesuConfigurationImpl commonPluginConfiguration = new BesuConfigurationImpl(); commonPluginConfiguration.init( @@ -156,7 +158,11 @@ public void startNode(final BesuNode node) { node, n -> buildPluginContext( - node, storageService, securityModuleService, commonPluginConfiguration)); + node, + storageService, + securityModuleService, + transactionSelectionServiceImpl, + commonPluginConfiguration)); GlobalOpenTelemetry.resetForTest(); final ObservableMetricsSystem metricsSystem = @@ -193,8 +199,8 @@ public void startNode(final BesuNode node) { final int maxPeers = 25; - final Optional transactionSelectorFactory = - getTransactionSelectorFactory(besuPluginContext); + final TransactionSelectionService transactionSelectorService = + getTransactionSelectorService(besuPluginContext); final PluginTransactionValidatorFactory pluginTransactionValidatorFactory = getPluginTransactionValidatorFactory(besuPluginContext); @@ -220,7 +226,7 @@ public void startNode(final BesuNode node) { .maxRemotelyInitiatedPeers(15) .networkConfiguration(node.getNetworkingConfiguration()) .randomPeerPriority(false) - .transactionSelectorFactory(transactionSelectorFactory) + .transactionSelectorService(transactionSelectorService) .pluginTransactionValidatorFactory(pluginTransactionValidatorFactory); node.getGenesisConfig() @@ -332,11 +338,9 @@ public String getConsoleContents() { throw new RuntimeException("Console contents can only be captured in process execution"); } - private Optional getTransactionSelectorFactory( + private TransactionSelectionService getTransactionSelectorService( final BesuPluginContextImpl besuPluginContext) { - final Optional txSelectionService = - besuPluginContext.getService(TransactionSelectionService.class); - return txSelectionService.isPresent() ? txSelectionService.get().get() : Optional.empty(); + return besuPluginContext.getService(TransactionSelectionService.class).orElseThrow(); } private PluginTransactionValidatorFactory getPluginTransactionValidatorFactory( diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 6429339dbb4..d254cc0b0df 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -174,7 +174,6 @@ import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.storage.PrivacyKeyValueStorageFactory; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBPlugin; -import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; import org.hyperledger.besu.plugin.services.txvalidator.PluginTransactionValidatorFactory; import org.hyperledger.besu.services.BesuConfigurationImpl; import org.hyperledger.besu.services.BesuEventsImpl; @@ -224,7 +223,6 @@ import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; -import javax.annotation.Nonnull; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; @@ -1793,7 +1791,7 @@ public BesuControllerBuilder getControllerBuilder() { .synchronizerConfiguration(buildSyncConfig()) .ethProtocolConfiguration(unstableEthProtocolOptions.toDomainObject()) .networkConfiguration(unstableNetworkingOptions.toDomainObject()) - .transactionSelectorFactory(getTransactionSelectorFactory()) + .transactionSelectorService(getTransactionSelectorService()) .pluginTransactionValidatorFactory(getPluginTransactionValidatorFactory()) .dataDirectory(dataDir()) .dataStorageConfiguration(getDataStorageConfiguration()) @@ -1825,11 +1823,8 @@ public BesuControllerBuilder getControllerBuilder() { .cacheLastBlocks(numberOfblocksToCache); } - @Nonnull - private Optional getTransactionSelectorFactory() { - final Optional txSelectionService = - besuPluginContext.getService(TransactionSelectionService.class); - return txSelectionService.isPresent() ? txSelectionService.get().get() : Optional.empty(); + private TransactionSelectionService getTransactionSelectorService() { + return besuPluginContext.getService(TransactionSelectionService.class).orElseThrow(); } private PluginTransactionValidatorFactory getPluginTransactionValidatorFactory() { @@ -2147,6 +2142,7 @@ private MiningParameters getMiningParameters() { if (miningParameters == null) { miningOptions.setGenesisBlockPeriodSeconds( getGenesisBlockPeriodSeconds(getActualGenesisConfigOptions())); + miningOptions.setTransactionSelectionService(transactionSelectionServiceImpl); miningParameters = miningOptions.toDomainObject(); initMiningParametersMetrics(miningParameters); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java index 2e5a239f16d..976be808495 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java @@ -14,6 +14,7 @@ */ package org.hyperledger.besu.cli.options; +import static com.google.common.base.Preconditions.checkNotNull; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; @@ -37,6 +38,7 @@ import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters; import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.MutableInitValues; import org.hyperledger.besu.ethereum.core.MiningParameters; +import org.hyperledger.besu.plugin.services.TransactionSelectionService; import org.hyperledger.besu.util.number.PositiveNumber; import java.util.List; @@ -190,6 +192,7 @@ static class Unstable { } private OptionalInt maybeGenesisBlockPeriodSeconds; + private TransactionSelectionService transactionSelectionService; private MiningOptions() {} @@ -212,6 +215,16 @@ public void setGenesisBlockPeriodSeconds(final OptionalInt genesisBlockPeriodSec maybeGenesisBlockPeriodSeconds = genesisBlockPeriodSeconds; } + /** + * Set the transaction selection service + * + * @param transactionSelectionService the transaction selection service + */ + public void setTransactionSelectionService( + final TransactionSelectionService transactionSelectionService) { + this.transactionSelectionService = transactionSelectionService; + } + /** * Validate that there are no inconsistencies in the specified options. For example that the * options are valid for the selected implementation. @@ -299,6 +312,7 @@ public void validate( static MiningOptions fromConfig(final MiningParameters miningParameters) { final MiningOptions miningOptions = MiningOptions.create(); miningOptions.setGenesisBlockPeriodSeconds(miningParameters.getGenesisBlockPeriodSeconds()); + miningOptions.setTransactionSelectionService(miningParameters.getTransactionSelectionService()); miningOptions.isMiningEnabled = miningParameters.isMiningEnabled(); miningOptions.iStratumMiningEnabled = miningParameters.isStratumMiningEnabled(); miningOptions.stratumNetworkInterface = miningParameters.getStratumNetworkInterface(); @@ -333,10 +347,12 @@ static MiningOptions fromConfig(final MiningParameters miningParameters) { @Override public MiningParameters toDomainObject() { - if (maybeGenesisBlockPeriodSeconds == null) { - throw new IllegalStateException( - "genesisBlockPeriodSeconds must be set before using this object"); - } + checkNotNull( + maybeGenesisBlockPeriodSeconds, + "genesisBlockPeriodSeconds must be set before using this object"); + checkNotNull( + transactionSelectionService, + "transactionSelectionService must be set before using this object"); final var updatableInitValuesBuilder = MutableInitValues.builder() @@ -355,6 +371,7 @@ public MiningParameters toDomainObject() { return ImmutableMiningParameters.builder() .genesisBlockPeriodSeconds(maybeGenesisBlockPeriodSeconds) + .transactionSelectionService(transactionSelectionService) .mutableInitValues(updatableInitValuesBuilder.build()) .isStratumMiningEnabled(iStratumMiningEnabled) .stratumNetworkInterface(stratumNetworkInterface) diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 2ca4a1d12ec..1a74e4996e2 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -97,9 +97,9 @@ import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.TransactionSelectionService; import org.hyperledger.besu.plugin.services.permissioning.NodeMessagePermissioningProvider; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; -import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; import org.hyperledger.besu.plugin.services.txvalidator.PluginTransactionValidatorFactory; import java.io.Closeable; @@ -187,7 +187,6 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides private NetworkingConfiguration networkingConfiguration; private Boolean randomPeerPriority; - private Optional transactionSelectorFactory = Optional.empty(); /** the Dagger configured context that can provide dependencies */ protected Optional besuComponent = Optional.empty(); @@ -535,14 +534,13 @@ public BesuControllerBuilder randomPeerPriority(final Boolean randomPeerPriority } /** - * sets the transactionSelectorFactory in the builder + * sets the transactionSelectionService in the builder * - * @param transactionSelectorFactory the optional transaction selector factory + * @param transactionSelectionService the transaction selector service * @return the besu controller builder */ - public BesuControllerBuilder transactionSelectorFactory( - final Optional transactionSelectorFactory) { - this.transactionSelectorFactory = transactionSelectorFactory; + public BesuControllerBuilder transactionSelectorService( + final TransactionSelectionService transactionSelectionService) { return this; } @@ -617,11 +615,7 @@ public BesuController build() { final ProtocolContext protocolContext = createProtocolContext( - blockchain, - worldStateArchive, - protocolSchedule, - this::createConsensusContext, - transactionSelectorFactory); + blockchain, worldStateArchive, protocolSchedule, this::createConsensusContext); validateContext(protocolContext); if (chainPrunerConfiguration.getChainPruningEnabled()) { @@ -1057,22 +1051,15 @@ protected EthProtocolManager createEthProtocolManager( * @param worldStateArchive the world state archive * @param protocolSchedule the protocol schedule * @param consensusContextFactory the consensus context factory - * @param transactionSelectorFactory optional transaction selector factory * @return the protocol context */ protected ProtocolContext createProtocolContext( final MutableBlockchain blockchain, final WorldStateArchive worldStateArchive, final ProtocolSchedule protocolSchedule, - final ConsensusContextFactory consensusContextFactory, - final Optional transactionSelectorFactory) { + final ConsensusContextFactory consensusContextFactory) { return ProtocolContext.init( - blockchain, - worldStateArchive, - protocolSchedule, - consensusContextFactory, - transactionSelectorFactory, - badBlockManager); + blockchain, worldStateArchive, protocolSchedule, consensusContextFactory, badBlockManager); } private Optional createSnapProtocolManager( diff --git a/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java index 0754f96ff11..a41e9bb3ee0 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java @@ -62,7 +62,6 @@ import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.permissioning.NodeMessagePermissioningProvider; -import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; import java.math.BigInteger; import java.nio.file.Path; @@ -175,15 +174,9 @@ protected ProtocolContext createProtocolContext( final MutableBlockchain blockchain, final WorldStateArchive worldStateArchive, final ProtocolSchedule protocolSchedule, - final ConsensusContextFactory consensusContextFactory, - final Optional transactionSelectorFactory) { + final ConsensusContextFactory consensusContextFactory) { return MigratingProtocolContext.init( - blockchain, - worldStateArchive, - protocolSchedule, - consensusContextFactory, - transactionSelectorFactory, - badBlockManager); + blockchain, worldStateArchive, protocolSchedule, consensusContextFactory, badBlockManager); } @Override diff --git a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java index 65aadd46c63..a9de9de7e9f 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java @@ -60,7 +60,6 @@ import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.permissioning.NodeMessagePermissioningProvider; -import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; import java.math.BigInteger; import java.nio.file.Path; @@ -189,15 +188,10 @@ protected ProtocolContext createProtocolContext( final MutableBlockchain blockchain, final WorldStateArchive worldStateArchive, final ProtocolSchedule protocolSchedule, - final ConsensusContextFactory consensusContextFactory, - final Optional transactionSelectorFactory) { + final ConsensusContextFactory consensusContextFactory) { final ProtocolContext protocolContext = super.createProtocolContext( - blockchain, - worldStateArchive, - protocolSchedule, - consensusContextFactory, - transactionSelectorFactory); + blockchain, worldStateArchive, protocolSchedule, consensusContextFactory); transitionProtocolSchedule.setProtocolContext(protocolContext); return protocolContext; } diff --git a/besu/src/main/java/org/hyperledger/besu/services/TransactionSelectionServiceImpl.java b/besu/src/main/java/org/hyperledger/besu/services/TransactionSelectionServiceImpl.java index 3175fe731f9..c6d2125e7d5 100644 --- a/besu/src/main/java/org/hyperledger/besu/services/TransactionSelectionServiceImpl.java +++ b/besu/src/main/java/org/hyperledger/besu/services/TransactionSelectionServiceImpl.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.services; import org.hyperledger.besu.plugin.services.TransactionSelectionService; +import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelector; import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; import java.util.Optional; @@ -25,13 +26,15 @@ public class TransactionSelectionServiceImpl implements TransactionSelectionServ private Optional factory = Optional.empty(); @Override - public Optional get() { - return factory; + public PluginTransactionSelector createPluginTransactionSelector() { + return factory + .map(PluginTransactionSelectorFactory::create) + .orElse(PluginTransactionSelector.ACCEPT_ALL); } @Override - public void registerTransactionSelectorFactory( - final PluginTransactionSelectorFactory transactionSelectorFactory) { - factory = Optional.ofNullable(transactionSelectorFactory); + public void registerPluginTransactionSelectorFactory( + final PluginTransactionSelectorFactory pluginTransactionSelectorFactory) { + factory = Optional.ofNullable(pluginTransactionSelectorFactory); } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CascadingDefaultProviderTest.java b/besu/src/test/java/org/hyperledger/besu/cli/CascadingDefaultProviderTest.java index 008ae6a89b1..bb3849f4492 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/CascadingDefaultProviderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/CascadingDefaultProviderTest.java @@ -32,7 +32,7 @@ import org.hyperledger.besu.ethereum.api.graphql.GraphQLConfiguration; import org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration; import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration; -import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters; +import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.p2p.peers.EnodeURLImpl; @@ -198,14 +198,11 @@ public void envVariableOverridesValueFromConfigFile() { setEnvironmentVariable("BESU_MINER_COINBASE", expectedCoinbase); parseCommand("--config-file", configFile); - verify(mockControllerBuilder) - .miningParameters( - ImmutableMiningParameters.builder() - .mutableInitValues( - ImmutableMiningParameters.MutableInitValues.builder() - .coinbase(Address.fromHexString(expectedCoinbase)) - .build()) - .build()); + final var captMiningParameters = ArgumentCaptor.forClass(MiningParameters.class); + verify(mockControllerBuilder).miningParameters(captMiningParameters.capture()); + + assertThat(captMiningParameters.getValue().getCoinbase()) + .contains(Address.fromHexString(expectedCoinbase)); } /** @@ -220,14 +217,11 @@ public void cliOptionOverridesEnvVariableAndConfig() { setEnvironmentVariable("BESU_MINER_COINBASE", "0x0000000000000000000000000000000000000004"); parseCommand("--config-file", configFile, "--miner-coinbase", expectedCoinbase); - verify(mockControllerBuilder) - .miningParameters( - ImmutableMiningParameters.builder() - .mutableInitValues( - ImmutableMiningParameters.MutableInitValues.builder() - .coinbase(Address.fromHexString(expectedCoinbase)) - .build()) - .build()); + final var captMiningParameters = ArgumentCaptor.forClass(MiningParameters.class); + verify(mockControllerBuilder).miningParameters(captMiningParameters.capture()); + + assertThat(captMiningParameters.getValue().getCoinbase()) + .contains(Address.fromHexString(expectedCoinbase)); } /** diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java index c5e34295a5b..271ed1b8155 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java @@ -77,6 +77,7 @@ import org.hyperledger.besu.pki.config.PkiKeyStoreConfiguration; import org.hyperledger.besu.plugin.services.PicoCLIOptions; import org.hyperledger.besu.plugin.services.StorageService; +import org.hyperledger.besu.plugin.services.TransactionSelectionService; import org.hyperledger.besu.plugin.services.securitymodule.SecurityModule; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageFactory; import org.hyperledger.besu.plugin.services.storage.PrivacyKeyValueStorageFactory; @@ -206,6 +207,7 @@ public abstract class CommandTestAbstract { @Mock protected JsonBlockImporter jsonBlockImporter; @Mock protected RlpBlockImporter rlpBlockImporter; @Mock protected StorageServiceImpl storageService; + @Mock protected TransactionSelectionServiceImpl txSelectionService; @Mock protected SecurityModuleServiceImpl securityModuleService; @Mock protected SecurityModule securityModule; @Mock protected BesuConfigurationImpl commonPluginConfiguration; @@ -292,7 +294,7 @@ public void initMocks() throws Exception { when(mockControllerBuilder.maxPeers(anyInt())).thenReturn(mockControllerBuilder); when(mockControllerBuilder.maxRemotelyInitiatedPeers(anyInt())) .thenReturn(mockControllerBuilder); - when(mockControllerBuilder.transactionSelectorFactory(any())).thenReturn(mockControllerBuilder); + when(mockControllerBuilder.transactionSelectorService(any())).thenReturn(mockControllerBuilder); when(mockControllerBuilder.pluginTransactionValidatorFactory(any())) .thenReturn(mockControllerBuilder); when(mockControllerBuilder.besuComponent(any(BesuComponent.class))) @@ -380,6 +382,9 @@ public void initMocks() throws Exception { lenient() .when(mockBesuPluginContext.getService(StorageService.class)) .thenReturn(Optional.of(storageService)); + lenient() + .when(mockBesuPluginContext.getService(TransactionSelectionService.class)) + .thenReturn(Optional.of(txSelectionService)); lenient() .doReturn(mockPkiBlockCreationConfiguration) diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java index 74b696cdf9a..fda642d6ead 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java @@ -422,6 +422,6 @@ protected MiningOptions getOptionsFromBesuCommand(final TestBesuCommand besuComm @Override protected String[] getNonOptionFields() { - return new String[] {"maybeGenesisBlockPeriodSeconds"}; + return new String[] {"maybeGenesisBlockPeriodSeconds", "transactionSelectionService"}; } } diff --git a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/CliqueDifficultyCalculatorTest.java b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/CliqueDifficultyCalculatorTest.java index c3bf0a6def6..3c9da2520d9 100644 --- a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/CliqueDifficultyCalculatorTest.java +++ b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/CliqueDifficultyCalculatorTest.java @@ -32,7 +32,6 @@ import java.math.BigInteger; import java.util.List; -import java.util.Optional; import com.google.common.collect.Lists; import org.junit.jupiter.api.BeforeEach; @@ -59,8 +58,7 @@ public void setup() { when(validatorProvider.getValidatorsAfterBlock(any())).thenReturn(validatorList); final CliqueContext cliqueContext = new CliqueContext(validatorProvider, null, blockInterface); - cliqueProtocolContext = - new ProtocolContext(null, null, cliqueContext, Optional.empty(), new BadBlockManager()); + cliqueProtocolContext = new ProtocolContext(null, null, cliqueContext, new BadBlockManager()); blockHeaderBuilder = new BlockHeaderTestFixture(); } diff --git a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/NodeCanProduceNextBlockTest.java b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/NodeCanProduceNextBlockTest.java index 1f97cb88d9f..afd9651df61 100644 --- a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/NodeCanProduceNextBlockTest.java +++ b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/NodeCanProduceNextBlockTest.java @@ -38,7 +38,6 @@ import org.hyperledger.besu.ethereum.core.Util; import java.util.List; -import java.util.Optional; import com.google.common.collect.Lists; import org.junit.jupiter.api.BeforeEach; @@ -82,8 +81,7 @@ public void networkWithOneValidatorIsAllowedToCreateConsecutiveBlocks() { when(validatorProvider.getValidatorsAfterBlock(any())).thenReturn(validatorList); final CliqueContext cliqueContext = new CliqueContext(validatorProvider, null, blockInterface); cliqueProtocolContext = - new ProtocolContext( - blockChain, null, cliqueContext, Optional.empty(), new BadBlockManager()); + new ProtocolContext(blockChain, null, cliqueContext, new BadBlockManager()); headerBuilder.number(1).parentHash(genesisBlock.getHash()); final Block block_1 = createEmptyBlock(proposerKeyPair); @@ -108,8 +106,7 @@ public void networkWithTwoValidatorsIsAllowedToProduceBlockIfNotPreviousBlockPro when(validatorProvider.getValidatorsAfterBlock(any())).thenReturn(validatorList); final CliqueContext cliqueContext = new CliqueContext(validatorProvider, null, blockInterface); cliqueProtocolContext = - new ProtocolContext( - blockChain, null, cliqueContext, Optional.empty(), new BadBlockManager()); + new ProtocolContext(blockChain, null, cliqueContext, new BadBlockManager()); headerBuilder.number(1).parentHash(genesisBlock.getHash()); final Block block_1 = createEmptyBlock(proposerKeyPair); @@ -143,8 +140,7 @@ public void networkWithTwoValidatorsIsNotAllowedToProduceBlockIfIsPreviousBlockP when(validatorProvider.getValidatorsAfterBlock(any())).thenReturn(validatorList); final CliqueContext cliqueContext = new CliqueContext(validatorProvider, null, blockInterface); cliqueProtocolContext = - new ProtocolContext( - blockChain, null, cliqueContext, Optional.empty(), new BadBlockManager()); + new ProtocolContext(blockChain, null, cliqueContext, new BadBlockManager()); headerBuilder.parentHash(genesisBlock.getHash()).number(1); final Block block_1 = createEmptyBlock(proposerKeyPair); @@ -174,8 +170,7 @@ public void withThreeValidatorsMustHaveOneBlockBetweenSignings() { when(validatorProvider.getValidatorsAfterBlock(any())).thenReturn(validatorList); final CliqueContext cliqueContext = new CliqueContext(validatorProvider, null, blockInterface); cliqueProtocolContext = - new ProtocolContext( - blockChain, null, cliqueContext, Optional.empty(), new BadBlockManager()); + new ProtocolContext(blockChain, null, cliqueContext, new BadBlockManager()); headerBuilder.parentHash(genesisBlock.getHash()).number(1); final Block block_1 = createEmptyBlock(proposerKeyPair); @@ -220,8 +215,7 @@ public void signerIsValidIfInsufficientBlocksExistInHistory() { when(validatorProvider.getValidatorsAfterBlock(any())).thenReturn(validatorList); final CliqueContext cliqueContext = new CliqueContext(validatorProvider, null, blockInterface); cliqueProtocolContext = - new ProtocolContext( - blockChain, null, cliqueContext, Optional.empty(), new BadBlockManager()); + new ProtocolContext(blockChain, null, cliqueContext, new BadBlockManager()); headerBuilder.parentHash(genesisBlock.getHash()).number(1); final Block block_1 = createEmptyBlock(otherNodeKeyPair); @@ -250,8 +244,7 @@ public void exceptionIsThrownIfOnAnOrphanedChain() { when(validatorProvider.getValidatorsAfterBlock(any())).thenReturn(validatorList); final CliqueContext cliqueContext = new CliqueContext(validatorProvider, null, blockInterface); cliqueProtocolContext = - new ProtocolContext( - blockChain, null, cliqueContext, Optional.empty(), new BadBlockManager()); + new ProtocolContext(blockChain, null, cliqueContext, new BadBlockManager()); headerBuilder.parentHash(Hash.ZERO).number(3); final BlockHeader parentHeader = @@ -275,8 +268,7 @@ public void nonValidatorIsNotAllowedToCreateABlock() { when(validatorProvider.getValidatorsAfterBlock(any())).thenReturn(validatorList); final CliqueContext cliqueContext = new CliqueContext(validatorProvider, null, blockInterface); cliqueProtocolContext = - new ProtocolContext( - blockChain, null, cliqueContext, Optional.empty(), new BadBlockManager()); + new ProtocolContext(blockChain, null, cliqueContext, new BadBlockManager()); headerBuilder.parentHash(Hash.ZERO).number(3); final BlockHeader parentHeader = headerBuilder.buildHeader(); diff --git a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueBlockCreatorTest.java b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueBlockCreatorTest.java index 5b0fc0357bb..a98c9f7b95e 100644 --- a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueBlockCreatorTest.java +++ b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueBlockCreatorTest.java @@ -119,8 +119,7 @@ public void setup() { GenesisState.fromConfig(GenesisConfigFile.mainnet(), protocolSchedule).getBlock(); blockchain = createInMemoryBlockchain(genesis); protocolContext = - new ProtocolContext( - blockchain, stateArchive, cliqueContext, Optional.empty(), new BadBlockManager()); + new ProtocolContext(blockchain, stateArchive, cliqueContext, new BadBlockManager()); epochManager = new EpochManager(10); // Add a block above the genesis diff --git a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueBlockMinerTest.java b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueBlockMinerTest.java index 7f8d6a3babf..370847638f1 100644 --- a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueBlockMinerTest.java +++ b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueBlockMinerTest.java @@ -71,7 +71,7 @@ void doesNotMineBlockIfNoTransactionsWhenEmptyBlocksNotAllowed() throws Interrup final CliqueContext cliqueContext = new CliqueContext(validatorProvider, null, null); final ProtocolContext protocolContext = - new ProtocolContext(null, null, cliqueContext, Optional.empty(), new BadBlockManager()); + new ProtocolContext(null, null, cliqueContext, new BadBlockManager()); final CliqueBlockCreator blockCreator = mock(CliqueBlockCreator.class); final Function blockCreatorSupplier = @@ -126,7 +126,7 @@ void minesBlockIfHasTransactionsWhenEmptyBlocksNotAllowed() throws InterruptedEx final CliqueContext cliqueContext = new CliqueContext(validatorProvider, null, null); final ProtocolContext protocolContext = - new ProtocolContext(null, null, cliqueContext, Optional.empty(), new BadBlockManager()); + new ProtocolContext(null, null, cliqueContext, new BadBlockManager()); final CliqueBlockCreator blockCreator = mock(CliqueBlockCreator.class); final Function blockCreatorSupplier = diff --git a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueMinerExecutorTest.java b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueMinerExecutorTest.java index 5400f46ed94..a919360f5d5 100644 --- a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueMinerExecutorTest.java +++ b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueMinerExecutorTest.java @@ -95,8 +95,7 @@ public void setup() { when(validatorProvider.getValidatorsAfterBlock(any())).thenReturn(validatorList); final CliqueContext cliqueContext = new CliqueContext(validatorProvider, null, blockInterface); - cliqueProtocolContext = - new ProtocolContext(null, null, cliqueContext, Optional.empty(), new BadBlockManager()); + cliqueProtocolContext = new ProtocolContext(null, null, cliqueContext, new BadBlockManager()); cliqueProtocolSchedule = CliqueProtocolSchedule.create( GENESIS_CONFIG_OPTIONS, diff --git a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/headervalidationrules/CliqueDifficultyValidationRuleTest.java b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/headervalidationrules/CliqueDifficultyValidationRuleTest.java index 38f8c3536e0..d970d83d80e 100644 --- a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/headervalidationrules/CliqueDifficultyValidationRuleTest.java +++ b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/headervalidationrules/CliqueDifficultyValidationRuleTest.java @@ -35,7 +35,6 @@ import org.hyperledger.besu.ethereum.core.Util; import java.util.List; -import java.util.Optional; import com.google.common.collect.Lists; import org.junit.jupiter.api.BeforeEach; @@ -59,8 +58,7 @@ public void setup() { when(validatorProvider.getValidatorsAfterBlock(any())).thenReturn(validatorList); final CliqueContext cliqueContext = new CliqueContext(validatorProvider, null, blockInterface); - cliqueProtocolContext = - new ProtocolContext(null, null, cliqueContext, Optional.empty(), new BadBlockManager()); + cliqueProtocolContext = new ProtocolContext(null, null, cliqueContext, new BadBlockManager()); blockHeaderBuilder = new BlockHeaderTestFixture(); } diff --git a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/headervalidationrules/CliqueExtraDataValidationRuleTest.java b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/headervalidationrules/CliqueExtraDataValidationRuleTest.java index 8f2e8d8a9b9..0d30cdf655b 100644 --- a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/headervalidationrules/CliqueExtraDataValidationRuleTest.java +++ b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/headervalidationrules/CliqueExtraDataValidationRuleTest.java @@ -36,7 +36,6 @@ import org.hyperledger.besu.ethereum.core.Util; import java.util.List; -import java.util.Optional; import com.google.common.collect.Lists; import org.apache.tuweni.bytes.Bytes; @@ -63,8 +62,7 @@ public void setup() { when(validatorProvider.getValidatorsAfterBlock(any())).thenReturn(validatorList); final CliqueContext cliqueContext = new CliqueContext(validatorProvider, null, blockInterface); - cliqueProtocolContext = - new ProtocolContext(null, null, cliqueContext, Optional.empty(), new BadBlockManager()); + cliqueProtocolContext = new ProtocolContext(null, null, cliqueContext, new BadBlockManager()); } @Test diff --git a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingProtocolContext.java b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingProtocolContext.java index 2c4ff215cce..f34c101c299 100644 --- a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingProtocolContext.java +++ b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingProtocolContext.java @@ -21,9 +21,6 @@ import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; -import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; - -import java.util.Optional; /** The Migrating protocol context. */ public class MigratingProtocolContext extends ProtocolContext { @@ -36,16 +33,14 @@ public class MigratingProtocolContext extends ProtocolContext { * @param blockchain the blockchain * @param worldStateArchive the world state archive * @param consensusContextSchedule the consensus context schedule - * @param transactionSelectorFactory the optional transaction selector factory * @param badBlockManager the cache to use to keep invalid blocks */ public MigratingProtocolContext( final MutableBlockchain blockchain, final WorldStateArchive worldStateArchive, final ForksSchedule consensusContextSchedule, - final Optional transactionSelectorFactory, final BadBlockManager badBlockManager) { - super(blockchain, worldStateArchive, null, transactionSelectorFactory, badBlockManager); + super(blockchain, worldStateArchive, null, badBlockManager); this.consensusContextSchedule = consensusContextSchedule; } @@ -56,7 +51,6 @@ public MigratingProtocolContext( * @param worldStateArchive the world state archive * @param protocolSchedule the protocol schedule * @param consensusContextFactory the consensus context factory - * @param transactionSelectorFactory the optional transaction selector factory * @param badBlockManager the cache to use to keep invalid blocks * @return the protocol context */ @@ -65,7 +59,6 @@ public static ProtocolContext init( final WorldStateArchive worldStateArchive, final ProtocolSchedule protocolSchedule, final ConsensusContextFactory consensusContextFactory, - final Optional transactionSelectorFactory, final BadBlockManager badBlockManager) { final ConsensusContext consensusContext = consensusContextFactory.create(blockchain, worldStateArchive, protocolSchedule); @@ -74,7 +67,6 @@ public static ProtocolContext init( blockchain, worldStateArchive, migratingContext.getConsensusContextSchedule(), - transactionSelectorFactory, badBlockManager); } diff --git a/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/MigratingProtocolContextTest.java b/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/MigratingProtocolContextTest.java index 0017e79d59c..f1d19059920 100644 --- a/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/MigratingProtocolContextTest.java +++ b/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/MigratingProtocolContextTest.java @@ -24,7 +24,6 @@ import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import java.util.List; -import java.util.Optional; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -46,11 +45,7 @@ public void returnsContextForSpecificChainHeight() { new ForksSchedule<>(List.of(new ForkSpec<>(0L, context1), new ForkSpec<>(10L, context2))); final MigratingProtocolContext migratingProtocolContext = new MigratingProtocolContext( - blockchain, - worldStateArchive, - contextSchedule, - Optional.empty(), - new BadBlockManager()); + blockchain, worldStateArchive, contextSchedule, new BadBlockManager()); assertThat(migratingProtocolContext.getConsensusContext(ConsensusContext.class)) .isSameAs(context1); diff --git a/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/headervalidationrules/BftCoinbaseValidationRuleTest.java b/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/headervalidationrules/BftCoinbaseValidationRuleTest.java index 091141fcc9f..791edaf636d 100644 --- a/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/headervalidationrules/BftCoinbaseValidationRuleTest.java +++ b/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/headervalidationrules/BftCoinbaseValidationRuleTest.java @@ -28,7 +28,6 @@ import org.hyperledger.besu.ethereum.core.Util; import java.util.List; -import java.util.Optional; import com.google.common.collect.Lists; import org.junit.jupiter.api.Test; @@ -53,11 +52,7 @@ public void proposerInValidatorListPassesValidation() { final ProtocolContext context = new ProtocolContext( - null, - null, - setupContextWithValidators(validators), - Optional.empty(), - new BadBlockManager()); + null, null, setupContextWithValidators(validators), new BadBlockManager()); final BftCoinbaseValidationRule coinbaseValidationRule = new BftCoinbaseValidationRule(); @@ -78,11 +73,7 @@ public void proposerNotInValidatorListFailsValidation() { final ProtocolContext context = new ProtocolContext( - null, - null, - setupContextWithValidators(validators), - Optional.empty(), - new BadBlockManager()); + null, null, setupContextWithValidators(validators), new BadBlockManager()); final BftCoinbaseValidationRule coinbaseValidationRule = new BftCoinbaseValidationRule(); diff --git a/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/headervalidationrules/BftCommitSealsValidationRuleTest.java b/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/headervalidationrules/BftCommitSealsValidationRuleTest.java index ee1c44b7534..84d59397bcd 100644 --- a/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/headervalidationrules/BftCommitSealsValidationRuleTest.java +++ b/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/headervalidationrules/BftCommitSealsValidationRuleTest.java @@ -34,7 +34,6 @@ import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -60,7 +59,7 @@ public void correctlyConstructedHeaderPassesValidation() { final BftContext bftContext = setupContextWithValidators(committerAddresses); final ProtocolContext context = - new ProtocolContext(null, null, bftContext, Optional.empty(), new BadBlockManager()); + new ProtocolContext(null, null, bftContext, new BadBlockManager()); when(bftContext.getBlockInterface().getCommitters(any())).thenReturn(committerAddresses); assertThat(commitSealsValidationRule.validate(blockHeader, null, context)).isTrue(); @@ -75,7 +74,7 @@ public void insufficientCommitSealsFailsValidation() { final List
validators = singletonList(committerAddress); final BftContext bftContext = setupContextWithValidators(validators); final ProtocolContext context = - new ProtocolContext(null, null, bftContext, Optional.empty(), new BadBlockManager()); + new ProtocolContext(null, null, bftContext, new BadBlockManager()); when(bftContext.getBlockInterface().getCommitters(any())).thenReturn(emptyList()); assertThat(commitSealsValidationRule.validate(blockHeader, null, context)).isFalse(); @@ -93,7 +92,7 @@ public void committerNotInValidatorListFailsValidation() { final BftContext bftContext = setupContextWithValidators(validators); final ProtocolContext context = - new ProtocolContext(null, null, bftContext, Optional.empty(), new BadBlockManager()); + new ProtocolContext(null, null, bftContext, new BadBlockManager()); when(bftContext.getBlockInterface().getCommitters(any())) .thenReturn(singletonList(Util.publicKeyToAddress(nonValidatorNodeKey.getPublicKey()))); @@ -141,7 +140,7 @@ public void headerContainsDuplicateSealsFailsValidation() { final BftContext bftContext = setupContextWithValidators(validators); final ProtocolContext context = - new ProtocolContext(null, null, bftContext, Optional.empty(), new BadBlockManager()); + new ProtocolContext(null, null, bftContext, new BadBlockManager()); when(bftContext.getBlockInterface().getCommitters(any())) .thenReturn(List.of(committerAddress, committerAddress)); @@ -161,7 +160,7 @@ private boolean subExecution(final int validatorCount, final int committerCount) final BftContext bftContext = setupContextWithValidators(validators); final ProtocolContext context = - new ProtocolContext(null, null, bftContext, Optional.empty(), new BadBlockManager()); + new ProtocolContext(null, null, bftContext, new BadBlockManager()); when(bftContext.getBlockInterface().getCommitters(any())) .thenReturn(validators.subList(0, committerCount)); diff --git a/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/headervalidationrules/BftValidatorsValidationRuleTest.java b/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/headervalidationrules/BftValidatorsValidationRuleTest.java index 8fb065d740b..8ffd5633456 100644 --- a/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/headervalidationrules/BftValidatorsValidationRuleTest.java +++ b/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/headervalidationrules/BftValidatorsValidationRuleTest.java @@ -27,7 +27,6 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import java.util.List; -import java.util.Optional; import com.google.common.collect.Lists; import org.junit.jupiter.api.Test; @@ -50,7 +49,6 @@ public void correctlyConstructedHeaderPassesValidation() { null, null, setupContextWithBftExtraData(validators, bftExtraData), - Optional.empty(), new BadBlockManager()); when(bftExtraData.getValidators()).thenReturn(validators); @@ -69,7 +67,6 @@ public void validatorsInNonAscendingOrderFailValidation() { null, null, setupContextWithBftExtraData(validators, bftExtraData), - Optional.empty(), new BadBlockManager()); when(bftExtraData.getValidators()).thenReturn(Lists.reverse(validators)); @@ -91,7 +88,6 @@ public void mismatchingReportedValidatorsVsLocallyStoredListFailsValidation() { null, null, setupContextWithBftExtraData(storedValidators, bftExtraData), - Optional.empty(), new BadBlockManager()); when(bftExtraData.getValidators()).thenReturn(Lists.reverse(reportedValidators)); diff --git a/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/headervalidationrules/BftVanityDataValidationRuleTest.java b/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/headervalidationrules/BftVanityDataValidationRuleTest.java index 15da7c04e89..39f296a9937 100644 --- a/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/headervalidationrules/BftVanityDataValidationRuleTest.java +++ b/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/headervalidationrules/BftVanityDataValidationRuleTest.java @@ -25,8 +25,6 @@ import org.hyperledger.besu.ethereum.chain.BadBlockManager; import org.hyperledger.besu.ethereum.core.BlockHeader; -import java.util.Optional; - import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.Test; @@ -51,7 +49,6 @@ public boolean headerWithVanityDataOfSize(final int extraDataSize) { null, null, setupContextWithBftExtraData(emptyList(), extraData), - Optional.empty(), new BadBlockManager()); return validationRule.validate(blockHeader, null, context); } diff --git a/consensus/ibft/src/integration-test/java/org/hyperledger/besu/consensus/ibft/support/TestContextBuilder.java b/consensus/ibft/src/integration-test/java/org/hyperledger/besu/consensus/ibft/support/TestContextBuilder.java index 101581e4ec2..63c28901a8a 100644 --- a/consensus/ibft/src/integration-test/java/org/hyperledger/besu/consensus/ibft/support/TestContextBuilder.java +++ b/consensus/ibft/src/integration-test/java/org/hyperledger/besu/consensus/ibft/support/TestContextBuilder.java @@ -349,7 +349,6 @@ private static ControllerAndState createControllerAndFinalState( blockChain, worldStateArchive, new BftContext(validatorProvider, epochManager, blockInterface), - Optional.empty(), new BadBlockManager()); final TransactionPoolConfiguration poolConf = ImmutableTransactionPoolConfiguration.builder().txPoolMaxSize(1).build(); diff --git a/consensus/ibft/src/integration-test/java/org/hyperledger/besu/consensus/ibft/tests/round/IbftRoundIntegrationTest.java b/consensus/ibft/src/integration-test/java/org/hyperledger/besu/consensus/ibft/tests/round/IbftRoundIntegrationTest.java index 02a20b0ad32..5c43e086fb8 100644 --- a/consensus/ibft/src/integration-test/java/org/hyperledger/besu/consensus/ibft/tests/round/IbftRoundIntegrationTest.java +++ b/consensus/ibft/src/integration-test/java/org/hyperledger/besu/consensus/ibft/tests/round/IbftRoundIntegrationTest.java @@ -127,7 +127,6 @@ public void setup() { blockChain, worldStateArchive, setupContextWithBftExtraDataEncoder(emptyList(), bftExtraDataEncoder), - Optional.empty(), new BadBlockManager()); } diff --git a/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/IbftBlockHeaderValidationRulesetFactoryTest.java b/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/IbftBlockHeaderValidationRulesetFactoryTest.java index 8ed7f6aa87e..a7deb9e1973 100644 --- a/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/IbftBlockHeaderValidationRulesetFactoryTest.java +++ b/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/IbftBlockHeaderValidationRulesetFactoryTest.java @@ -52,7 +52,6 @@ private ProtocolContext protocolContext(final Collection
validators) { null, null, setupContextWithBftExtraDataEncoder(validators, new IbftExtraDataCodec()), - Optional.empty(), new BadBlockManager()); } diff --git a/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/IbftProtocolScheduleTest.java b/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/IbftProtocolScheduleTest.java index 8ba9f0be2fc..de8ead1d23e 100644 --- a/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/IbftProtocolScheduleTest.java +++ b/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/IbftProtocolScheduleTest.java @@ -48,7 +48,6 @@ import java.math.BigInteger; import java.util.Collection; import java.util.List; -import java.util.Optional; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -123,7 +122,6 @@ private ProtocolContext protocolContext(final Collection
validators) { null, null, setupContextWithBftExtraDataEncoder(BftContext.class, validators, bftExtraDataCodec), - Optional.empty(), new BadBlockManager()); } } diff --git a/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/blockcreation/BftBlockCreatorTest.java b/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/blockcreation/BftBlockCreatorTest.java index 8e1717b12f9..5481297dee7 100644 --- a/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/blockcreation/BftBlockCreatorTest.java +++ b/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/blockcreation/BftBlockCreatorTest.java @@ -126,7 +126,6 @@ public BlockHeaderValidator.Builder createBlockHeaderRuleset( blockchain, createInMemoryWorldStateArchive(), setupContextWithBftExtraDataEncoder(initialValidatorList, bftExtraDataEncoder), - Optional.empty(), new BadBlockManager()); final TransactionPoolConfiguration poolConf = diff --git a/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/statemachine/IbftBlockHeightManagerTest.java b/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/statemachine/IbftBlockHeightManagerTest.java index f1c0750a5d0..2521737f81c 100644 --- a/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/statemachine/IbftBlockHeightManagerTest.java +++ b/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/statemachine/IbftBlockHeightManagerTest.java @@ -169,11 +169,7 @@ public void setup() { protocolContext = new ProtocolContext( - blockchain, - null, - setupContextWithValidators(validators), - Optional.empty(), - new BadBlockManager()); + blockchain, null, setupContextWithValidators(validators), new BadBlockManager()); final ProtocolScheduleBuilder protocolScheduleBuilder = new ProtocolScheduleBuilder( diff --git a/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/statemachine/IbftRoundTest.java b/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/statemachine/IbftRoundTest.java index d7a79a9f0c5..5057b057409 100644 --- a/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/statemachine/IbftRoundTest.java +++ b/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/statemachine/IbftRoundTest.java @@ -113,7 +113,6 @@ public void setup() { blockChain, worldStateArchive, setupContextWithBftExtraDataEncoder(emptyList(), new IbftExtraDataCodec()), - Optional.empty(), new BadBlockManager()); lenient().when(messageValidator.validateProposal(any())).thenReturn(true); diff --git a/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/validation/MessageValidatorTest.java b/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/validation/MessageValidatorTest.java index c0927769b67..f7fb7af3e94 100644 --- a/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/validation/MessageValidatorTest.java +++ b/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/validation/MessageValidatorTest.java @@ -105,7 +105,6 @@ public void setup() { mock(MutableBlockchain.class), mock(WorldStateArchive.class), mockBftCtx, - Optional.empty(), new BadBlockManager()); lenient() diff --git a/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinatorTest.java b/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinatorTest.java index 552c93b8aa8..7f54bbc96d4 100644 --- a/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinatorTest.java +++ b/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinatorTest.java @@ -187,8 +187,7 @@ public void setUp() { .thenReturn(genesisState.getBlock().getHeader().getDifficulty().plus(1L)); protocolContext = - new ProtocolContext( - blockchain, worldStateArchive, mergeContext, Optional.empty(), badBlockManager); + new ProtocolContext(blockchain, worldStateArchive, mergeContext, badBlockManager); var mutable = worldStateArchive.getMutable(); genesisState.writeStateTo(mutable); mutable.persist(null); diff --git a/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeReorgTest.java b/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeReorgTest.java index 4462386a0dd..bd539123e04 100644 --- a/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeReorgTest.java +++ b/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeReorgTest.java @@ -76,8 +76,7 @@ public class MergeReorgTest implements MergeGenesisConfigHelper { private final MutableBlockchain blockchain = createInMemoryBlockchain(genesisState.getBlock()); private final EthScheduler ethScheduler = new DeterministicEthScheduler(); private final ProtocolContext protocolContext = - new ProtocolContext( - blockchain, worldStateArchive, mergeContext, Optional.empty(), new BadBlockManager()); + new ProtocolContext(blockchain, worldStateArchive, mergeContext, new BadBlockManager()); private final Address coinbase = genesisAllocations(getPowGenesisConfigFile()).findFirst().get(); private final BlockHeaderTestFixture headerGenerator = new BlockHeaderTestFixture(); diff --git a/consensus/qbft/src/integration-test/java/org/hyperledger/besu/consensus/qbft/support/TestContextBuilder.java b/consensus/qbft/src/integration-test/java/org/hyperledger/besu/consensus/qbft/support/TestContextBuilder.java index 1679575d14c..caffe3f2ba8 100644 --- a/consensus/qbft/src/integration-test/java/org/hyperledger/besu/consensus/qbft/support/TestContextBuilder.java +++ b/consensus/qbft/src/integration-test/java/org/hyperledger/besu/consensus/qbft/support/TestContextBuilder.java @@ -446,7 +446,6 @@ private static ControllerAndState createControllerAndFinalState( blockChain, worldStateArchive, new QbftContext(validatorProvider, epochManager, blockInterface, Optional.empty()), - Optional.empty(), new BadBlockManager()); final TransactionPoolConfiguration poolConf = diff --git a/consensus/qbft/src/integration-test/java/org/hyperledger/besu/consensus/qbft/test/round/QbftRoundIntegrationTest.java b/consensus/qbft/src/integration-test/java/org/hyperledger/besu/consensus/qbft/test/round/QbftRoundIntegrationTest.java index 45090e908be..52cdfa1ed04 100644 --- a/consensus/qbft/src/integration-test/java/org/hyperledger/besu/consensus/qbft/test/round/QbftRoundIntegrationTest.java +++ b/consensus/qbft/src/integration-test/java/org/hyperledger/besu/consensus/qbft/test/round/QbftRoundIntegrationTest.java @@ -58,7 +58,6 @@ import org.hyperledger.besu.util.Subscribers; import java.math.BigInteger; -import java.util.Optional; import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.BeforeEach; @@ -131,7 +130,6 @@ public void setup() { worldStateArchive, setupContextWithBftExtraDataEncoder( QbftContext.class, emptyList(), qbftExtraDataEncoder), - Optional.empty(), new BadBlockManager()); } diff --git a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/QbftBlockHeaderValidationRulesetFactoryTest.java b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/QbftBlockHeaderValidationRulesetFactoryTest.java index 41ed04ef78a..3a09295a990 100644 --- a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/QbftBlockHeaderValidationRulesetFactoryTest.java +++ b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/QbftBlockHeaderValidationRulesetFactoryTest.java @@ -47,7 +47,6 @@ private ProtocolContext protocolContext(final Collection
validators) { null, setupContextWithBftExtraDataEncoder( QbftContext.class, validators, new QbftExtraDataCodec()), - Optional.empty(), new BadBlockManager()); } diff --git a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/QbftProtocolScheduleTest.java b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/QbftProtocolScheduleTest.java index 65afc58e231..16a5f4ca31a 100644 --- a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/QbftProtocolScheduleTest.java +++ b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/QbftProtocolScheduleTest.java @@ -59,7 +59,6 @@ private ProtocolContext protocolContext(final Collection
validators) { null, setupContextWithBftExtraDataEncoder( QbftContext.class, validators, new QbftExtraDataCodec()), - Optional.empty(), new BadBlockManager()); } diff --git a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/headervalidationrules/QbftValidatorsValidationRuleTest.java b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/headervalidationrules/QbftValidatorsValidationRuleTest.java index 9c86df33fce..3e325142c4d 100644 --- a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/headervalidationrules/QbftValidatorsValidationRuleTest.java +++ b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/headervalidationrules/QbftValidatorsValidationRuleTest.java @@ -48,7 +48,6 @@ public void validationPassesIfValidatorsAndVoteAreEmpty() { null, null, setupContextWithBftExtraData(QbftContext.class, Collections.emptyList(), bftExtraData), - Optional.empty(), new BadBlockManager()); when(bftExtraData.getValidators()).thenReturn(Collections.emptyList()); when(bftExtraData.getVote()).thenReturn(Optional.empty()); @@ -68,7 +67,6 @@ public void validationIsDelegatedWhenConstructorFlagIsFalse() { null, null, setupContextWithBftExtraData(QbftContext.class, validators, bftExtraData), - Optional.empty(), new BadBlockManager()); when(bftExtraData.getValidators()).thenReturn(validators); assertThat(qbftValidatorsValidationRule.validate(blockHeader, null, context)).isTrue(); @@ -87,7 +85,6 @@ public void validationFailsIfValidatorsAreNotEmpty() { null, null, setupContextWithBftExtraData(QbftContext.class, validators, bftExtraData), - Optional.empty(), new BadBlockManager()); when(bftExtraData.getValidators()).thenReturn(validators); assertThat(qbftValidatorsValidationRule.validate(blockHeader, null, context)).isFalse(); @@ -102,7 +99,6 @@ public void validationFailsIfVoteIsPresent() { null, null, setupContextWithBftExtraData(QbftContext.class, Collections.emptyList(), bftExtraData), - Optional.empty(), new BadBlockManager()); when(bftExtraData.getValidators()).thenReturn(Collections.emptyList()); when(bftExtraData.getVote()).thenReturn(Optional.of(mock(Vote.class))); diff --git a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/statemachine/QbftBlockHeightManagerTest.java b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/statemachine/QbftBlockHeightManagerTest.java index 8d0a1628636..b582f1a9b88 100644 --- a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/statemachine/QbftBlockHeightManagerTest.java +++ b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/statemachine/QbftBlockHeightManagerTest.java @@ -169,7 +169,6 @@ public void setup() { null, setupContextWithBftExtraDataEncoder( QbftContext.class, validators, new QbftExtraDataCodec()), - Optional.empty(), new BadBlockManager()); final ProtocolScheduleBuilder protocolScheduleBuilder = diff --git a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/statemachine/QbftRoundTest.java b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/statemachine/QbftRoundTest.java index d369c155ddf..bf730bbf7e6 100644 --- a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/statemachine/QbftRoundTest.java +++ b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/statemachine/QbftRoundTest.java @@ -123,7 +123,6 @@ public void setup() { worldStateArchive, setupContextWithBftExtraDataEncoder( QbftContext.class, emptyList(), new QbftExtraDataCodec()), - Optional.empty(), new BadBlockManager()); when(messageValidator.validateProposal(any())).thenReturn(true); diff --git a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/validation/ProposalPayloadValidatorTest.java b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/validation/ProposalPayloadValidatorTest.java index 5a0660b59f6..e4687f3c104 100644 --- a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/validation/ProposalPayloadValidatorTest.java +++ b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/validation/ProposalPayloadValidatorTest.java @@ -88,7 +88,6 @@ public void setup() { blockChain, worldStateArchive, setupContextWithBftExtraDataEncoder(QbftContext.class, emptyList(), bftExtraDataCodec), - Optional.empty(), new BadBlockManager()); } @@ -242,8 +241,7 @@ public void validationForCmsFailsWhenCmsFailsValidation() { setupContextWithBftExtraDataEncoder(QbftContext.class, emptyList(), pkiQbftExtraDataCodec); final Bytes cms = Bytes.fromHexStringLenient("0x1"); final ProtocolContext protocolContext = - new ProtocolContext( - blockChain, worldStateArchive, qbftContext, Optional.empty(), new BadBlockManager()); + new ProtocolContext(blockChain, worldStateArchive, qbftContext, new BadBlockManager()); final ProposalPayloadValidator payloadValidator = new ProposalPayloadValidator( @@ -278,8 +276,7 @@ public void validationForCmsPassesWhenCmsIsValid() { setupContextWithBftExtraDataEncoder(QbftContext.class, emptyList(), pkiQbftExtraDataCodec); final Bytes cms = Bytes.fromHexStringLenient("0x1"); final ProtocolContext protocolContext = - new ProtocolContext( - blockChain, worldStateArchive, qbftContext, Optional.empty(), new BadBlockManager()); + new ProtocolContext(blockChain, worldStateArchive, qbftContext, new BadBlockManager()); final ProposalPayloadValidator payloadValidator = new ProposalPayloadValidator( diff --git a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/validation/ProposalValidatorTest.java b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/validation/ProposalValidatorTest.java index c7c2d6e316b..53876f5b87e 100644 --- a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/validation/ProposalValidatorTest.java +++ b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/validation/ProposalValidatorTest.java @@ -104,7 +104,6 @@ public void setup() { worldStateArchive, setupContextWithBftExtraDataEncoder( QbftContext.class, emptyList(), bftExtraDataEncoder), - Optional.empty(), new BadBlockManager()); // typically tests require the blockValidation to be successful diff --git a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/validation/RoundChangeMessageValidatorTest.java b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/validation/RoundChangeMessageValidatorTest.java index e850372951b..e02a668c2dc 100644 --- a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/validation/RoundChangeMessageValidatorTest.java +++ b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/validation/RoundChangeMessageValidatorTest.java @@ -86,7 +86,6 @@ public void setup() { worldStateArchive, setupContextWithBftExtraDataEncoder( QbftContext.class, emptyList(), bftExtraDataEncoder), - Optional.empty(), new BadBlockManager()); lenient().when(protocolSchedule.getByBlockHeader(any())).thenReturn(protocolSpec); diff --git a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestMethodsFactory.java b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestMethodsFactory.java index 8552b99d8bd..d31a12e0f68 100644 --- a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestMethodsFactory.java +++ b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestMethodsFactory.java @@ -79,9 +79,7 @@ public JsonRpcTestMethodsFactory(final BlockchainImporter importer) { this.blockchain = createInMemoryBlockchain(importer.getGenesisBlock()); this.stateArchive = createInMemoryWorldStateArchive(); this.importer.getGenesisState().writeStateTo(stateArchive.getMutable()); - this.context = - new ProtocolContext( - blockchain, stateArchive, null, Optional.empty(), new BadBlockManager()); + this.context = new ProtocolContext(blockchain, stateArchive, null, new BadBlockManager()); final ProtocolSchedule protocolSchedule = importer.getProtocolSchedule(); this.synchronizer = mock(Synchronizer.class); diff --git a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthGetBlockByNumberLatestDesyncIntegrationTest.java b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthGetBlockByNumberLatestDesyncIntegrationTest.java index 20d937f022b..927aff6404c 100644 --- a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthGetBlockByNumberLatestDesyncIntegrationTest.java +++ b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthGetBlockByNumberLatestDesyncIntegrationTest.java @@ -68,8 +68,7 @@ public static void setUpOnce() throws Exception { InMemoryKeyValueStorageProvider.createInMemoryBlockchain(importer.getGenesisBlock()); WorldStateArchive state = InMemoryKeyValueStorageProvider.createInMemoryWorldStateArchive(); importer.getGenesisState().writeStateTo(state.getMutable()); - ProtocolContext context = - new ProtocolContext(chain, state, null, Optional.empty(), new BadBlockManager()); + ProtocolContext context = new ProtocolContext(chain, state, null, new BadBlockManager()); for (final Block block : importer.getBlocks()) { final ProtocolSchedule protocolSchedule = importer.getProtocolSchedule(); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java index 771385af3c3..5565c9e4855 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java @@ -109,11 +109,7 @@ public void setupTest() throws Exception { final MutableBlockchain blockchain = blockchainSetupUtil.getBlockchain(); ProtocolContext context = new ProtocolContext( - blockchain, - blockchainSetupUtil.getWorldArchive(), - null, - Optional.empty(), - new BadBlockManager()); + blockchain, blockchainSetupUtil.getWorldArchive(), null, new BadBlockManager()); final BlockchainQueries blockchainQueries = new BlockchainQueries( context.getBlockchain(), diff --git a/ethereum/blockcreation/build.gradle b/ethereum/blockcreation/build.gradle index 7fc35a6ec10..5f426fd9320 100644 --- a/ethereum/blockcreation/build.gradle +++ b/ethereum/blockcreation/build.gradle @@ -27,6 +27,7 @@ dependencies { implementation 'io.tmio:tuweni-bytes' implementation 'io.tmio:tuweni-units' + testImplementation project(':besu') testImplementation project(path: ':config', configuration: 'testSupportArtifacts') testImplementation project(path: ':ethereum:core', configuration: 'testArtifacts') testImplementation project(path: ':ethereum:core', configuration: 'testSupportArtifacts') diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockCreator.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockCreator.java index deac72bc5da..f7503c65791 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockCreator.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockCreator.java @@ -24,7 +24,6 @@ import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.blockcreation.txselection.BlockTransactionSelector; import org.hyperledger.besu.ethereum.blockcreation.txselection.TransactionSelectionResults; -import org.hyperledger.besu.ethereum.blockcreation.txselection.selectors.AllAcceptingTransactionSelector; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockBody; import org.hyperledger.besu.ethereum.core.BlockHeader; @@ -60,7 +59,6 @@ import org.hyperledger.besu.plugin.services.securitymodule.SecurityModuleException; import org.hyperledger.besu.plugin.services.tracer.BlockAwareOperationTracer; import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelector; -import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; import java.math.BigInteger; import java.util.List; @@ -192,10 +190,7 @@ protected BlockCreationResult createBlock( throwIfStopped(); final PluginTransactionSelector pluginTransactionSelector = - protocolContext - .getTransactionSelectorFactory() - .map(PluginTransactionSelectorFactory::create) - .orElseGet(() -> AllAcceptingTransactionSelector.INSTANCE); + miningParameters.getTransactionSelectionService().createPluginTransactionSelector(); final BlockAwareOperationTracer operationTracer = pluginTransactionSelector.getOperationTracer(); diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/AllAcceptingTransactionSelector.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/AllAcceptingTransactionSelector.java deleted file mode 100644 index a2b8d57ff29..00000000000 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/AllAcceptingTransactionSelector.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright Hyperledger Besu Contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.blockcreation.txselection.selectors; - -import org.hyperledger.besu.datatypes.PendingTransaction; -import org.hyperledger.besu.plugin.data.TransactionProcessingResult; -import org.hyperledger.besu.plugin.data.TransactionSelectionResult; -import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelector; -import org.hyperledger.besu.plugin.services.txselection.TransactionEvaluationContext; - -/** A TransactionSelector that unconditionally selects all transactions. */ -public class AllAcceptingTransactionSelector implements PluginTransactionSelector { - public static final AllAcceptingTransactionSelector INSTANCE = - new AllAcceptingTransactionSelector(); - - private AllAcceptingTransactionSelector() {} - - /** - * Always selects the transaction in the pre-processing stage. - * - * @param evaluationContext The current selection context. - * @return Always SELECTED. - */ - @Override - public TransactionSelectionResult evaluateTransactionPreProcessing( - final TransactionEvaluationContext evaluationContext) { - return TransactionSelectionResult.SELECTED; - } - - /** - * Always selects the transaction in the post-processing stage. - * - * @param evaluationContext The current selection context. - * @param processingResult The result of the transaction processing. - * @return Always SELECTED. - */ - @Override - public TransactionSelectionResult evaluateTransactionPostProcessing( - final TransactionEvaluationContext evaluationContext, - final TransactionProcessingResult processingResult) { - return TransactionSelectionResult.SELECTED; - } -} diff --git a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java index 335e94aea6c..3fc970110ca 100644 --- a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java +++ b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java @@ -27,7 +27,6 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -43,7 +42,6 @@ import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.blockcreation.txselection.BlockTransactionSelector; import org.hyperledger.besu.ethereum.blockcreation.txselection.TransactionSelectionResults; -import org.hyperledger.besu.ethereum.blockcreation.txselection.selectors.AllAcceptingTransactionSelector; import org.hyperledger.besu.ethereum.chain.BadBlockManager; import org.hyperledger.besu.ethereum.chain.DefaultBlockchain; import org.hyperledger.besu.ethereum.chain.GenesisState; @@ -82,9 +80,11 @@ import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.data.TransactionSelectionResult; import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.TransactionSelectionService; import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelector; import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; import org.hyperledger.besu.plugin.services.txselection.TransactionEvaluationContext; +import org.hyperledger.besu.services.TransactionSelectionServiceImpl; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.hyperledger.besu.util.number.PositiveNumber; @@ -121,8 +121,6 @@ public abstract class AbstractBlockTransactionSelectorTest { protected static final double MIN_OCCUPANCY_80_PERCENT = 0.8; protected static final double MIN_OCCUPANCY_100_PERCENT = 1; - protected static final PluginTransactionSelectorFactory NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY = - () -> AllAcceptingTransactionSelector.INSTANCE; protected static final BigInteger CHAIN_ID = BigInteger.valueOf(42L); protected static final KeyPair keyPair = SignatureAlgorithmFactory.getInstance().generateKeyPair(); @@ -135,9 +133,8 @@ public abstract class AbstractBlockTransactionSelectorTest { protected TransactionPool transactionPool; protected MutableWorldState worldState; protected ProtocolSchedule protocolSchedule; - protected final MiningParameters defaultTestMiningParameters = - createMiningParameters( - Wei.ZERO, MIN_OCCUPANCY_80_PERCENT, DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME); + protected TransactionSelectionService transactionSelectionService; + protected MiningParameters defaultTestMiningParameters; @Mock protected EthScheduler ethScheduler; @@ -153,6 +150,14 @@ public abstract class AbstractBlockTransactionSelectorTest { public void setup() { genesisConfigFile = getGenesisConfigFile(); protocolSchedule = createProtocolSchedule(); + transactionSelectionService = new TransactionSelectionServiceImpl(); + defaultTestMiningParameters = + createMiningParameters( + transactionSelectionService, + Wei.ZERO, + MIN_OCCUPANCY_80_PERCENT, + DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME); + final Block genesisBlock = GenesisState.fromConfig(genesisConfigFile, protocolSchedule).getBlock(); @@ -228,7 +233,7 @@ public void emptyPendingTransactionsResultsInEmptyVettingResult() { blockHeader, miningBeneficiary, Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); final TransactionSelectionResults results = selector.buildTransactionListForBlock(); @@ -249,7 +254,7 @@ public void validPendingTransactionIsIncludedInTheBlock() { blockHeader, miningBeneficiary, Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); final Transaction transaction = createTransaction(1, Wei.of(7L), 100_000); transactionPool.addRemoteTransactions(List.of(transaction)); @@ -276,7 +281,7 @@ public void invalidTransactionsAreSkippedButBlockStillFills() { blockHeader, miningBeneficiary, Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); final List transactionsToInject = Lists.newArrayList(); for (int i = 0; i < 5; i++) { @@ -317,7 +322,7 @@ public void subsetOfPendingTransactionsIncludedWhenBlockGasLimitHit() { blockHeader, miningBeneficiary, Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); final List transactionsToInject = Lists.newArrayList(); for (int i = 0; i < 5; i++) { @@ -358,7 +363,7 @@ public void transactionTooLargeForBlockDoesNotPreventMoreBeingAddedIfBlockOccupa blockHeader, miningBeneficiary, Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); // Add 3 transactions to the Pending Transactions, 79% of block, 100% of block and 10% of block // should end up selecting the first and third only. @@ -393,7 +398,7 @@ public void transactionSelectionStopsWhenSufficientBlockOccupancyIsReached() { blockHeader, miningBeneficiary, Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); // Add 4 transactions to the Pending Transactions 15% (ok), 79% (ok), 25% (too large), 10% // (not included, it would fit, however previous transaction was too large and block was @@ -427,12 +432,15 @@ public void transactionSelectionStopsWhenBlockIsFull() { final BlockTransactionSelector selector = createBlockSelectorAndSetupTxPool( createMiningParameters( - Wei.ZERO, MIN_OCCUPANCY_100_PERCENT, DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME), + transactionSelectionService, + Wei.ZERO, + MIN_OCCUPANCY_100_PERCENT, + DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME), transactionProcessor, blockHeader, miningBeneficiary, Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); final long minTxGasCost = getGasCalculator().getMinimumTransactionCost(); @@ -486,12 +494,15 @@ public void transactionSelectionStopsWhenRemainingGasIsNotEnoughForAnyMoreTransa final BlockTransactionSelector selector = createBlockSelectorAndSetupTxPool( createMiningParameters( - Wei.ZERO, MIN_OCCUPANCY_100_PERCENT, DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME), + transactionSelectionService, + Wei.ZERO, + MIN_OCCUPANCY_100_PERCENT, + DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME), transactionProcessor, blockHeader, miningBeneficiary, Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); final long minTxGasCost = getGasCalculator().getMinimumTransactionCost(); @@ -543,7 +554,7 @@ public void shouldDiscardTransactionsThatFailValidation() { blockHeader, miningBeneficiary, Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); final Transaction validTransaction = createTransaction(0, Wei.of(10), 21_000); @@ -610,6 +621,8 @@ public TransactionSelectionResult evaluateTransactionPostProcessing( return SELECTED; } }; + transactionSelectionService.registerPluginTransactionSelectorFactory( + transactionSelectorFactory); final BlockTransactionSelector selector = createBlockSelectorAndSetupTxPool( @@ -618,7 +631,7 @@ public TransactionSelectionResult evaluateTransactionPostProcessing( blockHeader, miningBeneficiary, Wei.ZERO, - transactionSelectorFactory); + transactionSelectionService); transactionPool.addRemoteTransactions( List.of(selected, notSelectedTransient, notSelectedInvalid)); @@ -676,17 +689,22 @@ public TransactionSelectionResult evaluateTransactionPostProcessing( return SELECTED; } }; + transactionSelectionService.registerPluginTransactionSelectorFactory( + transactionSelectorFactory); final Address miningBeneficiary = AddressHelpers.ofValue(1); final BlockTransactionSelector selector = createBlockSelectorAndSetupTxPool( createMiningParameters( - Wei.ZERO, MIN_OCCUPANCY_80_PERCENT, DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME), + transactionSelectionService, + Wei.ZERO, + MIN_OCCUPANCY_80_PERCENT, + DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME), transactionProcessor, blockHeader, miningBeneficiary, Wei.ZERO, - transactionSelectorFactory); + transactionSelectionService); transactionPool.addRemoteTransactions(List.of(selected, notSelected, selected3)); @@ -703,9 +721,14 @@ public TransactionSelectionResult evaluateTransactionPostProcessing( public void transactionSelectionPluginShouldBeNotifiedWhenTransactionSelectionCompletes() { final PluginTransactionSelectorFactory transactionSelectorFactory = mock(PluginTransactionSelectorFactory.class); - PluginTransactionSelector transactionSelector = spy(AllAcceptingTransactionSelector.INSTANCE); + PluginTransactionSelector transactionSelector = mock(PluginTransactionSelector.class); + when(transactionSelector.evaluateTransactionPreProcessing(any())).thenReturn(SELECTED); + when(transactionSelector.evaluateTransactionPostProcessing(any(), any())).thenReturn(SELECTED); when(transactionSelectorFactory.create()).thenReturn(transactionSelector); + transactionSelectionService.registerPluginTransactionSelectorFactory( + transactionSelectorFactory); + final Transaction transaction = createTransaction(0, Wei.of(10), 21_000); ensureTransactionIsValid(transaction, 21_000, 0); @@ -720,7 +743,7 @@ public void transactionSelectionPluginShouldBeNotifiedWhenTransactionSelectionCo createBlock(300_000), AddressHelpers.ofValue(1), Wei.ZERO, - transactionSelectorFactory); + transactionSelectionService); transactionPool.addRemoteTransactions(List.of(transaction, invalidTransaction)); @@ -756,7 +779,7 @@ public void transactionWithIncorrectNonceRemainsInPoolAndNotSelected() { blockHeader, miningBeneficiary, Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); final Transaction futureTransaction = createTransaction(4, Wei.of(10), 100_000); @@ -792,7 +815,7 @@ public void increaseOfMinGasPriceAtRuntimeExcludeTxFromBeingSelected() { blockHeader, miningBeneficiary, Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); transactionPool.addRemoteTransactions(List.of(transaction)); @@ -828,7 +851,7 @@ public void decreaseOfMinGasPriceAtRuntimeIncludeTxThatWasPreviouslyNotSelected( blockHeader, miningBeneficiary, Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); transactionPool.addRemoteTransactions(List.of(transaction)); ensureTransactionIsValid(transaction, 0, 5); @@ -856,7 +879,7 @@ public void decreaseOfMinGasPriceAtRuntimeIncludeTxThatWasPreviouslyNotSelected( blockHeader, miningBeneficiary, Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); final TransactionSelectionResults results2 = selector2.buildTransactionListForBlock(); @@ -884,7 +907,7 @@ public void shouldNotSelectTransactionsWithPriorityFeeLessThanConfig() { blockHeader, AddressHelpers.ofValue(1), Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); transactionPool.addRemoteTransactions(List.of(txSelected, txNotSelected)); @@ -987,15 +1010,20 @@ private void internalBlockSelectionTimeoutSimulation( mock(PluginTransactionSelectorFactory.class); when(transactionSelectorFactory.create()).thenReturn(transactionSelector); + transactionSelectionService.registerPluginTransactionSelectorFactory( + transactionSelectorFactory); + final BlockTransactionSelector selector = createBlockSelectorAndSetupTxPool( isPoa ? createMiningParameters( + transactionSelectionService, Wei.ZERO, MIN_OCCUPANCY_100_PERCENT, poaGenesisBlockPeriod, PositiveNumber.fromInt(75)) : createMiningParameters( + transactionSelectionService, Wei.ZERO, MIN_OCCUPANCY_100_PERCENT, PositiveNumber.fromInt(blockTxsSelectionMaxTime)), @@ -1003,7 +1031,7 @@ private void internalBlockSelectionTimeoutSimulation( blockHeader, miningBeneficiary, Wei.ZERO, - transactionSelectorFactory); + transactionSelectionService); transactionPool.addRemoteTransactions(transactionsToInject); @@ -1136,15 +1164,20 @@ private void internalBlockSelectionTimeoutSimulationInvalidTxs( mock(PluginTransactionSelectorFactory.class); when(transactionSelectorFactory.create()).thenReturn(transactionSelector); + transactionSelectionService.registerPluginTransactionSelectorFactory( + transactionSelectorFactory); + final BlockTransactionSelector selector = createBlockSelectorAndSetupTxPool( isPoa ? createMiningParameters( + transactionSelectionService, Wei.ZERO, MIN_OCCUPANCY_100_PERCENT, poaGenesisBlockPeriod, PositiveNumber.fromInt(75)) : createMiningParameters( + transactionSelectionService, Wei.ZERO, MIN_OCCUPANCY_100_PERCENT, PositiveNumber.fromInt(blockTxsSelectionMaxTime)), @@ -1152,7 +1185,7 @@ private void internalBlockSelectionTimeoutSimulationInvalidTxs( blockHeader, miningBeneficiary, Wei.ZERO, - transactionSelectorFactory); + transactionSelectionService); transactionPool.addRemoteTransactions(transactionsToInject); @@ -1195,7 +1228,7 @@ protected BlockTransactionSelector createBlockSelectorAndSetupTxPool( final ProcessableBlockHeader blockHeader, final Address miningBeneficiary, final Wei blobGasPrice, - final PluginTransactionSelectorFactory transactionSelectorFactory) { + final TransactionSelectionService transactionSelectionService) { transactionPool = createTransactionPool(); @@ -1205,7 +1238,7 @@ protected BlockTransactionSelector createBlockSelectorAndSetupTxPool( blockHeader, miningBeneficiary, blobGasPrice, - transactionSelectorFactory); + transactionSelectionService); } protected BlockTransactionSelector createBlockSelector( @@ -1214,7 +1247,7 @@ protected BlockTransactionSelector createBlockSelector( final ProcessableBlockHeader blockHeader, final Address miningBeneficiary, final Wei blobGasPrice, - final PluginTransactionSelectorFactory transactionSelectorFactory) { + final TransactionSelectionService transactionSelectionService) { final BlockTransactionSelector selector = new BlockTransactionSelector( @@ -1231,7 +1264,7 @@ protected BlockTransactionSelector createBlockSelector( getFeeMarket(), new LondonGasCalculator(), GasLimitCalculator.constant(), - transactionSelectorFactory.create(), + transactionSelectionService.createPluginTransactionSelector(), ethScheduler); return selector; @@ -1344,6 +1377,7 @@ private BlockHeader blockHeader(final long number) { } protected MiningParameters createMiningParameters( + final TransactionSelectionService transactionSelectionService, final Wei minGasPrice, final double minBlockOccupancyRatio, final PositiveNumber txsSelectionMaxTime) { @@ -1353,11 +1387,13 @@ protected MiningParameters createMiningParameters( .minTransactionGasPrice(minGasPrice) .minBlockOccupancyRatio(minBlockOccupancyRatio) .build()) + .transactionSelectionService(transactionSelectionService) .nonPoaBlockTxsSelectionMaxTime(txsSelectionMaxTime) .build(); } protected MiningParameters createMiningParameters( + final TransactionSelectionService transactionSelectionService, final Wei minGasPrice, final double minBlockOccupancyRatio, final int genesisBlockPeriodSeconds, @@ -1368,6 +1404,7 @@ protected MiningParameters createMiningParameters( .minTransactionGasPrice(minGasPrice) .minBlockOccupancyRatio(minBlockOccupancyRatio) .build()) + .transactionSelectionService(transactionSelectionService) .genesisBlockPeriodSeconds(genesisBlockPeriodSeconds) .poaBlockTxsSelectionMaxTime(minBlockTimePercentage) .build(); diff --git a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/BlockMinerTest.java b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/BlockMinerTest.java index 6afa8ea593c..066fe5b2be3 100644 --- a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/BlockMinerTest.java +++ b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/BlockMinerTest.java @@ -58,7 +58,7 @@ public void blockCreatedIsAddedToBlockChain() throws InterruptedException { headerBuilder.buildHeader(), new BlockBody(Lists.newArrayList(), Lists.newArrayList())); final ProtocolContext protocolContext = - new ProtocolContext(null, null, null, Optional.empty(), new BadBlockManager()); + new ProtocolContext(null, null, null, new BadBlockManager()); final PoWBlockCreator blockCreator = mock(PoWBlockCreator.class); final Function blockCreatorSupplier = @@ -100,7 +100,7 @@ public void failureToImportDoesNotTriggerObservers() throws InterruptedException headerBuilder.buildHeader(), new BlockBody(Lists.newArrayList(), Lists.newArrayList())); final ProtocolContext protocolContext = - new ProtocolContext(null, null, null, Optional.empty(), new BadBlockManager()); + new ProtocolContext(null, null, null, new BadBlockManager()); final PoWBlockCreator blockCreator = mock(PoWBlockCreator.class); final Function blockCreatorSupplier = @@ -146,7 +146,7 @@ public void blockValidationFailureBeforeImportDoesNotImportBlock() throws Interr headerBuilder.buildHeader(), new BlockBody(Lists.newArrayList(), Lists.newArrayList())); final ProtocolContext protocolContext = - new ProtocolContext(null, null, null, Optional.empty(), new BadBlockManager()); + new ProtocolContext(null, null, null, new BadBlockManager()); final PoWBlockCreator blockCreator = mock(PoWBlockCreator.class); final Function blockCreatorSupplier = diff --git a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/LondonFeeMarketBlockTransactionSelectorTest.java b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/LondonFeeMarketBlockTransactionSelectorTest.java index 08e9092339a..32b3935018c 100644 --- a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/LondonFeeMarketBlockTransactionSelectorTest.java +++ b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/LondonFeeMarketBlockTransactionSelectorTest.java @@ -112,12 +112,15 @@ public void eip1559TransactionCurrentGasPriceLessThanMinimumIsSkippedAndKeptInTh final BlockTransactionSelector selector = createBlockSelectorAndSetupTxPool( createMiningParameters( - Wei.of(6), MIN_OCCUPANCY_80_PERCENT, DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME), + transactionSelectionService, + Wei.of(6), + MIN_OCCUPANCY_80_PERCENT, + DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME), transactionProcessor, blockHeader, miningBeneficiary, Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); // tx is willing to pay max 7 wei for gas, but current network condition (baseFee == 1) // result in it paying 2 wei, that is below the minimum accepted by the node, so it is skipped @@ -141,12 +144,15 @@ public void eip1559TransactionCurrentGasPriceGreaterThanMinimumIsSelected() { final BlockTransactionSelector selector = createBlockSelectorAndSetupTxPool( createMiningParameters( - Wei.of(6), MIN_OCCUPANCY_80_PERCENT, DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME), + transactionSelectionService, + Wei.of(6), + MIN_OCCUPANCY_80_PERCENT, + DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME), transactionProcessor, blockHeader, miningBeneficiary, Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); // tx is willing to pay max 7 wei for gas, and current network condition (baseFee == 5) // result in it paying the max, that is >= the minimum accepted by the node, so it is selected @@ -169,12 +175,15 @@ public void eip1559PriorityTransactionCurrentGasPriceLessThanMinimumIsSelected() final BlockTransactionSelector selector = createBlockSelectorAndSetupTxPool( createMiningParameters( - Wei.of(6), MIN_OCCUPANCY_80_PERCENT, DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME), + transactionSelectionService, + Wei.of(6), + MIN_OCCUPANCY_80_PERCENT, + DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME), transactionProcessor, blockHeader, miningBeneficiary, Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); // tx is willing to pay max 7 wei for gas, but current network condition (baseFee == 1) // result in it paying 2 wei, that is below the minimum accepted by the node, but since it is @@ -213,7 +222,7 @@ public void transactionFromSameSenderWithMixedTypes() { blockHeader, miningBeneficiary, Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); transactionPool.addRemoteTransactions(List.of(txFrontier1, txLondon1, txFrontier2, txLondon2)); @@ -254,7 +263,7 @@ public void shouldNotSelectTransactionsWithPriorityFeeLessThanConfig() { blockHeader, AddressHelpers.ofValue(1), Wei.ZERO, - NO_PLUGIN_TRANSACTION_SELECTOR_FACTORY); + transactionSelectionService); transactionPool.addRemoteTransactions( List.of(txSelected1, txNotSelected1, txSelected2, txNotSelected2)); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ProtocolContext.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ProtocolContext.java index f82bcbbecc5..1468c621026 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ProtocolContext.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ProtocolContext.java @@ -19,7 +19,6 @@ import org.hyperledger.besu.ethereum.core.Synchronizer; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; -import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; import java.util.Optional; @@ -33,7 +32,6 @@ public class ProtocolContext { private final WorldStateArchive worldStateArchive; private final BadBlockManager badBlockManager; private final ConsensusContext consensusContext; - private final Optional transactionSelectorFactory; private Optional synchronizer; @@ -42,20 +40,10 @@ public ProtocolContext( final WorldStateArchive worldStateArchive, final ConsensusContext consensusContext, final BadBlockManager badBlockManager) { - this(blockchain, worldStateArchive, consensusContext, Optional.empty(), badBlockManager); - } - - public ProtocolContext( - final MutableBlockchain blockchain, - final WorldStateArchive worldStateArchive, - final ConsensusContext consensusContext, - final Optional transactionSelectorFactory, - final BadBlockManager badBlockManager) { this.blockchain = blockchain; this.worldStateArchive = worldStateArchive; this.consensusContext = consensusContext; this.synchronizer = Optional.empty(); - this.transactionSelectorFactory = transactionSelectorFactory; this.badBlockManager = badBlockManager; } @@ -64,13 +52,11 @@ public static ProtocolContext init( final WorldStateArchive worldStateArchive, final ProtocolSchedule protocolSchedule, final ConsensusContextFactory consensusContextFactory, - final Optional transactionSelectorFactory, final BadBlockManager badBlockManager) { return new ProtocolContext( blockchain, worldStateArchive, consensusContextFactory.create(blockchain, worldStateArchive, protocolSchedule), - transactionSelectorFactory, badBlockManager); } @@ -103,8 +89,4 @@ public Optional safeConsensusContext(final Class .filter(c -> klass.isAssignableFrom(c.getClass())) .map(klass::cast); } - - public Optional getTransactionSelectorFactory() { - return transactionSelectorFactory; - } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MiningParameters.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MiningParameters.java index 7f543f370a3..0a7144d30d9 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MiningParameters.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MiningParameters.java @@ -16,6 +16,9 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.plugin.services.TransactionSelectionService; +import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelector; +import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; import org.hyperledger.besu.util.number.PositiveNumber; import java.time.Duration; @@ -144,6 +147,20 @@ public PositiveNumber getPoaBlockTxsSelectionMaxTime() { return DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME; } + @Value.Default + public TransactionSelectionService getTransactionSelectionService() { + return new TransactionSelectionService() { + @Override + public PluginTransactionSelector createPluginTransactionSelector() { + return PluginTransactionSelector.ACCEPT_ALL; + } + + @Override + public void registerPluginTransactionSelectorFactory( + final PluginTransactionSelectorFactory transactionSelectorFactory) {} + }; + } + public abstract OptionalInt getGenesisBlockPeriodSeconds(); @Value.Derived diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockchainSetupUtil.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockchainSetupUtil.java index 349c819ddac..cf142eb2ed8 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockchainSetupUtil.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockchainSetupUtil.java @@ -50,7 +50,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Optional; import com.google.common.io.Resources; @@ -157,7 +156,6 @@ public C as(final Class klass) { return null; } }, - Optional.empty(), new BadBlockManager()); } diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java index 03419637b66..25c65b357f8 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java @@ -36,7 +36,6 @@ import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.math.BigInteger; -import java.util.Optional; import java.util.function.Function; public class ExecutionContextTestFixture { @@ -71,8 +70,7 @@ private ExecutionContextTestFixture( this.stateArchive = createInMemoryWorldStateArchive(); this.protocolSchedule = protocolSchedule; this.protocolContext = - new ProtocolContext( - blockchain, stateArchive, null, Optional.empty(), new BadBlockManager()); + new ProtocolContext(blockchain, stateArchive, null, new BadBlockManager()); genesisState.writeStateTo(stateArchive.getMutable()); } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java index e14ab5969a5..9f061facac5 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java @@ -161,8 +161,7 @@ public void createStorage() { EvmConfiguration.DEFAULT); var ws = archive.getMutable(); genesisState.writeStateTo(ws); - protocolContext = - new ProtocolContext(blockchain, archive, null, Optional.empty(), new BadBlockManager()); + protocolContext = new ProtocolContext(blockchain, archive, null, new BadBlockManager()); ethContext = mock(EthContext.class, RETURNS_DEEP_STUBS); when(ethContext.getEthPeers().subscribeConnect(any())).thenReturn(1L); transactionPool = diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/AbstractBlockPropagationManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/AbstractBlockPropagationManagerTest.java index 4406d9a7ee5..a91b19c8e9e 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/AbstractBlockPropagationManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/AbstractBlockPropagationManagerTest.java @@ -65,7 +65,6 @@ import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; @@ -106,7 +105,6 @@ protected void setup(final DataStorageFormat dataStorageFormat) { blockchain, tempProtocolContext.getWorldStateArchive(), tempProtocolContext.getConsensusContext(ConsensusContext.class), - Optional.empty(), new BadBlockManager()); ethProtocolManager = EthProtocolManagerTestUtil.create( diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManagerTest.java index b4a2c8eaa69..d10be455ea5 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManagerTest.java @@ -39,7 +39,6 @@ import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; -import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.stream.Stream; @@ -77,8 +76,7 @@ public void setup(final DataStorageFormat storageFormat) { final ProtocolSchedule protocolSchedule = ProtocolScheduleFixture.MAINNET; final ProtocolContext protocolContext = - new ProtocolContext( - localBlockchain, localWorldState, null, Optional.empty(), new BadBlockManager()); + new ProtocolContext(localBlockchain, localWorldState, null, new BadBlockManager()); ethProtocolManager = EthProtocolManagerTestUtil.create( protocolSchedule, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskParameterizedTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskParameterizedTest.java index 6fbbf3ed889..71c0bcb5440 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskParameterizedTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskParameterizedTest.java @@ -44,7 +44,6 @@ import java.io.IOException; import java.util.List; -import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -151,8 +150,7 @@ public void searchesAgainstNetwork(final int headerRequestSize, final int common final EthContext ethContext = ethProtocolManager.ethContext(); final ProtocolContext protocolContext = - new ProtocolContext( - localBlockchain, worldStateArchive, null, Optional.empty(), new BadBlockManager()); + new ProtocolContext(localBlockchain, worldStateArchive, null, new BadBlockManager()); final EthTask task = DetermineCommonAncestorTask.create( diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskTest.java index 346dafe8ab4..1b19a076d2c 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskTest.java @@ -55,7 +55,6 @@ import org.hyperledger.besu.util.ExceptionUtils; import java.util.List; -import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicReference; @@ -89,8 +88,7 @@ public void setup() { EthProtocolConfiguration.defaultConfig()); ethContext = ethProtocolManager.ethContext(); protocolContext = - new ProtocolContext( - localBlockchain, worldStateArchive, null, Optional.empty(), new BadBlockManager()); + new ProtocolContext(localBlockchain, worldStateArchive, null, new BadBlockManager()); } @Test diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java index 2545eb818de..ff624baa761 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java @@ -125,8 +125,7 @@ public TestNode( final WorldStateArchive worldStateArchive = createInMemoryWorldStateArchive(); genesisState.writeStateTo(worldStateArchive.getMutable()); final ProtocolContext protocolContext = - new ProtocolContext( - blockchain, worldStateArchive, null, Optional.empty(), new BadBlockManager()); + new ProtocolContext(blockchain, worldStateArchive, null, new BadBlockManager()); final SyncState syncState = mock(SyncState.class); final SynchronizerConfiguration syncConfig = mock(SynchronizerConfiguration.class); diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java index 4c459fc2d06..985c977507f 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java @@ -109,8 +109,7 @@ public BlockchainReferenceTestCaseSpec( this.blockchain = buildBlockchain(genesisBlockHeader); this.sealEngine = sealEngine; this.protocolContext = - new ProtocolContext( - this.blockchain, this.worldStateArchive, null, Optional.empty(), new BadBlockManager()); + new ProtocolContext(this.blockchain, this.worldStateArchive, null, new BadBlockManager()); } public String getNetwork() { diff --git a/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java b/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java index bc08935037d..a78e689bcf4 100644 --- a/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java +++ b/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java @@ -177,8 +177,7 @@ private boolean buildContext( genesisState.writeStateTo(worldState); blockchain = createInMemoryBlockchain(genesisState.getBlock()); - protocolContext = - new ProtocolContext(blockchain, worldStateArchive, null, Optional.empty(), badBlockManager); + protocolContext = new ProtocolContext(blockchain, worldStateArchive, null, badBlockManager); blockchainQueries = new BlockchainQueries(blockchain, worldStateArchive, ethScheduler); diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 5e09c0cfa4f..bd3e0e43ef8 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -69,7 +69,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = 'jvIsInEUQ/NaxWCGkQIb72tuZsx5288Ownn2F6rsvjQ=' + knownHash = 'Jv/pqyKOoZo3wlxRRwmzPuLIJjEqyV1i55wFyIUV90A=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/TransactionSelectionService.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/TransactionSelectionService.java index aab6e717bad..b780015f0be 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/TransactionSelectionService.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/TransactionSelectionService.java @@ -16,26 +16,25 @@ package org.hyperledger.besu.plugin.services; import org.hyperledger.besu.plugin.Unstable; +import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelector; import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; -import java.util.Optional; - /** Transaction selection service interface */ @Unstable public interface TransactionSelectionService extends BesuService { /** - * Returns the (Optional) transaction selector factory + * Create a transaction selector plugin * - * @return the transaction selector factory + * @return the transaction selector plugin */ - Optional get(); + PluginTransactionSelector createPluginTransactionSelector(); /** * Registers the transaction selector factory with the service * * @param transactionSelectorFactory transaction selector factory to be used */ - void registerTransactionSelectorFactory( + void registerPluginTransactionSelectorFactory( PluginTransactionSelectorFactory transactionSelectorFactory); } diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/txselection/PluginTransactionSelector.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/txselection/PluginTransactionSelector.java index 3d152c370ca..c323f6071c7 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/txselection/PluginTransactionSelector.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/txselection/PluginTransactionSelector.java @@ -15,6 +15,8 @@ package org.hyperledger.besu.plugin.services.txselection; +import static org.hyperledger.besu.plugin.data.TransactionSelectionResult.SELECTED; + import org.hyperledger.besu.datatypes.PendingTransaction; import org.hyperledger.besu.plugin.Unstable; import org.hyperledger.besu.plugin.data.TransactionProcessingResult; @@ -24,6 +26,22 @@ /** Interface for the transaction selector */ @Unstable public interface PluginTransactionSelector { + /** Plugin transaction selector that unconditionally select every transaction */ + PluginTransactionSelector ACCEPT_ALL = + new PluginTransactionSelector() { + @Override + public TransactionSelectionResult evaluateTransactionPreProcessing( + TransactionEvaluationContext evaluationContext) { + return SELECTED; + } + + @Override + public TransactionSelectionResult evaluateTransactionPostProcessing( + TransactionEvaluationContext evaluationContext, + TransactionProcessingResult processingResult) { + return SELECTED; + } + }; /** * Method that returns an OperationTracer that will be used when executing transactions that are From 0c8dcfcde4fd4b28597ee9c8fdeea517b46a3a7f Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 27 Feb 2024 12:07:56 +0100 Subject: [PATCH 8/8] Make layered txpool aware of minGasPrice and minPriorityFeePerGas dynamic options (#6611) Signed-off-by: Fabio Di Fabio Co-authored-by: Simon Dudley --- CHANGELOG.md | 1 + .../controller/BesuControllerBuilder.java | 3 +- .../besu/services/BesuEventsImplTest.java | 4 +- .../trie/bonsai/AbstractIsolationTests.java | 3 +- .../transactions/TransactionPoolFactory.java | 28 ++++++--- .../AbstractPrioritizedTransactions.java | 6 +- .../BaseFeePrioritizedTransactions.java | 40 ++++++++++--- .../GasPricePrioritizedTransactions.java | 14 ++++- .../eth/manager/EthProtocolManagerTest.java | 4 +- .../ethtaskutils/AbstractMessageTaskTest.java | 4 +- .../AbstractTransactionPoolTest.java | 2 + .../ethereum/eth/transactions/TestNode.java | 4 +- .../TransactionPoolFactoryTest.java | 7 ++- ...stractPrioritizedTransactionsTestBase.java | 42 ++++++++++--- .../BaseFeePrioritizedTransactionsTest.java | 60 ++++++++++++++++--- .../layered/BaseTransactionPoolTest.java | 3 +- .../GasPricePrioritizedTransactionsTest.java | 18 ++++-- .../LayeredPendingTransactionsTest.java | 22 ++++--- .../LayeredTransactionPoolBaseFeeTest.java | 4 +- .../LayeredTransactionPoolGasPriceTest.java | 8 ++- .../eth/transactions/layered/LayersTest.java | 4 +- .../eth/transactions/layered/ReplayTest.java | 4 +- .../ethereum/retesteth/RetestethContext.java | 3 +- 23 files changed, 225 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 225b1efc382..bf9338d4f6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ ### Additions and Improvements - Extend `Blockchain` service [#6592](https://github.com/hyperledger/besu/pull/6592) - RocksDB database metadata refactoring [#6555](https://github.com/hyperledger/besu/pull/6555) +- Make layered txpool aware of minGasPrice and minPriorityFeePerGas dynamic options [#6611](https://github.com/hyperledger/besu/pull/6611) ### Bug fixes diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 1a74e4996e2..3dd221514a6 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -710,7 +710,8 @@ public BesuController build() { syncState, transactionPoolConfiguration, pluginTransactionValidatorFactory, - besuComponent.map(BesuComponent::getBlobCache).orElse(new BlobCache())); + besuComponent.map(BesuComponent::getBlobCache).orElse(new BlobCache()), + miningParameters); final List peerValidators = createPeerValidators(protocolSchedule); diff --git a/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java b/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java index 87b6255b593..8b21fa215fa 100644 --- a/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java +++ b/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java @@ -34,6 +34,7 @@ import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.Difficulty; +import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.TransactionReceipt; import org.hyperledger.besu.ethereum.core.TransactionTestFixture; @@ -167,7 +168,8 @@ public void setUp() { syncState, txPoolConfig, null, - new BlobCache()); + new BlobCache(), + MiningParameters.newDefault()); serviceImpl = new BesuEventsImpl(blockchain, blockBroadcaster, transactionPool, syncState); } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java index 9f061facac5..b6c9c14a632 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java @@ -132,7 +132,8 @@ public abstract class AbstractIsolationTests { new EndLayer(txPoolMetrics), txPoolMetrics, transactionReplacementTester, - new BlobCache())); + new BlobCache(), + MiningParameters.newDefault())); protected final List accounts = GenesisConfigFile.development() diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactory.java index 253d37a4758..0f9aaa002db 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactory.java @@ -17,6 +17,7 @@ import static org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration.Implementation.LAYERED; import org.hyperledger.besu.ethereum.ProtocolContext; +import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.messages.EthPV62; import org.hyperledger.besu.ethereum.eth.messages.EthPV65; @@ -55,7 +56,8 @@ public static TransactionPool createTransactionPool( final SyncState syncState, final TransactionPoolConfiguration transactionPoolConfiguration, final PluginTransactionValidatorFactory pluginTransactionValidatorFactory, - final BlobCache blobCache) { + final BlobCache blobCache, + final MiningParameters miningParameters) { final TransactionPoolMetrics metrics = new TransactionPoolMetrics(metricsSystem); @@ -78,7 +80,8 @@ public static TransactionPool createTransactionPool( transactionsMessageSender, newPooledTransactionHashesMessageSender, pluginTransactionValidatorFactory, - blobCache); + blobCache, + miningParameters); } static TransactionPool createTransactionPool( @@ -93,7 +96,8 @@ static TransactionPool createTransactionPool( final TransactionsMessageSender transactionsMessageSender, final NewPooledTransactionHashesMessageSender newPooledTransactionHashesMessageSender, final PluginTransactionValidatorFactory pluginTransactionValidatorFactory, - final BlobCache blobCache) { + final BlobCache blobCache, + final MiningParameters miningParameters) { final TransactionPool transactionPool = new TransactionPool( @@ -104,7 +108,8 @@ static TransactionPool createTransactionPool( clock, metrics, transactionPoolConfiguration, - blobCache), + blobCache, + miningParameters), protocolSchedule, protocolContext, new TransactionBroadcaster( @@ -233,7 +238,8 @@ private static PendingTransactions createPendingTransactions( final Clock clock, final TransactionPoolMetrics metrics, final TransactionPoolConfiguration transactionPoolConfiguration, - final BlobCache blobCache) { + final BlobCache blobCache, + final MiningParameters miningParameters) { boolean isFeeMarketImplementBaseFee = protocolSchedule.anyMatch( @@ -246,7 +252,8 @@ private static PendingTransactions createPendingTransactions( metrics, transactionPoolConfiguration, isFeeMarketImplementBaseFee, - blobCache); + blobCache, + miningParameters); } else { return createPendingTransactionSorter( protocolContext, @@ -284,7 +291,8 @@ private static PendingTransactions createLayeredPendingTransactions( final TransactionPoolMetrics metrics, final TransactionPoolConfiguration transactionPoolConfiguration, final boolean isFeeMarketImplementBaseFee, - final BlobCache blobCache) { + final BlobCache blobCache, + final MiningParameters miningParameters) { final TransactionPoolReplacementHandler transactionReplacementHandler = new TransactionPoolReplacementHandler(transactionPoolConfiguration.getPriceBump()); @@ -327,7 +335,8 @@ private static PendingTransactions createLayeredPendingTransactions( metrics, transactionReplacementTester, feeMarket, - blobCache); + blobCache, + miningParameters); } else { pendingTransactionsSorter = new GasPricePrioritizedTransactions( @@ -335,7 +344,8 @@ private static PendingTransactions createLayeredPendingTransactions( readyTransactions, metrics, transactionReplacementTester, - blobCache); + blobCache, + miningParameters); } return new LayeredPendingTransactions(transactionPoolConfiguration, pendingTransactionsSorter); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactions.java index 7d59f40afb7..b796eefaca4 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactions.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactions.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.eth.transactions.layered; import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.eth.transactions.BlobCache; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; import org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult; @@ -36,6 +37,7 @@ */ public abstract class AbstractPrioritizedTransactions extends AbstractSequentialTransactionsLayer { protected final TreeSet orderByFee; + protected final MiningParameters miningParameters; public AbstractPrioritizedTransactions( final TransactionPoolConfiguration poolConfig, @@ -43,9 +45,11 @@ public AbstractPrioritizedTransactions( final TransactionPoolMetrics metrics, final BiFunction transactionReplacementTester, - final BlobCache blobCache) { + final BlobCache blobCache, + final MiningParameters miningParameters) { super(poolConfig, prioritizedTransactions, transactionReplacementTester, metrics, blobCache); this.orderByFee = new TreeSet<>(this::compareByFee); + this.miningParameters = miningParameters; } @Override diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseFeePrioritizedTransactions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseFeePrioritizedTransactions.java index c1e9bec63d2..e1d10e9ee2a 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseFeePrioritizedTransactions.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseFeePrioritizedTransactions.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.eth.transactions.BlobCache; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; @@ -47,8 +48,10 @@ public BaseFeePrioritizedTransactions( final BiFunction transactionReplacementTester, final FeeMarket feeMarket, - final BlobCache blobCache) { - super(poolConfig, nextLayer, metrics, transactionReplacementTester, blobCache); + final BlobCache blobCache, + final MiningParameters miningParameters) { + super( + poolConfig, nextLayer, metrics, transactionReplacementTester, blobCache, miningParameters); this.nextBlockBaseFee = Optional.of(calculateNextBlockBaseFee(feeMarket, chainHeadHeaderSupplier.get())); } @@ -146,11 +149,34 @@ private Wei calculateNextBlockBaseFee(final FeeMarket feeMarket, final BlockHead @Override protected boolean promotionFilter(final PendingTransaction pendingTransaction) { - return nextBlockBaseFee - .map( - baseFee -> - pendingTransaction.getTransaction().getMaxGasPrice().greaterOrEqualThan(baseFee)) - .orElse(false); + // check if the tx is willing to pay at least the base fee + if (nextBlockBaseFee + .map(pendingTransaction.getTransaction().getMaxGasPrice()::lessThan) + .orElse(true)) { + return false; + } + + // priority txs are promoted even if they pay less + if (!pendingTransaction.hasPriority()) { + // check if max fee per gas is higher than the min gas price + if (pendingTransaction + .getTransaction() + .getMaxGasPrice() + .lessThan(miningParameters.getMinTransactionGasPrice())) { + return false; + } + + // check if enough priority fee is paid + if (!miningParameters.getMinPriorityFeePerGas().equals(Wei.ZERO)) { + final Wei priorityFeePerGas = + pendingTransaction.getTransaction().getEffectivePriorityFeePerGas(nextBlockBaseFee); + if (priorityFeePerGas.lessThan(miningParameters.getMinPriorityFeePerGas())) { + return false; + } + } + } + + return true; } @Override diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/GasPricePrioritizedTransactions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/GasPricePrioritizedTransactions.java index c6c402a1108..e6a939d660e 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/GasPricePrioritizedTransactions.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/GasPricePrioritizedTransactions.java @@ -17,6 +17,7 @@ import static java.util.Comparator.comparing; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.eth.transactions.BlobCache; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; @@ -39,8 +40,10 @@ public GasPricePrioritizedTransactions( final TransactionPoolMetrics metrics, final BiFunction transactionReplacementTester, - final BlobCache blobCache) { - super(poolConfig, nextLayer, metrics, transactionReplacementTester, blobCache); + final BlobCache blobCache, + final MiningParameters miningParameters) { + super( + poolConfig, nextLayer, metrics, transactionReplacementTester, blobCache, miningParameters); } @Override @@ -58,7 +61,12 @@ protected void internalBlockAdded(final BlockHeader blockHeader, final FeeMarket @Override protected boolean promotionFilter(final PendingTransaction pendingTransaction) { - return true; + return pendingTransaction.hasPriority() + || pendingTransaction + .getTransaction() + .getGasPrice() + .map(miningParameters.getMinTransactionGasPrice()::lessThan) + .orElse(false); } @Override diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java index c05bdcb1865..7602de7ab69 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java @@ -36,6 +36,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; import org.hyperledger.besu.ethereum.core.Difficulty; +import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.ProtocolScheduleFixture; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.TransactionReceipt; @@ -1118,7 +1119,8 @@ public void transactionMessagesGoToTheCorrectExecutor() { new SyncState(blockchain, ethManager.ethContext().getEthPeers()), TransactionPoolConfiguration.DEFAULT, null, - new BlobCache()) + new BlobCache(), + MiningParameters.newDefault()) .setEnabled(); // Send just a transaction message. diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/ethtaskutils/AbstractMessageTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/ethtaskutils/AbstractMessageTaskTest.java index 3ea2dc27f3c..5b39672ff04 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/ethtaskutils/AbstractMessageTaskTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/ethtaskutils/AbstractMessageTaskTest.java @@ -25,6 +25,7 @@ import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; +import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.eth.EthProtocol; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; import org.hyperledger.besu.ethereum.eth.manager.EthContext; @@ -136,7 +137,8 @@ public void setupTest() { syncState, TransactionPoolConfiguration.DEFAULT, null, - new BlobCache()); + new BlobCache(), + MiningParameters.newDefault()); transactionPool.setEnabled(); ethProtocolManager = diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/AbstractTransactionPoolTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/AbstractTransactionPoolTest.java index 6d6ff994b56..62a2e615b07 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/AbstractTransactionPoolTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/AbstractTransactionPoolTest.java @@ -128,6 +128,7 @@ public abstract class AbstractTransactionPoolTest { private static final KeyPair KEY_PAIR2 = SignatureAlgorithmFactory.getInstance().generateKeyPair(); protected static final Wei BASE_FEE_FLOOR = Wei.of(7L); + protected static final Wei DEFAULT_MIN_GAS_PRICE = Wei.of(50L); @Mock(answer = Answers.RETURNS_DEEP_STUBS) protected TransactionValidatorFactory transactionValidatorFactory; @@ -455,6 +456,7 @@ public void shouldNotReAddTransactionsThatAreInBothForksWhenReorgHappens() { } @Test + @EnabledIf("isBaseFeeMarket") public void shouldReAddBlobTxsWhenReorgHappens() { givenTransactionIsValid(transaction0); givenTransactionIsValid(transaction1); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java index ff624baa761..674dec68530 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java @@ -32,6 +32,7 @@ import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.BlockHeaderFunctions; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; +import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.difficulty.fixed.FixedDifficultyProtocolSchedule; import org.hyperledger.besu.ethereum.eth.EthProtocol; @@ -166,7 +167,8 @@ public boolean isMessagePermitted(final EnodeURL destinationEnode, final int cod syncState, TransactionPoolConfiguration.DEFAULT, null, - new BlobCache()); + new BlobCache(), + MiningParameters.newDefault()); final EthProtocolManager ethProtocolManager = new EthProtocolManager( diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java index 3d97f703c18..66419f62525 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java @@ -32,6 +32,7 @@ import org.hyperledger.besu.ethereum.chain.BlockAddedObserver; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; +import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; import org.hyperledger.besu.ethereum.eth.manager.EthContext; @@ -251,7 +252,8 @@ private void setupInitialSyncPhase(final boolean hasInitialSyncPhase) { transactionsMessageSender, newPooledTransactionHashesMessageSender, null, - new BlobCache()); + new BlobCache(), + MiningParameters.newDefault()); ethProtocolManager = new EthProtocolManager( @@ -359,7 +361,8 @@ private TransactionPool createTransactionPool( .build()) .build(), null, - new BlobCache()); + new BlobCache(), + MiningParameters.newDefault()); txPool.setEnabled(); return txPool; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactionsTestBase.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactionsTestBase.java index 86741de9146..10e40a6ed1f 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactionsTestBase.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactionsTestBase.java @@ -16,9 +16,11 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.ADDED; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.DROPPED; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; @@ -27,7 +29,6 @@ import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolReplacementHandler; -import java.math.BigInteger; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -40,19 +41,26 @@ public abstract class AbstractPrioritizedTransactionsTestBase extends BaseTransa protected static final int MAX_TRANSACTIONS = 5; protected final TransactionPoolMetrics txPoolMetrics = new TransactionPoolMetrics(metricsSystem); protected final EvictCollectorLayer evictCollector = new EvictCollectorLayer(txPoolMetrics); + protected final MiningParameters miningParameters = + MiningParameters.newDefault() + .setMinTransactionGasPrice(DEFAULT_MIN_GAS_PRICE) + .setMinPriorityFeePerGas(DEFAULT_MIN_PRIORITY_FEE); protected AbstractPrioritizedTransactions transactions = getSorter( ImmutableTransactionPoolConfiguration.builder() .maxPrioritizedTransactions(MAX_TRANSACTIONS) .maxFutureBySender(MAX_TRANSACTIONS) - .build()); + .build(), + miningParameters); - private AbstractPrioritizedTransactions getSorter(final TransactionPoolConfiguration poolConfig) { + private AbstractPrioritizedTransactions getSorter( + final TransactionPoolConfiguration poolConfig, final MiningParameters miningParameters) { return getSorter( poolConfig, evictCollector, txPoolMetrics, - (pt1, pt2) -> transactionReplacementTester(poolConfig, pt1, pt2)); + (pt1, pt2) -> transactionReplacementTester(poolConfig, pt1, pt2), + miningParameters); } abstract AbstractPrioritizedTransactions getSorter( @@ -60,7 +68,8 @@ abstract AbstractPrioritizedTransactions getSorter( final TransactionsLayer nextLayer, final TransactionPoolMetrics txPoolMetrics, final BiFunction - transactionReplacementTester); + transactionReplacementTester, + final MiningParameters miningParameters); abstract BlockHeader mockBlockHeader(); @@ -80,13 +89,13 @@ public void prioritizeLocalTransactionThenValue() { assertThat(prioritizeTransaction(localTransaction)).isEqualTo(ADDED); final List remoteTxs = new ArrayList<>(); - TransactionAddedResult prioritizeResult = null; + TransactionAddedResult prioritizeResult; for (int i = 0; i < MAX_TRANSACTIONS; i++) { final PendingTransaction highValueRemoteTx = createRemotePendingTransaction( createTransaction( 0, - Wei.of(BigInteger.valueOf(100).pow(i)), + Wei.of(DEFAULT_MIN_GAS_PRICE.multiply(2).toBigInteger().pow(i + 1)), SIGNATURE_ALGORITHM.get().generateKeyPair())); remoteTxs.add(highValueRemoteTx); prioritizeResult = prioritizeTransaction(highValueRemoteTx); @@ -123,6 +132,25 @@ public void shouldStartDroppingLocalTransactionsWhenPoolIsFullOfLocalTransaction assertTransactionNotPrioritized(lastLocalTransaction); } + @Test + public void txBelowCurrentMineableMinGasPriceIsNotPrioritized() { + final PendingTransaction lowGasPriceTx = + createRemotePendingTransaction( + createTransaction(0, DEFAULT_MIN_GAS_PRICE.subtract(1), KEYS1)); + assertThat(prioritizeTransaction(lowGasPriceTx)).isEqualTo(DROPPED); + assertEvicted(lowGasPriceTx); + assertTransactionNotPrioritized(lowGasPriceTx); + } + + @Test + public void txWithPriorityBelowCurrentMineableMinGasPriceIsPrioritized() { + final PendingTransaction lowGasPriceTx = + createRemotePendingTransaction( + createTransaction(0, DEFAULT_MIN_GAS_PRICE.subtract(1), KEYS1), true); + assertThat(prioritizeTransaction(lowGasPriceTx)).isEqualTo(ADDED); + assertTransactionPrioritized(lowGasPriceTx); + } + protected void shouldPrioritizeValueThenTimeAddedToPool( final Iterator lowValueTxSupplier, final PendingTransaction highValueTx, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseFeePrioritizedTransactionsTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseFeePrioritizedTransactionsTest.java index 4529f7f36e1..31785fa1898 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseFeePrioritizedTransactionsTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseFeePrioritizedTransactionsTest.java @@ -14,8 +14,11 @@ */ package org.hyperledger.besu.ethereum.eth.transactions.layered; +import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.datatypes.TransactionType.EIP1559; import static org.hyperledger.besu.datatypes.TransactionType.FRONTIER; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.ADDED; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.DROPPED; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -23,6 +26,7 @@ import org.hyperledger.besu.datatypes.TransactionType; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.TransactionTestFixture; import org.hyperledger.besu.ethereum.eth.transactions.BlobCache; @@ -33,6 +37,7 @@ import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Random; import java.util.function.BiFunction; @@ -42,7 +47,7 @@ import org.junit.jupiter.api.Test; public class BaseFeePrioritizedTransactionsTest extends AbstractPrioritizedTransactionsTestBase { - + private static final FeeMarket EIP1559_FEE_MARKET = FeeMarket.london(0L); private static final Random randomizeTxType = new Random(); @Override @@ -51,7 +56,8 @@ AbstractPrioritizedTransactions getSorter( final TransactionsLayer nextLayer, final TransactionPoolMetrics txPoolMetrics, final BiFunction - transactionReplacementTester) { + transactionReplacementTester, + final MiningParameters miningParameters) { return new BaseFeePrioritizedTransactions( poolConfig, @@ -59,14 +65,19 @@ AbstractPrioritizedTransactions getSorter( nextLayer, txPoolMetrics, transactionReplacementTester, - FeeMarket.london(0L), - new BlobCache()); + EIP1559_FEE_MARKET, + new BlobCache(), + miningParameters); } @Override protected BlockHeader mockBlockHeader() { + return mockBlockHeader(Wei.ONE); + } + + private BlockHeader mockBlockHeader(final Wei baseFee) { final BlockHeader blockHeader = mock(BlockHeader.class); - when(blockHeader.getBaseFee()).thenReturn(Optional.of(Wei.ONE)); + when(blockHeader.getBaseFee()).thenReturn(Optional.of(baseFee)); return blockHeader; } @@ -116,7 +127,8 @@ public void shouldPrioritizeEffectivePriorityFeeThenTimeAddedToPoolOnMixedTypes( final var nextBlockBaseFee = Optional.of(Wei.ONE); final PendingTransaction highGasPriceTransaction = - createRemotePendingTransaction(createTransaction(0, Wei.of(100), KEYS1)); + createRemotePendingTransaction( + createTransaction(0, DEFAULT_MIN_GAS_PRICE.multiply(2), KEYS1)); final List lowValueTxs = IntStream.range(0, MAX_TRANSACTIONS) @@ -124,7 +136,9 @@ public void shouldPrioritizeEffectivePriorityFeeThenTimeAddedToPoolOnMixedTypes( i -> new PendingTransaction.Remote( createTransaction( - 0, Wei.of(10), SIGNATURE_ALGORITHM.get().generateKeyPair()))) + 0, + DEFAULT_MIN_GAS_PRICE.add(1), + SIGNATURE_ALGORITHM.get().generateKeyPair()))) .collect(Collectors.toUnmodifiableList()); final var lowestPriorityFee = @@ -151,10 +165,34 @@ public void shouldPrioritizeEffectivePriorityFeeThenTimeAddedToPoolOnMixedTypes( lowValueTxs.iterator(), highGasPriceTransaction, firstLowValueTx); } + @Test + public void txBelowCurrentMineableMinPriorityFeeIsNotPrioritized() { + setBaseFee(DEFAULT_MIN_GAS_PRICE.subtract(2)); + miningParameters.setMinPriorityFeePerGas(Wei.of(5)); + final PendingTransaction lowPriorityFeeTx = + createRemotePendingTransaction( + createTransaction(0, DEFAULT_MIN_GAS_PRICE.subtract(1), KEYS1)); + assertThat(prioritizeTransaction(lowPriorityFeeTx)).isEqualTo(DROPPED); + assertEvicted(lowPriorityFeeTx); + assertTransactionNotPrioritized(lowPriorityFeeTx); + } + + @Test + public void txWithPriorityBelowCurrentMineableMinPriorityFeeIsPrioritized() { + setBaseFee(DEFAULT_MIN_GAS_PRICE.subtract(2)); + miningParameters.setMinPriorityFeePerGas(Wei.of(5)); + final PendingTransaction lowGasPriceTx = + createRemotePendingTransaction( + createTransaction(0, DEFAULT_MIN_GAS_PRICE.subtract(1), KEYS1), true); + assertThat(prioritizeTransaction(lowGasPriceTx)).isEqualTo(ADDED); + assertTransactionPrioritized(lowGasPriceTx); + } + private void shouldPrioritizePriorityFeeThenTimeAddedToPoolSameTypeTxs( final TransactionType transactionType) { final PendingTransaction highGasPriceTransaction = - createRemotePendingTransaction(createTransaction(0, Wei.of(100), KEYS1)); + createRemotePendingTransaction( + createTransaction(0, DEFAULT_MIN_GAS_PRICE.multiply(20), KEYS1)); final var lowValueTxs = IntStream.range(0, MAX_TRANSACTIONS) @@ -164,7 +202,7 @@ private void shouldPrioritizePriorityFeeThenTimeAddedToPoolSameTypeTxs( createTransaction( transactionType, 0, - Wei.of(10), + DEFAULT_MIN_GAS_PRICE.add(1), 0, SIGNATURE_ALGORITHM.get().generateKeyPair()))) .collect(Collectors.toUnmodifiableList()); @@ -172,4 +210,8 @@ private void shouldPrioritizePriorityFeeThenTimeAddedToPoolSameTypeTxs( shouldPrioritizeValueThenTimeAddedToPool( lowValueTxs.iterator(), highGasPriceTransaction, lowValueTxs.get(0)); } + + private void setBaseFee(final Wei baseFee) { + transactions.blockAdded(EIP1559_FEE_MARKET, mockBlockHeader(baseFee), Map.of()); + } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseTransactionPoolTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseTransactionPoolTest.java index 0d69dac4faa..160c60c66d9 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseTransactionPoolTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseTransactionPoolTest.java @@ -54,7 +54,8 @@ public class BaseTransactionPoolTest { protected static final KeyPair KEYS2 = SIGNATURE_ALGORITHM.get().generateKeyPair(); protected static final Address SENDER1 = Util.publicKeyToAddress(KEYS1.getPublicKey()); protected static final Address SENDER2 = Util.publicKeyToAddress(KEYS2.getPublicKey()); - + protected static final Wei DEFAULT_MIN_GAS_PRICE = Wei.of(50); + protected static final Wei DEFAULT_MIN_PRIORITY_FEE = Wei.ZERO; private static final Random randomizeTxType = new Random(); protected final Transaction transaction0 = createTransaction(0); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/GasPricePrioritizedTransactionsTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/GasPricePrioritizedTransactionsTest.java index 89e986678ff..c3aa720a618 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/GasPricePrioritizedTransactionsTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/GasPricePrioritizedTransactionsTest.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.crypto.KeyPair; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.TransactionTestFixture; import org.hyperledger.besu.ethereum.eth.transactions.BlobCache; @@ -42,10 +43,16 @@ AbstractPrioritizedTransactions getSorter( final TransactionsLayer nextLayer, final TransactionPoolMetrics txPoolMetrics, final BiFunction - transactionReplacementTester) { + transactionReplacementTester, + final MiningParameters miningParameters) { return new GasPricePrioritizedTransactions( - poolConfig, nextLayer, txPoolMetrics, transactionReplacementTester, new BlobCache()); + poolConfig, + nextLayer, + txPoolMetrics, + transactionReplacementTester, + new BlobCache(), + miningParameters); } @Override @@ -80,11 +87,14 @@ public void shouldPrioritizeGasPriceThenTimeAddedToPool() { i -> createRemotePendingTransaction( createTransaction( - 0, Wei.of(10), SIGNATURE_ALGORITHM.get().generateKeyPair()))) + 0, + DEFAULT_MIN_GAS_PRICE.add(1), + SIGNATURE_ALGORITHM.get().generateKeyPair()))) .toList(); final PendingTransaction highGasPriceTransaction = - createRemotePendingTransaction(createTransaction(0, Wei.of(100), KEYS1)); + createRemotePendingTransaction( + createTransaction(0, DEFAULT_MIN_GAS_PRICE.multiply(2), KEYS1)); shouldPrioritizeValueThenTimeAddedToPool( lowValueTxs.iterator(), highGasPriceTransaction, lowValueTxs.get(0)); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactionsTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactionsTest.java index 9f86cfc9207..1984c82c7d1 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactionsTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactionsTest.java @@ -39,6 +39,7 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.eth.transactions.BlobCache; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; @@ -136,7 +137,8 @@ private CreatedLayers createLayers(final TransactionPoolConfiguration poolConfig txPoolMetrics, transactionReplacementTester, FeeMarket.london(0L), - new BlobCache()); + new BlobCache(), + MiningParameters.newDefault().setMinTransactionGasPrice(DEFAULT_MIN_GAS_PRICE)); return new CreatedLayers( prioritizedTransactions, readyTransactions, sparseTransactions, evictCollector); } @@ -216,7 +218,7 @@ public void evictTransactionsWhenSizeLimitExceeded() { final var tx = createTransaction( i, - Wei.of((i + 1) * 100L), + DEFAULT_MIN_GAS_PRICE.multiply(2 * (i + 1)), (int) poolConf.getPendingTransactionsLayerMaxCapacityBytes() + 1, SIGNATURE_ALGORITHM.get().generateKeyPair()); pendingTransactions.addTransaction(createRemotePendingTransaction(tx), Optional.of(sender)); @@ -229,7 +231,7 @@ public void evictTransactionsWhenSizeLimitExceeded() { final Transaction lastBigTx = createTransaction( 0, - Wei.of(100_000L), + DEFAULT_MIN_GAS_PRICE.multiply(1000), (int) poolConf.getPendingTransactionsLayerMaxCapacityBytes(), SIGNATURE_ALGORITHM.get().generateKeyPair()); final Account lastSender = mock(Account.class); @@ -458,8 +460,10 @@ public void notForceNonceOrderWhenSendersDiffer() { final Account sender2 = mock(Account.class); when(sender2.getNonce()).thenReturn(1L); - final Transaction transactionSender1 = createTransaction(0, Wei.of(100), KEYS1); - final Transaction transactionSender2 = createTransaction(1, Wei.of(200), KEYS2); + final Transaction transactionSender1 = + createTransaction(0, DEFAULT_MIN_GAS_PRICE.multiply(2), KEYS1); + final Transaction transactionSender2 = + createTransaction(1, DEFAULT_MIN_GAS_PRICE.multiply(4), KEYS2); pendingTransactions.addTransaction( createLocalPendingTransaction(transactionSender1), Optional.empty()); @@ -523,9 +527,9 @@ public void returnEmptyOptionalAsMaximumNonceWhenNoTransactionsPresent() { @Test public void replaceTransactionWithSameSenderAndNonce() { - final Transaction transaction1 = createTransaction(0, Wei.of(200), KEYS1); + final Transaction transaction1 = createTransaction(0, DEFAULT_MIN_GAS_PRICE.multiply(4), KEYS1); final Transaction transaction1b = createTransactionReplacement(transaction1, KEYS1); - final Transaction transaction2 = createTransaction(1, Wei.of(100), KEYS1); + final Transaction transaction2 = createTransaction(1, DEFAULT_MIN_GAS_PRICE.multiply(2), KEYS1); assertThat( pendingTransactions.addTransaction( createRemotePendingTransaction(transaction1), Optional.empty())) @@ -646,8 +650,8 @@ public void replaceTransactionWithSameSenderAndNonce_multipleReplacements() { @Test public void notReplaceTransactionWithSameSenderAndNonceWhenGasPriceIsLower() { - final Transaction transaction1 = createTransaction(0, Wei.of(2)); - final Transaction transaction1b = createTransaction(0, Wei.ONE); + final Transaction transaction1 = createTransaction(0, DEFAULT_MIN_GAS_PRICE.add(1)); + final Transaction transaction1b = createTransaction(0, DEFAULT_MIN_GAS_PRICE); assertThat( pendingTransactions.addTransaction( createRemotePendingTransaction(transaction1), Optional.empty())) diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredTransactionPoolBaseFeeTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredTransactionPoolBaseFeeTest.java index b70f3c24ac5..61d513e447b 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredTransactionPoolBaseFeeTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredTransactionPoolBaseFeeTest.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.Difficulty; import org.hyperledger.besu.ethereum.core.ExecutionContextTestFixture; +import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.TransactionTestFixture; import org.hyperledger.besu.ethereum.eth.transactions.BlobCache; @@ -46,7 +47,8 @@ protected AbstractPrioritizedTransactions createPrioritizedTransactions( txPoolMetrics, transactionReplacementTester, FeeMarket.london(0L), - new BlobCache()); + new BlobCache(), + MiningParameters.newDefault()); } @Override diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredTransactionPoolGasPriceTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredTransactionPoolGasPriceTest.java index e0066e03703..296219c90fc 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredTransactionPoolGasPriceTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredTransactionPoolGasPriceTest.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.Difficulty; import org.hyperledger.besu.ethereum.core.ExecutionContextTestFixture; +import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.TransactionTestFixture; import org.hyperledger.besu.ethereum.eth.transactions.BlobCache; @@ -39,7 +40,12 @@ protected AbstractPrioritizedTransactions createPrioritizedTransactions( final BiFunction transactionReplacementTester) { return new GasPricePrioritizedTransactions( - poolConfig, nextLayer, txPoolMetrics, transactionReplacementTester, new BlobCache()); + poolConfig, + nextLayer, + txPoolMetrics, + transactionReplacementTester, + new BlobCache(), + MiningParameters.newDefault()); } @Override diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayersTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayersTest.java index 7a22683ec42..d7874cb46a8 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayersTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayersTest.java @@ -29,6 +29,7 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.Util; import org.hyperledger.besu.ethereum.eth.transactions.BlobCache; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; @@ -94,7 +95,8 @@ public class LayersTest extends BaseTransactionPoolTest { txPoolMetrics, this::transactionReplacementTester, FeeMarket.london(0L), - new BlobCache()); + new BlobCache(), + MiningParameters.newDefault()); private final LayeredPendingTransactions pendingTransactions = new LayeredPendingTransactions(poolConfig, prioritizedTransactions); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReplayTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReplayTest.java index 10595b6fd7a..0c0aa346b49 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReplayTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReplayTest.java @@ -24,6 +24,7 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.eth.transactions.BlobCache; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; @@ -202,7 +203,8 @@ private BaseFeePrioritizedTransactions createLayers( txPoolMetrics, txReplacementTester, baseFeeMarket, - new BlobCache()); + new BlobCache(), + MiningParameters.newDefault()); } // ToDo: commented since not always working, needs fix diff --git a/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java b/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java index a78e689bcf4..c927b6f692d 100644 --- a/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java +++ b/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java @@ -256,7 +256,8 @@ private boolean buildContext( syncState, transactionPoolConfiguration, null, - new BlobCache()); + new BlobCache(), + MiningParameters.newDefault()); if (LOG.isTraceEnabled()) { LOG.trace("Genesis Block {} ", genesisState.getBlock());